def test_basedata(self):
prometheusstack_config = PrometheusStackBuilder(kubragen=self.kg, options=PrometheusStackOptions({
'namespace': 'myns',
'basename': 'myprometheusstack',
'config': {
'prometheus': {
'prometheus_config': '',
},
},
'kubernetes': {
'volumes': {
'prometheus-data': {
'emptyDir': {},
}
}
}
}))
self.assertEqual(prometheusstack_config.object_name('prometheus-config'), 'myprometheusstack-prometheus-config')
self.assertEqual(prometheusstack_config.object_name('prometheus-statefulset'), 'myprometheusstack-prometheus')
FilterJSONPatches_Apply(items=prometheusstack_config.build(prometheusstack_config.BUILD_SERVICE), jsonpatches=[
FilterJSONPatch(filters=ObjectFilter(names=[prometheusstack_config.BUILDITEM_PROMETHEUS_SERVICE]), patches=[
{'op': 'check', 'path': '/metadata/name', 'cmp': 'equals', 'value': 'myprometheusstack-prometheus'},
{'op': 'check', 'path': '/metadata/namespace', 'cmp': 'equals', 'value': 'myns'},
]),
])
def test_basedata(self):
promtail_config = PromtailBuilder(kubragen=self.kg,
options=PromtailOptions({
'namespace':
'myns',
'basename':
'mypromtail',
}))
self.assertEqual(promtail_config.object_name('config'),
'mypromtail-config')
self.assertEqual(promtail_config.object_name('daemonset'),
'mypromtail')
FilterJSONPatches_Apply(
items=promtail_config.build(promtail_config.BUILD_SERVICE),
jsonpatches=[
FilterJSONPatch(filters=ObjectFilter(
names=[promtail_config.BUILDITEM_DAEMONSET]),
patches=[
{
'op': 'check',
'path': '/metadata/name',
'cmp': 'equals',
'value': 'mypromtail'
},
{
'op': 'check',
'path': '/metadata/namespace',
'cmp': 'equals',
'value': 'myns'
},
]),
])
def test_basedata(self):
kms_config = KubeResourceReportBuilder(
kubragen=self.kg,
options=KubeResourceReportOptions({
'namespace': 'myns',
'basename': 'mykms',
}))
self.assertEqual(kms_config.object_name('service'), 'mykms')
self.assertEqual(kms_config.object_name('deployment'), 'mykms')
FilterJSONPatches_Apply(
items=kms_config.build(kms_config.BUILD_SERVICE),
jsonpatches=[
FilterJSONPatch(
filters=ObjectFilter(names=[kms_config.BUILDITEM_SERVICE]),
patches=[
{
'op': 'check',
'path': '/metadata/name',
'cmp': 'equals',
'value': 'mykms'
},
{
'op': 'check',
'path': '/metadata/namespace',
'cmp': 'equals',
'value': 'myns'
},
]),
])
def test_merge(self):
data = [
Object({
'foo': 'bar',
'shin': {
'gami': 'hai',
'shami': 'nai',
},
},
name='x')
]
ret = FilterJSONPatches_Apply(items=data,
jsonpatches=[
FilterJSONPatch(
filters={'names': ['x']},
patches=[{
'op': 'add',
'path': '/shin/tari',
'value': 'bai'
}])
])
self.assertEqual(ret, [{
'foo': 'bar',
'shin': {
'gami': 'hai',
'shami': 'nai',
'tari': 'bai'
}
}])
def test_merge_filters_4(self):
data = [
Object({
'foo': 'bar',
'shin': 'gami',
},
name='x',
source='y',
instance='z')
]
ret = FilterJSONPatches_Apply(items=data,
jsonpatches=[
FilterJSONPatch(filters=[
lambda o: o.name == 'a',
lambda o: o.name == 'x'
],
patches=[{
'op':
'add',
'path':
'/tari',
'value':
'bai'
}])
])
self.assertEqual(ret, [{'foo': 'bar', 'shin': 'gami', 'tari': 'bai'}])
def test_basedata(self):
rabbit_config = RabbitMQOnlineBuilder(kubragen=self.kg,
options=RabbitMQOnlineOptions({
'namespace':
'myns',
'basename':
'myrabbit',
'config': {
'admin': {
'username':
'******',
'password':
'******',
}
},
'kubernetes': {
'volumes': {
'data': {
'emptyDir': {},
}
}
}
}))
self.assertEqual(rabbit_config.object_name('config'),
'myrabbit-config')
self.assertEqual(rabbit_config.object_name('statefulset'), 'myrabbit')
FilterJSONPatches_Apply(
items=rabbit_config.build(rabbit_config.BUILD_SERVICE),
jsonpatches=[
FilterJSONPatch(filters=ObjectFilter(
names=[rabbit_config.BUILDITEM_SERVICE]),
patches=[
{
'op': 'check',
'path': '/metadata/name',
'cmp': 'equals',
'value': 'myrabbit'
},
{
'op': 'check',
'path': '/metadata/namespace',
'cmp': 'equals',
'value': 'myns'
},
]),
])
def test_merge_str_to_helperstr(self):
data = [Object({
'foo': 'bar',
'shin': QuotedStr('gami'),
}, name='x')]
ret = FilterJSONPatches_Apply(items=data,
jsonpatches=[
FilterJSONPatch(
filters={'names': ['x']},
patches=[{
'op': 'replace',
'path': '/shin',
'value': 'bai'
}])
])
self.assertEqual(ret, [{'foo': 'bar', 'shin': 'bai'}])
self.assertNotIsInstance(ret[0]['shin'], HelperStr)
def test_basedata(self):
efk_config = EFKBuilder(kubragen=self.kg, options=EFKOptions({
'namespace': 'myns',
'basename': 'myefk',
'kubernetes': {
'volumes': {
'elasticsearch-data': {
'emptyDir': {},
}
}
}
}))
self.assertEqual(efk_config.object_name('elasticsearch-service'), 'myefk-elasticsearch')
self.assertEqual(efk_config.object_name('fluentd-daemonset'), 'myefk-fluentd')
FilterJSONPatches_Apply(items=efk_config.build(efk_config.BUILD_SERVICE), jsonpatches=[
FilterJSONPatch(filters=ObjectFilter(names=[efk_config.BUILDITEM_ELASTICSEARCH_SERVICE]), patches=[
{'op': 'check', 'path': '/metadata/name', 'cmp': 'equals', 'value': 'myefk-elasticsearch'},
{'op': 'check', 'path': '/metadata/namespace', 'cmp': 'equals', 'value': 'myns'},
]),
])
def test_basedata(self):
loki_config = LokiBuilder(kubragen=self.kg,
options=LokiOptions({
'namespace': 'myns',
'basename': 'myloki',
'kubernetes': {
'volumes': {
'data': {
'emptyDir': {},
}
}
}
}))
self.assertEqual(loki_config.object_name('config-secret'),
'myloki-config-secret')
self.assertEqual(loki_config.object_name('statefulset'), 'myloki')
FilterJSONPatches_Apply(
items=loki_config.build(loki_config.BUILD_SERVICE),
jsonpatches=[
FilterJSONPatch(filters=ObjectFilter(
names=[loki_config.BUILDITEM_SERVICE]),
patches=[
{
'op': 'check',
'path': '/metadata/name',
'cmp': 'equals',
'value': 'myloki'
},
{
'op': 'check',
'path': '/metadata/namespace',
'cmp': 'equals',
'value': 'myns'
},
]),
])
'apiVersion': 'v1',
'kind': 'Namespace',
'metadata': {
'name': 'app-monitoring',
},
},
name='ns-monitoring',
source='app'),
],
jsonpatches=[
FilterJSONPatch(filters=ObjectFilter(names=['ns-monitoring']),
patches=[
{
'op': 'add',
'path': '/metadata/annotations',
'value': {
'kubragen.github.io/patches':
QuotedStr('true'),
}
},
])
]))
shell_script.append(
OD_FileTemplate(f'kubectl apply -f ${{FILE_{file.fileid}}}'))
shell_script.append(
f'kubectl config set-context --current --namespace=app-default')
#
# SETUP: rabbitmq
def main():
parser = argparse.ArgumentParser(description='Kube Creator')
parser.add_argument('-p', '--provider', help='provider', required=True, choices=[
'google-gke',
'amazon-eks',
'digitalocean-kubernetes',
'k3d',
])
parser.add_argument('-o', '--output-path', help='output path', default='output')
args = parser.parse_args()
if args.provider == 'k3d':
kgprovider = ProviderK3DGeneric()
elif args.provider == 'google-gke':
kgprovider = ProviderGoogleGKE()
elif args.provider == 'digitalocean-kubernetes':
kgprovider = ProviderDigitalOceanKubernetes()
elif args.provider == 'amazon-eks':
kgprovider = ProviderAmazonEKS()
else:
raise Exception('Unknown target')
kg = KubraGen(provider=kgprovider, options=Options({
'namespaces': {
'default': 'default',
'mon': 'monitoring',
},
}))
if kgprovider.provider == PROVIDER_K3D:
kg.resources().persistentvolumeprofile_add('default', KRPersistentVolumeProfile_HostPath())
elif kgprovider.provider == PROVIDER_GOOGLE:
kg.resources().persistentvolumeprofile_add('default', KRPersistentVolumeProfile_GCEPersistentDisk())
elif kgprovider.provider == PROVIDER_DIGITALOCEAN:
kg.resources().persistentvolumeprofile_add('default', KRPersistentVolumeProfile_CSI_DOBS())
elif kgprovider.provider == PROVIDER_AMAZON:
kg.resources().persistentvolumeprofile_add('default', KRPersistentVolumeProfile_AWSElasticBlockStore())
if kgprovider.provider == PROVIDER_K3D:
kg.resources().persistentvolumeclaimprofile_add('default', KRPersistentVolumeClaimProfile_Basic(allow_selector=False))
else:
kg.resources().persistentvolumeclaimprofile_add('default', KRPersistentVolumeClaimProfile_Basic())
kg.resources().persistentvolume_add('elasticsearch-storage', 'default', {
'hostPath': {
'path': '/var/storage/elasticsearch'
},
'csi': {
'fsType': 'ext4',
},
}, {
'metadata': {
'labels': {
'pv.role': 'elasticsearch',
},
},
'spec': {
'persistentVolumeReclaimPolicy': 'Retain',
'capacity': {
'storage': '50Gi'
},
'accessModes': ['ReadWriteOnce'],
},
})
kg.resources().persistentvolumeclaim_add('elasticsearch-storage-claim', 'default', {
'namespace': 'monitoring',
'persistentVolume': 'elasticsearch-storage',
}, {
'spec': {
'selector': {
'matchLabels': {
'pv.role': 'elasticsearch',
}
},
}
})
out = OutputProject(kg)
shell_script = OutputFile_ShellScript('create_{}.sh'.format(args.provider))
out.append(shell_script)
shell_script.append('set -e')
#
# Provider setup
#
if kgprovider.provider == PROVIDER_K3D:
storage_directory = os.path.join(os.getcwd(), 'output', 'storage')
if not os.path.exists(storage_directory):
os.makedirs(storage_directory)
shell_script.append(f'# k3d cluster create kgsample-efk-stack --port 5051:80@loadbalancer --port 5052:443@loadbalancer -v {storage_directory}:/var/storage')
#
# OUTPUTFILE: namespace.yaml
#
file = OutputFile_Kubernetes('namespace.yaml')
file.append([{
'apiVersion': 'v1',
'kind': 'Namespace',
'metadata': {
'name': 'monitoring',
},
}])
out.append(file)
shell_script.append(OD_FileTemplate(f'kubectl apply -f ${{FILE_{file.fileid}}}'))
#
# OUTPUTFILE: storage.yaml
#
file = OutputFile_Kubernetes('storage.yaml')
file.append(kg.persistentvolume_build())
file.append(kg.persistentvolumeclaim_build())
out.append(file)
shell_script.append(OD_FileTemplate(f'kubectl apply -f ${{FILE_{file.fileid}}}'))
#
# SETUP: Traefik 2
#
traefik2_config = Traefik2Builder(kubragen=kg, options=Traefik2Options({
'namespace': OptionRoot('namespaces.default'),
'config': {
'traefik_args': [
'--api.dashboard=true',
'--api.insecure=false',
'--entrypoints.web.Address=:80',
'--entrypoints.api.Address=:8080',
'--providers.kubernetescrd',
f'--providers.kubernetescrd.namespaces=default,monitoring'
],
'ports': [
Traefik2OptionsPort(name='web', port_container=80, port_service=80),
Traefik2OptionsPort(name='api', port_container=8080, port_service=8080),
],
'create_traefik_crd': True,
},
})
)
traefik2_config.ensure_build_names(traefik2_config.BUILD_CRD, traefik2_config.BUILD_ACCESSCONTROL,
traefik2_config.BUILD_SERVICE)
#
# OUTPUTFILE: traefik-config-crd.yaml
#
file = OutputFile_Kubernetes('traefik-config-crd.yaml')
file.append(traefik2_config.build(traefik2_config.BUILD_CRD))
out.append(file)
shell_script.append(OD_FileTemplate(f'kubectl apply -f ${{FILE_{file.fileid}}}'))
#
# OUTPUTFILE: traefik-config.yaml
#
file = OutputFile_Kubernetes('traefik-config.yaml')
file.append(traefik2_config.build(traefik2_config.BUILD_ACCESSCONTROL))
file.append([{
'apiVersion': 'traefik.containo.us/v1alpha1',
'kind': 'IngressRoute',
'metadata': {
'name': 'traefik-api',
'namespace': kg.option_get('namespaces.default'),
},
'spec': {
'entryPoints': ['api'],
'routes': [{
'match': 'Method(`GET`)',
'kind': 'Rule',
'services': [{
'name': 'api@internal',
'kind': 'TraefikService'
}]
}]
}
}])
out.append(file)
shell_script.append(OD_FileTemplate(f'kubectl apply -f ${{FILE_{file.fileid}}}'))
#
# OUTPUTFILE: traefik.yaml
#
file = OutputFile_Kubernetes('traefik.yaml')
file.append(traefik2_config.build(traefik2_config.BUILDITEM_SERVICE))
file.append({
'apiVersion': 'traefik.containo.us/v1alpha1',
'kind': 'IngressRoute',
'metadata': {
'name': 'admin-traefik',
'namespace': kg.option_get('namespaces.default'),
},
'spec': {
'entryPoints': ['web'],
'routes': [{
'match': f'Host(`admin-traefik.localdomain`)',
'kind': 'Rule',
'services': [{
'name': traefik2_config.object_name('service'),
'port': 8080
}],
}]
}
})
out.append(file)
shell_script.append(OD_FileTemplate(f'kubectl apply -f ${{FILE_{file.fileid}}}'))
#
# SETUP: efk
#
efk_config = EFKBuilder(kubragen=kg, options=EFKOptions({
'namespace': OptionRoot('namespaces.mon'),
'config': {
'probes': False,
'elasticsearch': {
'replicas': 1 if kgprovider.provider == PROVIDER_K3D else 3,
},
'kibana': {
'service_port': 80,
},
},
'enable': {
'kibana': True,
},
'kubernetes': {
'volumes': {
'elasticsearch-data': {
'persistentVolumeClaim': {
'claimName': 'elasticsearch-storage-claim'
}
}
},
},
}))
efk_config.ensure_build_names(efk_config.BUILD_ACCESSCONTROL, efk_config.BUILD_CONFIG,
efk_config.BUILD_SERVICE)
#
# OUTPUTFILE: efk-config.yaml
#
file = OutputFile_Kubernetes('efk-config.yaml')
out.append(file)
file.append(efk_config.build(efk_config.BUILD_ACCESSCONTROL, efk_config.BUILD_CONFIG))
shell_script.append(OD_FileTemplate(f'kubectl apply -f ${{FILE_{file.fileid}}}'))
#
# OUTPUTFILE: efk.yaml
#
file = OutputFile_Kubernetes('efk.yaml')
out.append(file)
file.append(efk_config.build(efk_config.BUILD_SERVICE))
file.append([{
'apiVersion': 'traefik.containo.us/v1alpha1',
'kind': 'IngressRoute',
'metadata': {
'name': 'admin-kibana',
'namespace': kg.option_get('namespaces.mon'),
},
'spec': {
'entryPoints': ['web'],
'routes': [{
'match': f'Host(`admin-kibana.localdomain`)',
'kind': 'Rule',
'services': [{
'name': efk_config.object_name('kibana-service'),
'namespace': efk_config.namespace(),
'port': efk_config.option_get('config.kibana.service_port'),
}],
}]
}
}])
shell_script.append(OD_FileTemplate(f'kubectl apply -f ${{FILE_{file.fileid}}}'))
#
# OUTPUTFILE: http-echo.yaml
#
file = OutputFile_Kubernetes('http-echo.yaml')
out.append(file)
file.append([{
'apiVersion': 'apps/v1',
'kind': 'Deployment',
'metadata': {
'name': 'echo-deployment',
'namespace': kg.option_get('namespaces.default'),
'labels': {
'app': 'echo'
}
},
'spec': {
'replicas': 1,
'selector': {
'matchLabels': {
'app': 'echo'
}
},
'template': {
'metadata': {
'labels': {
'app': 'echo'
}
},
'spec': {
'containers': [{
'name': 'echo',
'image': 'mendhak/http-https-echo',
'ports': [{
'containerPort': 80
},
{
'containerPort': 443
}],
}]
}
}
}
},
{
'apiVersion': 'v1',
'kind': 'Service',
'metadata': {
'name': 'echo-service',
'namespace': kg.option_get('namespaces.default'),
},
'spec': {
'selector': {
'app': 'echo'
},
'ports': [{
'name': 'http',
'port': 80,
'targetPort': 80,
'protocol': 'TCP'
}]
}
}, {
'apiVersion': 'traefik.containo.us/v1alpha1',
'kind': 'IngressRoute',
'metadata': {
'name': 'http-echo',
'namespace': kg.option_get('namespaces.default'),
},
'spec': {
'entryPoints': ['web'],
'routes': [{
# 'match': f'Host(`http-echo.localdomain`)',
'match': f'PathPrefix(`/`)',
'kind': 'Rule',
'services': [{
'name': 'echo-service',
'port': 80,
}],
}]
}
}])
shell_script.append(OD_FileTemplate(f'kubectl apply -f ${{FILE_{file.fileid}}}'))
#
# OUTPUTFILE: ingress.yaml
#
file = OutputFile_Kubernetes('ingress.yaml')
http_path = '/'
if kgprovider.provider == PROVIDER_GOOGLE or kgprovider.provider == PROVIDER_AMAZON:
http_path = '/*'
file_data = [
Object({
'apiVersion': 'extensions/v1beta1',
'kind': 'Ingress',
'metadata': {
'name': 'ingress',
'namespace': kg.option_get('namespaces.default'),
},
'spec': {
'rules': [{
'http': {
'paths': [{
'path': http_path,
'backend': {
'serviceName': traefik2_config.object_name('service'),
'servicePort': 80,
}
}]
}
}]
}
}, name='ingress', source='app', instance='ingress')
]
if kgprovider.provider == PROVIDER_AMAZON:
FilterJSONPatches_Apply(file_data, jsonpatches=[
FilterJSONPatch(filters={'names': ['ingress']}, patches=[
{'op': 'merge', 'path': '/metadata', 'value': {'annotations': {
'kubernetes.io/ingress.class': 'alb',
'alb.ingress.kubernetes.io/scheme': 'internet-facing',
'alb.ingress.kubernetes.io/listen-ports': QuotedStr('[{"HTTP": 80}]'),
}}}
])
])
file.append(file_data)
out.append(file)
shell_script.append(OD_FileTemplate(f'kubectl apply -f ${{FILE_{file.fileid}}}'))
#
# OUTPUT
#
output_path = os.path.join(args.output_path, '{}-{}'.format(
args.provider, datetime.datetime.today().strftime("%Y%m%d-%H%M%S")))
print('Saving files to {}'.format(output_path))
if not os.path.exists(output_path):
os.makedirs(output_path)
out.output(OutputDriver_Directory(output_path))
def test_no_merge(self):
data = [
Object({
'foo': 'bar',
'shin': {
'gami': 'hai',
'shami': 'nai',
},
},
name='x',
source='y',
instance='z')
]
ret = FilterJSONPatches_Apply(items=data,
jsonpatches=[
FilterJSONPatch(
filters={'names': ['a']},
patches=[{
'op': 'add',
'path': '/shin/tari',
'value': 'bai'
}])
])
self.assertEqual(ret, [{
'foo': 'bar',
'shin': {
'gami': 'hai',
'shami': 'nai'
}
}])
ret = FilterJSONPatches_Apply(items=data,
jsonpatches=[
FilterJSONPatch(
filters={'sources': ['b']},
patches=[{
'op': 'add',
'path': '/shin/tari',
'value': 'bai'
}])
])
self.assertEqual(ret, [{
'foo': 'bar',
'shin': {
'gami': 'hai',
'shami': 'nai'
}
}])
ret = FilterJSONPatches_Apply(items=data,
jsonpatches=[
FilterJSONPatch(
filters={'instances': ['c']},
patches=[{
'op': 'add',
'path': '/shin/tari',
'value': 'bai'
}])
])
self.assertEqual(ret, [{
'foo': 'bar',
'shin': {
'gami': 'hai',
'shami': 'nai'
}
}])
ret = FilterJSONPatches_Apply(items=data,
jsonpatches=[
FilterJSONPatch(filters=ObjectFilter(
names='a',
sources='y',
instances='z'),
patches=[{
'op':
'add',
'path':
'/shin/tari',
'value':
'bai'
}])
])
self.assertEqual(ret, [{
'foo': 'bar',
'shin': {
'gami': 'hai',
'shami': 'nai'
}
}])
ret = FilterJSONPatches_Apply(items=data,
jsonpatches=[
FilterJSONPatch(filters=[
ObjectFilter(sources='h',
instances='j'),
lambda o: o.name == 'i'
],
patches=[{
'op':
'add',
'path':
'/shin/tari',
'value':
'bai'
}])
])
self.assertEqual(ret, [{
'foo': 'bar',
'shin': {
'gami': 'hai',
'shami': 'nai'
}
}])