def test_basedata(self): prometheusstack_config = PrometheusStackBuilder(kubragen=self.kg, options=PrometheusStackOptions({ 'namespace': 'myns', 'basename': 'myprometheusstack', 'config': { 'prometheus': { 'prometheus_config': '', }, }, 'kubernetes': { 'volumes': { 'prometheus-data': { 'emptyDir': {}, } } } })) self.assertEqual(prometheusstack_config.object_name('prometheus-config'), 'myprometheusstack-prometheus-config') self.assertEqual(prometheusstack_config.object_name('prometheus-statefulset'), 'myprometheusstack-prometheus') FilterJSONPatches_Apply(items=prometheusstack_config.build(prometheusstack_config.BUILD_SERVICE), jsonpatches=[ FilterJSONPatch(filters=ObjectFilter(names=[prometheusstack_config.BUILDITEM_PROMETHEUS_SERVICE]), patches=[ {'op': 'check', 'path': '/metadata/name', 'cmp': 'equals', 'value': 'myprometheusstack-prometheus'}, {'op': 'check', 'path': '/metadata/namespace', 'cmp': 'equals', 'value': 'myns'}, ]), ])
def test_basedata(self): promtail_config = PromtailBuilder(kubragen=self.kg, options=PromtailOptions({ 'namespace': 'myns', 'basename': 'mypromtail', })) self.assertEqual(promtail_config.object_name('config'), 'mypromtail-config') self.assertEqual(promtail_config.object_name('daemonset'), 'mypromtail') FilterJSONPatches_Apply( items=promtail_config.build(promtail_config.BUILD_SERVICE), jsonpatches=[ FilterJSONPatch(filters=ObjectFilter( names=[promtail_config.BUILDITEM_DAEMONSET]), patches=[ { 'op': 'check', 'path': '/metadata/name', 'cmp': 'equals', 'value': 'mypromtail' }, { 'op': 'check', 'path': '/metadata/namespace', 'cmp': 'equals', 'value': 'myns' }, ]), ])
def test_basedata(self): kms_config = KubeResourceReportBuilder( kubragen=self.kg, options=KubeResourceReportOptions({ 'namespace': 'myns', 'basename': 'mykms', })) self.assertEqual(kms_config.object_name('service'), 'mykms') self.assertEqual(kms_config.object_name('deployment'), 'mykms') FilterJSONPatches_Apply( items=kms_config.build(kms_config.BUILD_SERVICE), jsonpatches=[ FilterJSONPatch( filters=ObjectFilter(names=[kms_config.BUILDITEM_SERVICE]), patches=[ { 'op': 'check', 'path': '/metadata/name', 'cmp': 'equals', 'value': 'mykms' }, { 'op': 'check', 'path': '/metadata/namespace', 'cmp': 'equals', 'value': 'myns' }, ]), ])
def test_merge(self): data = [ Object({ 'foo': 'bar', 'shin': { 'gami': 'hai', 'shami': 'nai', }, }, name='x') ] ret = FilterJSONPatches_Apply(items=data, jsonpatches=[ FilterJSONPatch( filters={'names': ['x']}, patches=[{ 'op': 'add', 'path': '/shin/tari', 'value': 'bai' }]) ]) self.assertEqual(ret, [{ 'foo': 'bar', 'shin': { 'gami': 'hai', 'shami': 'nai', 'tari': 'bai' } }])
def test_merge_filters_4(self): data = [ Object({ 'foo': 'bar', 'shin': 'gami', }, name='x', source='y', instance='z') ] ret = FilterJSONPatches_Apply(items=data, jsonpatches=[ FilterJSONPatch(filters=[ lambda o: o.name == 'a', lambda o: o.name == 'x' ], patches=[{ 'op': 'add', 'path': '/tari', 'value': 'bai' }]) ]) self.assertEqual(ret, [{'foo': 'bar', 'shin': 'gami', 'tari': 'bai'}])
def test_basedata(self): rabbit_config = RabbitMQOnlineBuilder(kubragen=self.kg, options=RabbitMQOnlineOptions({ 'namespace': 'myns', 'basename': 'myrabbit', 'config': { 'admin': { 'username': '******', 'password': '******', } }, 'kubernetes': { 'volumes': { 'data': { 'emptyDir': {}, } } } })) self.assertEqual(rabbit_config.object_name('config'), 'myrabbit-config') self.assertEqual(rabbit_config.object_name('statefulset'), 'myrabbit') FilterJSONPatches_Apply( items=rabbit_config.build(rabbit_config.BUILD_SERVICE), jsonpatches=[ FilterJSONPatch(filters=ObjectFilter( names=[rabbit_config.BUILDITEM_SERVICE]), patches=[ { 'op': 'check', 'path': '/metadata/name', 'cmp': 'equals', 'value': 'myrabbit' }, { 'op': 'check', 'path': '/metadata/namespace', 'cmp': 'equals', 'value': 'myns' }, ]), ])
def test_merge_str_to_helperstr(self): data = [Object({ 'foo': 'bar', 'shin': QuotedStr('gami'), }, name='x')] ret = FilterJSONPatches_Apply(items=data, jsonpatches=[ FilterJSONPatch( filters={'names': ['x']}, patches=[{ 'op': 'replace', 'path': '/shin', 'value': 'bai' }]) ]) self.assertEqual(ret, [{'foo': 'bar', 'shin': 'bai'}]) self.assertNotIsInstance(ret[0]['shin'], HelperStr)
def test_basedata(self): efk_config = EFKBuilder(kubragen=self.kg, options=EFKOptions({ 'namespace': 'myns', 'basename': 'myefk', 'kubernetes': { 'volumes': { 'elasticsearch-data': { 'emptyDir': {}, } } } })) self.assertEqual(efk_config.object_name('elasticsearch-service'), 'myefk-elasticsearch') self.assertEqual(efk_config.object_name('fluentd-daemonset'), 'myefk-fluentd') FilterJSONPatches_Apply(items=efk_config.build(efk_config.BUILD_SERVICE), jsonpatches=[ FilterJSONPatch(filters=ObjectFilter(names=[efk_config.BUILDITEM_ELASTICSEARCH_SERVICE]), patches=[ {'op': 'check', 'path': '/metadata/name', 'cmp': 'equals', 'value': 'myefk-elasticsearch'}, {'op': 'check', 'path': '/metadata/namespace', 'cmp': 'equals', 'value': 'myns'}, ]), ])
def test_basedata(self): loki_config = LokiBuilder(kubragen=self.kg, options=LokiOptions({ 'namespace': 'myns', 'basename': 'myloki', 'kubernetes': { 'volumes': { 'data': { 'emptyDir': {}, } } } })) self.assertEqual(loki_config.object_name('config-secret'), 'myloki-config-secret') self.assertEqual(loki_config.object_name('statefulset'), 'myloki') FilterJSONPatches_Apply( items=loki_config.build(loki_config.BUILD_SERVICE), jsonpatches=[ FilterJSONPatch(filters=ObjectFilter( names=[loki_config.BUILDITEM_SERVICE]), patches=[ { 'op': 'check', 'path': '/metadata/name', 'cmp': 'equals', 'value': 'myloki' }, { 'op': 'check', 'path': '/metadata/namespace', 'cmp': 'equals', 'value': 'myns' }, ]), ])
FilterJSONPatches_Apply( [ Object( { 'apiVersion': 'v1', 'kind': 'Namespace', 'metadata': { 'name': 'app-default', 'annotations': { 'will-not-output': ValueData(value='anything', enabled=False), } }, }, name='ns-default', source='app'), Object( { 'apiVersion': 'v1', 'kind': 'Namespace', 'metadata': { 'name': 'app-monitoring', }, }, name='ns-monitoring', source='app'), ], jsonpatches=[ FilterJSONPatch(filters=ObjectFilter(names=['ns-monitoring']), patches=[ { 'op': 'add', 'path': '/metadata/annotations', 'value': { 'kubragen.github.io/patches': QuotedStr('true'), } }, ]) ]))
def main(): parser = argparse.ArgumentParser(description='Kube Creator') parser.add_argument('-p', '--provider', help='provider', required=True, choices=[ 'google-gke', 'amazon-eks', 'digitalocean-kubernetes', 'k3d', ]) parser.add_argument('-o', '--output-path', help='output path', default='output') args = parser.parse_args() if args.provider == 'k3d': kgprovider = ProviderK3DGeneric() elif args.provider == 'google-gke': kgprovider = ProviderGoogleGKE() elif args.provider == 'digitalocean-kubernetes': kgprovider = ProviderDigitalOceanKubernetes() elif args.provider == 'amazon-eks': kgprovider = ProviderAmazonEKS() else: raise Exception('Unknown target') kg = KubraGen(provider=kgprovider, options=Options({ 'namespaces': { 'default': 'default', 'mon': 'monitoring', }, })) if kgprovider.provider == PROVIDER_K3D: kg.resources().persistentvolumeprofile_add('default', KRPersistentVolumeProfile_HostPath()) elif kgprovider.provider == PROVIDER_GOOGLE: kg.resources().persistentvolumeprofile_add('default', KRPersistentVolumeProfile_GCEPersistentDisk()) elif kgprovider.provider == PROVIDER_DIGITALOCEAN: kg.resources().persistentvolumeprofile_add('default', KRPersistentVolumeProfile_CSI_DOBS()) elif kgprovider.provider == PROVIDER_AMAZON: kg.resources().persistentvolumeprofile_add('default', KRPersistentVolumeProfile_AWSElasticBlockStore()) if kgprovider.provider == PROVIDER_K3D: kg.resources().persistentvolumeclaimprofile_add('default', KRPersistentVolumeClaimProfile_Basic(allow_selector=False)) else: kg.resources().persistentvolumeclaimprofile_add('default', KRPersistentVolumeClaimProfile_Basic()) kg.resources().persistentvolume_add('elasticsearch-storage', 'default', { 'hostPath': { 'path': '/var/storage/elasticsearch' }, 'csi': { 'fsType': 'ext4', }, }, { 'metadata': { 'labels': { 'pv.role': 'elasticsearch', }, }, 'spec': { 'persistentVolumeReclaimPolicy': 'Retain', 'capacity': { 'storage': '50Gi' }, 'accessModes': ['ReadWriteOnce'], }, }) kg.resources().persistentvolumeclaim_add('elasticsearch-storage-claim', 'default', { 'namespace': 'monitoring', 'persistentVolume': 'elasticsearch-storage', }, { 'spec': { 'selector': { 'matchLabels': { 'pv.role': 'elasticsearch', } }, } }) out = OutputProject(kg) shell_script = OutputFile_ShellScript('create_{}.sh'.format(args.provider)) out.append(shell_script) shell_script.append('set -e') # # Provider setup # if kgprovider.provider == PROVIDER_K3D: storage_directory = os.path.join(os.getcwd(), 'output', 'storage') if not os.path.exists(storage_directory): os.makedirs(storage_directory) shell_script.append(f'# k3d cluster create kgsample-efk-stack --port 5051:80@loadbalancer --port 5052:443@loadbalancer -v {storage_directory}:/var/storage') # # OUTPUTFILE: namespace.yaml # file = OutputFile_Kubernetes('namespace.yaml') file.append([{ 'apiVersion': 'v1', 'kind': 'Namespace', 'metadata': { 'name': 'monitoring', }, }]) out.append(file) shell_script.append(OD_FileTemplate(f'kubectl apply -f ${{FILE_{file.fileid}}}')) # # OUTPUTFILE: storage.yaml # file = OutputFile_Kubernetes('storage.yaml') file.append(kg.persistentvolume_build()) file.append(kg.persistentvolumeclaim_build()) out.append(file) shell_script.append(OD_FileTemplate(f'kubectl apply -f ${{FILE_{file.fileid}}}')) # # SETUP: Traefik 2 # traefik2_config = Traefik2Builder(kubragen=kg, options=Traefik2Options({ 'namespace': OptionRoot('namespaces.default'), 'config': { 'traefik_args': [ '--api.dashboard=true', '--api.insecure=false', '--entrypoints.web.Address=:80', '--entrypoints.api.Address=:8080', '--providers.kubernetescrd', f'--providers.kubernetescrd.namespaces=default,monitoring' ], 'ports': [ Traefik2OptionsPort(name='web', port_container=80, port_service=80), Traefik2OptionsPort(name='api', port_container=8080, port_service=8080), ], 'create_traefik_crd': True, }, }) ) traefik2_config.ensure_build_names(traefik2_config.BUILD_CRD, traefik2_config.BUILD_ACCESSCONTROL, traefik2_config.BUILD_SERVICE) # # OUTPUTFILE: traefik-config-crd.yaml # file = OutputFile_Kubernetes('traefik-config-crd.yaml') file.append(traefik2_config.build(traefik2_config.BUILD_CRD)) out.append(file) shell_script.append(OD_FileTemplate(f'kubectl apply -f ${{FILE_{file.fileid}}}')) # # OUTPUTFILE: traefik-config.yaml # file = OutputFile_Kubernetes('traefik-config.yaml') file.append(traefik2_config.build(traefik2_config.BUILD_ACCESSCONTROL)) file.append([{ 'apiVersion': 'traefik.containo.us/v1alpha1', 'kind': 'IngressRoute', 'metadata': { 'name': 'traefik-api', 'namespace': kg.option_get('namespaces.default'), }, 'spec': { 'entryPoints': ['api'], 'routes': [{ 'match': 'Method(`GET`)', 'kind': 'Rule', 'services': [{ 'name': 'api@internal', 'kind': 'TraefikService' }] }] } }]) out.append(file) shell_script.append(OD_FileTemplate(f'kubectl apply -f ${{FILE_{file.fileid}}}')) # # OUTPUTFILE: traefik.yaml # file = OutputFile_Kubernetes('traefik.yaml') file.append(traefik2_config.build(traefik2_config.BUILDITEM_SERVICE)) file.append({ 'apiVersion': 'traefik.containo.us/v1alpha1', 'kind': 'IngressRoute', 'metadata': { 'name': 'admin-traefik', 'namespace': kg.option_get('namespaces.default'), }, 'spec': { 'entryPoints': ['web'], 'routes': [{ 'match': f'Host(`admin-traefik.localdomain`)', 'kind': 'Rule', 'services': [{ 'name': traefik2_config.object_name('service'), 'port': 8080 }], }] } }) out.append(file) shell_script.append(OD_FileTemplate(f'kubectl apply -f ${{FILE_{file.fileid}}}')) # # SETUP: efk # efk_config = EFKBuilder(kubragen=kg, options=EFKOptions({ 'namespace': OptionRoot('namespaces.mon'), 'config': { 'probes': False, 'elasticsearch': { 'replicas': 1 if kgprovider.provider == PROVIDER_K3D else 3, }, 'kibana': { 'service_port': 80, }, }, 'enable': { 'kibana': True, }, 'kubernetes': { 'volumes': { 'elasticsearch-data': { 'persistentVolumeClaim': { 'claimName': 'elasticsearch-storage-claim' } } }, }, })) efk_config.ensure_build_names(efk_config.BUILD_ACCESSCONTROL, efk_config.BUILD_CONFIG, efk_config.BUILD_SERVICE) # # OUTPUTFILE: efk-config.yaml # file = OutputFile_Kubernetes('efk-config.yaml') out.append(file) file.append(efk_config.build(efk_config.BUILD_ACCESSCONTROL, efk_config.BUILD_CONFIG)) shell_script.append(OD_FileTemplate(f'kubectl apply -f ${{FILE_{file.fileid}}}')) # # OUTPUTFILE: efk.yaml # file = OutputFile_Kubernetes('efk.yaml') out.append(file) file.append(efk_config.build(efk_config.BUILD_SERVICE)) file.append([{ 'apiVersion': 'traefik.containo.us/v1alpha1', 'kind': 'IngressRoute', 'metadata': { 'name': 'admin-kibana', 'namespace': kg.option_get('namespaces.mon'), }, 'spec': { 'entryPoints': ['web'], 'routes': [{ 'match': f'Host(`admin-kibana.localdomain`)', 'kind': 'Rule', 'services': [{ 'name': efk_config.object_name('kibana-service'), 'namespace': efk_config.namespace(), 'port': efk_config.option_get('config.kibana.service_port'), }], }] } }]) shell_script.append(OD_FileTemplate(f'kubectl apply -f ${{FILE_{file.fileid}}}')) # # OUTPUTFILE: http-echo.yaml # file = OutputFile_Kubernetes('http-echo.yaml') out.append(file) file.append([{ 'apiVersion': 'apps/v1', 'kind': 'Deployment', 'metadata': { 'name': 'echo-deployment', 'namespace': kg.option_get('namespaces.default'), 'labels': { 'app': 'echo' } }, 'spec': { 'replicas': 1, 'selector': { 'matchLabels': { 'app': 'echo' } }, 'template': { 'metadata': { 'labels': { 'app': 'echo' } }, 'spec': { 'containers': [{ 'name': 'echo', 'image': 'mendhak/http-https-echo', 'ports': [{ 'containerPort': 80 }, { 'containerPort': 443 }], }] } } } }, { 'apiVersion': 'v1', 'kind': 'Service', 'metadata': { 'name': 'echo-service', 'namespace': kg.option_get('namespaces.default'), }, 'spec': { 'selector': { 'app': 'echo' }, 'ports': [{ 'name': 'http', 'port': 80, 'targetPort': 80, 'protocol': 'TCP' }] } }, { 'apiVersion': 'traefik.containo.us/v1alpha1', 'kind': 'IngressRoute', 'metadata': { 'name': 'http-echo', 'namespace': kg.option_get('namespaces.default'), }, 'spec': { 'entryPoints': ['web'], 'routes': [{ # 'match': f'Host(`http-echo.localdomain`)', 'match': f'PathPrefix(`/`)', 'kind': 'Rule', 'services': [{ 'name': 'echo-service', 'port': 80, }], }] } }]) shell_script.append(OD_FileTemplate(f'kubectl apply -f ${{FILE_{file.fileid}}}')) # # OUTPUTFILE: ingress.yaml # file = OutputFile_Kubernetes('ingress.yaml') http_path = '/' if kgprovider.provider == PROVIDER_GOOGLE or kgprovider.provider == PROVIDER_AMAZON: http_path = '/*' file_data = [ Object({ 'apiVersion': 'extensions/v1beta1', 'kind': 'Ingress', 'metadata': { 'name': 'ingress', 'namespace': kg.option_get('namespaces.default'), }, 'spec': { 'rules': [{ 'http': { 'paths': [{ 'path': http_path, 'backend': { 'serviceName': traefik2_config.object_name('service'), 'servicePort': 80, } }] } }] } }, name='ingress', source='app', instance='ingress') ] if kgprovider.provider == PROVIDER_AMAZON: FilterJSONPatches_Apply(file_data, jsonpatches=[ FilterJSONPatch(filters={'names': ['ingress']}, patches=[ {'op': 'merge', 'path': '/metadata', 'value': {'annotations': { 'kubernetes.io/ingress.class': 'alb', 'alb.ingress.kubernetes.io/scheme': 'internet-facing', 'alb.ingress.kubernetes.io/listen-ports': QuotedStr('[{"HTTP": 80}]'), }}} ]) ]) file.append(file_data) out.append(file) shell_script.append(OD_FileTemplate(f'kubectl apply -f ${{FILE_{file.fileid}}}')) # # OUTPUT # output_path = os.path.join(args.output_path, '{}-{}'.format( args.provider, datetime.datetime.today().strftime("%Y%m%d-%H%M%S"))) print('Saving files to {}'.format(output_path)) if not os.path.exists(output_path): os.makedirs(output_path) out.output(OutputDriver_Directory(output_path))
def test_no_merge(self): data = [ Object({ 'foo': 'bar', 'shin': { 'gami': 'hai', 'shami': 'nai', }, }, name='x', source='y', instance='z') ] ret = FilterJSONPatches_Apply(items=data, jsonpatches=[ FilterJSONPatch( filters={'names': ['a']}, patches=[{ 'op': 'add', 'path': '/shin/tari', 'value': 'bai' }]) ]) self.assertEqual(ret, [{ 'foo': 'bar', 'shin': { 'gami': 'hai', 'shami': 'nai' } }]) ret = FilterJSONPatches_Apply(items=data, jsonpatches=[ FilterJSONPatch( filters={'sources': ['b']}, patches=[{ 'op': 'add', 'path': '/shin/tari', 'value': 'bai' }]) ]) self.assertEqual(ret, [{ 'foo': 'bar', 'shin': { 'gami': 'hai', 'shami': 'nai' } }]) ret = FilterJSONPatches_Apply(items=data, jsonpatches=[ FilterJSONPatch( filters={'instances': ['c']}, patches=[{ 'op': 'add', 'path': '/shin/tari', 'value': 'bai' }]) ]) self.assertEqual(ret, [{ 'foo': 'bar', 'shin': { 'gami': 'hai', 'shami': 'nai' } }]) ret = FilterJSONPatches_Apply(items=data, jsonpatches=[ FilterJSONPatch(filters=ObjectFilter( names='a', sources='y', instances='z'), patches=[{ 'op': 'add', 'path': '/shin/tari', 'value': 'bai' }]) ]) self.assertEqual(ret, [{ 'foo': 'bar', 'shin': { 'gami': 'hai', 'shami': 'nai' } }]) ret = FilterJSONPatches_Apply(items=data, jsonpatches=[ FilterJSONPatch(filters=[ ObjectFilter(sources='h', instances='j'), lambda o: o.name == 'i' ], patches=[{ 'op': 'add', 'path': '/shin/tari', 'value': 'bai' }]) ]) self.assertEqual(ret, [{ 'foo': 'bar', 'shin': { 'gami': 'hai', 'shami': 'nai' } }])