def _init_namespace(instance_id, dry_run=False): logs.debug('Initializing helm-based instance deployment namespace', namespace=instance_id) if kubectl.get('ns', instance_id, required=False): logs.info(f'instance namespace already exists ({instance_id})') else: logs.info(f'creating instance namespace ({instance_id})') kubectl.apply(kubectl.get_resource('v1', 'Namespace', instance_id, {}), dry_run=dry_run) service_account_name = f'ckan-{instance_id}-operator' logs.debug('Creating service account', service_account_name=service_account_name) if not dry_run: kubectl_rbac_driver.update_service_account( f'ckan-{instance_id}-operator', {}, namespace=instance_id) role_name = f'ckan-{instance_id}-operator-role' logs.debug('Creating role and binding to the service account', role_name=role_name) if not dry_run: kubectl_rbac_driver.update_role(role_name, {}, [{ "apiGroups": ["*"], "resources": ['secrets', 'pods', 'pods/exec', 'pods/portforward'], "verbs": ["list", "get", "create"] }], namespace=instance_id) kubectl_rbac_driver.update_role_binding( name=f'ckan-{instance_id}-operator-rolebinding', role_name=f'ckan-{instance_id}-operator-role', namespace=instance_id, service_account_name=f'ckan-{instance_id}-operator', labels={})
def deploy(): """Deploys a proxy inside the cluster which allows to access the centralized solr without authentication""" labels = {'app': 'ckan-cloud-solrcloud-proxy'} solr_url = parse_url(solr_manager.get_internal_http_endpoint()) scheme = solr_url.scheme hostname = solr_url.hostname port = solr_url.port solr_user, solr_password = solr_url.auth.split(':') if not port: port = '443' if scheme == 'https' else '8983' kubectl.update_secret( 'solrcloud-proxy', { 'SOLR_URL': f'{scheme}://{hostname}:{port}', 'SOLR_USER': solr_user, 'SOLR_PASSWORD': solr_password }) kubectl.apply( kubectl.get_deployment( 'solrcloud-proxy', labels, { 'replicas': 1, 'revisionHistoryLimit': 10, 'strategy': { 'type': 'RollingUpdate', }, 'template': { 'metadata': { 'labels': labels, 'annotations': { 'ckan-cloud/operator-timestamp': str(datetime.datetime.now()) } }, 'spec': { 'containers': [{ 'name': 'solrcloud-proxy', 'image': 'viderum/ckan-cloud-operator-solrcloud-proxy', 'envFrom': [{ 'secretRef': { 'name': 'solrcloud-proxy' } }], 'ports': [{ 'containerPort': 8983 }], }] } } })) service = kubectl.get_resource('v1', 'Service', 'solrcloud-proxy', labels) service['spec'] = { 'ports': [{ 'name': '8983', 'port': 8983 }], 'selector': labels } kubectl.apply(service)
def _apply_zookeeper_headless_service(dry_run=False): headless_service_name = _get_resource_name('zk-headless') kubectl.apply(kubectl.get_resource( 'v1', 'Service', headless_service_name, _get_resource_labels(suffix='zk-headless'), spec={ 'clusterIP': 'None', 'ports': [{ 'name': 'client', 'port': 2181, 'protocol': 'TCP', 'targetPort': 2181 }, { 'name': 'server', 'port': 2888, 'protocol': 'TCP', 'targetPort': 2888 }, { 'name': 'leader-election', 'port': 3888, 'protocol': 'TCP', 'targetPort': 3888 }], 'selector': { 'app': _get_resource_labels(for_deployment=True, suffix='zk')['app'] } }), dry_run=dry_run) return headless_service_name
def _apply_solrcloud_headless_service(dry_run=False): headless_service_name = _get_resource_name('sc-headless') kubectl.apply(kubectl.get_resource( 'v1', 'Service', headless_service_name, _get_resource_labels(suffix='sc-headless'), spec={ 'clusterIP': 'None', 'ports': [{ 'name': 'solr', 'port': 8983, 'protocol': 'TCP', 'targetPort': 8983 }, { 'name': 'stop', 'port': 7983, 'protocol': 'TCP', 'targetPort': 7983 }, { 'name': 'rmi', 'port': 18983, 'protocol': 'TCP', 'targetPort': 18983 }], 'selector': { 'app': _get_resource_labels(for_deployment=True, suffix='sc')['app'] } }), dry_run=dry_run) return headless_service_name
def get_resource(singular, name, extra_label_suffixes=None, **kwargs): crd_group = get_crd_group() _, kind_suffix = _get_plural_kind_suffix(singular) resource = kubectl.get_resource( f'{crd_group}/v1', get_resource_kind(singular), get_resource_name(singular, name), get_resource_labels(singular, name, extra_label_suffixes=extra_label_suffixes)) resource.update(**kwargs) return resource
def create(name, image, config, router_name=None): labels = _get_labels(name) datapusher = kubectl.get_resource('stable.viderum.com/v1', 'CkanCloudDatapusher', name, labels) datapusher['spec'] = {'image': image, 'config': config} kubectl.apply(datapusher) if router_name: routers_manager.create_subdomain_route(router_name, { 'target-type': 'datapusher', 'datapusher-name': name })
def create_subdomain_route(router_name, route_spec): target_type = route_spec['target-type'] sub_domain = route_spec.get('sub-domain') root_domain = route_spec.get('root-domain') if target_type == 'datapusher': target_resource_id = route_spec['datapusher-name'] elif target_type == 'deis-instance': target_resource_id = route_spec['deis-instance-id'] elif target_type == 'backend-url': target_resource_id = route_spec['target-resource-id'] else: raise Exception(f'Invalid route spec: {route_spec}') sub_domain, root_domain = _get_default_sub_root_domain( sub_domain, root_domain, target_resource_id) route_name = 'cc' + hashlib.sha3_224( f'{target_type} {target_resource_id} {root_domain} {sub_domain}'. encode()).hexdigest() router, spec, router_type, annotations, labels, router_type_config = _init_router( router_name) route_type = f'{target_type}-subdomain' labels.update( **{ 'ckan-cloud/route-type': route_type, 'ckan-cloud/route-root-domain': root_domain, 'ckan-cloud/route-sub-domain': sub_domain, 'ckan-cloud/route-target-type': target_type, 'ckan-cloud/route-target-resource-id': target_resource_id, }) spec = { 'name': route_name, 'type': route_type, 'root-domain': root_domain, 'sub-domain': sub_domain, 'router_name': router_name, 'router_type': router_type, 'route-target-type': target_type, 'route-target-resource-id': target_resource_id, } if target_type == 'datapusher': labels['ckan-cloud/route-datapusher-name'] = spec[ 'datapusher-name'] = route_spec['datapusher-name'] elif target_type == 'deis-instance': labels['ckan-cloud/route-deis-instance-id'] = spec[ 'deis-instance-id'] = route_spec['deis-instance-id'] elif target_type == 'backend-url': spec['backend-url'] = route_spec['backend-url'] route = kubectl.get_resource('stable.viderum.com/v1', 'CkanCloudRoute', route_name, labels, spec=spec) kubectl.apply(route)
def create(router_name, router_spec): router_type = router_spec.get('type') default_root_domain = router_spec.get('default-root-domain') assert router_type in ROUTER_TYPES and default_root_domain, f'Invalid router spec: {router_spec}' get(router_name, only_dns=True, failfast=True) print(f'Creating CkanCloudRouter {router_name} {router_spec}') labels = _get_labels(router_name, router_type) router = kubectl.get_resource('stable.viderum.com/v1', 'CkanCloudRouter', router_name, labels, spec=dict(router_spec, **{'type': router_type})) router_manager = ROUTER_TYPES[router_type]['manager'] router = router_manager.create(router) annotations = CkanRoutersAnnotations(router_name, router) annotations.json_annotate('default-root-domain', default_root_domain)
def update_service(name, labels): labels.update(**_get_labels(name)) service = kubectl.get_resource('v1', 'Service', get_service_name(name), labels) service['spec'] = { 'ports': [{ 'name': '8000', 'port': 8000 }], 'selector': { 'ckan-cloud/datapusher-name': name, 'app': 'datapusher', } } kubectl.apply(service)
def _apply_solrcloud_service(): service_name = _get_resource_name('sc') kubectl.apply(kubectl.get_resource( 'v1', 'Service', service_name, _get_resource_labels(suffix='sc'), spec={ 'ports': [ {'name': 'solr', 'port': 8983, 'protocol': 'TCP', 'targetPort': 8983}, ], 'selector': { 'app': _get_resource_labels(for_deployment=True, suffix='sc')['app'] } } )) return service_name
def deploy_ckan_infra_solr_proxy(): """Deploys a proxy inside the cluster which allows to access the centralized solr without authentication""" labels = {'app': 'ckan-cloud-solrcloud-proxy'} infra = cls() solr_url = urlparse(infra.SOLR_HTTP_ENDPOINT) scheme = solr_url.scheme hostname = solr_url.hostname port = solr_url.port if not port: port = '443' if scheme == 'https' else '8983' kubectl.update_secret('solrcloud-proxy', { 'SOLR_URL': f'{scheme}://{hostname}:{port}', 'SOLR_USER': infra.SOLR_USER, 'SOLR_PASSWORD': infra.SOLR_PASSWORD }) kubectl.apply(kubectl.get_deployment('solrcloud-proxy', labels, { 'replicas': 1, 'revisionHistoryLimit': 10, 'strategy': {'type': 'RollingUpdate', }, 'template': { 'metadata': { 'labels': labels, 'annotations': { 'ckan-cloud/operator-timestamp': str(datetime.datetime.now()) } }, 'spec': { 'containers': [ { 'name': 'solrcloud-proxy', 'image': 'orihoch/ckan-cloud-operator-solrcloud-proxy', 'envFrom': [{'secretRef': {'name': 'solrcloud-proxy'}}], 'ports': [{'containerPort': 8983}], } ] } } })) service = kubectl.get_resource('v1', 'Service', 'solrcloud-proxy', labels) service['spec'] = { 'ports': [ {'name': '8983', 'port': 8983} ], 'selector': labels } kubectl.apply(service)
def create(router_name, router_spec): from ckan_cloud_operator.providers.cluster.manager import get_or_create_multi_user_volume_claim from ckan_cloud_operator.routers.traefik.deployment import get_label_suffixes router_type = router_spec.get('type') default_root_domain = router_spec.get('default-root-domain') assert router_type in ROUTER_TYPES and default_root_domain, f'Invalid router spec: {router_spec}' get(router_name, only_dns=True, failfast=True) print(f'Creating CkanCloudRouter {router_name} {router_spec}') labels = _get_labels(router_name, router_type) router = kubectl.get_resource('stable.viderum.com/v1', 'CkanCloudRouter', router_name, labels, spec=dict(router_spec, **{'type': router_type})) router_manager = ROUTER_TYPES[router_type]['manager'] router = router_manager.create(router) get_or_create_multi_user_volume_claim(get_label_suffixes(router_name, router_type)) annotations = CkanRoutersAnnotations(router_name, router) annotations.json_annotate('default-root-domain', default_root_domain)
def _apply_service(storage_suffix=None, dry_run=False): kubectl.apply(kubectl.get_resource( 'v1', 'Service', _get_resource_name(suffix=storage_suffix), _get_resource_labels(suffix=storage_suffix), spec={ 'ports': [{ 'name': '9000', 'port': 9000 }], 'selector': { 'app': _get_resource_labels(for_deployment=True, suffix=storage_suffix)['app'] } }), dry_run=dry_run)
def pre_deployment_hook(route, labels): name, spec = _init_route(route) deis_instance_id = spec['deis-instance-id'] if kubectl.get(f'ns {deis_instance_id}', required=False): print( f'updating route name {name} for deis instance {deis_instance_id}') route_service = kubectl.get_resource('v1', 'Service', name, labels, namespace=deis_instance_id) route_service['spec'] = { 'ports': [{ 'name': '5000', 'port': 5000 }], 'selector': { 'app': 'ckan' } } kubectl.apply(route_service)
def _init_namespace(instance_id): if kubectl.get('ns', instance_id, required=False): logs.info(f'instance namespace already exists ({instance_id})') else: logs.info(f'creating instance namespace ({instance_id})') kubectl.apply(kubectl.get_resource('v1', 'Namespace', instance_id, {})) kubectl_rbac_driver.update_service_account( f'ckan-{instance_id}-operator', {}, namespace=instance_id) kubectl_rbac_driver.update_role( f'ckan-{instance_id}-operator-role', {}, [{ "apiGroups": ["*"], "resources": ['secrets', 'pods', 'pods/exec', 'pods/portforward'], "verbs": ["list", "get", "create"] }], namespace=instance_id) kubectl_rbac_driver.update_role_binding( name=f'ckan-{instance_id}-operator-rolebinding', role_name=f'ckan-{instance_id}-operator-role', namespace=instance_id, service_account_name=f'ckan-{instance_id}-operator', labels={})
def update_service_account(service_account_name, labels): kubectl.apply( kubectl.get_resource('v1', 'ServiceAccount', service_account_name, labels))
def deploy_gcs_minio_proxy(router_name): """Deploys a minio proxy (AKA gateway) for access to google storage""" labels = {'app': 'ckan-cloud-gcsminio-proxy'} if not kubectl.get('secret gcsminio-proxy-credentials', required=False): print('Creating minio credentials') minio_access_key = binascii.hexlify(os.urandom(8)).decode() minio_secret_key = binascii.hexlify(os.urandom(12)).decode() kubectl.update_secret( 'gcsminio-proxy-credentials', { 'MINIO_ACCESS_KEY': minio_access_key, 'MINIO_SECRET_KEY': minio_secret_key, }) kubectl.apply( kubectl.get_deployment( 'gcsminio-proxy', labels, { 'replicas': 1, 'revisionHistoryLimit': 10, 'strategy': { 'type': 'RollingUpdate', }, 'template': { 'metadata': { 'labels': labels, 'annotations': { 'ckan-cloud/operator-timestamp': str(datetime.datetime.now()) } }, 'spec': { 'containers': [{ 'name': 'minio', 'image': 'orihoch/ckan-cloud-operator-gcsminio-proxy', 'env': [{ 'name': 'GOOGLE_APPLICATION_CREDENTIALS', 'value': '/gcloud-credentials/credentials.json' }], 'envFrom': [{ 'secretRef': { 'name': 'gcsminio-proxy-credentials' } }], 'ports': [{ 'containerPort': 9000 }], 'volumeMounts': [ { 'name': 'gcloud-credentials', 'mountPath': '/gcloud-credentials/credentials.json', 'subPath': 'GCLOUD_SERVICE_ACCOUNT_JSON' }, ], }], 'volumes': [ { 'name': 'gcloud-credentials', 'secret': { 'secretName': 'ckan-infra' } }, ] } } })) service = kubectl.get_resource('v1', 'Service', 'gcsminio-proxy', labels) service['spec'] = { 'ports': [{ 'name': '9000', 'port': 9000 }], 'selector': labels } kubectl.apply(service) if not routers_manager.get_backend_url_routes('gcs-minio'): routers_manager.create_subdomain_route( router_name, { 'target-type': 'backend-url', 'target-resource-id': 'gcs-minio', 'backend-url': 'http://gcsminio-proxy.ckan-cloud:9000', 'sub-domain': 'default', 'root-domain': 'default', }) routers_manager.update(router_name, wait_ready=True)
def _update(router_name, spec, annotations, routes): resource_name = _get_resource_name(router_name) router_type = spec['type'] cloudflare_email, cloudflare_auth_key = get_cloudflare_credentials() external_domains = spec.get('external-domains') dns_provider = spec.get('dns-provider', 'cloudflare') logs.info('updating traefik deployment', resource_name=resource_name, router_type=router_type, cloudflare_email=cloudflare_email, cloudflare_auth_key_len=len(cloudflare_auth_key) if cloudflare_auth_key else 0, external_domains=external_domains, dns_provider=dns_provider) kubectl.apply( kubectl.get_configmap( resource_name, get_labels(router_name, router_type), { 'traefik.toml': toml.dumps( traefik_router_config.get( routes, cloudflare_email, enable_access_log=bool(spec.get('enable-access-log')), wildcard_ssl_domain=spec.get('wildcard-ssl-domain'), external_domains=external_domains, dns_provider=dns_provider, force=True)) })) domains = {} httpauth_secrets = [] for route in routes: root_domain, sub_domain = routes_manager.get_domain_parts(route) domains.setdefault(root_domain, []).append(sub_domain) routes_manager.pre_deployment_hook( route, get_labels(router_name, router_type)) if route['spec'].get('httpauth-secret') and route['spec'][ 'httpauth-secret'] not in httpauth_secrets: httpauth_secrets.append(route['spec']['httpauth-secret']) load_balancer = kubectl.get_resource('v1', 'Service', f'loadbalancer-{resource_name}', get_labels(router_name, router_type)) load_balancer['spec'] = { 'ports': [ { 'name': '80', 'port': 80 }, { 'name': '443', 'port': 443 }, ], 'selector': { 'app': get_labels(router_name, router_type, for_deployment=True)['app'] }, 'type': 'LoadBalancer' } kubectl.apply(load_balancer) load_balancer_ip = get_load_balancer_ip(router_name) print(f'load balancer ip: {load_balancer_ip}') from ckan_cloud_operator.providers.routers import manager as routers_manager if external_domains: from ckan_cloud_operator.providers.routers import manager as routers_manager external_domains_router_root_domain = routers_manager.get_default_root_domain( ) env_id = routers_manager.get_env_id() assert router_name.startswith( 'prod-'), f'invalid external domains router name: {router_name}' external_domains_router_sub_domain = f'cc-{env_id}-{router_name}' routers_manager.update_dns_record(dns_provider, external_domains_router_sub_domain, external_domains_router_root_domain, load_balancer_ip, cloudflare_email, cloudflare_auth_key) else: for root_domain, sub_domains in domains.items(): for sub_domain in sub_domains: routers_manager.update_dns_record(dns_provider, sub_domain, root_domain, load_balancer_ip, cloudflare_email, cloudflare_auth_key) kubectl.apply( kubectl.get_deployment( resource_name, get_labels(router_name, router_type, for_deployment=True), _get_deployment_spec( router_name, router_type, annotations, image=('traefik:1.7' if (external_domains or len(httpauth_secrets) > 0) else None), httpauth_secrets=httpauth_secrets, dns_provider=dns_provider)))
def update_service_account(service_account_name, labels, namespace=None): service_account = kubectl.get_resource('v1', 'ServiceAccount', service_account_name, labels) if namespace: service_account['metadata']['namespace'] = namespace kubectl.apply(service_account)
def _update(router_name, spec, annotations, routes): resource_name = _get_resource_name(router_name) router_type = spec['type'] cloudflare_email, cloudflare_auth_key = get_cloudflare_credentials() external_domains = spec.get('external-domains') kubectl.apply( kubectl.get_configmap( resource_name, get_labels(router_name, router_type), { 'traefik.toml': toml.dumps( traefik_router_config.get( routes, cloudflare_email, wildcard_ssl_domain=spec.get('wildcard-ssl-domain'), external_domains=external_domains)) })) domains = {} for route in routes: root_domain, sub_domain = routes_manager.get_domain_parts(route) domains.setdefault(root_domain, []).append(sub_domain) routes_manager.pre_deployment_hook( route, get_labels(router_name, router_type)) load_balancer = kubectl.get_resource('v1', 'Service', f'loadbalancer-{resource_name}', get_labels(router_name, router_type)) load_balancer['spec'] = { 'ports': [ { 'name': '80', 'port': 80 }, { 'name': '443', 'port': 443 }, ], 'selector': { 'app': get_labels(router_name, router_type, for_deployment=True)['app'] }, 'type': 'LoadBalancer' } kubectl.apply(load_balancer) load_balancer_ip = get_load_balancer_ip(router_name) print(f'load balancer ip: {load_balancer_ip}') if external_domains: from ckan_cloud_operator.providers.routers import manager as routers_manager external_domains_router_root_domain = routers_manager.get_default_root_domain( ) env_id = routers_manager.get_env_id() assert router_name.startswith( 'prod-'), f'invalid external domains router name: {router_name}' external_domains_router_sub_domain = f'cc-{env_id}-{router_name}' cloudflare.update_a_record( cloudflare_email, cloudflare_auth_key, external_domains_router_root_domain, f'{external_domains_router_sub_domain}.{external_domains_router_root_domain}', load_balancer_ip) else: for root_domain, sub_domains in domains.items(): for sub_domain in sub_domains: cloudflare.update_a_record(cloudflare_email, cloudflare_auth_key, root_domain, f'{sub_domain}.{root_domain}', load_balancer_ip) kubectl.apply( kubectl.get_deployment( resource_name, get_labels(router_name, router_type, for_deployment=True), _get_deployment_spec( router_name, router_type, annotations, image='traefik:1.7' if external_domains else None)))