def get_route_frontend_hostname(route): frontend_hostname = routes_manager.get_frontend_hostname(route) if frontend_hostname.endswith('.default'): return frontend_hostname.replace('.default', '.' + get_default_root_domain()) else: return frontend_hostname
def get_domain_routes(root_domain=None, sub_domain=None): if not root_domain or root_domain == get_default_root_domain(): root_domain = 'default' assert sub_domain or root_domain != 'default', 'cannot delete all routes from default root domain' labels = {'ckan-cloud/route-root-domain': root_domain} if sub_domain: labels['ckan-cloud/route-sub-domain']: sub_domain return kubectl.get_items_by_labels('CkanCloudRoute', labels, required=False)
def get_dns_data(router_name, router, failfast=False): external_domains = router['spec'].get('external-domains') data = { 'load-balancer-ip': get_load_balancer_ip(router_name, failfast=failfast), } if external_domains: from ckan_cloud_operator.providers.routers import manager as routers_manager external_domains_router_root_domain = routers_manager.get_default_root_domain() env_id = routers_manager.get_env_id() assert router_name.startswith('prod-'), f'invalid external domains router name: {router_name}' external_domains_router_sub_domain = f'cc-{env_id}-{router_name}' data['external-domain'] = f'{external_domains_router_sub_domain}.{external_domains_router_root_domain}' return data
def get_datapusher_url(instance_datapusher_url): if instance_datapusher_url and len(instance_datapusher_url) > 10: hostname = urlparse(instance_datapusher_url).hostname if hostname.endswith('.l3.ckan.io'): datapusher_name = hostname.replace('.l3.ckan.io', '') elif hostname.endswith('.ckan.io'): datapusher_name = hostname.replace('.ckan.io', '') else: logs.warning( f'failed to parse datapusher url from instance datapusher url: {instance_datapusher_url}' ) datapusher_name = None if datapusher_name: routes = kubectl.get( f'CkanCloudRoute -l ckan-cloud/route-datapusher-name={datapusher_name},ckan-cloud/route-type=datapusher-subdomain', required=False) if routes: routes = routes.get('items', []) if len(routes) > 0: assert len(routes) == 1 route = routes[0] sub_domain = route['spec']['sub-domain'] root_domain = route['spec']['root-domain'] assert sub_domain and sub_domain != 'default', f'invalid sub_domain: {sub_domain}' if not root_domain or root_domain == 'default': default_root_domain = routers_manager.get_default_root_domain( ) assert default_root_domain, 'missing routers default root domain' root_domain = default_root_domain return 'https://{}.{}/'.format(sub_domain, root_domain) else: logs.warning( f'failed to find route for datapusher: {datapusher_name}' ) else: logs.warning( f'failed to find route for datapusher: {datapusher_name}') return None
def _update(router_name, spec, annotations, routes): resource_name = _get_resource_name(router_name) router_type = spec['type'] cloudflare_email, cloudflare_auth_key = get_cloudflare_credentials() external_domains = spec.get('external-domains') dns_provider = spec.get('dns-provider', 'cloudflare') logs.info('updating traefik deployment', resource_name=resource_name, router_type=router_type, cloudflare_email=cloudflare_email, cloudflare_auth_key_len=len(cloudflare_auth_key) if cloudflare_auth_key else 0, external_domains=external_domains, dns_provider=dns_provider) kubectl.apply( kubectl.get_configmap( resource_name, get_labels(router_name, router_type), { 'traefik.toml': toml.dumps( traefik_router_config.get( routes, cloudflare_email, enable_access_log=bool(spec.get('enable-access-log')), wildcard_ssl_domain=spec.get('wildcard-ssl-domain'), external_domains=external_domains, dns_provider=dns_provider, force=True)) })) domains = {} httpauth_secrets = [] for route in routes: root_domain, sub_domain = routes_manager.get_domain_parts(route) domains.setdefault(root_domain, []).append(sub_domain) routes_manager.pre_deployment_hook( route, get_labels(router_name, router_type)) if route['spec'].get('httpauth-secret') and route['spec'][ 'httpauth-secret'] not in httpauth_secrets: httpauth_secrets.append(route['spec']['httpauth-secret']) load_balancer = kubectl.get_resource('v1', 'Service', f'loadbalancer-{resource_name}', get_labels(router_name, router_type)) load_balancer['spec'] = { 'ports': [ { 'name': '80', 'port': 80 }, { 'name': '443', 'port': 443 }, ], 'selector': { 'app': get_labels(router_name, router_type, for_deployment=True)['app'] }, 'type': 'LoadBalancer' } kubectl.apply(load_balancer) load_balancer_ip = get_load_balancer_ip(router_name) print(f'load balancer ip: {load_balancer_ip}') from ckan_cloud_operator.providers.routers import manager as routers_manager if external_domains: from ckan_cloud_operator.providers.routers import manager as routers_manager external_domains_router_root_domain = routers_manager.get_default_root_domain( ) env_id = routers_manager.get_env_id() assert router_name.startswith( 'prod-'), f'invalid external domains router name: {router_name}' external_domains_router_sub_domain = f'cc-{env_id}-{router_name}' routers_manager.update_dns_record(dns_provider, external_domains_router_sub_domain, external_domains_router_root_domain, load_balancer_ip, cloudflare_email, cloudflare_auth_key) else: for root_domain, sub_domains in domains.items(): for sub_domain in sub_domains: routers_manager.update_dns_record(dns_provider, sub_domain, root_domain, load_balancer_ip, cloudflare_email, cloudflare_auth_key) kubectl.apply( kubectl.get_deployment( resource_name, get_labels(router_name, router_type, for_deployment=True), _get_deployment_spec( router_name, router_type, annotations, image=('traefik:1.7' if (external_domains or len(httpauth_secrets) > 0) else None), httpauth_secrets=httpauth_secrets, dns_provider=dns_provider)))
def migrate_from_deis(old_site_id, new_instance_id, router_name, deis_instance_class, skip_gitlab=False, db_migration_name=None, recreate=False, skip_routes=False, skip_solr=False, skip_deployment=False, no_db_proxy=False): assert db_migration_name, 'migration without a db migration is not supported yet' log_labels = {'instance': new_instance_id} if recreate: from ckan_cloud_operator.deis_ckan.instance import DeisCkanInstance DeisCkanInstance(new_instance_id).delete( force=True, wait_deleted=not db_migration_name) logs.info( f'Migrating from old site id {old_site_id} to new instance id {new_instance_id}', **log_labels) instance_kind = ckan_manager.instance_kind() values = kubectl.get(f'{instance_kind} {new_instance_id}', required=False) if values: logs.info('instance already exists', **log_labels) else: logs.info('creating instance', **log_labels) path_to_old_cluster_kubeconfig = get_path_to_old_cluster_kubeconfig() solr_config = get_solr_config(old_site_id, path_to_old_cluster_kubeconfig) assert solr_config, 'failed to get solr config name' instance_env = get_instance_env(old_site_id, path_to_old_cluster_kubeconfig) gitlab_repo = f'viderum/cloud-{old_site_id}' if not skip_gitlab: CkanGitlab().initialize(gitlab_repo) gitlab_registry = f'registry.gitlab.com/{gitlab_repo}' old_bucket_name = instance_env.get( 'CKANEXT__S3FILESTORE__AWS_BUCKET_NAME') old_storage_path = instance_env.get( 'CKANEXT__S3FILESTORE__AWS_STORAGE_PATH') assert old_bucket_name == 'ckan' assert old_storage_path and len(old_storage_path) > 1 storage_path = f'/ckan/{old_storage_path}' deis_instance_class.create('from-gcloud-envvars', instance_env, gitlab_registry, solr_config, storage_path, new_instance_id, db_migration_name=db_migration_name) routers_env_id = routers_provider.get_env_id() default_root_domain = routers_provider.get_default_root_domain() assert routers_env_id and default_root_domain ckan_site_url = f'https://cc-{routers_env_id}-{new_instance_id}.{default_root_domain}' logs.info( f'updating instance and setting ckan site url to {ckan_site_url}', **log_labels) deis_instance_class(new_instance_id, override_spec={ 'envvars': { 'CKAN_SITE_URL': ckan_site_url }, **({ 'db': { 'no-db-proxy': 'yes' }, 'datastore': { 'no-db-proxy': 'yes' } } if no_db_proxy else {}) }, persist_overrides=True).update( wait_ready=True, skip_solr=skip_solr, skip_deployment=skip_deployment) if routers_manager.get_deis_instance_routes(new_instance_id): logs.info('default instance route already exists', **log_labels) else: logs.info('creating instance route', **log_labels) routers_manager.create_subdomain_route( router_name, { 'target-type': 'deis-instance', 'deis-instance-id': new_instance_id, 'root-domain': 'default', 'sub-domain': 'default' }) if not skip_routes: routers_manager.update(router_name, wait_ready=True) if not skip_solr: logs.info('Rebuilding solr search index', **log_labels) deis_instance_class(new_instance_id).ckan.paster( 'search-index rebuild --force')
def _update(router_name, spec, annotations, routes): resource_name = _get_resource_name(router_name) router_type = spec['type'] cloudflare_email, cloudflare_auth_key = get_cloudflare_credentials() external_domains = spec.get('external-domains') kubectl.apply( kubectl.get_configmap( resource_name, get_labels(router_name, router_type), { 'traefik.toml': toml.dumps( traefik_router_config.get( routes, cloudflare_email, wildcard_ssl_domain=spec.get('wildcard-ssl-domain'), external_domains=external_domains)) })) domains = {} for route in routes: root_domain, sub_domain = routes_manager.get_domain_parts(route) domains.setdefault(root_domain, []).append(sub_domain) routes_manager.pre_deployment_hook( route, get_labels(router_name, router_type)) load_balancer = kubectl.get_resource('v1', 'Service', f'loadbalancer-{resource_name}', get_labels(router_name, router_type)) load_balancer['spec'] = { 'ports': [ { 'name': '80', 'port': 80 }, { 'name': '443', 'port': 443 }, ], 'selector': { 'app': get_labels(router_name, router_type, for_deployment=True)['app'] }, 'type': 'LoadBalancer' } kubectl.apply(load_balancer) load_balancer_ip = get_load_balancer_ip(router_name) print(f'load balancer ip: {load_balancer_ip}') if external_domains: from ckan_cloud_operator.providers.routers import manager as routers_manager external_domains_router_root_domain = routers_manager.get_default_root_domain( ) env_id = routers_manager.get_env_id() assert router_name.startswith( 'prod-'), f'invalid external domains router name: {router_name}' external_domains_router_sub_domain = f'cc-{env_id}-{router_name}' cloudflare.update_a_record( cloudflare_email, cloudflare_auth_key, external_domains_router_root_domain, f'{external_domains_router_sub_domain}.{external_domains_router_root_domain}', load_balancer_ip) else: for root_domain, sub_domains in domains.items(): for sub_domain in sub_domains: cloudflare.update_a_record(cloudflare_email, cloudflare_auth_key, root_domain, f'{sub_domain}.{root_domain}', load_balancer_ip) kubectl.apply( kubectl.get_deployment( resource_name, get_labels(router_name, router_type, for_deployment=True), _get_deployment_spec( router_name, router_type, annotations, image='traefik:1.7' if external_domains else None)))
def get_default_root_domain(): from ckan_cloud_operator.providers.routers import manager as routers_manager default_root_domain = routers_manager.get_default_root_domain() assert default_root_domain, 'missing ckan-infra ROUTERS_DEFAULT_ROOT_DOMAIN' return default_root_domain