def _validate_sub_root_domain(sub_domain, root_domain): env_id = get_env_id() assert env_id, 'missing env id value' assert env_id == 'p' or root_domain == 'default', 'non-default root domain is not allowed for non production environmnets' assert env_id == 'p' or sub_domain.startswith( f'cc-{env_id}-' ), 'non-default sub-domains are not allowed for non production environments'
def update(self, site_url): from ckan_cloud_operator.providers.routers import manager as routers_manager env_id = routers_manager.get_env_id() website_name = f'{env_id}-{self.instance_id}' data = { "WebsiteName": website_name, "WebsiteURL": site_url, "CheckRate": 300, "TestType": "HTTP", "ContactGroup": self.statuscake_group } test_id = self.get_test_id(website_name) if test_id is None: try: self.insert_test(data) except Exception as e: print('Failed to create StatusCake test, skipping') print(e) else: data['TestID'] = test_id try: self.update_test(data) except Exception as e: print('Failed to update StatusCake test, skipping') print(e)
def _get_default_sub_root_domain(sub_domain, root_domain, default_sub_domain_suffix): if not root_domain or root_domain == 'default': root_domain = 'default' if not sub_domain or sub_domain == 'default': env_id = get_env_id() assert env_id, 'missing env id value' sub_domain = f'cc-{env_id}-{default_sub_domain_suffix}' _validate_sub_root_domain(sub_domain, root_domain) return sub_domain, root_domain
def get_dns_data(router_name, router, failfast=False): external_domains = router['spec'].get('external-domains') data = { 'load-balancer-ip': get_load_balancer_ip(router_name, failfast=failfast), } if external_domains: from ckan_cloud_operator.providers.routers import manager as routers_manager external_domains_router_root_domain = routers_manager.get_default_root_domain() env_id = routers_manager.get_env_id() assert router_name.startswith('prod-'), f'invalid external domains router name: {router_name}' external_domains_router_sub_domain = f'cc-{env_id}-{router_name}' data['external-domain'] = f'{external_domains_router_sub_domain}.{external_domains_router_root_domain}' return data
def _update(router_name, spec, annotations, routes): resource_name = _get_resource_name(router_name) router_type = spec['type'] cloudflare_email, cloudflare_auth_key = get_cloudflare_credentials() external_domains = spec.get('external-domains') dns_provider = spec.get('dns-provider', 'cloudflare') logs.info('updating traefik deployment', resource_name=resource_name, router_type=router_type, cloudflare_email=cloudflare_email, cloudflare_auth_key_len=len(cloudflare_auth_key) if cloudflare_auth_key else 0, external_domains=external_domains, dns_provider=dns_provider) kubectl.apply( kubectl.get_configmap( resource_name, get_labels(router_name, router_type), { 'traefik.toml': toml.dumps( traefik_router_config.get( routes, cloudflare_email, enable_access_log=bool(spec.get('enable-access-log')), wildcard_ssl_domain=spec.get('wildcard-ssl-domain'), external_domains=external_domains, dns_provider=dns_provider, force=True)) })) domains = {} httpauth_secrets = [] for route in routes: root_domain, sub_domain = routes_manager.get_domain_parts(route) domains.setdefault(root_domain, []).append(sub_domain) routes_manager.pre_deployment_hook( route, get_labels(router_name, router_type)) if route['spec'].get('httpauth-secret') and route['spec'][ 'httpauth-secret'] not in httpauth_secrets: httpauth_secrets.append(route['spec']['httpauth-secret']) load_balancer = kubectl.get_resource('v1', 'Service', f'loadbalancer-{resource_name}', get_labels(router_name, router_type)) load_balancer['spec'] = { 'ports': [ { 'name': '80', 'port': 80 }, { 'name': '443', 'port': 443 }, ], 'selector': { 'app': get_labels(router_name, router_type, for_deployment=True)['app'] }, 'type': 'LoadBalancer' } kubectl.apply(load_balancer) load_balancer_ip = get_load_balancer_ip(router_name) print(f'load balancer ip: {load_balancer_ip}') from ckan_cloud_operator.providers.routers import manager as routers_manager if external_domains: from ckan_cloud_operator.providers.routers import manager as routers_manager external_domains_router_root_domain = routers_manager.get_default_root_domain( ) env_id = routers_manager.get_env_id() assert router_name.startswith( 'prod-'), f'invalid external domains router name: {router_name}' external_domains_router_sub_domain = f'cc-{env_id}-{router_name}' routers_manager.update_dns_record(dns_provider, external_domains_router_sub_domain, external_domains_router_root_domain, load_balancer_ip, cloudflare_email, cloudflare_auth_key) else: for root_domain, sub_domains in domains.items(): for sub_domain in sub_domains: routers_manager.update_dns_record(dns_provider, sub_domain, root_domain, load_balancer_ip, cloudflare_email, cloudflare_auth_key) kubectl.apply( kubectl.get_deployment( resource_name, get_labels(router_name, router_type, for_deployment=True), _get_deployment_spec( router_name, router_type, annotations, image=('traefik:1.7' if (external_domains or len(httpauth_secrets) > 0) else None), httpauth_secrets=httpauth_secrets, dns_provider=dns_provider)))
def migrate_from_deis(old_site_id, new_instance_id, router_name, deis_instance_class, skip_gitlab=False, db_migration_name=None, recreate=False, skip_routes=False, skip_solr=False, skip_deployment=False, no_db_proxy=False): assert db_migration_name, 'migration without a db migration is not supported yet' log_labels = {'instance': new_instance_id} if recreate: from ckan_cloud_operator.deis_ckan.instance import DeisCkanInstance DeisCkanInstance(new_instance_id).delete( force=True, wait_deleted=not db_migration_name) logs.info( f'Migrating from old site id {old_site_id} to new instance id {new_instance_id}', **log_labels) instance_kind = ckan_manager.instance_kind() values = kubectl.get(f'{instance_kind} {new_instance_id}', required=False) if values: logs.info('instance already exists', **log_labels) else: logs.info('creating instance', **log_labels) path_to_old_cluster_kubeconfig = get_path_to_old_cluster_kubeconfig() solr_config = get_solr_config(old_site_id, path_to_old_cluster_kubeconfig) assert solr_config, 'failed to get solr config name' instance_env = get_instance_env(old_site_id, path_to_old_cluster_kubeconfig) gitlab_repo = f'viderum/cloud-{old_site_id}' if not skip_gitlab: CkanGitlab().initialize(gitlab_repo) gitlab_registry = f'registry.gitlab.com/{gitlab_repo}' old_bucket_name = instance_env.get( 'CKANEXT__S3FILESTORE__AWS_BUCKET_NAME') old_storage_path = instance_env.get( 'CKANEXT__S3FILESTORE__AWS_STORAGE_PATH') assert old_bucket_name == 'ckan' assert old_storage_path and len(old_storage_path) > 1 storage_path = f'/ckan/{old_storage_path}' deis_instance_class.create('from-gcloud-envvars', instance_env, gitlab_registry, solr_config, storage_path, new_instance_id, db_migration_name=db_migration_name) routers_env_id = routers_provider.get_env_id() default_root_domain = routers_provider.get_default_root_domain() assert routers_env_id and default_root_domain ckan_site_url = f'https://cc-{routers_env_id}-{new_instance_id}.{default_root_domain}' logs.info( f'updating instance and setting ckan site url to {ckan_site_url}', **log_labels) deis_instance_class(new_instance_id, override_spec={ 'envvars': { 'CKAN_SITE_URL': ckan_site_url }, **({ 'db': { 'no-db-proxy': 'yes' }, 'datastore': { 'no-db-proxy': 'yes' } } if no_db_proxy else {}) }, persist_overrides=True).update( wait_ready=True, skip_solr=skip_solr, skip_deployment=skip_deployment) if routers_manager.get_deis_instance_routes(new_instance_id): logs.info('default instance route already exists', **log_labels) else: logs.info('creating instance route', **log_labels) routers_manager.create_subdomain_route( router_name, { 'target-type': 'deis-instance', 'deis-instance-id': new_instance_id, 'root-domain': 'default', 'sub-domain': 'default' }) if not skip_routes: routers_manager.update(router_name, wait_ready=True) if not skip_solr: logs.info('Rebuilding solr search index', **log_labels) deis_instance_class(new_instance_id).ckan.paster( 'search-index rebuild --force')
def _update(router_name, spec, annotations, routes): resource_name = _get_resource_name(router_name) router_type = spec['type'] cloudflare_email, cloudflare_auth_key = get_cloudflare_credentials() external_domains = spec.get('external-domains') kubectl.apply( kubectl.get_configmap( resource_name, get_labels(router_name, router_type), { 'traefik.toml': toml.dumps( traefik_router_config.get( routes, cloudflare_email, wildcard_ssl_domain=spec.get('wildcard-ssl-domain'), external_domains=external_domains)) })) domains = {} for route in routes: root_domain, sub_domain = routes_manager.get_domain_parts(route) domains.setdefault(root_domain, []).append(sub_domain) routes_manager.pre_deployment_hook( route, get_labels(router_name, router_type)) load_balancer = kubectl.get_resource('v1', 'Service', f'loadbalancer-{resource_name}', get_labels(router_name, router_type)) load_balancer['spec'] = { 'ports': [ { 'name': '80', 'port': 80 }, { 'name': '443', 'port': 443 }, ], 'selector': { 'app': get_labels(router_name, router_type, for_deployment=True)['app'] }, 'type': 'LoadBalancer' } kubectl.apply(load_balancer) load_balancer_ip = get_load_balancer_ip(router_name) print(f'load balancer ip: {load_balancer_ip}') if external_domains: from ckan_cloud_operator.providers.routers import manager as routers_manager external_domains_router_root_domain = routers_manager.get_default_root_domain( ) env_id = routers_manager.get_env_id() assert router_name.startswith( 'prod-'), f'invalid external domains router name: {router_name}' external_domains_router_sub_domain = f'cc-{env_id}-{router_name}' cloudflare.update_a_record( cloudflare_email, cloudflare_auth_key, external_domains_router_root_domain, f'{external_domains_router_sub_domain}.{external_domains_router_root_domain}', load_balancer_ip) else: for root_domain, sub_domains in domains.items(): for sub_domain in sub_domains: cloudflare.update_a_record(cloudflare_email, cloudflare_auth_key, root_domain, f'{sub_domain}.{root_domain}', load_balancer_ip) kubectl.apply( kubectl.get_deployment( resource_name, get_labels(router_name, router_type, for_deployment=True), _get_deployment_spec( router_name, router_type, annotations, image='traefik:1.7' if external_domains else None)))