def get_route_frontend_hostname(route):
frontend_hostname = routes_manager.get_frontend_hostname(route)
if frontend_hostname.endswith('.default'):
return frontend_hostname.replace('.default',
'.' + get_default_root_domain())
else:
return frontend_hostname
def get_domain_routes(root_domain=None, sub_domain=None):
if not root_domain or root_domain == get_default_root_domain():
root_domain = 'default'
assert sub_domain or root_domain != 'default', 'cannot delete all routes from default root domain'
labels = {'ckan-cloud/route-root-domain': root_domain}
if sub_domain:
labels['ckan-cloud/route-sub-domain']: sub_domain
return kubectl.get_items_by_labels('CkanCloudRoute', labels, required=False)
def get_dns_data(router_name, router, failfast=False):
external_domains = router['spec'].get('external-domains')
data = {
'load-balancer-ip': get_load_balancer_ip(router_name, failfast=failfast),
}
if external_domains:
from ckan_cloud_operator.providers.routers import manager as routers_manager
external_domains_router_root_domain = routers_manager.get_default_root_domain()
env_id = routers_manager.get_env_id()
assert router_name.startswith('prod-'), f'invalid external domains router name: {router_name}'
external_domains_router_sub_domain = f'cc-{env_id}-{router_name}'
data['external-domain'] = f'{external_domains_router_sub_domain}.{external_domains_router_root_domain}'
return data
def get_datapusher_url(instance_datapusher_url):
if instance_datapusher_url and len(instance_datapusher_url) > 10:
hostname = urlparse(instance_datapusher_url).hostname
if hostname.endswith('.l3.ckan.io'):
datapusher_name = hostname.replace('.l3.ckan.io', '')
elif hostname.endswith('.ckan.io'):
datapusher_name = hostname.replace('.ckan.io', '')
else:
logs.warning(
f'failed to parse datapusher url from instance datapusher url: {instance_datapusher_url}'
)
datapusher_name = None
if datapusher_name:
routes = kubectl.get(
f'CkanCloudRoute -l ckan-cloud/route-datapusher-name={datapusher_name},ckan-cloud/route-type=datapusher-subdomain',
required=False)
if routes:
routes = routes.get('items', [])
if len(routes) > 0:
assert len(routes) == 1
route = routes[0]
sub_domain = route['spec']['sub-domain']
root_domain = route['spec']['root-domain']
assert sub_domain and sub_domain != 'default', f'invalid sub_domain: {sub_domain}'
if not root_domain or root_domain == 'default':
default_root_domain = routers_manager.get_default_root_domain(
)
assert default_root_domain, 'missing routers default root domain'
root_domain = default_root_domain
return 'https://{}.{}/'.format(sub_domain, root_domain)
else:
logs.warning(
f'failed to find route for datapusher: {datapusher_name}'
)
else:
logs.warning(
f'failed to find route for datapusher: {datapusher_name}')
return None
def _update(router_name, spec, annotations, routes):
resource_name = _get_resource_name(router_name)
router_type = spec['type']
cloudflare_email, cloudflare_auth_key = get_cloudflare_credentials()
external_domains = spec.get('external-domains')
dns_provider = spec.get('dns-provider', 'cloudflare')
logs.info('updating traefik deployment',
resource_name=resource_name,
router_type=router_type,
cloudflare_email=cloudflare_email,
cloudflare_auth_key_len=len(cloudflare_auth_key)
if cloudflare_auth_key else 0,
external_domains=external_domains,
dns_provider=dns_provider)
kubectl.apply(
kubectl.get_configmap(
resource_name, get_labels(router_name, router_type), {
'traefik.toml':
toml.dumps(
traefik_router_config.get(
routes,
cloudflare_email,
enable_access_log=bool(spec.get('enable-access-log')),
wildcard_ssl_domain=spec.get('wildcard-ssl-domain'),
external_domains=external_domains,
dns_provider=dns_provider,
force=True))
}))
domains = {}
httpauth_secrets = []
for route in routes:
root_domain, sub_domain = routes_manager.get_domain_parts(route)
domains.setdefault(root_domain, []).append(sub_domain)
routes_manager.pre_deployment_hook(
route, get_labels(router_name, router_type))
if route['spec'].get('httpauth-secret') and route['spec'][
'httpauth-secret'] not in httpauth_secrets:
httpauth_secrets.append(route['spec']['httpauth-secret'])
load_balancer = kubectl.get_resource('v1', 'Service',
f'loadbalancer-{resource_name}',
get_labels(router_name, router_type))
load_balancer['spec'] = {
'ports': [
{
'name': '80',
'port': 80
},
{
'name': '443',
'port': 443
},
],
'selector': {
'app': get_labels(router_name, router_type,
for_deployment=True)['app']
},
'type': 'LoadBalancer'
}
kubectl.apply(load_balancer)
load_balancer_ip = get_load_balancer_ip(router_name)
print(f'load balancer ip: {load_balancer_ip}')
from ckan_cloud_operator.providers.routers import manager as routers_manager
if external_domains:
from ckan_cloud_operator.providers.routers import manager as routers_manager
external_domains_router_root_domain = routers_manager.get_default_root_domain(
)
env_id = routers_manager.get_env_id()
assert router_name.startswith(
'prod-'), f'invalid external domains router name: {router_name}'
external_domains_router_sub_domain = f'cc-{env_id}-{router_name}'
routers_manager.update_dns_record(dns_provider,
external_domains_router_sub_domain,
external_domains_router_root_domain,
load_balancer_ip, cloudflare_email,
cloudflare_auth_key)
else:
for root_domain, sub_domains in domains.items():
for sub_domain in sub_domains:
routers_manager.update_dns_record(dns_provider, sub_domain,
root_domain,
load_balancer_ip,
cloudflare_email,
cloudflare_auth_key)
kubectl.apply(
kubectl.get_deployment(
resource_name,
get_labels(router_name, router_type, for_deployment=True),
_get_deployment_spec(
router_name,
router_type,
annotations,
image=('traefik:1.7' if
(external_domains
or len(httpauth_secrets) > 0) else None),
httpauth_secrets=httpauth_secrets,
dns_provider=dns_provider)))
def migrate_from_deis(old_site_id,
new_instance_id,
router_name,
deis_instance_class,
skip_gitlab=False,
db_migration_name=None,
recreate=False,
skip_routes=False,
skip_solr=False,
skip_deployment=False,
no_db_proxy=False):
assert db_migration_name, 'migration without a db migration is not supported yet'
log_labels = {'instance': new_instance_id}
if recreate:
from ckan_cloud_operator.deis_ckan.instance import DeisCkanInstance
DeisCkanInstance(new_instance_id).delete(
force=True, wait_deleted=not db_migration_name)
logs.info(
f'Migrating from old site id {old_site_id} to new instance id {new_instance_id}',
**log_labels)
instance_kind = ckan_manager.instance_kind()
values = kubectl.get(f'{instance_kind} {new_instance_id}', required=False)
if values:
logs.info('instance already exists', **log_labels)
else:
logs.info('creating instance', **log_labels)
path_to_old_cluster_kubeconfig = get_path_to_old_cluster_kubeconfig()
solr_config = get_solr_config(old_site_id,
path_to_old_cluster_kubeconfig)
assert solr_config, 'failed to get solr config name'
instance_env = get_instance_env(old_site_id,
path_to_old_cluster_kubeconfig)
gitlab_repo = f'viderum/cloud-{old_site_id}'
if not skip_gitlab:
CkanGitlab().initialize(gitlab_repo)
gitlab_registry = f'registry.gitlab.com/{gitlab_repo}'
old_bucket_name = instance_env.get(
'CKANEXT__S3FILESTORE__AWS_BUCKET_NAME')
old_storage_path = instance_env.get(
'CKANEXT__S3FILESTORE__AWS_STORAGE_PATH')
assert old_bucket_name == 'ckan'
assert old_storage_path and len(old_storage_path) > 1
storage_path = f'/ckan/{old_storage_path}'
deis_instance_class.create('from-gcloud-envvars',
instance_env,
gitlab_registry,
solr_config,
storage_path,
new_instance_id,
db_migration_name=db_migration_name)
routers_env_id = routers_provider.get_env_id()
default_root_domain = routers_provider.get_default_root_domain()
assert routers_env_id and default_root_domain
ckan_site_url = f'https://cc-{routers_env_id}-{new_instance_id}.{default_root_domain}'
logs.info(
f'updating instance and setting ckan site url to {ckan_site_url}',
**log_labels)
deis_instance_class(new_instance_id,
override_spec={
'envvars': {
'CKAN_SITE_URL': ckan_site_url
},
**({
'db': {
'no-db-proxy': 'yes'
},
'datastore': {
'no-db-proxy': 'yes'
}
} if no_db_proxy else {})
},
persist_overrides=True).update(
wait_ready=True,
skip_solr=skip_solr,
skip_deployment=skip_deployment)
if routers_manager.get_deis_instance_routes(new_instance_id):
logs.info('default instance route already exists', **log_labels)
else:
logs.info('creating instance route', **log_labels)
routers_manager.create_subdomain_route(
router_name, {
'target-type': 'deis-instance',
'deis-instance-id': new_instance_id,
'root-domain': 'default',
'sub-domain': 'default'
})
if not skip_routes:
routers_manager.update(router_name, wait_ready=True)
if not skip_solr:
logs.info('Rebuilding solr search index', **log_labels)
deis_instance_class(new_instance_id).ckan.paster(
'search-index rebuild --force')
def _update(router_name, spec, annotations, routes):
resource_name = _get_resource_name(router_name)
router_type = spec['type']
cloudflare_email, cloudflare_auth_key = get_cloudflare_credentials()
external_domains = spec.get('external-domains')
kubectl.apply(
kubectl.get_configmap(
resource_name, get_labels(router_name, router_type), {
'traefik.toml':
toml.dumps(
traefik_router_config.get(
routes,
cloudflare_email,
wildcard_ssl_domain=spec.get('wildcard-ssl-domain'),
external_domains=external_domains))
}))
domains = {}
for route in routes:
root_domain, sub_domain = routes_manager.get_domain_parts(route)
domains.setdefault(root_domain, []).append(sub_domain)
routes_manager.pre_deployment_hook(
route, get_labels(router_name, router_type))
load_balancer = kubectl.get_resource('v1', 'Service',
f'loadbalancer-{resource_name}',
get_labels(router_name, router_type))
load_balancer['spec'] = {
'ports': [
{
'name': '80',
'port': 80
},
{
'name': '443',
'port': 443
},
],
'selector': {
'app': get_labels(router_name, router_type,
for_deployment=True)['app']
},
'type': 'LoadBalancer'
}
kubectl.apply(load_balancer)
load_balancer_ip = get_load_balancer_ip(router_name)
print(f'load balancer ip: {load_balancer_ip}')
if external_domains:
from ckan_cloud_operator.providers.routers import manager as routers_manager
external_domains_router_root_domain = routers_manager.get_default_root_domain(
)
env_id = routers_manager.get_env_id()
assert router_name.startswith(
'prod-'), f'invalid external domains router name: {router_name}'
external_domains_router_sub_domain = f'cc-{env_id}-{router_name}'
cloudflare.update_a_record(
cloudflare_email, cloudflare_auth_key,
external_domains_router_root_domain,
f'{external_domains_router_sub_domain}.{external_domains_router_root_domain}',
load_balancer_ip)
else:
for root_domain, sub_domains in domains.items():
for sub_domain in sub_domains:
cloudflare.update_a_record(cloudflare_email,
cloudflare_auth_key, root_domain,
f'{sub_domain}.{root_domain}',
load_balancer_ip)
kubectl.apply(
kubectl.get_deployment(
resource_name,
get_labels(router_name, router_type, for_deployment=True),
_get_deployment_spec(
router_name,
router_type,
annotations,
image='traefik:1.7' if external_domains else None)))
def get_default_root_domain():
from ckan_cloud_operator.providers.routers import manager as routers_manager
default_root_domain = routers_manager.get_default_root_domain()
assert default_root_domain, 'missing ckan-infra ROUTERS_DEFAULT_ROOT_DOMAIN'
return default_root_domain
