def update(instance_id_or_name, override_spec=None, persist_overrides=False, wait_ready=False, skip_deployment=False, skip_route=False):
instance_id, instance_type, instance = _get_instance_id_and_type(instance_id_or_name)
if override_spec:
for k, v in override_spec.items():
logs.info(f'Applying override spec {k}={v}')
instance['spec'][k] = v
assert instance['spec'].get('useCentralizedInfra'), 'non-centralized instances are not supported'
# full domain to route to the instance
instance_domain = instance['spec'].get('domain')
# instance is added to router only if this is true, as all routers must use SSL and may use sans SSL too
with_sans_ssl = instance['spec'].get('withSansSSL')
# subdomain to register on the default root domain
register_subdomain = instance['spec'].get('registerSubdomain')
if persist_overrides:
logs.info('Persisting overrides')
kubectl.apply(instance)
if not skip_deployment:
deployment_manager.update(instance_id, instance_type, instance)
if wait_ready:
wait_instance_ready(instance_id_or_name)
if not skip_route:
if instance_domain:
assert with_sans_ssl, 'withSansSSL must be set to true to add routes'
assert '.'.join(instance_domain.split('.')[1:]) == routers_manager.get_default_root_domain(), f'invalid root domain ({instance_domain})'
assert instance_domain.split('.')[0] == register_subdomain, f'invalid register_subdomain ({register_subdomain})'
logs.info(f'adding instance route to {instance_domain}')
routers_manager.create_subdomain_route('instances-default', {
'target-type': 'ckan-instance',
'ckan-instance-id': instance_id,
'root-domain': routers_manager.get_default_root_domain(),
'sub-domain': register_subdomain
})
routers_manager.update('instances-default', wait_ready)
else:
assert not register_subdomain, 'subdomain registration is only supported with instance_domain'
def initialize(interactive=False):
ckan_db_migration_manager.initialize(interactive=interactive)
if config_manager.get('enable-deis-ckan',
configmap_name='global-ckan-config') == 'y':
ckan_infra = CkanInfra(required=False)
config_manager.interactive_set(
{
'deis-kubeconfig': ckan_infra.DEIS_KUBECONFIG,
},
from_file=True,
secret_name='ckan-migration-secrets',
interactive=interactive)
config_manager.interactive_set(
{
'gitlab-token': ckan_infra.GITLAB_TOKEN_PASSWORD,
},
secret_name='ckan-migration-secrets',
interactive=interactive)
config_manager.interactive_set(
{
'docker-server': ckan_infra.DOCKER_REGISTRY_SERVER,
'docker-username': ckan_infra.DOCKER_REGISTRY_USERNAME,
'docker-password': ckan_infra.DOCKER_REGISTRY_PASSWORD,
'docker-email': ckan_infra.DOCKER_REGISTRY_EMAIL,
},
secret_name='ckan-docker-registry',
interactive=interactive)
crds_manager.install_crd(INSTANCE_CRD_SINGULAR, INSTANCE_CRD_PLURAL_SUFFIX,
INSTANCE_CRD_KIND_SUFFIX)
from ckan_cloud_operator.datapushers import initialize as datapusher_initialize
datapusher_initialize()
from ckan_cloud_operator.routers import manager as routers_manager
router_name = get_default_instances_router_name()
wildcard_ssl_domain = routers_manager.get_default_root_domain()
allow_wildcard_ssl = (routers_manager.get_env_id() == 'p'
and routers_manager.get_default_root_domain()
== wildcard_ssl_domain)
router = routers_manager.get(router_name, required=False)
if router:
assert ((allow_wildcard_ssl
and router.get('wildcard-ssl-domain') == wildcard_ssl_domain)
or (not allow_wildcard_ssl
and not router.get('wildcard-ssl-domain'))
), f'invalid router wildcard ssl config: {router}'
else:
routers_manager.create(
router_name,
routers_manager.get_traefik_router_spec(
wildcard_ssl_domain=wildcard_ssl_domain))
from .storage.manager import initialize as ckan_storage_initialize
ckan_storage_initialize(interactive=interactive)
def update(instance_id_or_name, override_spec=None, persist_overrides=False, wait_ready=False, skip_deployment=False,
skip_route=False, force=False, dry_run=False):
instance_id, instance = _get_instance(instance_id_or_name, required=not dry_run)
if dry_run:
logs.info('update instance', instance_id=instance_id, instance_id_or_name=instance_id_or_name,
override_spec=override_spec, persist_overrides=persist_overrides, wait_ready=wait_ready,
skip_deployment=skip_deployment, skip_route=skip_route, force=force, dry_run=dry_run)
else:
pre_update_hook_data = deployment_manager.pre_update_hook(instance_id, instance, override_spec,
skip_route)
if persist_overrides:
logs.info('Persisting overrides')
kubectl.apply(instance)
if not skip_deployment:
deployment_manager.update(instance_id, instance)
if wait_ready:
wait_instance_ready(instance_id_or_name)
if not skip_route and pre_update_hook_data.get('sub-domain'):
root_domain = pre_update_hook_data.get('root-domain')
sub_domain = pre_update_hook_data['sub-domain']
assert root_domain == routers_manager.get_default_root_domain(), \
'invalid domain, must use default root domain'
logs.info(f'adding instance default route to {sub_domain}.{root_domain}')
routers_manager.create_subdomain_route('instances-default', {
'target-type': 'app-instance',
'app-instance-id': instance_id,
'root-domain': root_domain,
'sub-domain': sub_domain
})
routers_manager.update('instances-default', wait_ready)
else:
logs.info('skipping route creation', skip_route=skip_route,
sub_domain=pre_update_hook_data.get('sub-domain'))
logs.info('Instance is ready', instance_id=instance_id,
instance_name=(instance_id_or_name if instance_id_or_name != instance_id else None))
def create_ckan_admin_user(instance_id_or_name,
name,
email=None,
password=None,
dry_run=False):
if not email:
default_root_domain = routers_manager.get_default_root_domain()
email = f'{name}@{instance_id_or_name}.{default_root_domain}'
if not password:
password = _generate_password(8)
instance_id, instance_type, instance = _get_instance_id_and_type(
instance_id_or_name)
user = {'name': name, 'email': email, 'password': password}
if not dry_run:
deployment_manager.create_ckan_admin_user(instance_id, instance_type,
instance, user)
return {
'instance-id': instance_id,
'instance-type': instance_type,
**{f'admin-{k}': v
for k, v in user.items()},
**({
'dry-run': True
} if dry_run else {}),
}
def _pre_update_hook_route(instance_id, skip_route, instance, res, dry_run=False):
root_domain = routers_manager.get_default_root_domain()
sub_domain = f'ckan-cloud-{instance_id}'
if not skip_route:
# full domain to route to the instance
instance_domain = instance['spec'].get('domain')
if instance_domain and instance_domain != f'{sub_domain}.{root_domain}':
logs.warning(f'instance domain was changed from {instance_domain} to {sub_domain}.{root_domain}')
_pre_update_hook_modify_spec(instance_id, instance,
lambda i: i.update(domain=f'{sub_domain}.{root_domain}'),
dry_run=dry_run)
# instance is added to router only if this is true, as all routers must use SSL and may use sans SSL too
with_sans_ssl = instance['spec'].get('withSansSSL')
if not with_sans_ssl:
logs.warning(f'forcing with_sans_ssl, even though withSansSSL is disabled')
_pre_update_hook_modify_spec(instance_id, instance,
lambda i: i.update(withSansSSL=True),
dry_run=dry_run)
# subdomain to register on the default root domain
register_subdomain = instance['spec'].get('registerSubdomain')
if register_subdomain != sub_domain:
logs.warning(f'instance register sub domain was changed from {register_subdomain} to {sub_domain}')
_pre_update_hook_modify_spec(instance_id, instance,
lambda i: i.update(registerSubdomain=sub_domain),
dry_run=dry_run)
res.update(**{'root-domain': root_domain, 'sub-domain': sub_domain})
site_url = instance['spec'].get('siteUrl')
if site_url != f'https://{sub_domain}.{root_domain}':
logs.warning(f'instance siteUrl was changed from {site_url} to https://{sub_domain}.{root_domain}')
_pre_update_hook_modify_spec(instance_id, instance,
lambda i: i.update(siteUrl=f'https://{sub_domain}.{root_domain}'),
dry_run=dry_run)
return sub_domain, root_domain
def create_ckan_admin_user(instance_id_or_name, name, email=None, password=None, dry_run=False, use_paster=False):
if not email:
default_root_domain = routers_manager.get_default_root_domain()
email = f'{name}@{instance_id_or_name}.{default_root_domain}'
if not password:
password = _generate_password(8)
instance_id, instance_type, instance = _get_instance_id_and_type(instance_id_or_name)
user = {
'name': name,
'email': email,
'password': password
}
if not dry_run:
pod_name = _get_running_pod_name(instance_id)
name, password, email = [user[k] for k in ['name', 'password', 'email']]
logs.info(f'Creating CKAN admin user with {name} ({email}) on pod {pod_name}')
if use_paster:
logs.subprocess_check_call(
f'echo y | kubectl -n {instance_id} exec -i {pod_name} -- ckan-paster --plugin=ckan sysadmin -c /etc/ckan/production.ini add {name} password={password} email={email}',
shell=True
)
else:
logs.subprocess_check_call(
f'echo y | kubectl -n {instance_id} exec -i {pod_name} -- ckan --config /etc/ckan/production.ini sysadmin add {name} password={password} email={email}',
shell=True
)
return {
'instance-id': instance_id,
'instance-type': instance_type,
**{f'admin-{k}': v for k, v in user.items()},
**({'dry-run': True} if dry_run else {}),
}
def update(instance_id_or_name, override_spec=None, persist_overrides=False, wait_ready=False, skip_deployment=False,
skip_route=False, force=False, dry_run=False, skip_solr=False):
instance_id, instance_type, instance = _get_instance_id_and_type(instance_id_or_name, required=not dry_run)
if dry_run:
logs.info('update instance', instance_id=instance_id, instance_id_or_name=instance_id_or_name,
override_spec=override_spec, persist_overrides=persist_overrides, wait_ready=wait_ready,
skip_deployment=skip_deployment, skip_route=skip_route, force=force, dry_run=dry_run)
else:
pre_update_hook_data = deployment_manager.pre_update_hook(instance_id, instance_type, instance, override_spec,
skip_route)
bucket_credentials = instance['spec'].get('ckanStorageBucket', {}).get(get_storage_provider_id())
use_cloud_storage = bucket_credentials and config_manager.get('use-cloud-native-storage', secret_name=CONFIG_NAME)
if use_cloud_storage and bucket_credentials:
config_manager.set(
values=bucket_credentials,
secret_name='bucket-credentials',
namespace=instance_id
)
if persist_overrides:
logs.info('Persisting overrides')
kubectl.apply(instance)
if not skip_deployment:
deployment_manager.update(instance_id, instance_type, instance, force=force, skip_solr=skip_solr)
if wait_ready:
wait_instance_ready(instance_id_or_name)
if not skip_route and pre_update_hook_data.get('sub-domain'):
root_domain = pre_update_hook_data.get('root-domain')
sub_domain = pre_update_hook_data['sub-domain']
assert root_domain == routers_manager.get_default_root_domain(), 'invalid domain, must use default root domain'
logs.info(f'adding instance default route to {sub_domain}.{root_domain}')
routers_manager.create_subdomain_route('instances-default', {
'target-type': 'ckan-instance',
'ckan-instance-id': instance_id,
'root-domain': root_domain,
'sub-domain': sub_domain
})
logs.info(f'updating routers_manager wait_ready: {wait_ready}')
routers_manager.update('instances-default', wait_ready)
else:
logs.info('skipping route creation', skip_route=skip_route, sub_domain=pre_update_hook_data.get('sub-domain'))
logs.info('creating ckan admin')
# Need to set in values.yaml
if pre_update_hook_data.get('create-sysadmin'):
ckan_admin_email = pre_update_hook_data.get('ckan-admin-email')
ckan_admin_password = pre_update_hook_data.get('ckan-admin-password')
ckan_admin_name = pre_update_hook_data.get('ckan-admin-name', 'admin')
res = create_ckan_admin_user(instance_id, ckan_admin_name, ckan_admin_email, ckan_admin_password)
logs.info(**res)
logs.info('Instance is ready', instance_id=instance_id, instance_name=(instance_id_or_name if instance_id_or_name != instance_id else None))
def check_cluster_url(url, verify_site_url=False, deis_instance=None):
from ckan_cloud_operator.deis_ckan.ckan import DeisCkanInstanceCKAN
from ckan_cloud_operator.providers.routers import manager as routers_manager
env_id = routers_manager.get_env_id()
root_domain = routers_manager.get_default_root_domain()
assert url.startswith(f'https://cc-{env_id}-') and url.rstrip('/').endswith(root_domain), f'invalid cluster url: {url}'
if verify_site_url:
logs.info('checking site url in running instance...')
site_url = None
for line in DeisCkanInstanceCKAN(deis_instance).exec('env', check_output=True).decode().splitlines():
if line.startswith('CKAN_SITE_URL='):
site_url = line.replace('CKAN_SITE_URL=', '').strip()
assert site_url == url, f'mismatch between envvar and instance env: {site_url} != {url}'
def initialize(interactive=False):
ckan_db_migration_manager.initialize(interactive=interactive)
if config_manager.get('enable-deis-ckan',
configmap_name='global-ckan-config') == 'y':
ckan_infra = CkanInfra(required=False)
config_manager.interactive_set(
{
'deis-kubeconfig': ckan_infra.DEIS_KUBECONFIG,
},
from_file=True,
secret_name='ckan-migration-secrets',
interactive=interactive)
config_manager.interactive_set(
{
'gitlab-token': ckan_infra.GITLAB_TOKEN_PASSWORD,
},
secret_name='ckan-migration-secrets',
interactive=interactive)
config_manager.interactive_set(
{
'docker-server': ckan_infra.DOCKER_REGISTRY_SERVER,
'docker-username': ckan_infra.DOCKER_REGISTRY_USERNAME,
'docker-password': ckan_infra.DOCKER_REGISTRY_PASSWORD,
'docker-email': ckan_infra.DOCKER_REGISTRY_EMAIL,
},
secret_name='ckan-docker-registry',
interactive=interactive)
crds_manager.install_crd(INSTANCE_CRD_SINGULAR, INSTANCE_CRD_PLURAL_SUFFIX,
INSTANCE_CRD_KIND_SUFFIX)
crds_manager.install_crd(INSTANCE_NAME_CRD_SINGULAR,
INSTANCE_NAME_CRD_PLURAL_SUFFIX,
INSTANCE_NAME_CRD_KIND_SUFFIX)
from ckan_cloud_operator.providers.solr.manager import zk_list_configs, zk_put_configs
print('Checking CKAN Solr config in ZooKeeper')
if 'ckan_default' in zk_list_configs():
print('Found ckan_default Solr config')
else:
print(
'No default Solr config found. Putting CKAN 2.8 config for Solr to ZooKeeper as ckan_default...'
)
file_path = os.path.dirname(os.path.abspath(__file__))
root_path = Path(file_path).parent.parent
configs_dir = os.path.join(root_path, 'data', 'solr')
zk_put_configs(configs_dir)
if config_manager.get('disable-centralized-datapushers',
configmap_name='global-ckan-config',
required=False) != 'yes':
from ckan_cloud_operator.datapushers import initialize as datapusher_initialize
datapusher_initialize()
from ckan_cloud_operator.routers import manager as routers_manager
router_name = get_default_instances_router_name()
wildcard_ssl_domain = routers_manager.get_default_root_domain()
print(f'wildcard_ssl_domain={wildcard_ssl_domain}')
allow_wildcard_ssl = routers_manager.get_env_id() == 'p'
router = routers_manager.get(router_name, required=False)
if router:
assert ((allow_wildcard_ssl and router.get(
'spec', {}).get('wildcard-ssl-domain') == wildcard_ssl_domain)
or (not allow_wildcard_ssl and not router.get('spec', {}))
), f'invalid router wildcard ssl config: {router}'
else:
routers_manager.create(
router_name,
routers_manager.get_traefik_router_spec(
wildcard_ssl_domain=wildcard_ssl_domain))
from .storage.manager import initialize as ckan_storage_initialize
ckan_storage_initialize(interactive=interactive)
from .deployment.manager import initialize as ckan_deployment_initialize
ckan_deployment_initialize(interactive=interactive)
def update(instance_id_or_name,
override_spec=None,
persist_overrides=False,
wait_ready=False,
skip_deployment=False,
skip_route=False,
force=False,
dry_run=False):
instance_id, instance_type, instance = _get_instance_id_and_type(
instance_id_or_name, required=not dry_run)
if dry_run:
logs.info('update instance',
instance_id=instance_id,
instance_id_or_name=instance_id_or_name,
override_spec=override_spec,
persist_overrides=persist_overrides,
wait_ready=wait_ready,
skip_deployment=skip_deployment,
skip_route=skip_route,
force=force,
dry_run=dry_run)
else:
pre_update_hook_data = deployment_manager.pre_update_hook(
instance_id, instance_type, instance, override_spec, skip_route)
bucket_credentials = instance['spec'].get('ckanStorageBucket', {}).get(
get_storage_provider_id())
use_cloud_storage = bucket_credentials and config_manager.get(
'use-cloud-native-storage', secret_name=CONFIG_NAME)
if use_cloud_storage:
cluster_provider_id = cluster_manager.get_provider_id()
if bucket_credentials:
literal = []
config_manager.set(values=bucket_credentials,
secret_name='bucket-credentials',
namespace=instance_id)
if instance['spec'].get('operatorCopySecrets'):
for target_secret_name, source_secret_config in json.loads(
instance['spec']['operatorCopySecrets']).items():
for k, v in source_secret_config.items():
source_secret_config[k] = v.replace(
"__INSTANCE_NAME__", instance_id_or_name)
kubectl.update_secret(
target_secret_name,
kubectl.decode_secret(
kubectl.get('secret',
source_secret_config["fromName"],
namespace=source_secret_config.get(
"fromNamespace", "ckan-cloud"))),
namespace=instance_id)
if persist_overrides:
logs.info('Persisting overrides')
kubectl.apply(instance)
if not skip_deployment:
deployment_manager.update(instance_id,
instance_type,
instance,
force=force)
if wait_ready:
wait_instance_ready(instance_id_or_name)
if not skip_route and pre_update_hook_data.get('sub-domain'):
root_domain = pre_update_hook_data.get('root-domain')
sub_domain = pre_update_hook_data['sub-domain']
assert root_domain == routers_manager.get_default_root_domain(
), 'invalid domain, must use default root domain'
logs.info(
f'adding instance default route to {sub_domain}.{root_domain}')
routers_manager.create_subdomain_route(
'instances-default', {
'target-type': 'ckan-instance',
'ckan-instance-id': instance_id,
'root-domain': root_domain,
'sub-domain': sub_domain
})
logs.info(f'updating routers_manager wait_ready: {wait_ready}')
routers_manager.update('instances-default', wait_ready)
else:
logs.info('skipping route creation',
skip_route=skip_route,
sub_domain=pre_update_hook_data.get('sub-domain'))
if not instance['spec'].get('skipCreateCkanAdmin', False):
logs.info('creating ckan admin')
ckan_admin_email = instance['spec'].get(
'ckanAdminEmail', pre_update_hook_data.get('ckan-admin-email'))
ckan_admin_password = pre_update_hook_data.get(
'ckan-admin-password')
ckan_admin_name = instance['spec'].get(
'ckanAdminName',
pre_update_hook_data.get('ckan-admin-name', 'admin'))
res = create_ckan_admin_user(instance_id, ckan_admin_name,
ckan_admin_email, ckan_admin_password)
logs.info(**res)
logs.info('Instance is ready',
instance_id=instance_id,
instance_name=(instance_id_or_name if
instance_id_or_name != instance_id else None))
