def update(instance_id_or_name, override_spec=None, persist_overrides=False, wait_ready=False, skip_deployment=False, skip_route=False): instance_id, instance_type, instance = _get_instance_id_and_type(instance_id_or_name) if override_spec: for k, v in override_spec.items(): logs.info(f'Applying override spec {k}={v}') instance['spec'][k] = v assert instance['spec'].get('useCentralizedInfra'), 'non-centralized instances are not supported' # full domain to route to the instance instance_domain = instance['spec'].get('domain') # instance is added to router only if this is true, as all routers must use SSL and may use sans SSL too with_sans_ssl = instance['spec'].get('withSansSSL') # subdomain to register on the default root domain register_subdomain = instance['spec'].get('registerSubdomain') if persist_overrides: logs.info('Persisting overrides') kubectl.apply(instance) if not skip_deployment: deployment_manager.update(instance_id, instance_type, instance) if wait_ready: wait_instance_ready(instance_id_or_name) if not skip_route: if instance_domain: assert with_sans_ssl, 'withSansSSL must be set to true to add routes' assert '.'.join(instance_domain.split('.')[1:]) == routers_manager.get_default_root_domain(), f'invalid root domain ({instance_domain})' assert instance_domain.split('.')[0] == register_subdomain, f'invalid register_subdomain ({register_subdomain})' logs.info(f'adding instance route to {instance_domain}') routers_manager.create_subdomain_route('instances-default', { 'target-type': 'ckan-instance', 'ckan-instance-id': instance_id, 'root-domain': routers_manager.get_default_root_domain(), 'sub-domain': register_subdomain }) routers_manager.update('instances-default', wait_ready) else: assert not register_subdomain, 'subdomain registration is only supported with instance_domain'
def initialize(interactive=False): ckan_db_migration_manager.initialize(interactive=interactive) if config_manager.get('enable-deis-ckan', configmap_name='global-ckan-config') == 'y': ckan_infra = CkanInfra(required=False) config_manager.interactive_set( { 'deis-kubeconfig': ckan_infra.DEIS_KUBECONFIG, }, from_file=True, secret_name='ckan-migration-secrets', interactive=interactive) config_manager.interactive_set( { 'gitlab-token': ckan_infra.GITLAB_TOKEN_PASSWORD, }, secret_name='ckan-migration-secrets', interactive=interactive) config_manager.interactive_set( { 'docker-server': ckan_infra.DOCKER_REGISTRY_SERVER, 'docker-username': ckan_infra.DOCKER_REGISTRY_USERNAME, 'docker-password': ckan_infra.DOCKER_REGISTRY_PASSWORD, 'docker-email': ckan_infra.DOCKER_REGISTRY_EMAIL, }, secret_name='ckan-docker-registry', interactive=interactive) crds_manager.install_crd(INSTANCE_CRD_SINGULAR, INSTANCE_CRD_PLURAL_SUFFIX, INSTANCE_CRD_KIND_SUFFIX) from ckan_cloud_operator.datapushers import initialize as datapusher_initialize datapusher_initialize() from ckan_cloud_operator.routers import manager as routers_manager router_name = get_default_instances_router_name() wildcard_ssl_domain = routers_manager.get_default_root_domain() allow_wildcard_ssl = (routers_manager.get_env_id() == 'p' and routers_manager.get_default_root_domain() == wildcard_ssl_domain) router = routers_manager.get(router_name, required=False) if router: assert ((allow_wildcard_ssl and router.get('wildcard-ssl-domain') == wildcard_ssl_domain) or (not allow_wildcard_ssl and not router.get('wildcard-ssl-domain')) ), f'invalid router wildcard ssl config: {router}' else: routers_manager.create( router_name, routers_manager.get_traefik_router_spec( wildcard_ssl_domain=wildcard_ssl_domain)) from .storage.manager import initialize as ckan_storage_initialize ckan_storage_initialize(interactive=interactive)
def update(instance_id_or_name, override_spec=None, persist_overrides=False, wait_ready=False, skip_deployment=False, skip_route=False, force=False, dry_run=False): instance_id, instance = _get_instance(instance_id_or_name, required=not dry_run) if dry_run: logs.info('update instance', instance_id=instance_id, instance_id_or_name=instance_id_or_name, override_spec=override_spec, persist_overrides=persist_overrides, wait_ready=wait_ready, skip_deployment=skip_deployment, skip_route=skip_route, force=force, dry_run=dry_run) else: pre_update_hook_data = deployment_manager.pre_update_hook(instance_id, instance, override_spec, skip_route) if persist_overrides: logs.info('Persisting overrides') kubectl.apply(instance) if not skip_deployment: deployment_manager.update(instance_id, instance) if wait_ready: wait_instance_ready(instance_id_or_name) if not skip_route and pre_update_hook_data.get('sub-domain'): root_domain = pre_update_hook_data.get('root-domain') sub_domain = pre_update_hook_data['sub-domain'] assert root_domain == routers_manager.get_default_root_domain(), \ 'invalid domain, must use default root domain' logs.info(f'adding instance default route to {sub_domain}.{root_domain}') routers_manager.create_subdomain_route('instances-default', { 'target-type': 'app-instance', 'app-instance-id': instance_id, 'root-domain': root_domain, 'sub-domain': sub_domain }) routers_manager.update('instances-default', wait_ready) else: logs.info('skipping route creation', skip_route=skip_route, sub_domain=pre_update_hook_data.get('sub-domain')) logs.info('Instance is ready', instance_id=instance_id, instance_name=(instance_id_or_name if instance_id_or_name != instance_id else None))
def create_ckan_admin_user(instance_id_or_name, name, email=None, password=None, dry_run=False): if not email: default_root_domain = routers_manager.get_default_root_domain() email = f'{name}@{instance_id_or_name}.{default_root_domain}' if not password: password = _generate_password(8) instance_id, instance_type, instance = _get_instance_id_and_type( instance_id_or_name) user = {'name': name, 'email': email, 'password': password} if not dry_run: deployment_manager.create_ckan_admin_user(instance_id, instance_type, instance, user) return { 'instance-id': instance_id, 'instance-type': instance_type, **{f'admin-{k}': v for k, v in user.items()}, **({ 'dry-run': True } if dry_run else {}), }
def _pre_update_hook_route(instance_id, skip_route, instance, res, dry_run=False): root_domain = routers_manager.get_default_root_domain() sub_domain = f'ckan-cloud-{instance_id}' if not skip_route: # full domain to route to the instance instance_domain = instance['spec'].get('domain') if instance_domain and instance_domain != f'{sub_domain}.{root_domain}': logs.warning(f'instance domain was changed from {instance_domain} to {sub_domain}.{root_domain}') _pre_update_hook_modify_spec(instance_id, instance, lambda i: i.update(domain=f'{sub_domain}.{root_domain}'), dry_run=dry_run) # instance is added to router only if this is true, as all routers must use SSL and may use sans SSL too with_sans_ssl = instance['spec'].get('withSansSSL') if not with_sans_ssl: logs.warning(f'forcing with_sans_ssl, even though withSansSSL is disabled') _pre_update_hook_modify_spec(instance_id, instance, lambda i: i.update(withSansSSL=True), dry_run=dry_run) # subdomain to register on the default root domain register_subdomain = instance['spec'].get('registerSubdomain') if register_subdomain != sub_domain: logs.warning(f'instance register sub domain was changed from {register_subdomain} to {sub_domain}') _pre_update_hook_modify_spec(instance_id, instance, lambda i: i.update(registerSubdomain=sub_domain), dry_run=dry_run) res.update(**{'root-domain': root_domain, 'sub-domain': sub_domain}) site_url = instance['spec'].get('siteUrl') if site_url != f'https://{sub_domain}.{root_domain}': logs.warning(f'instance siteUrl was changed from {site_url} to https://{sub_domain}.{root_domain}') _pre_update_hook_modify_spec(instance_id, instance, lambda i: i.update(siteUrl=f'https://{sub_domain}.{root_domain}'), dry_run=dry_run) return sub_domain, root_domain
def create_ckan_admin_user(instance_id_or_name, name, email=None, password=None, dry_run=False, use_paster=False): if not email: default_root_domain = routers_manager.get_default_root_domain() email = f'{name}@{instance_id_or_name}.{default_root_domain}' if not password: password = _generate_password(8) instance_id, instance_type, instance = _get_instance_id_and_type(instance_id_or_name) user = { 'name': name, 'email': email, 'password': password } if not dry_run: pod_name = _get_running_pod_name(instance_id) name, password, email = [user[k] for k in ['name', 'password', 'email']] logs.info(f'Creating CKAN admin user with {name} ({email}) on pod {pod_name}') if use_paster: logs.subprocess_check_call( f'echo y | kubectl -n {instance_id} exec -i {pod_name} -- ckan-paster --plugin=ckan sysadmin -c /etc/ckan/production.ini add {name} password={password} email={email}', shell=True ) else: logs.subprocess_check_call( f'echo y | kubectl -n {instance_id} exec -i {pod_name} -- ckan --config /etc/ckan/production.ini sysadmin add {name} password={password} email={email}', shell=True ) return { 'instance-id': instance_id, 'instance-type': instance_type, **{f'admin-{k}': v for k, v in user.items()}, **({'dry-run': True} if dry_run else {}), }
def update(instance_id_or_name, override_spec=None, persist_overrides=False, wait_ready=False, skip_deployment=False, skip_route=False, force=False, dry_run=False, skip_solr=False): instance_id, instance_type, instance = _get_instance_id_and_type(instance_id_or_name, required=not dry_run) if dry_run: logs.info('update instance', instance_id=instance_id, instance_id_or_name=instance_id_or_name, override_spec=override_spec, persist_overrides=persist_overrides, wait_ready=wait_ready, skip_deployment=skip_deployment, skip_route=skip_route, force=force, dry_run=dry_run) else: pre_update_hook_data = deployment_manager.pre_update_hook(instance_id, instance_type, instance, override_spec, skip_route) bucket_credentials = instance['spec'].get('ckanStorageBucket', {}).get(get_storage_provider_id()) use_cloud_storage = bucket_credentials and config_manager.get('use-cloud-native-storage', secret_name=CONFIG_NAME) if use_cloud_storage and bucket_credentials: config_manager.set( values=bucket_credentials, secret_name='bucket-credentials', namespace=instance_id ) if persist_overrides: logs.info('Persisting overrides') kubectl.apply(instance) if not skip_deployment: deployment_manager.update(instance_id, instance_type, instance, force=force, skip_solr=skip_solr) if wait_ready: wait_instance_ready(instance_id_or_name) if not skip_route and pre_update_hook_data.get('sub-domain'): root_domain = pre_update_hook_data.get('root-domain') sub_domain = pre_update_hook_data['sub-domain'] assert root_domain == routers_manager.get_default_root_domain(), 'invalid domain, must use default root domain' logs.info(f'adding instance default route to {sub_domain}.{root_domain}') routers_manager.create_subdomain_route('instances-default', { 'target-type': 'ckan-instance', 'ckan-instance-id': instance_id, 'root-domain': root_domain, 'sub-domain': sub_domain }) logs.info(f'updating routers_manager wait_ready: {wait_ready}') routers_manager.update('instances-default', wait_ready) else: logs.info('skipping route creation', skip_route=skip_route, sub_domain=pre_update_hook_data.get('sub-domain')) logs.info('creating ckan admin') # Need to set in values.yaml if pre_update_hook_data.get('create-sysadmin'): ckan_admin_email = pre_update_hook_data.get('ckan-admin-email') ckan_admin_password = pre_update_hook_data.get('ckan-admin-password') ckan_admin_name = pre_update_hook_data.get('ckan-admin-name', 'admin') res = create_ckan_admin_user(instance_id, ckan_admin_name, ckan_admin_email, ckan_admin_password) logs.info(**res) logs.info('Instance is ready', instance_id=instance_id, instance_name=(instance_id_or_name if instance_id_or_name != instance_id else None))
def check_cluster_url(url, verify_site_url=False, deis_instance=None): from ckan_cloud_operator.deis_ckan.ckan import DeisCkanInstanceCKAN from ckan_cloud_operator.providers.routers import manager as routers_manager env_id = routers_manager.get_env_id() root_domain = routers_manager.get_default_root_domain() assert url.startswith(f'https://cc-{env_id}-') and url.rstrip('/').endswith(root_domain), f'invalid cluster url: {url}' if verify_site_url: logs.info('checking site url in running instance...') site_url = None for line in DeisCkanInstanceCKAN(deis_instance).exec('env', check_output=True).decode().splitlines(): if line.startswith('CKAN_SITE_URL='): site_url = line.replace('CKAN_SITE_URL=', '').strip() assert site_url == url, f'mismatch between envvar and instance env: {site_url} != {url}'
def initialize(interactive=False): ckan_db_migration_manager.initialize(interactive=interactive) if config_manager.get('enable-deis-ckan', configmap_name='global-ckan-config') == 'y': ckan_infra = CkanInfra(required=False) config_manager.interactive_set( { 'deis-kubeconfig': ckan_infra.DEIS_KUBECONFIG, }, from_file=True, secret_name='ckan-migration-secrets', interactive=interactive) config_manager.interactive_set( { 'gitlab-token': ckan_infra.GITLAB_TOKEN_PASSWORD, }, secret_name='ckan-migration-secrets', interactive=interactive) config_manager.interactive_set( { 'docker-server': ckan_infra.DOCKER_REGISTRY_SERVER, 'docker-username': ckan_infra.DOCKER_REGISTRY_USERNAME, 'docker-password': ckan_infra.DOCKER_REGISTRY_PASSWORD, 'docker-email': ckan_infra.DOCKER_REGISTRY_EMAIL, }, secret_name='ckan-docker-registry', interactive=interactive) crds_manager.install_crd(INSTANCE_CRD_SINGULAR, INSTANCE_CRD_PLURAL_SUFFIX, INSTANCE_CRD_KIND_SUFFIX) crds_manager.install_crd(INSTANCE_NAME_CRD_SINGULAR, INSTANCE_NAME_CRD_PLURAL_SUFFIX, INSTANCE_NAME_CRD_KIND_SUFFIX) from ckan_cloud_operator.providers.solr.manager import zk_list_configs, zk_put_configs print('Checking CKAN Solr config in ZooKeeper') if 'ckan_default' in zk_list_configs(): print('Found ckan_default Solr config') else: print( 'No default Solr config found. Putting CKAN 2.8 config for Solr to ZooKeeper as ckan_default...' ) file_path = os.path.dirname(os.path.abspath(__file__)) root_path = Path(file_path).parent.parent configs_dir = os.path.join(root_path, 'data', 'solr') zk_put_configs(configs_dir) if config_manager.get('disable-centralized-datapushers', configmap_name='global-ckan-config', required=False) != 'yes': from ckan_cloud_operator.datapushers import initialize as datapusher_initialize datapusher_initialize() from ckan_cloud_operator.routers import manager as routers_manager router_name = get_default_instances_router_name() wildcard_ssl_domain = routers_manager.get_default_root_domain() print(f'wildcard_ssl_domain={wildcard_ssl_domain}') allow_wildcard_ssl = routers_manager.get_env_id() == 'p' router = routers_manager.get(router_name, required=False) if router: assert ((allow_wildcard_ssl and router.get( 'spec', {}).get('wildcard-ssl-domain') == wildcard_ssl_domain) or (not allow_wildcard_ssl and not router.get('spec', {})) ), f'invalid router wildcard ssl config: {router}' else: routers_manager.create( router_name, routers_manager.get_traefik_router_spec( wildcard_ssl_domain=wildcard_ssl_domain)) from .storage.manager import initialize as ckan_storage_initialize ckan_storage_initialize(interactive=interactive) from .deployment.manager import initialize as ckan_deployment_initialize ckan_deployment_initialize(interactive=interactive)
def update(instance_id_or_name, override_spec=None, persist_overrides=False, wait_ready=False, skip_deployment=False, skip_route=False, force=False, dry_run=False): instance_id, instance_type, instance = _get_instance_id_and_type( instance_id_or_name, required=not dry_run) if dry_run: logs.info('update instance', instance_id=instance_id, instance_id_or_name=instance_id_or_name, override_spec=override_spec, persist_overrides=persist_overrides, wait_ready=wait_ready, skip_deployment=skip_deployment, skip_route=skip_route, force=force, dry_run=dry_run) else: pre_update_hook_data = deployment_manager.pre_update_hook( instance_id, instance_type, instance, override_spec, skip_route) bucket_credentials = instance['spec'].get('ckanStorageBucket', {}).get( get_storage_provider_id()) use_cloud_storage = bucket_credentials and config_manager.get( 'use-cloud-native-storage', secret_name=CONFIG_NAME) if use_cloud_storage: cluster_provider_id = cluster_manager.get_provider_id() if bucket_credentials: literal = [] config_manager.set(values=bucket_credentials, secret_name='bucket-credentials', namespace=instance_id) if instance['spec'].get('operatorCopySecrets'): for target_secret_name, source_secret_config in json.loads( instance['spec']['operatorCopySecrets']).items(): for k, v in source_secret_config.items(): source_secret_config[k] = v.replace( "__INSTANCE_NAME__", instance_id_or_name) kubectl.update_secret( target_secret_name, kubectl.decode_secret( kubectl.get('secret', source_secret_config["fromName"], namespace=source_secret_config.get( "fromNamespace", "ckan-cloud"))), namespace=instance_id) if persist_overrides: logs.info('Persisting overrides') kubectl.apply(instance) if not skip_deployment: deployment_manager.update(instance_id, instance_type, instance, force=force) if wait_ready: wait_instance_ready(instance_id_or_name) if not skip_route and pre_update_hook_data.get('sub-domain'): root_domain = pre_update_hook_data.get('root-domain') sub_domain = pre_update_hook_data['sub-domain'] assert root_domain == routers_manager.get_default_root_domain( ), 'invalid domain, must use default root domain' logs.info( f'adding instance default route to {sub_domain}.{root_domain}') routers_manager.create_subdomain_route( 'instances-default', { 'target-type': 'ckan-instance', 'ckan-instance-id': instance_id, 'root-domain': root_domain, 'sub-domain': sub_domain }) logs.info(f'updating routers_manager wait_ready: {wait_ready}') routers_manager.update('instances-default', wait_ready) else: logs.info('skipping route creation', skip_route=skip_route, sub_domain=pre_update_hook_data.get('sub-domain')) if not instance['spec'].get('skipCreateCkanAdmin', False): logs.info('creating ckan admin') ckan_admin_email = instance['spec'].get( 'ckanAdminEmail', pre_update_hook_data.get('ckan-admin-email')) ckan_admin_password = pre_update_hook_data.get( 'ckan-admin-password') ckan_admin_name = instance['spec'].get( 'ckanAdminName', pre_update_hook_data.get('ckan-admin-name', 'admin')) res = create_ckan_admin_user(instance_id, ckan_admin_name, ckan_admin_email, ckan_admin_password) logs.info(**res) logs.info('Instance is ready', instance_id=instance_id, instance_name=(instance_id_or_name if instance_id_or_name != instance_id else None))