def update(instance_id_or_name, override_spec=None, persist_overrides=False, wait_ready=False, skip_deployment=False, skip_route=False, force=False, dry_run=False): instance_id, instance = _get_instance(instance_id_or_name, required=not dry_run) if dry_run: logs.info('update instance', instance_id=instance_id, instance_id_or_name=instance_id_or_name, override_spec=override_spec, persist_overrides=persist_overrides, wait_ready=wait_ready, skip_deployment=skip_deployment, skip_route=skip_route, force=force, dry_run=dry_run) else: pre_update_hook_data = deployment_manager.pre_update_hook(instance_id, instance, override_spec, skip_route) if persist_overrides: logs.info('Persisting overrides') kubectl.apply(instance) if not skip_deployment: deployment_manager.update(instance_id, instance) if wait_ready: wait_instance_ready(instance_id_or_name) if not skip_route and pre_update_hook_data.get('sub-domain'): root_domain = pre_update_hook_data.get('root-domain') sub_domain = pre_update_hook_data['sub-domain'] assert root_domain == routers_manager.get_default_root_domain(), \ 'invalid domain, must use default root domain' logs.info(f'adding instance default route to {sub_domain}.{root_domain}') routers_manager.create_subdomain_route('instances-default', { 'target-type': 'app-instance', 'app-instance-id': instance_id, 'root-domain': root_domain, 'sub-domain': sub_domain }) routers_manager.update('instances-default', wait_ready) else: logs.info('skipping route creation', skip_route=skip_route, sub_domain=pre_update_hook_data.get('sub-domain')) logs.info('Instance is ready', instance_id=instance_id, instance_name=(instance_id_or_name if instance_id_or_name != instance_id else None))
def initialize(): install_crds() datapusher_envvars = {'PORT': '8000'} router_name = 'datapushers' if not routers_manager.get(router_name, required=False): routers_manager.create(router_name, routers_manager.get_traefik_router_spec()) create( 'datapusher-1', 'registry.gitlab.com/viderum/docker-datapusher:cloud-datapusher-1-v9', datapusher_envvars, router_name) create( 'datapusher-de', 'registry.gitlab.com/viderum/docker-datapusher:cloud-de-git-943fc3e0', datapusher_envvars, router_name) create( 'datapusher-giga', 'registry.gitlab.com/viderum/docker-datapusher:cloud-giga-git-2b05b22d', datapusher_envvars, router_name) create( 'datapusher-increased-max-length', 'registry.gitlab.com/viderum/docker-datapusher:cloud-increased-max-length-git-84e86116', datapusher_envvars, router_name) update('datapusher-1') update('datapusher-de') update('datapusher-giga') update('datapusher-increased-max-length') routers_manager.update(router_name)
def update(instance_id_or_name, override_spec=None, persist_overrides=False, wait_ready=False, skip_deployment=False, skip_route=False): instance_id, instance_type, instance = _get_instance_id_and_type(instance_id_or_name) if override_spec: for k, v in override_spec.items(): logs.info(f'Applying override spec {k}={v}') instance['spec'][k] = v assert instance['spec'].get('useCentralizedInfra'), 'non-centralized instances are not supported' # full domain to route to the instance instance_domain = instance['spec'].get('domain') # instance is added to router only if this is true, as all routers must use SSL and may use sans SSL too with_sans_ssl = instance['spec'].get('withSansSSL') # subdomain to register on the default root domain register_subdomain = instance['spec'].get('registerSubdomain') if persist_overrides: logs.info('Persisting overrides') kubectl.apply(instance) if not skip_deployment: deployment_manager.update(instance_id, instance_type, instance) if wait_ready: wait_instance_ready(instance_id_or_name) if not skip_route: if instance_domain: assert with_sans_ssl, 'withSansSSL must be set to true to add routes' assert '.'.join(instance_domain.split('.')[1:]) == routers_manager.get_default_root_domain(), f'invalid root domain ({instance_domain})' assert instance_domain.split('.')[0] == register_subdomain, f'invalid register_subdomain ({register_subdomain})' logs.info(f'adding instance route to {instance_domain}') routers_manager.create_subdomain_route('instances-default', { 'target-type': 'ckan-instance', 'ckan-instance-id': instance_id, 'root-domain': routers_manager.get_default_root_domain(), 'sub-domain': register_subdomain }) routers_manager.update('instances-default', wait_ready) else: assert not register_subdomain, 'subdomain registration is only supported with instance_domain'
def routers_create_backend_url_subdomain_route(router_name, target_resource_id, backend_url, sub_domain, root_domain, wait_ready): routers_manager.create_subdomain_route(router_name, { 'target-type': 'backend-url', 'target-resource-id': target_resource_id, 'backend-url': backend_url, 'sub-domain': sub_domain, 'root-domain': root_domain, }) routers_manager.update(router_name, wait_ready) great_success()
def routers_create_datapusher_subdomain_route(router_name, datapusher_name, sub_domain, root_domain, wait_ready): routers_manager.create_subdomain_route(router_name, { 'target-type': 'datapusher', 'datapusher-name': datapusher_name, 'root-domain': root_domain, 'sub-domain': sub_domain }) routers_manager.update(router_name, wait_ready) great_success()
def routers_create_deis_instance_subdomain_route(router_name, deis_instance_id, sub_domain, root_domain, wait_ready): routers_manager.create_subdomain_route(router_name, { 'target-type': 'deis-instance', 'deis-instance-id': deis_instance_id, 'root-domain': root_domain, 'sub-domain': sub_domain }) routers_manager.update(router_name, wait_ready) great_success()
def routers_create(traefik_router_name, default_root_domain, cloudflare_email, cloudflare_api_key, external_domains): """Create a Traefik router, with domain registration and let's encrypt based on Cloudflare""" routers_manager.create( traefik_router_name, routers_manager.get_traefik_router_spec( default_root_domain, cloudflare_email, cloudflare_api_key, external_domains=external_domains ) ) routers_manager.update(traefik_router_name) great_success()
def update(instance_id_or_name, override_spec=None, persist_overrides=False, wait_ready=False, skip_deployment=False, skip_route=False, force=False, dry_run=False, skip_solr=False): instance_id, instance_type, instance = _get_instance_id_and_type(instance_id_or_name, required=not dry_run) if dry_run: logs.info('update instance', instance_id=instance_id, instance_id_or_name=instance_id_or_name, override_spec=override_spec, persist_overrides=persist_overrides, wait_ready=wait_ready, skip_deployment=skip_deployment, skip_route=skip_route, force=force, dry_run=dry_run) else: pre_update_hook_data = deployment_manager.pre_update_hook(instance_id, instance_type, instance, override_spec, skip_route) bucket_credentials = instance['spec'].get('ckanStorageBucket', {}).get(get_storage_provider_id()) use_cloud_storage = bucket_credentials and config_manager.get('use-cloud-native-storage', secret_name=CONFIG_NAME) if use_cloud_storage and bucket_credentials: config_manager.set( values=bucket_credentials, secret_name='bucket-credentials', namespace=instance_id ) if persist_overrides: logs.info('Persisting overrides') kubectl.apply(instance) if not skip_deployment: deployment_manager.update(instance_id, instance_type, instance, force=force, skip_solr=skip_solr) if wait_ready: wait_instance_ready(instance_id_or_name) if not skip_route and pre_update_hook_data.get('sub-domain'): root_domain = pre_update_hook_data.get('root-domain') sub_domain = pre_update_hook_data['sub-domain'] assert root_domain == routers_manager.get_default_root_domain(), 'invalid domain, must use default root domain' logs.info(f'adding instance default route to {sub_domain}.{root_domain}') routers_manager.create_subdomain_route('instances-default', { 'target-type': 'ckan-instance', 'ckan-instance-id': instance_id, 'root-domain': root_domain, 'sub-domain': sub_domain }) logs.info(f'updating routers_manager wait_ready: {wait_ready}') routers_manager.update('instances-default', wait_ready) else: logs.info('skipping route creation', skip_route=skip_route, sub_domain=pre_update_hook_data.get('sub-domain')) logs.info('creating ckan admin') # Need to set in values.yaml if pre_update_hook_data.get('create-sysadmin'): ckan_admin_email = pre_update_hook_data.get('ckan-admin-email') ckan_admin_password = pre_update_hook_data.get('ckan-admin-password') ckan_admin_name = pre_update_hook_data.get('ckan-admin-name', 'admin') res = create_ckan_admin_user(instance_id, ckan_admin_name, ckan_admin_email, ckan_admin_password) logs.info(**res) logs.info('Instance is ready', instance_id=instance_id, instance_name=(instance_id_or_name if instance_id_or_name != instance_id else None))
def routers_create_backend_url_subdomain_route(router_name, target_resource_id, backend_url, sub_domain, root_domain, wait_ready, httpauth_secret): routers_manager.create_subdomain_route(router_name, { 'target-type': 'backend-url', 'target-resource-id': target_resource_id, 'backend-url': backend_url, 'sub-domain': sub_domain, 'root-domain': root_domain, **({'httpauth-secret': httpauth_secret} if httpauth_secret else {}), }) routers_manager.update(router_name, wait_ready) great_success()
def test_update(self, list, traefik_manager, _init_router): _init_router.return_value = 'router', { 'update': True }, 'traefik', {}, {}, { 'manager': traefik_manager } list.return_value = 'router' manager.update('datapusher', wait_ready=True) _init_router.assert_called_once_with('datapusher') list.assert_called_once_with({}) traefik_manager.update.assert_called_once_with('datapusher', True, {'update': True}, {}, 'router', dry_run=False)
def _update_route(storage_suffix=None): backend_url_target_id = _get_backend_url_target_id( storage_suffix=storage_suffix) router_name = _config_get('router-name', required=True, suffix=storage_suffix) if not routers_manager.get_backend_url_routes(backend_url_target_id): deployment_name = _get_resource_name(suffix=storage_suffix) namespace = _get_namespace() routers_manager.create_subdomain_route( router_name, { 'target-type': 'backend-url', 'target-resource-id': backend_url_target_id, 'backend-url': f'http://{deployment_name}.{namespace}:9000', }) routers_manager.update(router_name, wait_ready=True)
def main(old_instance_id, new_instance_id, dry_run): dry_run = (dry_run == 'yes') router_names = set() for route in routers_manager.get_deis_instance_routes(old_instance_id): for label in [ 'ckan-cloud/route-deis-instance-id', 'ckan-cloud/route-target-resource-id' ]: _assert_set(route['metadata']['labels'], label, old_instance_id, new_instance_id) for attr in ['deis-instance-id', 'route-target-resource-id']: _assert_set(route['spec'], attr, old_instance_id, new_instance_id) kubectl.apply(route, dry_run=dry_run) router_names.add(route['spec']['router_name']) logs.info('updating routers', router_names=router_names) if not dry_run: for router_name in router_names: routers_manager.update(router_name)
def deploy_gcs_minio_proxy(router_name): """Deploys a minio proxy (AKA gateway) for access to google storage""" labels = {'app': 'ckan-cloud-gcsminio-proxy'} if not kubectl.get('secret gcsminio-proxy-credentials', required=False): print('Creating minio credentials') minio_access_key = binascii.hexlify(os.urandom(8)).decode() minio_secret_key = binascii.hexlify(os.urandom(12)).decode() kubectl.update_secret( 'gcsminio-proxy-credentials', { 'MINIO_ACCESS_KEY': minio_access_key, 'MINIO_SECRET_KEY': minio_secret_key, }) kubectl.apply( kubectl.get_deployment( 'gcsminio-proxy', labels, { 'replicas': 1, 'revisionHistoryLimit': 10, 'strategy': { 'type': 'RollingUpdate', }, 'template': { 'metadata': { 'labels': labels, 'annotations': { 'ckan-cloud/operator-timestamp': str(datetime.datetime.now()) } }, 'spec': { 'containers': [{ 'name': 'minio', 'image': 'orihoch/ckan-cloud-operator-gcsminio-proxy', 'env': [{ 'name': 'GOOGLE_APPLICATION_CREDENTIALS', 'value': '/gcloud-credentials/credentials.json' }], 'envFrom': [{ 'secretRef': { 'name': 'gcsminio-proxy-credentials' } }], 'ports': [{ 'containerPort': 9000 }], 'volumeMounts': [ { 'name': 'gcloud-credentials', 'mountPath': '/gcloud-credentials/credentials.json', 'subPath': 'GCLOUD_SERVICE_ACCOUNT_JSON' }, ], }], 'volumes': [ { 'name': 'gcloud-credentials', 'secret': { 'secretName': 'ckan-infra' } }, ] } } })) service = kubectl.get_resource('v1', 'Service', 'gcsminio-proxy', labels) service['spec'] = { 'ports': [{ 'name': '9000', 'port': 9000 }], 'selector': labels } kubectl.apply(service) if not routers_manager.get_backend_url_routes('gcs-minio'): routers_manager.create_subdomain_route( router_name, { 'target-type': 'backend-url', 'target-resource-id': 'gcs-minio', 'backend-url': 'http://gcsminio-proxy.ckan-cloud:9000', 'sub-domain': 'default', 'root-domain': 'default', }) routers_manager.update(router_name, wait_ready=True)
def migrate_from_deis(old_site_id, new_instance_id, router_name, deis_instance_class, skip_gitlab=False, db_migration_name=None, recreate=False, skip_routes=False, skip_solr=False, skip_deployment=False, no_db_proxy=False): assert db_migration_name, 'migration without a db migration is not supported yet' log_labels = {'instance': new_instance_id} if recreate: from ckan_cloud_operator.deis_ckan.instance import DeisCkanInstance DeisCkanInstance(new_instance_id).delete( force=True, wait_deleted=not db_migration_name) logs.info( f'Migrating from old site id {old_site_id} to new instance id {new_instance_id}', **log_labels) instance_kind = ckan_manager.instance_kind() values = kubectl.get(f'{instance_kind} {new_instance_id}', required=False) if values: logs.info('instance already exists', **log_labels) else: logs.info('creating instance', **log_labels) path_to_old_cluster_kubeconfig = get_path_to_old_cluster_kubeconfig() solr_config = get_solr_config(old_site_id, path_to_old_cluster_kubeconfig) assert solr_config, 'failed to get solr config name' instance_env = get_instance_env(old_site_id, path_to_old_cluster_kubeconfig) gitlab_repo = f'viderum/cloud-{old_site_id}' if not skip_gitlab: CkanGitlab().initialize(gitlab_repo) gitlab_registry = f'registry.gitlab.com/{gitlab_repo}' old_bucket_name = instance_env.get( 'CKANEXT__S3FILESTORE__AWS_BUCKET_NAME') old_storage_path = instance_env.get( 'CKANEXT__S3FILESTORE__AWS_STORAGE_PATH') assert old_bucket_name == 'ckan' assert old_storage_path and len(old_storage_path) > 1 storage_path = f'/ckan/{old_storage_path}' deis_instance_class.create('from-gcloud-envvars', instance_env, gitlab_registry, solr_config, storage_path, new_instance_id, db_migration_name=db_migration_name) routers_env_id = routers_provider.get_env_id() default_root_domain = routers_provider.get_default_root_domain() assert routers_env_id and default_root_domain ckan_site_url = f'https://cc-{routers_env_id}-{new_instance_id}.{default_root_domain}' logs.info( f'updating instance and setting ckan site url to {ckan_site_url}', **log_labels) deis_instance_class(new_instance_id, override_spec={ 'envvars': { 'CKAN_SITE_URL': ckan_site_url }, **({ 'db': { 'no-db-proxy': 'yes' }, 'datastore': { 'no-db-proxy': 'yes' } } if no_db_proxy else {}) }, persist_overrides=True).update( wait_ready=True, skip_solr=skip_solr, skip_deployment=skip_deployment) if routers_manager.get_deis_instance_routes(new_instance_id): logs.info('default instance route already exists', **log_labels) else: logs.info('creating instance route', **log_labels) routers_manager.create_subdomain_route( router_name, { 'target-type': 'deis-instance', 'deis-instance-id': new_instance_id, 'root-domain': 'default', 'sub-domain': 'default' }) if not skip_routes: routers_manager.update(router_name, wait_ready=True) if not skip_solr: logs.info('Rebuilding solr search index', **log_labels) deis_instance_class(new_instance_id).ckan.paster( 'search-index rebuild --force')
def routers_update(router_name, wait_ready): """Update a router to latest resource spec""" routers_manager.update(router_name, wait_ready) great_success()
def update(instance_id_or_name, override_spec=None, persist_overrides=False, wait_ready=False, skip_deployment=False, skip_route=False, force=False, dry_run=False): instance_id, instance_type, instance = _get_instance_id_and_type( instance_id_or_name, required=not dry_run) if dry_run: logs.info('update instance', instance_id=instance_id, instance_id_or_name=instance_id_or_name, override_spec=override_spec, persist_overrides=persist_overrides, wait_ready=wait_ready, skip_deployment=skip_deployment, skip_route=skip_route, force=force, dry_run=dry_run) else: pre_update_hook_data = deployment_manager.pre_update_hook( instance_id, instance_type, instance, override_spec, skip_route) bucket_credentials = instance['spec'].get('ckanStorageBucket', {}).get( get_storage_provider_id()) use_cloud_storage = bucket_credentials and config_manager.get( 'use-cloud-native-storage', secret_name=CONFIG_NAME) if use_cloud_storage: cluster_provider_id = cluster_manager.get_provider_id() if bucket_credentials: literal = [] config_manager.set(values=bucket_credentials, secret_name='bucket-credentials', namespace=instance_id) if instance['spec'].get('operatorCopySecrets'): for target_secret_name, source_secret_config in json.loads( instance['spec']['operatorCopySecrets']).items(): for k, v in source_secret_config.items(): source_secret_config[k] = v.replace( "__INSTANCE_NAME__", instance_id_or_name) kubectl.update_secret( target_secret_name, kubectl.decode_secret( kubectl.get('secret', source_secret_config["fromName"], namespace=source_secret_config.get( "fromNamespace", "ckan-cloud"))), namespace=instance_id) if persist_overrides: logs.info('Persisting overrides') kubectl.apply(instance) if not skip_deployment: deployment_manager.update(instance_id, instance_type, instance, force=force) if wait_ready: wait_instance_ready(instance_id_or_name) if not skip_route and pre_update_hook_data.get('sub-domain'): root_domain = pre_update_hook_data.get('root-domain') sub_domain = pre_update_hook_data['sub-domain'] assert root_domain == routers_manager.get_default_root_domain( ), 'invalid domain, must use default root domain' logs.info( f'adding instance default route to {sub_domain}.{root_domain}') routers_manager.create_subdomain_route( 'instances-default', { 'target-type': 'ckan-instance', 'ckan-instance-id': instance_id, 'root-domain': root_domain, 'sub-domain': sub_domain }) logs.info(f'updating routers_manager wait_ready: {wait_ready}') routers_manager.update('instances-default', wait_ready) else: logs.info('skipping route creation', skip_route=skip_route, sub_domain=pre_update_hook_data.get('sub-domain')) if not instance['spec'].get('skipCreateCkanAdmin', False): logs.info('creating ckan admin') ckan_admin_email = instance['spec'].get( 'ckanAdminEmail', pre_update_hook_data.get('ckan-admin-email')) ckan_admin_password = pre_update_hook_data.get( 'ckan-admin-password') ckan_admin_name = instance['spec'].get( 'ckanAdminName', pre_update_hook_data.get('ckan-admin-name', 'admin')) res = create_ckan_admin_user(instance_id, ckan_admin_name, ckan_admin_email, ckan_admin_password) logs.info(**res) logs.info('Instance is ready', instance_id=instance_id, instance_name=(instance_id_or_name if instance_id_or_name != instance_id else None))