def get_deis_instance_credentials(instance_id, is_datastore=False, is_datastore_readonly=False, required=True, with_db_prefix=False): none = (None, None, None, None) if with_db_prefix else (None, None, None) instance_kind = ckan_manager.instance_kind() instance = kubectl.get(f'{instance_kind} {instance_id}', required=required) if not instance: return none secret = kubectl.get(f'secret {instance_id}-annotations', namespace=instance_id, required=required) if not secret: return none secret = kubectl.decode_secret(secret) if is_datastore or is_datastore_readonly: db_name = user = instance['spec'].get('datastore', {}).get('name') if is_datastore_readonly: user = secret.get('datastoreReadonlyUser') password = secret.get('datatastoreReadonlyPassword') else: password = secret.get('datastorePassword') else: db_name = user = instance['spec'].get('db', {}).get('name') password = secret.get('databasePassword') res = [user, password, db_name] if all(res): if with_db_prefix: res.append( get_deis_instance_db_prefix_from_instance( instance, is_datastore or is_datastore_readonly)) return res else: assert not required, 'missing some db values' return none
def list(cls, full=False, quick=False, return_list=False): res = [] data = kubectl.get(ckan_manager.instance_kind(), required=False) if not data: data = {'items': []} for item in data['items']: if quick: data = { 'id': item['metadata']['name'], 'ready': None } if full: data['item'] = item else: try: instance = DeisCkanInstance(item['metadata']['name'], values=item) data = instance.get() if not full: data = {'id': instance.id, 'ready': data['ready']} except Exception: data = {'id': item['metadata']['name'], 'ready': False, 'error': traceback.format_exc()} if return_list: res.append(data) else: print(yaml.dump([data], default_flow_style=False)) if return_list: return res
def deis_instance_edit(instance_id, editor): """Launch an editor to modify and update an instance""" instance_kind = ckan_manager.instance_kind() subprocess.call( f'EDITOR={editor} kubectl -n ckan-cloud edit {instance_kind}/{instance_id}', shell=True) cls(instance_id).update() great_success()
def get_deis_instance_db_prefix(instance_id, is_datastore=False): instance_kind = ckan_manager.instance_kind() instance = kubectl.get(f'{instance_kind} {instance_id}', required=False) return get_deis_instance_db_prefix_from_instance(instance, is_datastore)
def create(cls, *args, **kwargs): create_type = args[0] instance_id = args[-1] from ckan_cloud_operator.providers.db.manager import get_default_db_prefix db_prefix = kwargs['db_prefix'] if kwargs.get('db_prefix') else get_default_db_prefix() if create_type == 'from-gitlab': gitlab_repo = args[1] solr_config = args[2] db_name = instance_id datastore_name = f'{instance_id}-datastore' storage_path = kwargs.get('storage_path') or f'/ckan/{instance_id}' from_db_backups = kwargs.get('from_db_backups') logs.info(f'Creating Deis CKAN instance {instance_id}', gitlab_repo=gitlab_repo, solr_config=solr_config, db_name=db_name, datastore_name=datastore_name, storage_path=storage_path, from_db_backups=from_db_backups) if kwargs.get('use_private_gitlab_repo'): deploy_token_server = input('Gitlab registry url [default: registry.gitlab.com]: ') or 'registry.gitlab.com' deploy_token_username = input('Gitlab deploy token username: '******'Gitlab deploy token password: '******'delete secret private-gitlab-registry', namespace=instance_id) kubectl.call(f'create secret docker-registry private-gitlab-registry --docker-server={deploy_token_server} --docker-username={deploy_token_username} --docker-password={deploy_token_password}', namespace=instance_id) if from_db_backups: db_import_url, datastore_import_url = from_db_backups.split(',') migration_name = None success = False for event in ckan_db_migration_manager.migrate_deis_dbs(None, db_name, datastore_name, db_import_url=db_import_url, datastore_import_url=datastore_import_url, rerun=kwargs.get('rerun'), force=kwargs.get('force'), recreate_dbs=kwargs.get('recreate_dbs'), db_prefix=db_prefix): migration_name = ckan_db_migration_manager.get_event_migration_created_name(event) or migration_name success = ckan_db_migration_manager.print_event_exit_on_complete( event, f'DBs import {from_db_backups} -> {db_name}, {datastore_name}', soft_exit=True ) if success is not None: break assert success, f'Invalid DB migration success value ({success})' else: migration_name = None spec = { 'ckanPodSpec': {}, 'ckanContainerSpec': {'imageFromGitlab': gitlab_repo}, 'envvars': {'fromGitlab': gitlab_repo}, 'solrCloudCollection': { 'name': kwargs.get('solr_collection') or instance_id, 'configName': solr_config }, 'db': { 'name': db_name, **({'fromDbMigration': migration_name} if migration_name else {}), **({'dbPrefix': db_prefix} if db_prefix else {}) }, 'datastore': { 'name': datastore_name, **({'fromDbMigration': migration_name} if migration_name else {}), **({'dbPrefix': db_prefix} if db_prefix else {}) }, 'storage': { 'path': storage_path, } } if kwargs.get('use_private_gitlab_repo'): spec['ckanContainerSpec']['imagePullSecrets'] = [{'name': 'private-gitlab-registry'}] elif create_type == 'from-gcloud-envvars': print(f'Creating Deis CKAN instance {instance_id} from gcloud envvars import') instance_env_yaml, image, solr_config, storage_path, instance_id = args[1:] db_migration_name = kwargs.get('db_migration_name') assert db_migration_name, 'creating from gcloud envvars without a db migration is not supported yet' if type(instance_env_yaml) == str: logs.info(f'Creating {instance_id}-envvars secret from file: {instance_env_yaml}') subprocess.check_call( f'kubectl -n ckan-cloud create secret generic {instance_id}-envvars --from-file=envvars.yaml={instance_env_yaml}', shell=True ) else: logs.info(f'Creating {instance_id}-envvars secret from inline string') kubectl.update_secret(f'{instance_id}-envvars', {'envvars.yaml': yaml.dump(instance_env_yaml, default_flow_style=False)}) spec = { 'ckanPodSpec': {}, 'ckanContainerSpec': {'image': image}, 'envvars': {'fromSecret': f'{instance_id}-envvars'}, 'solrCloudCollection': { 'name': instance_id, 'configName': solr_config }, 'db': { 'name': instance_id, 'fromDbMigration':db_migration_name, **({'dbPrefix': db_prefix} if db_prefix else {}) }, 'datastore': { 'name': f'{instance_id}-datastore', 'fromDbMigration': db_migration_name, **({'dbPrefix': db_prefix} if db_prefix else {}) }, 'storage': { 'path': storage_path } } else: raise NotImplementedError(f'invalid create type: {create_type}') instance_kind = ckan_manager.instance_kind() instance = { 'apiVersion': f'stable.viderum.com/v1', 'kind': instance_kind, 'metadata': { 'name': instance_id, 'namespace': 'ckan-cloud', 'finalizers': ['finalizer.stable.viderum.com'] }, 'spec': spec } subprocess.run('kubectl apply -f -', input=yaml.dump(instance).encode(), shell=True, check=True) return cls(instance_id, values=instance)
def kind(self): return ckan_manager.instance_kind()
def migrate_from_deis(old_site_id, new_instance_id, router_name, deis_instance_class, skip_gitlab=False, db_migration_name=None, recreate=False, skip_routes=False, skip_solr=False, skip_deployment=False, no_db_proxy=False): assert db_migration_name, 'migration without a db migration is not supported yet' log_labels = {'instance': new_instance_id} if recreate: from ckan_cloud_operator.deis_ckan.instance import DeisCkanInstance DeisCkanInstance(new_instance_id).delete( force=True, wait_deleted=not db_migration_name) logs.info( f'Migrating from old site id {old_site_id} to new instance id {new_instance_id}', **log_labels) instance_kind = ckan_manager.instance_kind() values = kubectl.get(f'{instance_kind} {new_instance_id}', required=False) if values: logs.info('instance already exists', **log_labels) else: logs.info('creating instance', **log_labels) path_to_old_cluster_kubeconfig = get_path_to_old_cluster_kubeconfig() solr_config = get_solr_config(old_site_id, path_to_old_cluster_kubeconfig) assert solr_config, 'failed to get solr config name' instance_env = get_instance_env(old_site_id, path_to_old_cluster_kubeconfig) gitlab_repo = f'viderum/cloud-{old_site_id}' if not skip_gitlab: CkanGitlab().initialize(gitlab_repo) gitlab_registry = f'registry.gitlab.com/{gitlab_repo}' old_bucket_name = instance_env.get( 'CKANEXT__S3FILESTORE__AWS_BUCKET_NAME') old_storage_path = instance_env.get( 'CKANEXT__S3FILESTORE__AWS_STORAGE_PATH') assert old_bucket_name == 'ckan' assert old_storage_path and len(old_storage_path) > 1 storage_path = f'/ckan/{old_storage_path}' deis_instance_class.create('from-gcloud-envvars', instance_env, gitlab_registry, solr_config, storage_path, new_instance_id, db_migration_name=db_migration_name) routers_env_id = routers_provider.get_env_id() default_root_domain = routers_provider.get_default_root_domain() assert routers_env_id and default_root_domain ckan_site_url = f'https://cc-{routers_env_id}-{new_instance_id}.{default_root_domain}' logs.info( f'updating instance and setting ckan site url to {ckan_site_url}', **log_labels) deis_instance_class(new_instance_id, override_spec={ 'envvars': { 'CKAN_SITE_URL': ckan_site_url }, **({ 'db': { 'no-db-proxy': 'yes' }, 'datastore': { 'no-db-proxy': 'yes' } } if no_db_proxy else {}) }, persist_overrides=True).update( wait_ready=True, skip_solr=skip_solr, skip_deployment=skip_deployment) if routers_manager.get_deis_instance_routes(new_instance_id): logs.info('default instance route already exists', **log_labels) else: logs.info('creating instance route', **log_labels) routers_manager.create_subdomain_route( router_name, { 'target-type': 'deis-instance', 'deis-instance-id': new_instance_id, 'root-domain': 'default', 'sub-domain': 'default' }) if not skip_routes: routers_manager.update(router_name, wait_ready=True) if not skip_solr: logs.info('Rebuilding solr search index', **log_labels) deis_instance_class(new_instance_id).ckan.paster( 'search-index rebuild --force')
**({'fromDbMigration': migration_name} if migration_name else {}), **({'dbPrefix': new_db_prefix} if new_db_prefix else {}) }, 'datastore': { 'name': new_datastore_name, **({'fromDbMigration': migration_name} if migration_name else {}), **({'dbPrefix': new_db_prefix} if new_db_prefix else {}) }, 'storage': { 'path': new_storage_path, } } print('spec', spec, file=sys.__stderr__) if os.environ.get('SKIP_CREATE') != 'yes': instance_kind = ckan_manager.instance_kind() instance = { 'apiVersion': f'stable.viderum.com/v1', 'kind': instance_kind, 'metadata': { 'name': new_instance_id, 'namespace': 'ckan-cloud', 'finalizers': ['finalizer.stable.viderum.com'] }, 'spec': spec } kubectl.apply(instance, dry_run=dry_run) if not dry_run: DeisCkanInstance(new_instance_id, values=instance).update() if os.environ.get('SKIP_ROUTER') != 'yes':