def update(self): db_migration_name = self.db_spec.get('fromDbMigration') if not db_migration_name: db_migration_name = f'new-{self.db_type}-{self.instance.id}' db_migration_spec = { 'type': f'new-{self.db_type}', f'{self.db_type}-name': self.db_spec['name'], } migration = ckan_db_migration_manager.create(db_migration_name, db_migration_spec, exists_ok=True) for event in ckan_db_migration_manager.update(migration): ckan_db_migration_manager.print_event_exit_on_complete( event, '') database_password, datastore_password, datastore_readonly_password = ckan_db_migration_manager.get_dbs_passwords( db_migration_name) datastore_readonly_user = ckan_db_migration_manager.get_datastore_raedonly_user_name( db_migration_name) if self.db_type == 'db': if database_password: self.instance.annotations.set_secrets( {'databasePassword': database_password}) elif self.db_type == 'datastore': if datastore_password and datastore_readonly_password: self.instance.annotations.set_secrets({ 'datastorePassword': datastore_password, 'datastoreReadonlyUser': datastore_readonly_user, 'datatastoreReadonlyPassword': datastore_readonly_password, })
def migrate_deis_dbs(old_site_id, db_name, datastore_name, force, rerun, recreate_dbs, dbs_suffix, skip_create_dbs, skip_datastore_import): migration_generator = db_migration_manager.migrate_deis_dbs( old_site_id, db_name, datastore_name, force=force, rerun=rerun, recreate_dbs=recreate_dbs, dbs_suffix=dbs_suffix, skip_create_dbs=skip_create_dbs, skip_datastore_import=skip_datastore_import ) for event in migration_generator: db_migration_manager.print_event_exit_on_complete( event, f'{old_site_id} -> {db_name}, {datastore_name}' )
def create(cls, *args, **kwargs): create_type = args[0] instance_id = args[-1] from ckan_cloud_operator.providers.db.manager import get_default_db_prefix db_prefix = kwargs['db_prefix'] if kwargs.get('db_prefix') else get_default_db_prefix() if create_type == 'from-gitlab': gitlab_repo = args[1] solr_config = args[2] db_name = instance_id datastore_name = f'{instance_id}-datastore' storage_path = kwargs.get('storage_path') or f'/ckan/{instance_id}' from_db_backups = kwargs.get('from_db_backups') logs.info(f'Creating Deis CKAN instance {instance_id}', gitlab_repo=gitlab_repo, solr_config=solr_config, db_name=db_name, datastore_name=datastore_name, storage_path=storage_path, from_db_backups=from_db_backups) if kwargs.get('use_private_gitlab_repo'): deploy_token_server = input('Gitlab registry url [default: registry.gitlab.com]: ') or 'registry.gitlab.com' deploy_token_username = input('Gitlab deploy token username: '******'Gitlab deploy token password: '******'delete secret private-gitlab-registry', namespace=instance_id) kubectl.call(f'create secret docker-registry private-gitlab-registry --docker-server={deploy_token_server} --docker-username={deploy_token_username} --docker-password={deploy_token_password}', namespace=instance_id) if from_db_backups: db_import_url, datastore_import_url = from_db_backups.split(',') migration_name = None success = False for event in ckan_db_migration_manager.migrate_deis_dbs(None, db_name, datastore_name, db_import_url=db_import_url, datastore_import_url=datastore_import_url, rerun=kwargs.get('rerun'), force=kwargs.get('force'), recreate_dbs=kwargs.get('recreate_dbs'), db_prefix=db_prefix): migration_name = ckan_db_migration_manager.get_event_migration_created_name(event) or migration_name success = ckan_db_migration_manager.print_event_exit_on_complete( event, f'DBs import {from_db_backups} -> {db_name}, {datastore_name}', soft_exit=True ) if success is not None: break assert success, f'Invalid DB migration success value ({success})' else: migration_name = None spec = { 'ckanPodSpec': {}, 'ckanContainerSpec': {'imageFromGitlab': gitlab_repo}, 'envvars': {'fromGitlab': gitlab_repo}, 'solrCloudCollection': { 'name': kwargs.get('solr_collection') or instance_id, 'configName': solr_config }, 'db': { 'name': db_name, **({'fromDbMigration': migration_name} if migration_name else {}), **({'dbPrefix': db_prefix} if db_prefix else {}) }, 'datastore': { 'name': datastore_name, **({'fromDbMigration': migration_name} if migration_name else {}), **({'dbPrefix': db_prefix} if db_prefix else {}) }, 'storage': { 'path': storage_path, } } if kwargs.get('use_private_gitlab_repo'): spec['ckanContainerSpec']['imagePullSecrets'] = [{'name': 'private-gitlab-registry'}] elif create_type == 'from-gcloud-envvars': print(f'Creating Deis CKAN instance {instance_id} from gcloud envvars import') instance_env_yaml, image, solr_config, storage_path, instance_id = args[1:] db_migration_name = kwargs.get('db_migration_name') assert db_migration_name, 'creating from gcloud envvars without a db migration is not supported yet' if type(instance_env_yaml) == str: logs.info(f'Creating {instance_id}-envvars secret from file: {instance_env_yaml}') subprocess.check_call( f'kubectl -n ckan-cloud create secret generic {instance_id}-envvars --from-file=envvars.yaml={instance_env_yaml}', shell=True ) else: logs.info(f'Creating {instance_id}-envvars secret from inline string') kubectl.update_secret(f'{instance_id}-envvars', {'envvars.yaml': yaml.dump(instance_env_yaml, default_flow_style=False)}) spec = { 'ckanPodSpec': {}, 'ckanContainerSpec': {'image': image}, 'envvars': {'fromSecret': f'{instance_id}-envvars'}, 'solrCloudCollection': { 'name': instance_id, 'configName': solr_config }, 'db': { 'name': instance_id, 'fromDbMigration':db_migration_name, **({'dbPrefix': db_prefix} if db_prefix else {}) }, 'datastore': { 'name': f'{instance_id}-datastore', 'fromDbMigration': db_migration_name, **({'dbPrefix': db_prefix} if db_prefix else {}) }, 'storage': { 'path': storage_path } } else: raise NotImplementedError(f'invalid create type: {create_type}') instance_kind = ckan_manager.instance_kind() instance = { 'apiVersion': f'stable.viderum.com/v1', 'kind': instance_kind, 'metadata': { 'name': instance_id, 'namespace': 'ckan-cloud', 'finalizers': ['finalizer.stable.viderum.com'] }, 'spec': spec } subprocess.run('kubectl apply -f -', input=yaml.dump(instance).encode(), shell=True, check=True) return cls(instance_id, values=instance)
print('new_db_name', new_db_name, file=sys.__stderr__) print('new_datastore_name', new_datastore_name, file=sys.__stderr__) migration_name = None if os.environ.get('USE_EXISTING_MIGRATION') == 'yes': for migration in ckan_db_migration.get()['items']: if migration.get('spec', {}).get('db-name') == new_db_name: migration_name = migration['spec']['name'] break elif not dry_run: success = False for event in ckan_db_migration.migrate_deis_dbs(None, new_db_name, new_datastore_name, db_import_url=old_db_url, datastore_import_url=old_datastore_url, db_prefix=new_db_prefix): migration_name = ckan_db_migration.get_event_migration_created_name(event) or migration_name success = ckan_db_migration.print_event_exit_on_complete(event, f'DBs import -> {new_db_name}, {new_datastore_name}', soft_exit=True) if success is not None: break assert success, f'Invalid DB migration success value ({success})' print('migration_name', migration_name, file=sys.__stderr__) if not new_solr_collection_name: new_solr_collection_name = new_instance_id logs.info(new_solr_collection_name=new_solr_collection_name) spec = { 'ckanPodSpec': {}, 'ckanContainerSpec': {'imageFromGitlab': new_gitlab_repo}, 'envvars': {'fromGitlab': new_gitlab_repo},