def update(self):
db_migration_name = self.db_spec.get('fromDbMigration')
if not db_migration_name:
db_migration_name = f'new-{self.db_type}-{self.instance.id}'
db_migration_spec = {
'type': f'new-{self.db_type}',
f'{self.db_type}-name': self.db_spec['name'],
}
migration = ckan_db_migration_manager.create(db_migration_name,
db_migration_spec,
exists_ok=True)
for event in ckan_db_migration_manager.update(migration):
ckan_db_migration_manager.print_event_exit_on_complete(
event, '')
database_password, datastore_password, datastore_readonly_password = ckan_db_migration_manager.get_dbs_passwords(
db_migration_name)
datastore_readonly_user = ckan_db_migration_manager.get_datastore_raedonly_user_name(
db_migration_name)
if self.db_type == 'db':
if database_password:
self.instance.annotations.set_secrets(
{'databasePassword': database_password})
elif self.db_type == 'datastore':
if datastore_password and datastore_readonly_password:
self.instance.annotations.set_secrets({
'datastorePassword':
datastore_password,
'datastoreReadonlyUser':
datastore_readonly_user,
'datatastoreReadonlyPassword':
datastore_readonly_password,
})
def migrate_deis_dbs(old_site_id, db_name, datastore_name, force, rerun, recreate_dbs, dbs_suffix, skip_create_dbs,
skip_datastore_import):
migration_generator = db_migration_manager.migrate_deis_dbs(
old_site_id, db_name, datastore_name, force=force, rerun=rerun, recreate_dbs=recreate_dbs, dbs_suffix=dbs_suffix,
skip_create_dbs=skip_create_dbs, skip_datastore_import=skip_datastore_import
)
for event in migration_generator:
db_migration_manager.print_event_exit_on_complete(
event,
f'{old_site_id} -> {db_name}, {datastore_name}'
)
def create(cls, *args, **kwargs):
create_type = args[0]
instance_id = args[-1]
from ckan_cloud_operator.providers.db.manager import get_default_db_prefix
db_prefix = kwargs['db_prefix'] if kwargs.get('db_prefix') else get_default_db_prefix()
if create_type == 'from-gitlab':
gitlab_repo = args[1]
solr_config = args[2]
db_name = instance_id
datastore_name = f'{instance_id}-datastore'
storage_path = kwargs.get('storage_path') or f'/ckan/{instance_id}'
from_db_backups = kwargs.get('from_db_backups')
logs.info(f'Creating Deis CKAN instance {instance_id}', gitlab_repo=gitlab_repo, solr_config=solr_config,
db_name=db_name, datastore_name=datastore_name, storage_path=storage_path,
from_db_backups=from_db_backups)
if kwargs.get('use_private_gitlab_repo'):
deploy_token_server = input('Gitlab registry url [default: registry.gitlab.com]: ') or 'registry.gitlab.com'
deploy_token_username = input('Gitlab deploy token username: '******'Gitlab deploy token password: '******'delete secret private-gitlab-registry', namespace=instance_id)
kubectl.call(f'create secret docker-registry private-gitlab-registry --docker-server={deploy_token_server} --docker-username={deploy_token_username} --docker-password={deploy_token_password}', namespace=instance_id)
if from_db_backups:
db_import_url, datastore_import_url = from_db_backups.split(',')
migration_name = None
success = False
for event in ckan_db_migration_manager.migrate_deis_dbs(None, db_name, datastore_name,
db_import_url=db_import_url,
datastore_import_url=datastore_import_url,
rerun=kwargs.get('rerun'),
force=kwargs.get('force'),
recreate_dbs=kwargs.get('recreate_dbs'),
db_prefix=db_prefix):
migration_name = ckan_db_migration_manager.get_event_migration_created_name(event) or migration_name
success = ckan_db_migration_manager.print_event_exit_on_complete(
event,
f'DBs import {from_db_backups} -> {db_name}, {datastore_name}',
soft_exit=True
)
if success is not None:
break
assert success, f'Invalid DB migration success value ({success})'
else:
migration_name = None
spec = {
'ckanPodSpec': {},
'ckanContainerSpec': {'imageFromGitlab': gitlab_repo},
'envvars': {'fromGitlab': gitlab_repo},
'solrCloudCollection': {
'name': kwargs.get('solr_collection') or instance_id,
'configName': solr_config
},
'db': {
'name': db_name,
**({'fromDbMigration': migration_name} if migration_name else {}),
**({'dbPrefix': db_prefix} if db_prefix else {})
},
'datastore': {
'name': datastore_name,
**({'fromDbMigration': migration_name} if migration_name else {}),
**({'dbPrefix': db_prefix} if db_prefix else {})
},
'storage': {
'path': storage_path,
}
}
if kwargs.get('use_private_gitlab_repo'):
spec['ckanContainerSpec']['imagePullSecrets'] = [{'name': 'private-gitlab-registry'}]
elif create_type == 'from-gcloud-envvars':
print(f'Creating Deis CKAN instance {instance_id} from gcloud envvars import')
instance_env_yaml, image, solr_config, storage_path, instance_id = args[1:]
db_migration_name = kwargs.get('db_migration_name')
assert db_migration_name, 'creating from gcloud envvars without a db migration is not supported yet'
if type(instance_env_yaml) == str:
logs.info(f'Creating {instance_id}-envvars secret from file: {instance_env_yaml}')
subprocess.check_call(
f'kubectl -n ckan-cloud create secret generic {instance_id}-envvars --from-file=envvars.yaml={instance_env_yaml}',
shell=True
)
else:
logs.info(f'Creating {instance_id}-envvars secret from inline string')
kubectl.update_secret(f'{instance_id}-envvars', {'envvars.yaml': yaml.dump(instance_env_yaml, default_flow_style=False)})
spec = {
'ckanPodSpec': {},
'ckanContainerSpec': {'image': image},
'envvars': {'fromSecret': f'{instance_id}-envvars'},
'solrCloudCollection': {
'name': instance_id,
'configName': solr_config
},
'db': {
'name': instance_id,
'fromDbMigration':db_migration_name,
**({'dbPrefix': db_prefix} if db_prefix else {})
},
'datastore': {
'name': f'{instance_id}-datastore',
'fromDbMigration': db_migration_name,
**({'dbPrefix': db_prefix} if db_prefix else {})
},
'storage': {
'path': storage_path
}
}
else:
raise NotImplementedError(f'invalid create type: {create_type}')
instance_kind = ckan_manager.instance_kind()
instance = {
'apiVersion': f'stable.viderum.com/v1',
'kind': instance_kind,
'metadata': {
'name': instance_id,
'namespace': 'ckan-cloud',
'finalizers': ['finalizer.stable.viderum.com']
},
'spec': spec
}
subprocess.run('kubectl apply -f -', input=yaml.dump(instance).encode(), shell=True, check=True)
return cls(instance_id, values=instance)
print('new_db_name', new_db_name, file=sys.__stderr__)
print('new_datastore_name', new_datastore_name, file=sys.__stderr__)
migration_name = None
if os.environ.get('USE_EXISTING_MIGRATION') == 'yes':
for migration in ckan_db_migration.get()['items']:
if migration.get('spec', {}).get('db-name') == new_db_name:
migration_name = migration['spec']['name']
break
elif not dry_run:
success = False
for event in ckan_db_migration.migrate_deis_dbs(None, new_db_name, new_datastore_name, db_import_url=old_db_url,
datastore_import_url=old_datastore_url, db_prefix=new_db_prefix):
migration_name = ckan_db_migration.get_event_migration_created_name(event) or migration_name
success = ckan_db_migration.print_event_exit_on_complete(event,
f'DBs import -> {new_db_name}, {new_datastore_name}',
soft_exit=True)
if success is not None:
break
assert success, f'Invalid DB migration success value ({success})'
print('migration_name', migration_name, file=sys.__stderr__)
if not new_solr_collection_name:
new_solr_collection_name = new_instance_id
logs.info(new_solr_collection_name=new_solr_collection_name)
spec = {
'ckanPodSpec': {},
'ckanContainerSpec': {'imageFromGitlab': new_gitlab_repo},
'envvars': {'fromGitlab': new_gitlab_repo},