def get_all_dbs_users():
from ckan_cloud_operator.deis_ckan.instance import DeisCkanInstance
from ckan_cloud_operator.providers.db import manager as db_manager
dbs, users = [], []
db_host, db_port = db_manager.get_internal_unproxied_db_host_port()
instance_db_names = []
instance_user_names = []
for item in DeisCkanInstance.list(quick=True, return_list=True):
instance_id = item['id']
instance = DeisCkanInstance(instance_id)
spec = instance.spec
db_name = spec.db['name']
db_password = instance.annotations.get_secret('databasePassword')
datastore_name = spec.datastore['name']
datastore_password = instance.annotations.get_secret('datastorePassword')
datastore_ro_user = instance.annotations.get_secret('datastoreReadonlyUser')
datastore_ro_password = instance.annotations.get_secret('datatastoreReadonlyPassword')
if all([db_name, db_password, datastore_name, datastore_password, datastore_ro_user, datastore_ro_password]):
dbs.append((db_name, db_host, db_port))
dbs.append((datastore_name, db_host, db_port))
instance_db_names += [db_name, datastore_name]
users.append((db_name, db_password))
users.append((datastore_name, datastore_password))
users.append((datastore_ro_user, datastore_ro_password))
instance_user_names += [db_name, datastore_name, datastore_ro_user]
migration_dbs, migration_users = ckan_db_migration_manager.get_all_dbs_users()
for migration_db in migration_dbs:
if migration_db[0] and migration_db[0] not in instance_db_names:
dbs.append(migration_db)
for migration_user in migration_users:
if migration_user[0] and migration_user[0] not in instance_user_names:
users.append(migration_user)
return dbs, users
def post_create_checks(instance_id):
from ckan_cloud_operator.deis_ckan.instance import DeisCkanInstance
from ckan_cloud_operator.deis_ckan.envvars import DeisCkanInstanceEnvvars
instance = DeisCkanInstance(instance_id)
envvars = DeisCkanInstanceEnvvars(instance).get(full=True)
assert envvars['ready']
envvars = envvars['envvars']
check_envvars(envvars, instance)
def migrate_deis_instance(old_site_id,
new_instance_id=None,
router_name=None,
skip_gitlab=False,
force=False,
rerun=False,
recreate_dbs=False,
recreate=False,
recreate_instance=False,
skip_routes=False,
skip_solr=False,
skip_deployment=False,
no_db_proxy=False):
"""Run a full end-to-end migration of an instasnce"""
from ckan_cloud_operator.deis_ckan.instance import DeisCkanInstance
if not router_name:
router_name = get_default_instances_router_name()
if not new_instance_id:
new_instance_id = old_site_id
if recreate or recreate_instance:
DeisCkanInstance(new_instance_id).delete(force=True, wait_deleted=True)
success = False
db_name = f'{new_instance_id}'
datastore_name = f'{new_instance_id}-datastore'
migration_name = None
for event in ckan_db_migration_manager.migrate_deis_dbs(
old_site_id,
db_name,
datastore_name,
force=force or recreate,
rerun=rerun or recreate,
recreate_dbs=recreate_dbs or recreate):
migration_name = ckan_db_migration_manager.get_event_migration_created_name(
event) or migration_name
success = ckan_db_migration_manager.print_event_exit_on_complete(
event,
f'DBs Migration from old site ID: {old_site_id}',
soft_exit=True)
if success is not None:
break
assert success, f'Invalid DB migration success value ({success})'
logs.info(
'DB migration completed successfully, continuing with instance migration'
)
migrate_from_deis(old_site_id,
new_instance_id,
router_name,
DeisCkanInstance,
skip_gitlab=skip_gitlab,
db_migration_name=migration_name,
recreate=recreate or recreate_instance,
skip_routes=skip_routes,
skip_solr=skip_solr,
skip_deployment=skip_deployment,
no_db_proxy=no_db_proxy)
post_create_checks(new_instance_id)
def ckan_admin_credentials(instance_id):
from ckan_cloud_operator.deis_ckan.instance import DeisCkanInstance
from ckan_cloud_operator.deis_ckan.ckan import DeisCkanInstanceCKAN
return DeisCkanInstanceCKAN(
DeisCkanInstance(instance_id)).admin_credentials()
def setUp(self):
self.ckan = DeisCkanInstanceCKAN(DeisCkanInstance('montreal'))
def setUp(self):
self.instance = DeisCkanInstance('montreal')
self.instance._spec = Mock(datastore={'name': 'test'})
self.db = DeisCkanInstanceDb(self.instance, 'db')
self.datastore = DeisCkanInstanceDb(self.instance, 'datastore')
def migrate_from_deis(old_site_id,
new_instance_id,
router_name,
deis_instance_class,
skip_gitlab=False,
db_migration_name=None,
recreate=False,
skip_routes=False,
skip_solr=False,
skip_deployment=False,
no_db_proxy=False):
assert db_migration_name, 'migration without a db migration is not supported yet'
log_labels = {'instance': new_instance_id}
if recreate:
from ckan_cloud_operator.deis_ckan.instance import DeisCkanInstance
DeisCkanInstance(new_instance_id).delete(
force=True, wait_deleted=not db_migration_name)
logs.info(
f'Migrating from old site id {old_site_id} to new instance id {new_instance_id}',
**log_labels)
instance_kind = ckan_manager.instance_kind()
values = kubectl.get(f'{instance_kind} {new_instance_id}', required=False)
if values:
logs.info('instance already exists', **log_labels)
else:
logs.info('creating instance', **log_labels)
path_to_old_cluster_kubeconfig = get_path_to_old_cluster_kubeconfig()
solr_config = get_solr_config(old_site_id,
path_to_old_cluster_kubeconfig)
assert solr_config, 'failed to get solr config name'
instance_env = get_instance_env(old_site_id,
path_to_old_cluster_kubeconfig)
gitlab_repo = f'viderum/cloud-{old_site_id}'
if not skip_gitlab:
CkanGitlab().initialize(gitlab_repo)
gitlab_registry = f'registry.gitlab.com/{gitlab_repo}'
old_bucket_name = instance_env.get(
'CKANEXT__S3FILESTORE__AWS_BUCKET_NAME')
old_storage_path = instance_env.get(
'CKANEXT__S3FILESTORE__AWS_STORAGE_PATH')
assert old_bucket_name == 'ckan'
assert old_storage_path and len(old_storage_path) > 1
storage_path = f'/ckan/{old_storage_path}'
deis_instance_class.create('from-gcloud-envvars',
instance_env,
gitlab_registry,
solr_config,
storage_path,
new_instance_id,
db_migration_name=db_migration_name)
routers_env_id = routers_provider.get_env_id()
default_root_domain = routers_provider.get_default_root_domain()
assert routers_env_id and default_root_domain
ckan_site_url = f'https://cc-{routers_env_id}-{new_instance_id}.{default_root_domain}'
logs.info(
f'updating instance and setting ckan site url to {ckan_site_url}',
**log_labels)
deis_instance_class(new_instance_id,
override_spec={
'envvars': {
'CKAN_SITE_URL': ckan_site_url
},
**({
'db': {
'no-db-proxy': 'yes'
},
'datastore': {
'no-db-proxy': 'yes'
}
} if no_db_proxy else {})
},
persist_overrides=True).update(
wait_ready=True,
skip_solr=skip_solr,
skip_deployment=skip_deployment)
if routers_manager.get_deis_instance_routes(new_instance_id):
logs.info('default instance route already exists', **log_labels)
else:
logs.info('creating instance route', **log_labels)
routers_manager.create_subdomain_route(
router_name, {
'target-type': 'deis-instance',
'deis-instance-id': new_instance_id,
'root-domain': 'default',
'sub-domain': 'default'
})
if not skip_routes:
routers_manager.update(router_name, wait_ready=True)
if not skip_solr:
logs.info('Rebuilding solr search index', **log_labels)
deis_instance_class(new_instance_id).ckan.paster(
'search-index rebuild --force')
from ckan_cloud_operator.providers.ckan import manager as ckan_manager
old_instance_id = os.environ['OLD_INSTANCE_ID']
new_instance_id = os.environ['NEW_INSTANCE_ID']
new_gitlab_repo = os.environ['NEW_GITLAB_REPO']
new_db_prefix = os.environ['NEW_DB_PREFIX']
new_solr_collection_name = os.environ['NEW_SOLR_COLLECTION_NAME']
new_storage_path = os.environ['NEW_STORAGE_PATH']
database_import_url = os.environ['DATABASE_IMPORT_URL']
datastore_import_url = os.environ['DATASTORE_IMPORT_URL']
dry_run = (os.environ['DRY_RUN'] == 'yes')
gs_base_url = config_manager.get(secret_name='ckan-cloud-provider-db-gcloudsql-credentials', key='backups-gs-base-url')
logs.info(gs_base_url=gs_base_url)
old_instance = DeisCkanInstance(old_instance_id)
old_db_prefix = old_instance.spec.db.get('dbPrefix', '')
if not old_db_prefix or old_db_prefix == 'prod1':
dbprefixpath = ''
else:
dbprefixpath = '/' + old_db_prefix
assert old_db_prefix == old_instance.spec.datastore.get('dbPrefix',
''), 'different db prefix for db and datastore is not supported'
old_db_name = old_instance.spec.db['name']
old_datastore_name = old_instance.spec.datastore['name']
logs.info(old_db_name=old_db_name, old_datastore_name=old_datastore_name, old_db_prefix=old_db_prefix)
old_db_url, old_datastore_url = None, None
@main.group()
def ckan_infra():
"""Manage the centralized infrastructure"""
pass
CkanInfra.add_cli_commands(click, ckan_infra, great_success)
@main.group()
def deis_instance():
"""Manage Deis CKAN instance resources"""
pass
DeisCkanInstance.add_cli_commands(click, deis_instance, great_success)
@main.group()
def routers():
"""Manage CKAN Cloud routers"""
pass
ckan_cloud_operator.routers.cli.add_cli_commands(click, routers, great_success)
@main.group()
def datapushers():
"""Manage centralized CKAN DataPushers"""
pass
