def get_deis_instance_credentials(instance_id,
is_datastore=False,
is_datastore_readonly=False,
required=True,
with_db_prefix=False):
none = (None, None, None, None) if with_db_prefix else (None, None, None)
instance_kind = ckan_manager.instance_kind()
instance = kubectl.get(f'{instance_kind} {instance_id}', required=required)
if not instance: return none
secret = kubectl.get(f'secret {instance_id}-annotations',
namespace=instance_id,
required=required)
if not secret: return none
secret = kubectl.decode_secret(secret)
if is_datastore or is_datastore_readonly:
db_name = user = instance['spec'].get('datastore', {}).get('name')
if is_datastore_readonly:
user = secret.get('datastoreReadonlyUser')
password = secret.get('datatastoreReadonlyPassword')
else:
password = secret.get('datastorePassword')
else:
db_name = user = instance['spec'].get('db', {}).get('name')
password = secret.get('databasePassword')
res = [user, password, db_name]
if all(res):
if with_db_prefix:
res.append(
get_deis_instance_db_prefix_from_instance(
instance, is_datastore or is_datastore_readonly))
return res
else:
assert not required, 'missing some db values'
return none
def list(cls, full=False, quick=False, return_list=False):
res = []
data = kubectl.get(ckan_manager.instance_kind(), required=False)
if not data: data = {'items': []}
for item in data['items']:
if quick:
data = {
'id': item['metadata']['name'],
'ready': None
}
if full:
data['item'] = item
else:
try:
instance = DeisCkanInstance(item['metadata']['name'], values=item)
data = instance.get()
if not full:
data = {'id': instance.id, 'ready': data['ready']}
except Exception:
data = {'id': item['metadata']['name'], 'ready': False, 'error': traceback.format_exc()}
if return_list:
res.append(data)
else:
print(yaml.dump([data], default_flow_style=False))
if return_list:
return res
def deis_instance_edit(instance_id, editor):
"""Launch an editor to modify and update an instance"""
instance_kind = ckan_manager.instance_kind()
subprocess.call(
f'EDITOR={editor} kubectl -n ckan-cloud edit {instance_kind}/{instance_id}',
shell=True)
cls(instance_id).update()
great_success()
def get_deis_instance_db_prefix(instance_id, is_datastore=False):
instance_kind = ckan_manager.instance_kind()
instance = kubectl.get(f'{instance_kind} {instance_id}', required=False)
return get_deis_instance_db_prefix_from_instance(instance, is_datastore)
def create(cls, *args, **kwargs):
create_type = args[0]
instance_id = args[-1]
from ckan_cloud_operator.providers.db.manager import get_default_db_prefix
db_prefix = kwargs['db_prefix'] if kwargs.get('db_prefix') else get_default_db_prefix()
if create_type == 'from-gitlab':
gitlab_repo = args[1]
solr_config = args[2]
db_name = instance_id
datastore_name = f'{instance_id}-datastore'
storage_path = kwargs.get('storage_path') or f'/ckan/{instance_id}'
from_db_backups = kwargs.get('from_db_backups')
logs.info(f'Creating Deis CKAN instance {instance_id}', gitlab_repo=gitlab_repo, solr_config=solr_config,
db_name=db_name, datastore_name=datastore_name, storage_path=storage_path,
from_db_backups=from_db_backups)
if kwargs.get('use_private_gitlab_repo'):
deploy_token_server = input('Gitlab registry url [default: registry.gitlab.com]: ') or 'registry.gitlab.com'
deploy_token_username = input('Gitlab deploy token username: '******'Gitlab deploy token password: '******'delete secret private-gitlab-registry', namespace=instance_id)
kubectl.call(f'create secret docker-registry private-gitlab-registry --docker-server={deploy_token_server} --docker-username={deploy_token_username} --docker-password={deploy_token_password}', namespace=instance_id)
if from_db_backups:
db_import_url, datastore_import_url = from_db_backups.split(',')
migration_name = None
success = False
for event in ckan_db_migration_manager.migrate_deis_dbs(None, db_name, datastore_name,
db_import_url=db_import_url,
datastore_import_url=datastore_import_url,
rerun=kwargs.get('rerun'),
force=kwargs.get('force'),
recreate_dbs=kwargs.get('recreate_dbs'),
db_prefix=db_prefix):
migration_name = ckan_db_migration_manager.get_event_migration_created_name(event) or migration_name
success = ckan_db_migration_manager.print_event_exit_on_complete(
event,
f'DBs import {from_db_backups} -> {db_name}, {datastore_name}',
soft_exit=True
)
if success is not None:
break
assert success, f'Invalid DB migration success value ({success})'
else:
migration_name = None
spec = {
'ckanPodSpec': {},
'ckanContainerSpec': {'imageFromGitlab': gitlab_repo},
'envvars': {'fromGitlab': gitlab_repo},
'solrCloudCollection': {
'name': kwargs.get('solr_collection') or instance_id,
'configName': solr_config
},
'db': {
'name': db_name,
**({'fromDbMigration': migration_name} if migration_name else {}),
**({'dbPrefix': db_prefix} if db_prefix else {})
},
'datastore': {
'name': datastore_name,
**({'fromDbMigration': migration_name} if migration_name else {}),
**({'dbPrefix': db_prefix} if db_prefix else {})
},
'storage': {
'path': storage_path,
}
}
if kwargs.get('use_private_gitlab_repo'):
spec['ckanContainerSpec']['imagePullSecrets'] = [{'name': 'private-gitlab-registry'}]
elif create_type == 'from-gcloud-envvars':
print(f'Creating Deis CKAN instance {instance_id} from gcloud envvars import')
instance_env_yaml, image, solr_config, storage_path, instance_id = args[1:]
db_migration_name = kwargs.get('db_migration_name')
assert db_migration_name, 'creating from gcloud envvars without a db migration is not supported yet'
if type(instance_env_yaml) == str:
logs.info(f'Creating {instance_id}-envvars secret from file: {instance_env_yaml}')
subprocess.check_call(
f'kubectl -n ckan-cloud create secret generic {instance_id}-envvars --from-file=envvars.yaml={instance_env_yaml}',
shell=True
)
else:
logs.info(f'Creating {instance_id}-envvars secret from inline string')
kubectl.update_secret(f'{instance_id}-envvars', {'envvars.yaml': yaml.dump(instance_env_yaml, default_flow_style=False)})
spec = {
'ckanPodSpec': {},
'ckanContainerSpec': {'image': image},
'envvars': {'fromSecret': f'{instance_id}-envvars'},
'solrCloudCollection': {
'name': instance_id,
'configName': solr_config
},
'db': {
'name': instance_id,
'fromDbMigration':db_migration_name,
**({'dbPrefix': db_prefix} if db_prefix else {})
},
'datastore': {
'name': f'{instance_id}-datastore',
'fromDbMigration': db_migration_name,
**({'dbPrefix': db_prefix} if db_prefix else {})
},
'storage': {
'path': storage_path
}
}
else:
raise NotImplementedError(f'invalid create type: {create_type}')
instance_kind = ckan_manager.instance_kind()
instance = {
'apiVersion': f'stable.viderum.com/v1',
'kind': instance_kind,
'metadata': {
'name': instance_id,
'namespace': 'ckan-cloud',
'finalizers': ['finalizer.stable.viderum.com']
},
'spec': spec
}
subprocess.run('kubectl apply -f -', input=yaml.dump(instance).encode(), shell=True, check=True)
return cls(instance_id, values=instance)
def kind(self):
return ckan_manager.instance_kind()
def migrate_from_deis(old_site_id,
new_instance_id,
router_name,
deis_instance_class,
skip_gitlab=False,
db_migration_name=None,
recreate=False,
skip_routes=False,
skip_solr=False,
skip_deployment=False,
no_db_proxy=False):
assert db_migration_name, 'migration without a db migration is not supported yet'
log_labels = {'instance': new_instance_id}
if recreate:
from ckan_cloud_operator.deis_ckan.instance import DeisCkanInstance
DeisCkanInstance(new_instance_id).delete(
force=True, wait_deleted=not db_migration_name)
logs.info(
f'Migrating from old site id {old_site_id} to new instance id {new_instance_id}',
**log_labels)
instance_kind = ckan_manager.instance_kind()
values = kubectl.get(f'{instance_kind} {new_instance_id}', required=False)
if values:
logs.info('instance already exists', **log_labels)
else:
logs.info('creating instance', **log_labels)
path_to_old_cluster_kubeconfig = get_path_to_old_cluster_kubeconfig()
solr_config = get_solr_config(old_site_id,
path_to_old_cluster_kubeconfig)
assert solr_config, 'failed to get solr config name'
instance_env = get_instance_env(old_site_id,
path_to_old_cluster_kubeconfig)
gitlab_repo = f'viderum/cloud-{old_site_id}'
if not skip_gitlab:
CkanGitlab().initialize(gitlab_repo)
gitlab_registry = f'registry.gitlab.com/{gitlab_repo}'
old_bucket_name = instance_env.get(
'CKANEXT__S3FILESTORE__AWS_BUCKET_NAME')
old_storage_path = instance_env.get(
'CKANEXT__S3FILESTORE__AWS_STORAGE_PATH')
assert old_bucket_name == 'ckan'
assert old_storage_path and len(old_storage_path) > 1
storage_path = f'/ckan/{old_storage_path}'
deis_instance_class.create('from-gcloud-envvars',
instance_env,
gitlab_registry,
solr_config,
storage_path,
new_instance_id,
db_migration_name=db_migration_name)
routers_env_id = routers_provider.get_env_id()
default_root_domain = routers_provider.get_default_root_domain()
assert routers_env_id and default_root_domain
ckan_site_url = f'https://cc-{routers_env_id}-{new_instance_id}.{default_root_domain}'
logs.info(
f'updating instance and setting ckan site url to {ckan_site_url}',
**log_labels)
deis_instance_class(new_instance_id,
override_spec={
'envvars': {
'CKAN_SITE_URL': ckan_site_url
},
**({
'db': {
'no-db-proxy': 'yes'
},
'datastore': {
'no-db-proxy': 'yes'
}
} if no_db_proxy else {})
},
persist_overrides=True).update(
wait_ready=True,
skip_solr=skip_solr,
skip_deployment=skip_deployment)
if routers_manager.get_deis_instance_routes(new_instance_id):
logs.info('default instance route already exists', **log_labels)
else:
logs.info('creating instance route', **log_labels)
routers_manager.create_subdomain_route(
router_name, {
'target-type': 'deis-instance',
'deis-instance-id': new_instance_id,
'root-domain': 'default',
'sub-domain': 'default'
})
if not skip_routes:
routers_manager.update(router_name, wait_ready=True)
if not skip_solr:
logs.info('Rebuilding solr search index', **log_labels)
deis_instance_class(new_instance_id).ckan.paster(
'search-index rebuild --force')
**({'fromDbMigration': migration_name} if migration_name else {}),
**({'dbPrefix': new_db_prefix} if new_db_prefix else {})
},
'datastore': {
'name': new_datastore_name,
**({'fromDbMigration': migration_name} if migration_name else {}),
**({'dbPrefix': new_db_prefix} if new_db_prefix else {})
},
'storage': {
'path': new_storage_path,
}
}
print('spec', spec, file=sys.__stderr__)
if os.environ.get('SKIP_CREATE') != 'yes':
instance_kind = ckan_manager.instance_kind()
instance = {
'apiVersion': f'stable.viderum.com/v1',
'kind': instance_kind,
'metadata': {
'name': new_instance_id,
'namespace': 'ckan-cloud',
'finalizers': ['finalizer.stable.viderum.com']
},
'spec': spec
}
kubectl.apply(instance, dry_run=dry_run)
if not dry_run:
DeisCkanInstance(new_instance_id, values=instance).update()
if os.environ.get('SKIP_ROUTER') != 'yes':
