def _get_instance(instance_id_or_name=None, instance_id=None, required=True):
if instance_id:
logs.debug(f'Getting instance using instance_id', instance_id=instance_id)
instance = crds_manager.get(APP_CRD_SINGULAR, name=instance_id, required=False)
instance_name = None
else:
logs.debug(f'Attempting to get instance using id', instance_id_or_name=instance_id_or_name)
instance = crds_manager.get(APP_CRD_SINGULAR, name=instance_id_or_name, required=False)
if instance:
instance_id = instance_id_or_name
instance_name = None
else:
logs.debug(f'Attempting to get instance from instance name', instance_id_or_name=instance_id_or_name)
instance_name = crds_manager.get(APP_NAME_CRD_SINGULAR, name=instance_id_or_name, required=False)
if instance_name:
instance_id = instance_name['spec'].get('latest-instance-id')
logs.debug(instance_id=instance_id)
instance = crds_manager.get(APP_CRD_SINGULAR, name=instance_id, required=False)
instance_name = instance_id_or_name
else:
instance_name = None
logs.debug_yaml_dump(instance_name=instance_name, instance_id=instance_id, instance=bool(instance))
if required:
assert instance_id and len(instance) > 2, \
f'Failed to find instance (instance_id_or_name={instance_id_or_name}, instance_id={instance_id})'
return instance_id, instance
def _get_instance_id_and_type(instance_id_or_name=None, instance_id=None, required=True):
if instance_id:
logs.debug(f'Getting instance type using instance_id', instance_id=instance_id)
instance = crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id, required=False)
instance_name = None
else:
logs.debug(f'Attempting to get instance type using id', instance_id_or_name=instance_id_or_name)
instance = crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id_or_name, required=False)
if instance:
instance_id = instance_id_or_name
instance_name = None
else:
logs.debug(f'Attempting to get instance type from instance name', instance_id_or_name=instance_id_or_name)
instance_name = crds_manager.get(INSTANCE_NAME_CRD_SINGULAR, name=instance_id_or_name, required=False)
if instance_name:
instance_id = instance_name['spec'].get('latest-instance-id')
logs.debug(instance_id=instance_id)
instance = crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id, required=False)
instance_name = instance_id_or_name
else:
instance_name = None
if instance:
instance_type = instance['metadata']['labels'].get('{}/instance-type'.format(labels_manager.get_label_prefix()))
else:
instance_type = None
logs.debug_yaml_dump(instance_name=instance_name, instance_id=instance_id, instance_type=instance_type,
instance=bool(instance))
if required:
assert instance_id and instance_type and len(instance) > 2, f'Failed to find instance (instance_id_or_name={instance_id_or_name}, instance_id={instance_id})'
return instance_id, instance_type, instance
def get_all_instance_id_names():
instance_names = crds_manager.get(INSTANCE_NAME_CRD_SINGULAR, required=False)
instance_name_ids = {}
if instance_names:
for instance_name in instance_names['items']:
instance_name_ids[instance_name['spec']['latest-instance-id']] = instance_name['spec']['name']
label_prefix = labels_manager.get_label_prefix()
for instance in crds_manager.get(INSTANCE_CRD_SINGULAR)['items']:
instance_id = instance['metadata']['labels'][f'{label_prefix}/crd-ckaninstance-name']
instance_name = instance_name_ids.pop(instance_id, None)
yield {'id': instance_id, 'name': instance_name}
for instance_id, instance_name in instance_name_ids.items():
yield {'id': instance_id, 'name': instance_name}
def _check_instance_events(instance_id, force_update_events=False):
status = get(instance_id)
errors = []
ckan_cloud_logs = []
ckan_cloud_events = set()
pod_names = []
for app, app_status in status.get('helm_app_statuses', {}).items():
for kind, kind_items in app_status.items():
for item in kind_items:
for error in item.get("errors", []):
errors.append(dict(error, kind=kind, app=app, name=item.get("name")))
for logdata in item.get("ckan-cloud-logs", []):
ckan_cloud_logs.append(dict(logdata, kind=kind, app=app, name=item.get("name")))
if "event" in logdata:
ckan_cloud_events.add(logdata["event"])
if kind == "pods":
pod_names.append(item["name"])
instance = crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id)
if force_update_events or annotations_manager.get_status(instance, 'helm', 'created'):
logs.debug('expecting update events')
expected_events = {
"ckan-env-vars-exists", "ckan-secrets-exists", "got-ckan-secrets",
"ckan-entrypoint-initialized", "ckan-entrypoint-db-init-success",
"ckan-entrypoint-extra-init-success"
}
else:
logs.debug('expecting create events')
expected_events = {
"ckan-env-vars-created", "ckan-secrets-created", "got-ckan-secrets", "ckan-db-initialized",
"ckan-datastore-db-initialized", "ckan-entrypoint-initialized", "ckan-entrypoint-db-init-success",
"ckan-entrypoint-extra-init-success"
}
logs.debug(ckan_cloud_events=ckan_cloud_events)
return expected_events.difference(ckan_cloud_events)
def create(instance_type, instance_id=None, instance_name=None, values=None, values_filename=None, exists_ok=False, dry_run=False):
if not instance_id:
if instance_name:
instance_id = '{}-{}'.format(instance_name, _generate_password(6))
else:
instance_id = _generate_password(12)
if values_filename:
assert values is None
with open(values_filename) as f:
values = yaml.load(f.read())
if not exists_ok and crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id, required=False):
raise Exception('instance already exists')
values_id = values.get('id')
if values_id:
assert values_id == instance_id, f'instance spec has conflicting instance_id ({values_id} != {instance_id})'
values['id'] = instance_id
logs.info('Creating instance', instance_id=instance_id)
kubectl.apply(crds_manager.get_resource(
INSTANCE_CRD_SINGULAR, instance_id,
extra_label_suffixes={'instance-type': instance_type},
spec=values
), dry_run=dry_run)
if instance_name:
set_name(instance_id, instance_name, dry_run=dry_run)
return instance_id
def set_name(instance_id, instance_name, dry_run=False):
resource = crds_manager.get(APP_NAME_CRD_SINGULAR,
name=instance_name,
required=False)
if resource:
resource['spec']['latest-instance-id'] = instance_id
if not resource['spec']['instance-ids'].get(instance_id):
resource['spec']['instance-ids'][instance_id] = {
'added': datetime.datetime.now()
}
else:
resource = crds_manager.get_resource(
APP_NAME_CRD_SINGULAR,
instance_name,
spec={
'name': instance_name,
'latest-instance-id': instance_id,
'instance-ids': {
instance_id: {
'added': datetime.datetime.now()
}
}
})
if dry_run:
logs.print_yaml_dump(resource)
else:
kubectl.apply(resource)
def update(instance_id, instance):
tiller_namespace_name = _get_resource_name()
_init_namespace(instance_id)
_init_ckan_infra_secret(instance_id)
ckan_helm_chart_repo = instance['spec'].get(
"ckanHelmChartRepo",
"https://raw.githubusercontent.com/ViderumGlobal/ckan-cloud-helm/master/charts_repository"
)
ckan_helm_chart_version = instance['spec'].get("ckanHelmChartVersion", "")
ckan_helm_release_name = f'ckan-cloud-{instance_id}'
instance['spec']['centralizedSolrHost'], instance['spec'][
'centralizedSolrPort'] = _init_solr(instance_id)
if annotations_manager.get_status(instance, 'helm', 'created'):
values = instance['spec']
else:
values = {
**instance['spec'], "replicas": 1,
"nginxReplicas": 1,
"disableJobs": True,
"noProbes": True
}
_helm_deploy(values, tiller_namespace_name, ckan_helm_chart_repo,
ckan_helm_chart_version, ckan_helm_release_name, instance_id)
_wait_instance_events(instance_id)
instance = crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id)
if not annotations_manager.get_status(instance, 'helm', 'created'):
annotations_manager.set_status(instance, 'helm', 'created')
_helm_deploy(instance['spec'], tiller_namespace_name,
ckan_helm_chart_repo, ckan_helm_chart_version,
ckan_helm_release_name, instance_id)
def _get_instance_and_type(instance_id):
instance = crds_manager.get(INSTANCE_CRD_SINGULAR,
name=instance_id,
required=False)
instance_type = instance['metadata']['labels'].get(
'{}/instance-type'.format(labels_manager.get_label_prefix()))
return instance, instance_type
def create(instance_type, instance_id=None, instance_name=None, values=None, values_filename=None, exists_ok=False,
dry_run=False, update_=False, wait_ready=False, skip_deployment=False, skip_route=False, force=False):
if not instance_id:
if instance_name:
instance_id = '{}-{}'.format(instance_name, _generate_password(6))
else:
instance_id = _generate_password(12)
if values_filename:
assert values is None
if values_filename != '-':
with open(values_filename) as f:
values = yaml.load(f.read())
else:
values = yaml.load(sys.stdin.read())
if not exists_ok and crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id, required=False):
raise Exception('instance already exists')
values_id = values.get('id')
if values_id and values_id != instance_id:
logs.warning(f'changing instance id in spec from {values_id} to the instance id {instance_id}')
values['id'] = instance_id
logs.info('Creating instance', instance_id=instance_id)
kubectl.apply(crds_manager.get_resource(
INSTANCE_CRD_SINGULAR, instance_id,
extra_label_suffixes={'instance-type': instance_type},
spec=values
), dry_run=dry_run)
if instance_name:
set_name(instance_id, instance_name, dry_run=dry_run)
if update_:
update(instance_id, wait_ready=wait_ready, skip_deployment=skip_deployment, skip_route=skip_route, force=force,
dry_run=dry_run)
return instance_id
def _pre_update_hook_modify_spec(instance_id, instance, callback, dry_run=False):
# applies changes to both the non-persistent spec and persists the changes on latest instance spec
latest_instance = crds_manager.get(INSTANCE_CRD_SINGULAR, crds_manager.get_resource_name(
INSTANCE_CRD_SINGULAR, instance_id
), required=True)
callback(instance)
callback(latest_instance)
kubectl.apply(latest_instance, dry_run=dry_run)
def _get_instance_id_and_type(instance_id_or_name=None, instance_id=None):
if instance_id:
instance = crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id, required=False)
else:
instance = crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id_or_name, required=False)
if instance:
instance_id = instance_id_or_name
if not instance:
assert not instance_id
instance_name = crds_manager.get(INSTANCE_NAME_CRD_SINGULAR, name=instance_id_or_name, required=False)
if instance_name:
instance_id = instance_name['spec'].get('latest-instance-id')
instance = crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id, required=False)
else:
instance_id = None
instance_type = instance['metadata']['labels'].get('{}/instance-type'.format(labels_manager.get_label_prefix())) if instance else None
return instance_id, instance_type, instance
def update(instance_id, instance, force=False, dry_run=False):
tiller_namespace_name = _get_resource_name()
logs.debug('Updating helm-based instance deployment',
instance_id=instance_id,
tiller_namespace_name=tiller_namespace_name)
_create_private_container_registry_secret(instance_id)
_init_ckan_infra_secret(instance_id, dry_run=dry_run)
ckan_helm_chart_repo = instance['spec'].get(
"ckanHelmChartRepo",
"https://raw.githubusercontent.com/ViderumGlobal/ckan-cloud-helm/master/charts_repository"
)
ckan_helm_chart_version = instance['spec'].get("ckanHelmChartVersion", "")
ckan_helm_release_name = f'ckan-cloud-{instance_id}'
solr_schema = instance['spec'].get("ckanSolrSchema", "ckan_default")
solr_host, solr_port = _init_solr(
instance_id,
solr_schema,
dry_run=dry_run,
)
logs.debug(ckan_helm_chart_repo=ckan_helm_chart_repo,
ckan_helm_chart_version=ckan_helm_chart_version,
ckan_helm_release_name=ckan_helm_release_name,
solr_host=solr_host,
solr_port=solr_port)
instance['spec']['centralizedSolrHost'], instance['spec'][
'centralizedSolrPort'] = solr_host, solr_port
if annotations_manager.get_status(instance, 'helm', 'created'):
logs.info('Updating existing instance')
values = instance['spec']
else:
logs.info(
'New instance, deploying first with 1 replica and disabled probes and jobs'
)
values = {
**instance['spec'], "replicas": 1,
"nginxReplicas": 1,
"disableJobs": True,
"noProbes": True,
"enableHarvesterNG": False
}
_helm_deploy(values,
tiller_namespace_name,
ckan_helm_chart_repo,
ckan_helm_chart_version,
ckan_helm_release_name,
instance_id,
dry_run=dry_run)
if not dry_run:
_wait_instance_events(instance_id)
instance = crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id)
if not annotations_manager.get_status(instance, 'helm', 'created'):
annotations_manager.set_status(instance, 'helm', 'created')
_helm_deploy(instance['spec'], tiller_namespace_name,
ckan_helm_chart_repo, ckan_helm_chart_version,
ckan_helm_release_name, instance_id)
_scale_down_scale_up(namespace=instance_id,
replicas=values.get('replicas', 1))
def load(cluster_id):
cluster = crds_manager.get(
'kamateracluster',
crds_manager.get_resource_name('kamateracluster', cluster_id))['spec']
cluster['servers'] = []
servers_path = os.path.expanduser(f'~/cluster-{cluster_id}-servers')
os.makedirs(servers_path, exist_ok=True)
for server_name in cluster['server_names']:
server = crds_manager.get(
'kamateraserver',
crds_manager.get_resource_name('kamateraserver',
server_name))['spec']
server_secrets = crds_manager.config_get('kamateraserver',
server_name,
is_secret=True)
password_filename = f'{servers_path}/server_{server_name}_password.txt'
private_key_filename = f'{servers_path}/server_{server_name}_id_rsa'
public_key_filename = private_key_filename + '.pub'
with open(password_filename, 'w') as f:
f.write(server_secrets['password'])
with open(private_key_filename, 'w') as f:
f.write(server_secrets['private_key'])
with open(public_key_filename, 'w') as f:
f.write(server_secrets['public_key'])
server['passwordfile'] = password_filename
server['keyfile'] = private_key_filename
cluster['servers'].append(server)
del cluster['server_names']
cluster_path = get_cluster_path(cluster_id)
os.makedirs(cluster_path, exist_ok=True)
with open(f'{cluster_path}/cluster.json', 'w') as f:
json.dump(cluster, f)
cluster_secrets = crds_manager.config_get('kamateracluster',
cluster_id,
is_secret=True)
with open(f'{cluster_path}/kube_config_rke-cluster.yml', 'w') as f:
f.write(cluster_secrets['kube_config_rke-cluster.yml'])
def set_storage(instance_id, instance_name, dry_run=False):
from ckan_cloud_operator.providers.storage.manager import get_provider, get_provider_id
resource = crds_manager.get(INSTANCE_NAME_CRD_SINGULAR, name=instance_name, required=False)
storage_provider = get_provider(default=None, provider_id=get_provider_id())
bucket = storage_provider.create_bucket(instance_id, exists_ok=True, dry_run=dry_run)
resource['spec']['ckanStorageBucket'] = {
storage_provider.PROVIDER_ID: bucket
}
if dry_run:
logs.print_yaml_dump(resource)
else:
kubectl.apply(resource)
def get_all_dbs_users():
dbs, users = [], []
for migration in crds_manager.get(CRD_SINGULAR)['items']:
migration_name = migration['spec']['name']
spec = migration['spec']
if spec.get('type') == 'deis-ckan':
db_name = spec['db-name']
datastore_ro_name = get_datastore_raedonly_user_name(
migration_name, required=False)
datastore_name = spec['datastore-name']
db_password, datastore_password, datastore_ro_password = get_dbs_passwords(
migration_name, required=False)
if all([
db_password, datastore_password, datastore_ro_password,
db_name, datastore_name, datastore_ro_name
]):
db_host, db_port = db_manager.get_internal_unproxied_db_host_port(
)
dbs.append((db_name, db_host, db_port))
dbs.append((datastore_name, db_host, db_port))
users.append((db_name, db_password))
users.append((datastore_name, datastore_password))
users.append((datastore_ro_name, datastore_ro_password))
elif spec.get('type') == 'new-db':
db_name = spec['db-name']
db_password, _, _ = get_dbs_passwords(migration_name,
required=False)
if db_password:
db_host, db_port = db_manager.get_internal_unproxied_db_host_port(
)
dbs.append((db_name, db_host, db_port))
users.append((db_name, db_password))
elif spec.get('type') == 'new-datastore':
datastore_ro_name = get_datastore_raedonly_user_name(
migration_name, required=False)
datastore_name = spec['datastore-name']
_, datastore_password, datastore_ro_password = get_dbs_passwords(
migration_name, required=False)
if datastore_password and datastore_ro_password:
db_host, db_port = db_manager.get_internal_unproxied_db_host_port(
)
dbs.append((datastore_name, db_host, db_port))
users.append((datastore_name, datastore_password))
users.append((datastore_ro_name, datastore_ro_password))
return dbs, users
def create_instance(instance_id,
instance_type,
values=None,
values_filename=None,
exists_ok=False,
dry_run=False):
if values_filename:
assert values is None
with open(values_filename) as f:
values = yaml.load(f.read())
if not exists_ok and crds_manager.get(
INSTANCE_CRD_SINGULAR, name=instance_id, required=False):
raise Exception('instance already exists')
return kubectl.apply(crds_manager.get_resource(
INSTANCE_CRD_SINGULAR,
instance_id,
extra_label_suffixes={'instance-type': instance_type},
spec=values),
dry_run=dry_run)
def delete(name, delete_dbs=False):
migration = crds_manager.get(CRD_SINGULAR, name=name, required=False) or {}
if delete_dbs:
admin_connection_string = db_manager.get_external_admin_connection_string(
)
db_name = migration.get('spec', {}).get('datastore-name')
datastore_name = migration.get('spec', {}).get('db-name')
datastore_ro_name = crds_manager.config_get(
CRD_SINGULAR,
name,
key='datastore-readonly-user-name',
is_secret=True,
required=False)
if db_name or datastore_name or datastore_ro_name:
with postgres_driver.connect(
admin_connection_string) as admin_conn:
_delete_dbs(admin_conn, db_name, datastore_name,
datastore_ro_name)
crds_manager.delete(CRD_SINGULAR, name)
def create(deployment_provider, instance_id=None, instance_name=None, values=None, values_filename=None, exists_ok=False,
dry_run=False, update_=False, wait_ready=False, skip_deployment=False, skip_route=False, force=False):
assert deployment_provider in ['helm']
if not instance_id:
if instance_name:
instance_id = '{}-{}'.format(instance_name, _generate_password(6))
logs.info('Generated instance id based on instance name', instance_name=instance_name, instance_id=instance_id)
else:
instance_id = _generate_password(12)
logs.info('Generated instance id', instance_id=instance_id)
if values_filename:
assert values is None
with open(values_filename) as f:
values = yaml.load(f.read())
if not exists_ok and crds_manager.get(APP_CRD_SINGULAR, name=instance_id, required=False):
raise Exception('instance already exists')
values_id = values.get('id')
if values_id and values_id != instance_id:
logs.warning(f'changing instance id in spec from {values_id} to the instance id {instance_id}')
values.update(id=instance_id)
logs.info('Creating instance', instance_id=instance_id)
instance = crds_manager.get_resource(
APP_CRD_SINGULAR, instance_id,
extra_label_suffixes={'deployment-provider': deployment_provider},
spec=values
)
label_prefix = labels_manager.get_label_prefix()
ckan_cloud_annotations = {
f'{label_prefix}/deployment-provider': deployment_provider,
f'{label_prefix}/instance-id': instance_id
}
logs.info('setting ckan-cloud annotations', ckan_cloud_annotations=ckan_cloud_annotations)
instance['metadata'].setdefault('annotations', {}).update(**ckan_cloud_annotations)
kubectl.apply(instance, dry_run=dry_run)
if instance_name:
set_name(instance_id, instance_name, dry_run=dry_run)
if update_:
update(instance_id, wait_ready=wait_ready, skip_deployment=skip_deployment, skip_route=skip_route, force=force,
dry_run=dry_run)
return instance_id
def get_all_instances():
return crds_manager.get(INSTANCE_CRD_SINGULAR)['items']
def get(name):
return crds_manager.get(CRD_SINGULAR, name=name)
def get(name=None, required=True):
return crds_manager.get(CRD_SINGULAR, name=name, required=required)