def _get_instance(instance_id_or_name=None, instance_id=None, required=True): if instance_id: logs.debug(f'Getting instance using instance_id', instance_id=instance_id) instance = crds_manager.get(APP_CRD_SINGULAR, name=instance_id, required=False) instance_name = None else: logs.debug(f'Attempting to get instance using id', instance_id_or_name=instance_id_or_name) instance = crds_manager.get(APP_CRD_SINGULAR, name=instance_id_or_name, required=False) if instance: instance_id = instance_id_or_name instance_name = None else: logs.debug(f'Attempting to get instance from instance name', instance_id_or_name=instance_id_or_name) instance_name = crds_manager.get(APP_NAME_CRD_SINGULAR, name=instance_id_or_name, required=False) if instance_name: instance_id = instance_name['spec'].get('latest-instance-id') logs.debug(instance_id=instance_id) instance = crds_manager.get(APP_CRD_SINGULAR, name=instance_id, required=False) instance_name = instance_id_or_name else: instance_name = None logs.debug_yaml_dump(instance_name=instance_name, instance_id=instance_id, instance=bool(instance)) if required: assert instance_id and len(instance) > 2, \ f'Failed to find instance (instance_id_or_name={instance_id_or_name}, instance_id={instance_id})' return instance_id, instance
def _get_instance_id_and_type(instance_id_or_name=None, instance_id=None, required=True): if instance_id: logs.debug(f'Getting instance type using instance_id', instance_id=instance_id) instance = crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id, required=False) instance_name = None else: logs.debug(f'Attempting to get instance type using id', instance_id_or_name=instance_id_or_name) instance = crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id_or_name, required=False) if instance: instance_id = instance_id_or_name instance_name = None else: logs.debug(f'Attempting to get instance type from instance name', instance_id_or_name=instance_id_or_name) instance_name = crds_manager.get(INSTANCE_NAME_CRD_SINGULAR, name=instance_id_or_name, required=False) if instance_name: instance_id = instance_name['spec'].get('latest-instance-id') logs.debug(instance_id=instance_id) instance = crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id, required=False) instance_name = instance_id_or_name else: instance_name = None if instance: instance_type = instance['metadata']['labels'].get('{}/instance-type'.format(labels_manager.get_label_prefix())) else: instance_type = None logs.debug_yaml_dump(instance_name=instance_name, instance_id=instance_id, instance_type=instance_type, instance=bool(instance)) if required: assert instance_id and instance_type and len(instance) > 2, f'Failed to find instance (instance_id_or_name={instance_id_or_name}, instance_id={instance_id})' return instance_id, instance_type, instance
def get_all_instance_id_names(): instance_names = crds_manager.get(INSTANCE_NAME_CRD_SINGULAR, required=False) instance_name_ids = {} if instance_names: for instance_name in instance_names['items']: instance_name_ids[instance_name['spec']['latest-instance-id']] = instance_name['spec']['name'] label_prefix = labels_manager.get_label_prefix() for instance in crds_manager.get(INSTANCE_CRD_SINGULAR)['items']: instance_id = instance['metadata']['labels'][f'{label_prefix}/crd-ckaninstance-name'] instance_name = instance_name_ids.pop(instance_id, None) yield {'id': instance_id, 'name': instance_name} for instance_id, instance_name in instance_name_ids.items(): yield {'id': instance_id, 'name': instance_name}
def _check_instance_events(instance_id, force_update_events=False): status = get(instance_id) errors = [] ckan_cloud_logs = [] ckan_cloud_events = set() pod_names = [] for app, app_status in status.get('helm_app_statuses', {}).items(): for kind, kind_items in app_status.items(): for item in kind_items: for error in item.get("errors", []): errors.append(dict(error, kind=kind, app=app, name=item.get("name"))) for logdata in item.get("ckan-cloud-logs", []): ckan_cloud_logs.append(dict(logdata, kind=kind, app=app, name=item.get("name"))) if "event" in logdata: ckan_cloud_events.add(logdata["event"]) if kind == "pods": pod_names.append(item["name"]) instance = crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id) if force_update_events or annotations_manager.get_status(instance, 'helm', 'created'): logs.debug('expecting update events') expected_events = { "ckan-env-vars-exists", "ckan-secrets-exists", "got-ckan-secrets", "ckan-entrypoint-initialized", "ckan-entrypoint-db-init-success", "ckan-entrypoint-extra-init-success" } else: logs.debug('expecting create events') expected_events = { "ckan-env-vars-created", "ckan-secrets-created", "got-ckan-secrets", "ckan-db-initialized", "ckan-datastore-db-initialized", "ckan-entrypoint-initialized", "ckan-entrypoint-db-init-success", "ckan-entrypoint-extra-init-success" } logs.debug(ckan_cloud_events=ckan_cloud_events) return expected_events.difference(ckan_cloud_events)
def create(instance_type, instance_id=None, instance_name=None, values=None, values_filename=None, exists_ok=False, dry_run=False): if not instance_id: if instance_name: instance_id = '{}-{}'.format(instance_name, _generate_password(6)) else: instance_id = _generate_password(12) if values_filename: assert values is None with open(values_filename) as f: values = yaml.load(f.read()) if not exists_ok and crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id, required=False): raise Exception('instance already exists') values_id = values.get('id') if values_id: assert values_id == instance_id, f'instance spec has conflicting instance_id ({values_id} != {instance_id})' values['id'] = instance_id logs.info('Creating instance', instance_id=instance_id) kubectl.apply(crds_manager.get_resource( INSTANCE_CRD_SINGULAR, instance_id, extra_label_suffixes={'instance-type': instance_type}, spec=values ), dry_run=dry_run) if instance_name: set_name(instance_id, instance_name, dry_run=dry_run) return instance_id
def set_name(instance_id, instance_name, dry_run=False): resource = crds_manager.get(APP_NAME_CRD_SINGULAR, name=instance_name, required=False) if resource: resource['spec']['latest-instance-id'] = instance_id if not resource['spec']['instance-ids'].get(instance_id): resource['spec']['instance-ids'][instance_id] = { 'added': datetime.datetime.now() } else: resource = crds_manager.get_resource( APP_NAME_CRD_SINGULAR, instance_name, spec={ 'name': instance_name, 'latest-instance-id': instance_id, 'instance-ids': { instance_id: { 'added': datetime.datetime.now() } } }) if dry_run: logs.print_yaml_dump(resource) else: kubectl.apply(resource)
def update(instance_id, instance): tiller_namespace_name = _get_resource_name() _init_namespace(instance_id) _init_ckan_infra_secret(instance_id) ckan_helm_chart_repo = instance['spec'].get( "ckanHelmChartRepo", "https://raw.githubusercontent.com/ViderumGlobal/ckan-cloud-helm/master/charts_repository" ) ckan_helm_chart_version = instance['spec'].get("ckanHelmChartVersion", "") ckan_helm_release_name = f'ckan-cloud-{instance_id}' instance['spec']['centralizedSolrHost'], instance['spec'][ 'centralizedSolrPort'] = _init_solr(instance_id) if annotations_manager.get_status(instance, 'helm', 'created'): values = instance['spec'] else: values = { **instance['spec'], "replicas": 1, "nginxReplicas": 1, "disableJobs": True, "noProbes": True } _helm_deploy(values, tiller_namespace_name, ckan_helm_chart_repo, ckan_helm_chart_version, ckan_helm_release_name, instance_id) _wait_instance_events(instance_id) instance = crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id) if not annotations_manager.get_status(instance, 'helm', 'created'): annotations_manager.set_status(instance, 'helm', 'created') _helm_deploy(instance['spec'], tiller_namespace_name, ckan_helm_chart_repo, ckan_helm_chart_version, ckan_helm_release_name, instance_id)
def _get_instance_and_type(instance_id): instance = crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id, required=False) instance_type = instance['metadata']['labels'].get( '{}/instance-type'.format(labels_manager.get_label_prefix())) return instance, instance_type
def create(instance_type, instance_id=None, instance_name=None, values=None, values_filename=None, exists_ok=False, dry_run=False, update_=False, wait_ready=False, skip_deployment=False, skip_route=False, force=False): if not instance_id: if instance_name: instance_id = '{}-{}'.format(instance_name, _generate_password(6)) else: instance_id = _generate_password(12) if values_filename: assert values is None if values_filename != '-': with open(values_filename) as f: values = yaml.load(f.read()) else: values = yaml.load(sys.stdin.read()) if not exists_ok and crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id, required=False): raise Exception('instance already exists') values_id = values.get('id') if values_id and values_id != instance_id: logs.warning(f'changing instance id in spec from {values_id} to the instance id {instance_id}') values['id'] = instance_id logs.info('Creating instance', instance_id=instance_id) kubectl.apply(crds_manager.get_resource( INSTANCE_CRD_SINGULAR, instance_id, extra_label_suffixes={'instance-type': instance_type}, spec=values ), dry_run=dry_run) if instance_name: set_name(instance_id, instance_name, dry_run=dry_run) if update_: update(instance_id, wait_ready=wait_ready, skip_deployment=skip_deployment, skip_route=skip_route, force=force, dry_run=dry_run) return instance_id
def _pre_update_hook_modify_spec(instance_id, instance, callback, dry_run=False): # applies changes to both the non-persistent spec and persists the changes on latest instance spec latest_instance = crds_manager.get(INSTANCE_CRD_SINGULAR, crds_manager.get_resource_name( INSTANCE_CRD_SINGULAR, instance_id ), required=True) callback(instance) callback(latest_instance) kubectl.apply(latest_instance, dry_run=dry_run)
def _get_instance_id_and_type(instance_id_or_name=None, instance_id=None): if instance_id: instance = crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id, required=False) else: instance = crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id_or_name, required=False) if instance: instance_id = instance_id_or_name if not instance: assert not instance_id instance_name = crds_manager.get(INSTANCE_NAME_CRD_SINGULAR, name=instance_id_or_name, required=False) if instance_name: instance_id = instance_name['spec'].get('latest-instance-id') instance = crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id, required=False) else: instance_id = None instance_type = instance['metadata']['labels'].get('{}/instance-type'.format(labels_manager.get_label_prefix())) if instance else None return instance_id, instance_type, instance
def update(instance_id, instance, force=False, dry_run=False): tiller_namespace_name = _get_resource_name() logs.debug('Updating helm-based instance deployment', instance_id=instance_id, tiller_namespace_name=tiller_namespace_name) _create_private_container_registry_secret(instance_id) _init_ckan_infra_secret(instance_id, dry_run=dry_run) ckan_helm_chart_repo = instance['spec'].get( "ckanHelmChartRepo", "https://raw.githubusercontent.com/ViderumGlobal/ckan-cloud-helm/master/charts_repository" ) ckan_helm_chart_version = instance['spec'].get("ckanHelmChartVersion", "") ckan_helm_release_name = f'ckan-cloud-{instance_id}' solr_schema = instance['spec'].get("ckanSolrSchema", "ckan_default") solr_host, solr_port = _init_solr( instance_id, solr_schema, dry_run=dry_run, ) logs.debug(ckan_helm_chart_repo=ckan_helm_chart_repo, ckan_helm_chart_version=ckan_helm_chart_version, ckan_helm_release_name=ckan_helm_release_name, solr_host=solr_host, solr_port=solr_port) instance['spec']['centralizedSolrHost'], instance['spec'][ 'centralizedSolrPort'] = solr_host, solr_port if annotations_manager.get_status(instance, 'helm', 'created'): logs.info('Updating existing instance') values = instance['spec'] else: logs.info( 'New instance, deploying first with 1 replica and disabled probes and jobs' ) values = { **instance['spec'], "replicas": 1, "nginxReplicas": 1, "disableJobs": True, "noProbes": True, "enableHarvesterNG": False } _helm_deploy(values, tiller_namespace_name, ckan_helm_chart_repo, ckan_helm_chart_version, ckan_helm_release_name, instance_id, dry_run=dry_run) if not dry_run: _wait_instance_events(instance_id) instance = crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id) if not annotations_manager.get_status(instance, 'helm', 'created'): annotations_manager.set_status(instance, 'helm', 'created') _helm_deploy(instance['spec'], tiller_namespace_name, ckan_helm_chart_repo, ckan_helm_chart_version, ckan_helm_release_name, instance_id) _scale_down_scale_up(namespace=instance_id, replicas=values.get('replicas', 1))
def load(cluster_id): cluster = crds_manager.get( 'kamateracluster', crds_manager.get_resource_name('kamateracluster', cluster_id))['spec'] cluster['servers'] = [] servers_path = os.path.expanduser(f'~/cluster-{cluster_id}-servers') os.makedirs(servers_path, exist_ok=True) for server_name in cluster['server_names']: server = crds_manager.get( 'kamateraserver', crds_manager.get_resource_name('kamateraserver', server_name))['spec'] server_secrets = crds_manager.config_get('kamateraserver', server_name, is_secret=True) password_filename = f'{servers_path}/server_{server_name}_password.txt' private_key_filename = f'{servers_path}/server_{server_name}_id_rsa' public_key_filename = private_key_filename + '.pub' with open(password_filename, 'w') as f: f.write(server_secrets['password']) with open(private_key_filename, 'w') as f: f.write(server_secrets['private_key']) with open(public_key_filename, 'w') as f: f.write(server_secrets['public_key']) server['passwordfile'] = password_filename server['keyfile'] = private_key_filename cluster['servers'].append(server) del cluster['server_names'] cluster_path = get_cluster_path(cluster_id) os.makedirs(cluster_path, exist_ok=True) with open(f'{cluster_path}/cluster.json', 'w') as f: json.dump(cluster, f) cluster_secrets = crds_manager.config_get('kamateracluster', cluster_id, is_secret=True) with open(f'{cluster_path}/kube_config_rke-cluster.yml', 'w') as f: f.write(cluster_secrets['kube_config_rke-cluster.yml'])
def set_storage(instance_id, instance_name, dry_run=False): from ckan_cloud_operator.providers.storage.manager import get_provider, get_provider_id resource = crds_manager.get(INSTANCE_NAME_CRD_SINGULAR, name=instance_name, required=False) storage_provider = get_provider(default=None, provider_id=get_provider_id()) bucket = storage_provider.create_bucket(instance_id, exists_ok=True, dry_run=dry_run) resource['spec']['ckanStorageBucket'] = { storage_provider.PROVIDER_ID: bucket } if dry_run: logs.print_yaml_dump(resource) else: kubectl.apply(resource)
def get_all_dbs_users(): dbs, users = [], [] for migration in crds_manager.get(CRD_SINGULAR)['items']: migration_name = migration['spec']['name'] spec = migration['spec'] if spec.get('type') == 'deis-ckan': db_name = spec['db-name'] datastore_ro_name = get_datastore_raedonly_user_name( migration_name, required=False) datastore_name = spec['datastore-name'] db_password, datastore_password, datastore_ro_password = get_dbs_passwords( migration_name, required=False) if all([ db_password, datastore_password, datastore_ro_password, db_name, datastore_name, datastore_ro_name ]): db_host, db_port = db_manager.get_internal_unproxied_db_host_port( ) dbs.append((db_name, db_host, db_port)) dbs.append((datastore_name, db_host, db_port)) users.append((db_name, db_password)) users.append((datastore_name, datastore_password)) users.append((datastore_ro_name, datastore_ro_password)) elif spec.get('type') == 'new-db': db_name = spec['db-name'] db_password, _, _ = get_dbs_passwords(migration_name, required=False) if db_password: db_host, db_port = db_manager.get_internal_unproxied_db_host_port( ) dbs.append((db_name, db_host, db_port)) users.append((db_name, db_password)) elif spec.get('type') == 'new-datastore': datastore_ro_name = get_datastore_raedonly_user_name( migration_name, required=False) datastore_name = spec['datastore-name'] _, datastore_password, datastore_ro_password = get_dbs_passwords( migration_name, required=False) if datastore_password and datastore_ro_password: db_host, db_port = db_manager.get_internal_unproxied_db_host_port( ) dbs.append((datastore_name, db_host, db_port)) users.append((datastore_name, datastore_password)) users.append((datastore_ro_name, datastore_ro_password)) return dbs, users
def create_instance(instance_id, instance_type, values=None, values_filename=None, exists_ok=False, dry_run=False): if values_filename: assert values is None with open(values_filename) as f: values = yaml.load(f.read()) if not exists_ok and crds_manager.get( INSTANCE_CRD_SINGULAR, name=instance_id, required=False): raise Exception('instance already exists') return kubectl.apply(crds_manager.get_resource( INSTANCE_CRD_SINGULAR, instance_id, extra_label_suffixes={'instance-type': instance_type}, spec=values), dry_run=dry_run)
def delete(name, delete_dbs=False): migration = crds_manager.get(CRD_SINGULAR, name=name, required=False) or {} if delete_dbs: admin_connection_string = db_manager.get_external_admin_connection_string( ) db_name = migration.get('spec', {}).get('datastore-name') datastore_name = migration.get('spec', {}).get('db-name') datastore_ro_name = crds_manager.config_get( CRD_SINGULAR, name, key='datastore-readonly-user-name', is_secret=True, required=False) if db_name or datastore_name or datastore_ro_name: with postgres_driver.connect( admin_connection_string) as admin_conn: _delete_dbs(admin_conn, db_name, datastore_name, datastore_ro_name) crds_manager.delete(CRD_SINGULAR, name)
def create(deployment_provider, instance_id=None, instance_name=None, values=None, values_filename=None, exists_ok=False, dry_run=False, update_=False, wait_ready=False, skip_deployment=False, skip_route=False, force=False): assert deployment_provider in ['helm'] if not instance_id: if instance_name: instance_id = '{}-{}'.format(instance_name, _generate_password(6)) logs.info('Generated instance id based on instance name', instance_name=instance_name, instance_id=instance_id) else: instance_id = _generate_password(12) logs.info('Generated instance id', instance_id=instance_id) if values_filename: assert values is None with open(values_filename) as f: values = yaml.load(f.read()) if not exists_ok and crds_manager.get(APP_CRD_SINGULAR, name=instance_id, required=False): raise Exception('instance already exists') values_id = values.get('id') if values_id and values_id != instance_id: logs.warning(f'changing instance id in spec from {values_id} to the instance id {instance_id}') values.update(id=instance_id) logs.info('Creating instance', instance_id=instance_id) instance = crds_manager.get_resource( APP_CRD_SINGULAR, instance_id, extra_label_suffixes={'deployment-provider': deployment_provider}, spec=values ) label_prefix = labels_manager.get_label_prefix() ckan_cloud_annotations = { f'{label_prefix}/deployment-provider': deployment_provider, f'{label_prefix}/instance-id': instance_id } logs.info('setting ckan-cloud annotations', ckan_cloud_annotations=ckan_cloud_annotations) instance['metadata'].setdefault('annotations', {}).update(**ckan_cloud_annotations) kubectl.apply(instance, dry_run=dry_run) if instance_name: set_name(instance_id, instance_name, dry_run=dry_run) if update_: update(instance_id, wait_ready=wait_ready, skip_deployment=skip_deployment, skip_route=skip_route, force=force, dry_run=dry_run) return instance_id
def get_all_instances(): return crds_manager.get(INSTANCE_CRD_SINGULAR)['items']
def get(name): return crds_manager.get(CRD_SINGULAR, name=name)
def get(name=None, required=True): return crds_manager.get(CRD_SINGULAR, name=name, required=required)