def update(instance_id_or_name, override_spec=None, persist_overrides=False, wait_ready=False, skip_deployment=False,
skip_route=False, force=False, dry_run=False):
instance_id, instance = _get_instance(instance_id_or_name, required=not dry_run)
if dry_run:
logs.info('update instance', instance_id=instance_id, instance_id_or_name=instance_id_or_name,
override_spec=override_spec, persist_overrides=persist_overrides, wait_ready=wait_ready,
skip_deployment=skip_deployment, skip_route=skip_route, force=force, dry_run=dry_run)
else:
pre_update_hook_data = deployment_manager.pre_update_hook(instance_id, instance, override_spec,
skip_route)
if persist_overrides:
logs.info('Persisting overrides')
kubectl.apply(instance)
if not skip_deployment:
deployment_manager.update(instance_id, instance)
if wait_ready:
wait_instance_ready(instance_id_or_name)
if not skip_route and pre_update_hook_data.get('sub-domain'):
root_domain = pre_update_hook_data.get('root-domain')
sub_domain = pre_update_hook_data['sub-domain']
assert root_domain == routers_manager.get_default_root_domain(), \
'invalid domain, must use default root domain'
logs.info(f'adding instance default route to {sub_domain}.{root_domain}')
routers_manager.create_subdomain_route('instances-default', {
'target-type': 'app-instance',
'app-instance-id': instance_id,
'root-domain': root_domain,
'sub-domain': sub_domain
})
routers_manager.update('instances-default', wait_ready)
else:
logs.info('skipping route creation', skip_route=skip_route,
sub_domain=pre_update_hook_data.get('sub-domain'))
logs.info('Instance is ready', instance_id=instance_id,
instance_name=(instance_id_or_name if instance_id_or_name != instance_id else None))
def initialize():
install_crds()
datapusher_envvars = {'PORT': '8000'}
router_name = 'datapushers'
if not routers_manager.get(router_name, required=False):
routers_manager.create(router_name,
routers_manager.get_traefik_router_spec())
create(
'datapusher-1',
'registry.gitlab.com/viderum/docker-datapusher:cloud-datapusher-1-v9',
datapusher_envvars, router_name)
create(
'datapusher-de',
'registry.gitlab.com/viderum/docker-datapusher:cloud-de-git-943fc3e0',
datapusher_envvars, router_name)
create(
'datapusher-giga',
'registry.gitlab.com/viderum/docker-datapusher:cloud-giga-git-2b05b22d',
datapusher_envvars, router_name)
create(
'datapusher-increased-max-length',
'registry.gitlab.com/viderum/docker-datapusher:cloud-increased-max-length-git-84e86116',
datapusher_envvars, router_name)
update('datapusher-1')
update('datapusher-de')
update('datapusher-giga')
update('datapusher-increased-max-length')
routers_manager.update(router_name)
def update(instance_id_or_name, override_spec=None, persist_overrides=False, wait_ready=False, skip_deployment=False, skip_route=False):
instance_id, instance_type, instance = _get_instance_id_and_type(instance_id_or_name)
if override_spec:
for k, v in override_spec.items():
logs.info(f'Applying override spec {k}={v}')
instance['spec'][k] = v
assert instance['spec'].get('useCentralizedInfra'), 'non-centralized instances are not supported'
# full domain to route to the instance
instance_domain = instance['spec'].get('domain')
# instance is added to router only if this is true, as all routers must use SSL and may use sans SSL too
with_sans_ssl = instance['spec'].get('withSansSSL')
# subdomain to register on the default root domain
register_subdomain = instance['spec'].get('registerSubdomain')
if persist_overrides:
logs.info('Persisting overrides')
kubectl.apply(instance)
if not skip_deployment:
deployment_manager.update(instance_id, instance_type, instance)
if wait_ready:
wait_instance_ready(instance_id_or_name)
if not skip_route:
if instance_domain:
assert with_sans_ssl, 'withSansSSL must be set to true to add routes'
assert '.'.join(instance_domain.split('.')[1:]) == routers_manager.get_default_root_domain(), f'invalid root domain ({instance_domain})'
assert instance_domain.split('.')[0] == register_subdomain, f'invalid register_subdomain ({register_subdomain})'
logs.info(f'adding instance route to {instance_domain}')
routers_manager.create_subdomain_route('instances-default', {
'target-type': 'ckan-instance',
'ckan-instance-id': instance_id,
'root-domain': routers_manager.get_default_root_domain(),
'sub-domain': register_subdomain
})
routers_manager.update('instances-default', wait_ready)
else:
assert not register_subdomain, 'subdomain registration is only supported with instance_domain'
def routers_create_backend_url_subdomain_route(router_name, target_resource_id, backend_url,
sub_domain, root_domain, wait_ready):
routers_manager.create_subdomain_route(router_name, {
'target-type': 'backend-url',
'target-resource-id': target_resource_id,
'backend-url': backend_url,
'sub-domain': sub_domain,
'root-domain': root_domain,
})
routers_manager.update(router_name, wait_ready)
great_success()
def routers_create_datapusher_subdomain_route(router_name, datapusher_name,
sub_domain, root_domain,
wait_ready):
routers_manager.create_subdomain_route(router_name, {
'target-type': 'datapusher',
'datapusher-name': datapusher_name,
'root-domain': root_domain,
'sub-domain': sub_domain
})
routers_manager.update(router_name, wait_ready)
great_success()
def routers_create_deis_instance_subdomain_route(router_name, deis_instance_id,
sub_domain, root_domain,
wait_ready):
routers_manager.create_subdomain_route(router_name, {
'target-type': 'deis-instance',
'deis-instance-id': deis_instance_id,
'root-domain': root_domain,
'sub-domain': sub_domain
})
routers_manager.update(router_name, wait_ready)
great_success()
def routers_create(traefik_router_name, default_root_domain, cloudflare_email, cloudflare_api_key, external_domains):
"""Create a Traefik router, with domain registration and let's encrypt based on Cloudflare"""
routers_manager.create(
traefik_router_name,
routers_manager.get_traefik_router_spec(
default_root_domain, cloudflare_email, cloudflare_api_key,
external_domains=external_domains
)
)
routers_manager.update(traefik_router_name)
great_success()
def update(instance_id_or_name, override_spec=None, persist_overrides=False, wait_ready=False, skip_deployment=False,
skip_route=False, force=False, dry_run=False, skip_solr=False):
instance_id, instance_type, instance = _get_instance_id_and_type(instance_id_or_name, required=not dry_run)
if dry_run:
logs.info('update instance', instance_id=instance_id, instance_id_or_name=instance_id_or_name,
override_spec=override_spec, persist_overrides=persist_overrides, wait_ready=wait_ready,
skip_deployment=skip_deployment, skip_route=skip_route, force=force, dry_run=dry_run)
else:
pre_update_hook_data = deployment_manager.pre_update_hook(instance_id, instance_type, instance, override_spec,
skip_route)
bucket_credentials = instance['spec'].get('ckanStorageBucket', {}).get(get_storage_provider_id())
use_cloud_storage = bucket_credentials and config_manager.get('use-cloud-native-storage', secret_name=CONFIG_NAME)
if use_cloud_storage and bucket_credentials:
config_manager.set(
values=bucket_credentials,
secret_name='bucket-credentials',
namespace=instance_id
)
if persist_overrides:
logs.info('Persisting overrides')
kubectl.apply(instance)
if not skip_deployment:
deployment_manager.update(instance_id, instance_type, instance, force=force, skip_solr=skip_solr)
if wait_ready:
wait_instance_ready(instance_id_or_name)
if not skip_route and pre_update_hook_data.get('sub-domain'):
root_domain = pre_update_hook_data.get('root-domain')
sub_domain = pre_update_hook_data['sub-domain']
assert root_domain == routers_manager.get_default_root_domain(), 'invalid domain, must use default root domain'
logs.info(f'adding instance default route to {sub_domain}.{root_domain}')
routers_manager.create_subdomain_route('instances-default', {
'target-type': 'ckan-instance',
'ckan-instance-id': instance_id,
'root-domain': root_domain,
'sub-domain': sub_domain
})
logs.info(f'updating routers_manager wait_ready: {wait_ready}')
routers_manager.update('instances-default', wait_ready)
else:
logs.info('skipping route creation', skip_route=skip_route, sub_domain=pre_update_hook_data.get('sub-domain'))
logs.info('creating ckan admin')
# Need to set in values.yaml
if pre_update_hook_data.get('create-sysadmin'):
ckan_admin_email = pre_update_hook_data.get('ckan-admin-email')
ckan_admin_password = pre_update_hook_data.get('ckan-admin-password')
ckan_admin_name = pre_update_hook_data.get('ckan-admin-name', 'admin')
res = create_ckan_admin_user(instance_id, ckan_admin_name, ckan_admin_email, ckan_admin_password)
logs.info(**res)
logs.info('Instance is ready', instance_id=instance_id, instance_name=(instance_id_or_name if instance_id_or_name != instance_id else None))
def routers_create_backend_url_subdomain_route(router_name, target_resource_id, backend_url,
sub_domain, root_domain, wait_ready,
httpauth_secret):
routers_manager.create_subdomain_route(router_name, {
'target-type': 'backend-url',
'target-resource-id': target_resource_id,
'backend-url': backend_url,
'sub-domain': sub_domain,
'root-domain': root_domain,
**({'httpauth-secret': httpauth_secret} if httpauth_secret else {}),
})
routers_manager.update(router_name, wait_ready)
great_success()
def test_update(self, list, traefik_manager, _init_router):
_init_router.return_value = 'router', {
'update': True
}, 'traefik', {}, {}, {
'manager': traefik_manager
}
list.return_value = 'router'
manager.update('datapusher', wait_ready=True)
_init_router.assert_called_once_with('datapusher')
list.assert_called_once_with({})
traefik_manager.update.assert_called_once_with('datapusher',
True, {'update': True},
{},
'router',
dry_run=False)
def _update_route(storage_suffix=None):
backend_url_target_id = _get_backend_url_target_id(
storage_suffix=storage_suffix)
router_name = _config_get('router-name',
required=True,
suffix=storage_suffix)
if not routers_manager.get_backend_url_routes(backend_url_target_id):
deployment_name = _get_resource_name(suffix=storage_suffix)
namespace = _get_namespace()
routers_manager.create_subdomain_route(
router_name, {
'target-type': 'backend-url',
'target-resource-id': backend_url_target_id,
'backend-url': f'http://{deployment_name}.{namespace}:9000',
})
routers_manager.update(router_name, wait_ready=True)
def main(old_instance_id, new_instance_id, dry_run):
dry_run = (dry_run == 'yes')
router_names = set()
for route in routers_manager.get_deis_instance_routes(old_instance_id):
for label in [
'ckan-cloud/route-deis-instance-id',
'ckan-cloud/route-target-resource-id'
]:
_assert_set(route['metadata']['labels'], label, old_instance_id,
new_instance_id)
for attr in ['deis-instance-id', 'route-target-resource-id']:
_assert_set(route['spec'], attr, old_instance_id, new_instance_id)
kubectl.apply(route, dry_run=dry_run)
router_names.add(route['spec']['router_name'])
logs.info('updating routers', router_names=router_names)
if not dry_run:
for router_name in router_names:
routers_manager.update(router_name)
def deploy_gcs_minio_proxy(router_name):
"""Deploys a minio proxy (AKA gateway) for access to google storage"""
labels = {'app': 'ckan-cloud-gcsminio-proxy'}
if not kubectl.get('secret gcsminio-proxy-credentials', required=False):
print('Creating minio credentials')
minio_access_key = binascii.hexlify(os.urandom(8)).decode()
minio_secret_key = binascii.hexlify(os.urandom(12)).decode()
kubectl.update_secret(
'gcsminio-proxy-credentials', {
'MINIO_ACCESS_KEY': minio_access_key,
'MINIO_SECRET_KEY': minio_secret_key,
})
kubectl.apply(
kubectl.get_deployment(
'gcsminio-proxy', labels, {
'replicas': 1,
'revisionHistoryLimit': 10,
'strategy': {
'type': 'RollingUpdate',
},
'template': {
'metadata': {
'labels': labels,
'annotations': {
'ckan-cloud/operator-timestamp':
str(datetime.datetime.now())
}
},
'spec': {
'containers': [{
'name':
'minio',
'image':
'orihoch/ckan-cloud-operator-gcsminio-proxy',
'env': [{
'name':
'GOOGLE_APPLICATION_CREDENTIALS',
'value':
'/gcloud-credentials/credentials.json'
}],
'envFrom': [{
'secretRef': {
'name': 'gcsminio-proxy-credentials'
}
}],
'ports': [{
'containerPort': 9000
}],
'volumeMounts': [
{
'name': 'gcloud-credentials',
'mountPath':
'/gcloud-credentials/credentials.json',
'subPath': 'GCLOUD_SERVICE_ACCOUNT_JSON'
},
],
}],
'volumes': [
{
'name': 'gcloud-credentials',
'secret': {
'secretName': 'ckan-infra'
}
},
]
}
}
}))
service = kubectl.get_resource('v1', 'Service', 'gcsminio-proxy', labels)
service['spec'] = {
'ports': [{
'name': '9000',
'port': 9000
}],
'selector': labels
}
kubectl.apply(service)
if not routers_manager.get_backend_url_routes('gcs-minio'):
routers_manager.create_subdomain_route(
router_name, {
'target-type': 'backend-url',
'target-resource-id': 'gcs-minio',
'backend-url': 'http://gcsminio-proxy.ckan-cloud:9000',
'sub-domain': 'default',
'root-domain': 'default',
})
routers_manager.update(router_name, wait_ready=True)
def migrate_from_deis(old_site_id,
new_instance_id,
router_name,
deis_instance_class,
skip_gitlab=False,
db_migration_name=None,
recreate=False,
skip_routes=False,
skip_solr=False,
skip_deployment=False,
no_db_proxy=False):
assert db_migration_name, 'migration without a db migration is not supported yet'
log_labels = {'instance': new_instance_id}
if recreate:
from ckan_cloud_operator.deis_ckan.instance import DeisCkanInstance
DeisCkanInstance(new_instance_id).delete(
force=True, wait_deleted=not db_migration_name)
logs.info(
f'Migrating from old site id {old_site_id} to new instance id {new_instance_id}',
**log_labels)
instance_kind = ckan_manager.instance_kind()
values = kubectl.get(f'{instance_kind} {new_instance_id}', required=False)
if values:
logs.info('instance already exists', **log_labels)
else:
logs.info('creating instance', **log_labels)
path_to_old_cluster_kubeconfig = get_path_to_old_cluster_kubeconfig()
solr_config = get_solr_config(old_site_id,
path_to_old_cluster_kubeconfig)
assert solr_config, 'failed to get solr config name'
instance_env = get_instance_env(old_site_id,
path_to_old_cluster_kubeconfig)
gitlab_repo = f'viderum/cloud-{old_site_id}'
if not skip_gitlab:
CkanGitlab().initialize(gitlab_repo)
gitlab_registry = f'registry.gitlab.com/{gitlab_repo}'
old_bucket_name = instance_env.get(
'CKANEXT__S3FILESTORE__AWS_BUCKET_NAME')
old_storage_path = instance_env.get(
'CKANEXT__S3FILESTORE__AWS_STORAGE_PATH')
assert old_bucket_name == 'ckan'
assert old_storage_path and len(old_storage_path) > 1
storage_path = f'/ckan/{old_storage_path}'
deis_instance_class.create('from-gcloud-envvars',
instance_env,
gitlab_registry,
solr_config,
storage_path,
new_instance_id,
db_migration_name=db_migration_name)
routers_env_id = routers_provider.get_env_id()
default_root_domain = routers_provider.get_default_root_domain()
assert routers_env_id and default_root_domain
ckan_site_url = f'https://cc-{routers_env_id}-{new_instance_id}.{default_root_domain}'
logs.info(
f'updating instance and setting ckan site url to {ckan_site_url}',
**log_labels)
deis_instance_class(new_instance_id,
override_spec={
'envvars': {
'CKAN_SITE_URL': ckan_site_url
},
**({
'db': {
'no-db-proxy': 'yes'
},
'datastore': {
'no-db-proxy': 'yes'
}
} if no_db_proxy else {})
},
persist_overrides=True).update(
wait_ready=True,
skip_solr=skip_solr,
skip_deployment=skip_deployment)
if routers_manager.get_deis_instance_routes(new_instance_id):
logs.info('default instance route already exists', **log_labels)
else:
logs.info('creating instance route', **log_labels)
routers_manager.create_subdomain_route(
router_name, {
'target-type': 'deis-instance',
'deis-instance-id': new_instance_id,
'root-domain': 'default',
'sub-domain': 'default'
})
if not skip_routes:
routers_manager.update(router_name, wait_ready=True)
if not skip_solr:
logs.info('Rebuilding solr search index', **log_labels)
deis_instance_class(new_instance_id).ckan.paster(
'search-index rebuild --force')
def routers_update(router_name, wait_ready):
"""Update a router to latest resource spec"""
routers_manager.update(router_name, wait_ready)
great_success()
def update(instance_id_or_name,
override_spec=None,
persist_overrides=False,
wait_ready=False,
skip_deployment=False,
skip_route=False,
force=False,
dry_run=False):
instance_id, instance_type, instance = _get_instance_id_and_type(
instance_id_or_name, required=not dry_run)
if dry_run:
logs.info('update instance',
instance_id=instance_id,
instance_id_or_name=instance_id_or_name,
override_spec=override_spec,
persist_overrides=persist_overrides,
wait_ready=wait_ready,
skip_deployment=skip_deployment,
skip_route=skip_route,
force=force,
dry_run=dry_run)
else:
pre_update_hook_data = deployment_manager.pre_update_hook(
instance_id, instance_type, instance, override_spec, skip_route)
bucket_credentials = instance['spec'].get('ckanStorageBucket', {}).get(
get_storage_provider_id())
use_cloud_storage = bucket_credentials and config_manager.get(
'use-cloud-native-storage', secret_name=CONFIG_NAME)
if use_cloud_storage:
cluster_provider_id = cluster_manager.get_provider_id()
if bucket_credentials:
literal = []
config_manager.set(values=bucket_credentials,
secret_name='bucket-credentials',
namespace=instance_id)
if instance['spec'].get('operatorCopySecrets'):
for target_secret_name, source_secret_config in json.loads(
instance['spec']['operatorCopySecrets']).items():
for k, v in source_secret_config.items():
source_secret_config[k] = v.replace(
"__INSTANCE_NAME__", instance_id_or_name)
kubectl.update_secret(
target_secret_name,
kubectl.decode_secret(
kubectl.get('secret',
source_secret_config["fromName"],
namespace=source_secret_config.get(
"fromNamespace", "ckan-cloud"))),
namespace=instance_id)
if persist_overrides:
logs.info('Persisting overrides')
kubectl.apply(instance)
if not skip_deployment:
deployment_manager.update(instance_id,
instance_type,
instance,
force=force)
if wait_ready:
wait_instance_ready(instance_id_or_name)
if not skip_route and pre_update_hook_data.get('sub-domain'):
root_domain = pre_update_hook_data.get('root-domain')
sub_domain = pre_update_hook_data['sub-domain']
assert root_domain == routers_manager.get_default_root_domain(
), 'invalid domain, must use default root domain'
logs.info(
f'adding instance default route to {sub_domain}.{root_domain}')
routers_manager.create_subdomain_route(
'instances-default', {
'target-type': 'ckan-instance',
'ckan-instance-id': instance_id,
'root-domain': root_domain,
'sub-domain': sub_domain
})
logs.info(f'updating routers_manager wait_ready: {wait_ready}')
routers_manager.update('instances-default', wait_ready)
else:
logs.info('skipping route creation',
skip_route=skip_route,
sub_domain=pre_update_hook_data.get('sub-domain'))
if not instance['spec'].get('skipCreateCkanAdmin', False):
logs.info('creating ckan admin')
ckan_admin_email = instance['spec'].get(
'ckanAdminEmail', pre_update_hook_data.get('ckan-admin-email'))
ckan_admin_password = pre_update_hook_data.get(
'ckan-admin-password')
ckan_admin_name = instance['spec'].get(
'ckanAdminName',
pre_update_hook_data.get('ckan-admin-name', 'admin'))
res = create_ckan_admin_user(instance_id, ckan_admin_name,
ckan_admin_email, ckan_admin_password)
logs.info(**res)
logs.info('Instance is ready',
instance_id=instance_id,
instance_name=(instance_id_or_name if
instance_id_or_name != instance_id else None))
