def create(router):
router_name = router['metadata']['name']
router_spec = router['spec']
cloudflare_spec = router_spec.get('cloudflare', {})
cloudflare_email = cloudflare_spec.get('email')
cloudflare_api_key = cloudflare_spec.get('api-key')
default_root_domain = router_spec.get('default-root-domain')
dns_provider = router_spec.get('dns-provider')
from ckan_cloud_operator.providers.cluster import manager as cluster_manager
default_dns_provider = 'route53' if cluster_manager.get_provider_id(
) == 'aws' else 'cloudflare'
logs.info(dns_provider=dns_provider,
default_dns_provider=default_dns_provider)
if not dns_provider:
dns_provider = default_dns_provider
router_spec['dns-provider'] = dns_provider
assert all([default_root_domain,
dns_provider]), f'invalid traefik router spec: {router_spec}'
if dns_provider == 'cloudflare':
assert cloudflare_email and cloudflare_api_key, 'invalid traefik router spec for cloudflare dns provider'
# cloudflare credentials are stored in a secret, not in the spec
if 'cloudflare' in router_spec:
del router_spec['cloudflare']
kubectl.apply(router)
annotations = CkanRoutersAnnotations(router_name, router)
if dns_provider == 'cloudflare':
annotations.update_flag(
'letsencryptCloudflareEnabled',
lambda: annotations.set_secrets(
{
'LETSENCRYPT_CLOUDFLARE_EMAIL': cloudflare_email,
'LETSENCRYPT_CLOUDFLARE_API_KEY': cloudflare_api_key
}),
force_update=True)
return router
def get_load_balancer_ip(router_name, failfast=False):
resource_name = _get_resource_name(router_name)
RETRIES = 10
for retries in range(RETRIES):
load_balancer = kubectl.get(f'service loadbalancer-{resource_name}', required=False)
if not load_balancer:
if failfast:
return None
else:
continue
ingresses = load_balancer.get('status', {}).get('loadBalancer', {}).get('ingress', [])
if len(ingresses) == 0: continue
assert len(ingresses) == 1
if cluster_manager.get_provider_id() == 'aws':
load_balancer_hostname = ingresses[0].get('hostname')
if load_balancer_hostname:
return load_balancer_hostname
logs.warning('Failed to get hostname, retrying %r' % ingresses[0])
else:
load_balancer_ip = ingresses[0].get('ip')
if load_balancer_ip:
return load_balancer_ip
logs.warning('Failed to get ip, retrying %r' % ingresses[0])
time.sleep(60)
assert retries < RETRIES - 1, "Gave up on waiting for load balancer IP"
def initialize(log_kwargs=None, interactive=False):
log_kwargs = log_kwargs or {}
logs.info(f'Installing crds', **log_kwargs)
crds_manager.install_crd(CRD_SINGULAR,
CRD_PLURAL,
CRD_KIND,
hash_names=True)
if cluster_manager.get_provider_id() == 'gcloud':
ckan_infra = CkanInfra(required=False)
if interactive:
providers_manager.config_interactive_set(
PROVIDER_SUBMODULE,
default_values={
'gcloud-storage-import-bucket':
ckan_infra.GCLOUD_SQL_DEIS_IMPORT_BUCKET
},
suffix='deis-migration')
else:
if not providers_manager.config_get(
PROVIDER_SUBMODULE,
key='gcloud-storage-import-bucket',
suffix='deis-migration',
required=False):
providers_manager.config_set(
PROVIDER_SUBMODULE,
values={
'gcloud-storage-import-bucket':
ckan_infra.GCLOUD_SQL_DEIS_IMPORT_BUCKET
},
suffix='deis-migration')
def initialize(interactive=False):
config_manager.interactive_set(
{
'env-id':
None,
'default-root-domain':
None,
'dns-provider':
'route53' if cluster_manager.get_provider_id() == 'aws' else None
},
configmap_name='routers-config',
interactive=interactive)
dns_provider = get_dns_provider()
if dns_provider.lower() == 'none':
if cluster_manager.get_provider_id() != 'minikube':
return
logs.info(dns_provider=dns_provider)
if dns_provider == 'cloudflare':
config_manager.interactive_set(
{
'cloudflare-email': None,
'cloudflare-api-key': None
},
configmap_name='routers-config',
interactive=interactive)
routers_manager.install_crds()
infra_router_name = routers_manager.get_default_infra_router_name()
default_root_domain = config_manager.get('default-root-domain',
configmap_name='routers-config',
required=True)
logs.info('Creating infra router',
infra_router_name=infra_router_name,
default_root_domain=default_root_domain)
routers_manager.create(
infra_router_name,
routers_manager.get_traefik_router_spec(
default_root_domain,
config_manager.get('cloudflare-email',
configmap_name='routers-config',
required=False,
default=None),
config_manager.get('cloudflare-api-key',
configmap_name='routers-config',
required=False,
default=None),
dns_provider=dns_provider))
def initialize(default_provider=None, interactive=False):
crds_manager.install_crd(CRD_SINGULAR, CRD_PLURAL, CRD_KIND)
if not default_provider:
default_provider = 'gcloud' if cluster_manager.get_provider_id(
) == 'gcloud' else 'rancher'
providers_manager.get_provider(
users_provider_submodule, required=False,
default=default_provider).initialize(interactive=interactive)
def initialize(interactive=False, dry_run=False):
if cluster_manager.get_provider_id() == 'minikube':
config_manager.set(
'container-spec-overrides',
'{"resources":{"limits":{"memory":"1Gi"}}}',
configmap_name='ckan-cloud-provider-solr-solrcloud-sc-config')
zk_host_names = initialize_zookeeper(interactive, dry_run=dry_run)
_config_set('zk-host-names',
yaml.dump(zk_host_names, default_flow_style=False))
logs.info(f'Initialized zookeeper: {zk_host_names}')
zoonavigator_deployment_name = _apply_zoonavigator_deployment(
dry_run=dry_run)
logs.info(f'Initialized zoonavigator: {zoonavigator_deployment_name}')
sc_host_names = initialize_solrcloud(zk_host_names,
pause_deployment=False,
interactive=interactive,
dry_run=dry_run)
_config_set('sc-host-names',
yaml.dump(sc_host_names, default_flow_style=False))
logs.info(f'Initialized solrcloud: {sc_host_names}')
solrcloud_host_name = _apply_solrcloud_service(dry_run=dry_run)
_config_set('sc-main-host-name', solrcloud_host_name)
logs.info(f'Initialized solrcloud service: {solrcloud_host_name}')
# TODO - need to check the pod names and ensure these are solrcloud ones
# TODO - actual number is _+ 1_ and not _+ 2_
expected_running = len(sc_host_names) + len(zk_host_names) + 2
RETRIES = 40 # ~20 minutes
for retry in range(RETRIES):
pods = kubectl.get('pods')
running = len(
[x for x in pods['items'] if x['status']['phase'] == 'Running'])
time.sleep(30)
logs.info('Waiting for SolrCloud to start... %d/%d' %
(running, expected_running))
for x in pods['items']:
logs.info(' - %-10s | %s: %s' %
(x['metadata'].get('labels', {}).get('app'),
x['metadata']['name'], x['status']['phase']))
if running == expected_running:
break
assert retry < RETRIES - 1, 'Gave up on waiting for SolrCloud'
_set_provider()
def get_provider_id():
from ckan_cloud_operator.providers.cluster import manager as cluster_manager
from ckan_cloud_operator.providers.cluster.gcloud.constants import PROVIDER_ID as gcloud_provider_id
from ckan_cloud_operator.providers.cluster.aws.constants import PROVIDER_ID as aws_provider_id
from ckan_cloud_operator.providers.cluster.azure.constants import PROVIDER_ID as azure_provider_id
from .s3.constants import PROVIDER_ID as s3_provider_id
from .gcloud.constants import PROVIDER_ID as gcloud_storage_provider_id
from .azure.constants import PROVIDER_ID as azure_storage_provider_id
cluster_provider_id = cluster_manager.get_provider_id()
if cluster_provider_id == gcloud_provider_id:
return gcloud_storage_provider_id
elif cluster_provider_id == aws_provider_id:
return s3_provider_id
elif cluster_provider_id == azure_provider_id:
return azure_storage_provider_id
return 'minio'
def get_load_balancer_ip(router_name, failfast=False):
resource_name = _get_resource_name(router_name)
while True:
time.sleep(.2)
load_balancer = kubectl.get(f'service loadbalancer-{resource_name}',
required=False)
if not load_balancer:
if failfast:
return None
else:
continue
ingresses = load_balancer.get('status', {}).get('loadBalancer',
{}).get('ingress', [])
if len(ingresses) == 0: continue
assert len(ingresses) == 1
if cluster_manager.get_provider_id() == 'aws':
load_balancer_hostname = ingresses[0].get('hostname')
if load_balancer_hostname:
return load_balancer_hostname
else:
load_balancer_ip = ingresses[0].get('ip')
if load_balancer_ip:
return load_balancer_ip
def _get_sc_suffixes():
return config_manager.get(
'sc-suffixes',
secret_name='solr-config',
default=DEFAULT_SC_SUFFIXES if
cluster_manager.get_provider_id() != 'minikube' else "sc-3").split(" ")
def initialize(interactive=False, dry_run=False):
if cluster_manager.get_provider_id() == 'minikube':
config_manager.set(
'container-spec-overrides',
'{"resources":{"limits":{"memory":"1Gi"}}}',
configmap_name='ckan-cloud-provider-solr-solrcloud-sc-config')
config_manager.interactive_set(
{
'sc-cpu': '1',
'sc-mem': '1Gi',
'zk-cpu': '0.2',
'zk-mem': '200Mi',
'zn-cpu': '0.01',
'zn-mem': '0.01Gi',
'sc-cpu-limit': '2.5',
'sc-mem-limit': '8Gi',
'zk-cpu-limit': '0.5',
'zk-mem-limit': '200Mi',
'zn-cpu-limit': '0.5',
'zn-mem-limit': '0.5Gi',
'sc-suffixes': DEFAULT_SC_SUFFIXES,
'zk-suffixes': DEFAULT_ZK_SUFFIXES
},
secret_name='solr-config',
interactive=interactive)
zk_host_names = initialize_zookeeper(interactive, dry_run=dry_run)
_config_set('zk-host-names',
yaml.dump(zk_host_names, default_flow_style=False))
logs.info(f'Initialized zookeeper: {zk_host_names}')
zoonavigator_deployment_name = _apply_zoonavigator_deployment(
dry_run=dry_run)
logs.info(f'Initialized zoonavigator: {zoonavigator_deployment_name}')
sc_host_names = initialize_solrcloud(zk_host_names,
pause_deployment=False,
interactive=interactive,
dry_run=dry_run)
_config_set('sc-host-names',
yaml.dump(sc_host_names, default_flow_style=False))
logs.info(f'Initialized solrcloud: {sc_host_names}')
solrcloud_host_name = _apply_solrcloud_service(dry_run=dry_run)
_config_set('sc-main-host-name', solrcloud_host_name)
logs.info(f'Initialized solrcloud service: {solrcloud_host_name}')
expected_running = len(sc_host_names) + len(zk_host_names) + 1
RETRIES = 40 # ~20 minutes
for retry in range(RETRIES):
pods = kubectl.get('pods')
running = len([
x for x in pods['items'] if x['status']['phase'] == 'Running'
and x['metadata']['labels']['app'].startswith(
_get_resource_labels(for_deployment=True)['app'])
])
time.sleep(45)
logs.info('Waiting for SolrCloud to start... %d/%d' %
(running, expected_running))
for x in pods['items']:
logs.info(' - %-10s | %s: %s' %
(x['metadata'].get('labels', {}).get('app'),
x['metadata']['name'], x['status']['phase']))
if running == expected_running:
break
assert retry < RETRIES - 1, 'Gave up on waiting for SolrCloud'
_set_provider()
def _get_sc_suffixes():
if cluster_manager.get_provider_id() != 'minikube':
return ['sc-3', 'sc-4', 'sc-5']
else:
return ['sc-3']
def _get_zk_suffixes():
if cluster_manager.get_provider_id() != 'minikube':
return ['zk-0', 'zk-1', 'zk-2']
else:
return ['zk-0']
def initialize(interactive=False, dry_run=False):
if cluster_manager.get_provider_id() == 'minikube':
config_manager.set(
'container-spec-overrides',
'{"resources":{"limits":{"memory":"1Gi"}}}',
configmap_name='ckan-cloud-provider-solr-solrcloud-sc-config')
# This is for setting solr's configurations and their defaults
# This can be modified from interactive,yaml or with prompt answers on cluster initialize
config_manager.interactive_set(
{
'sc-cpu': '1',
'sc-mem': '1Gi',
'zk-cpu': '0.2',
'zk-mem': '200Mi',
'zn-cpu': '0.01',
'zn-mem': '0.01Gi',
'sc-cpu-limit': '2.5',
'sc-mem-limit': '8Gi',
'sc-image': 'solr:5.5.5',
'sc-init-image': 'alpine',
'zk-cpu-limit': '0.5',
'zk-mem-limit': '200Mi',
'zk-image': 'gcr.io/google_samples/k8szk:v3',
'zn-cpu-limit': '0.5',
'zn-mem-limit': '0.5Gi',
},
secret_name='solr-config',
interactive=interactive)
registry_secrets = config_manager.interactive_set(
{'private-registry': 'n'},
secret_name='ckan-docker-registry',
interactive=interactive)
if registry_secrets.get('private-registry') == 'y':
registry_secrets = config_manager.interactive_set(
{
'docker-server': None,
'docker-username': None,
'docker-password': None,
'docker-email': None,
'docker-image-pull-secret-name': 'container-registry',
},
secret_name='ckan-docker-registry',
interactive=interactive)
helm_manager._create_private_container_registry_secret('ckan-cloud')
zk_host_names = initialize_zookeeper(interactive, dry_run=dry_run)
_config_set('zk-host-names',
yaml.dump(zk_host_names, default_flow_style=False))
logs.info(f'Initialized zookeeper: {zk_host_names}')
zoonavigator_deployment_name = _apply_zoonavigator_deployment(
dry_run=dry_run)
logs.info(f'Initialized zoonavigator: {zoonavigator_deployment_name}')
sc_host_names = initialize_solrcloud(zk_host_names,
pause_deployment=False,
interactive=interactive,
dry_run=dry_run)
_config_set('sc-host-names',
yaml.dump(sc_host_names, default_flow_style=False))
logs.info(f'Initialized solrcloud: {sc_host_names}')
solrcloud_host_name = _apply_solrcloud_service(dry_run=dry_run)
_config_set('sc-main-host-name', solrcloud_host_name)
logs.info(f'Initialized solrcloud service: {solrcloud_host_name}')
expected_running = len(sc_host_names) + len(zk_host_names) + 1
RETRIES = 40 # ~20 minutes
for retry in range(RETRIES):
pods = kubectl.get('pods')
running = len([
x for x in pods['items'] if x['status']['phase'] == 'Running'
and x['metadata']['labels']['app'].startswith(
_get_resource_labels(for_deployment=True)['app'])
])
time.sleep(45)
logs.info('Waiting for SolrCloud to start... %d/%d' %
(running, expected_running))
for x in pods['items']:
logs.info(' - %-10s | %s: %s' %
(x['metadata'].get('labels', {}).get('app'),
x['metadata']['name'], x['status']['phase']))
if running == expected_running:
break
assert retry < RETRIES - 1, 'Gave up on waiting for SolrCloud'
_set_provider()
def update(instance_id_or_name,
override_spec=None,
persist_overrides=False,
wait_ready=False,
skip_deployment=False,
skip_route=False,
force=False,
dry_run=False):
instance_id, instance_type, instance = _get_instance_id_and_type(
instance_id_or_name, required=not dry_run)
if dry_run:
logs.info('update instance',
instance_id=instance_id,
instance_id_or_name=instance_id_or_name,
override_spec=override_spec,
persist_overrides=persist_overrides,
wait_ready=wait_ready,
skip_deployment=skip_deployment,
skip_route=skip_route,
force=force,
dry_run=dry_run)
else:
pre_update_hook_data = deployment_manager.pre_update_hook(
instance_id, instance_type, instance, override_spec, skip_route)
bucket_credentials = instance['spec'].get('ckanStorageBucket', {}).get(
get_storage_provider_id())
use_cloud_storage = bucket_credentials and config_manager.get(
'use-cloud-native-storage', secret_name=CONFIG_NAME)
if use_cloud_storage:
cluster_provider_id = cluster_manager.get_provider_id()
if bucket_credentials:
literal = []
config_manager.set(values=bucket_credentials,
secret_name='bucket-credentials',
namespace=instance_id)
if persist_overrides:
logs.info('Persisting overrides')
kubectl.apply(instance)
if not skip_deployment:
deployment_manager.update(instance_id,
instance_type,
instance,
force=force)
if wait_ready:
wait_instance_ready(instance_id_or_name)
if not skip_route and pre_update_hook_data.get('sub-domain'):
root_domain = pre_update_hook_data.get('root-domain')
sub_domain = pre_update_hook_data['sub-domain']
assert root_domain == routers_manager.get_default_root_domain(
), 'invalid domain, must use default root domain'
logs.info(
f'adding instance default route to {sub_domain}.{root_domain}')
routers_manager.create_subdomain_route(
'instances-default', {
'target-type': 'ckan-instance',
'ckan-instance-id': instance_id,
'root-domain': root_domain,
'sub-domain': sub_domain
})
logs.info(f'updating routers_manager wait_ready: {wait_ready}')
routers_manager.update('instances-default', wait_ready)
else:
logs.info('skipping route creation',
skip_route=skip_route,
sub_domain=pre_update_hook_data.get('sub-domain'))
logs.info('creating ckan admin')
# Need to set in values.yaml
if pre_update_hook_data.get('create-sysadmin'):
ckan_admin_email = pre_update_hook_data.get('ckan-admin-email')
ckan_admin_password = pre_update_hook_data.get(
'ckan-admin-password')
ckan_admin_name = pre_update_hook_data.get('ckan-admin-name',
'admin')
res = create_ckan_admin_user(instance_id, ckan_admin_name,
ckan_admin_email, ckan_admin_password)
logs.info(**res)
logs.info('Instance is ready',
instance_id=instance_id,
instance_name=(instance_id_or_name if
instance_id_or_name != instance_id else None))
def initialize(interactive=False, skip=None):
ckan_db_migration_manager.initialize(interactive=interactive)
registry_secrets = config_manager.interactive_set(
{'disable-centralized-datapushers': 'no'},
configmap_name='global-ckan-config',
interactive=interactive)
if config_manager.get('enable-deis-ckan',
configmap_name='global-ckan-config') == 'y':
ckan_infra = CkanInfra(required=False)
config_manager.interactive_set(
{
'deis-kubeconfig': ckan_infra.DEIS_KUBECONFIG,
},
from_file=True,
secret_name='ckan-migration-secrets',
interactive=interactive)
config_manager.interactive_set(
{
'gitlab-token': ckan_infra.GITLAB_TOKEN_PASSWORD,
},
secret_name='ckan-migration-secrets',
interactive=interactive)
config_manager.interactive_set(
{
'docker-server': ckan_infra.DOCKER_REGISTRY_SERVER,
'docker-username': ckan_infra.DOCKER_REGISTRY_USERNAME,
'docker-password': ckan_infra.DOCKER_REGISTRY_PASSWORD,
'docker-email': ckan_infra.DOCKER_REGISTRY_EMAIL,
},
secret_name='ckan-docker-registry',
interactive=interactive)
crds_manager.install_crd(INSTANCE_CRD_SINGULAR, INSTANCE_CRD_PLURAL_SUFFIX,
INSTANCE_CRD_KIND_SUFFIX)
crds_manager.install_crd(INSTANCE_NAME_CRD_SINGULAR,
INSTANCE_NAME_CRD_PLURAL_SUFFIX,
INSTANCE_NAME_CRD_KIND_SUFFIX)
if skip != 'solr':
from ckan_cloud_operator.providers.solr.manager import zk_list_configs, zk_put_configs
logs.info('Checking CKAN Solr config in ZooKeeper')
if 'ckan_default' in zk_list_configs():
logs.info('Found ckan_default Solr config')
else:
logs.info(
'No default Solr config found. Putting CKAN 2.8 config for Solr to ZooKeeper as ckan_default...'
)
file_path = os.path.dirname(os.path.abspath(__file__))
root_path = Path(file_path).parent.parent
configs_dir = os.path.join(root_path, 'data', 'solr')
zk_put_configs(configs_dir)
if config_manager.get('disable-centralized-datapushers',
configmap_name='global-ckan-config',
required=False) != 'yes':
from ckan_cloud_operator.datapushers import initialize as datapusher_initialize
datapusher_initialize()
from ckan_cloud_operator.routers import manager as routers_manager
from ckan_cloud_operator.providers.routers import manager as router_provider_manager
router_name = get_default_instances_router_name()
wildcard_ssl_domain = routers_manager.get_default_root_domain()
dns_provider = router_provider_manager.get_dns_provider()
logs.info(
f'wildcard_ssl_domain={wildcard_ssl_domain} dns_provider={dns_provider}'
)
allow_wildcard_ssl = routers_manager.get_env_id() == 'p'
router = routers_manager.get(router_name, required=False)
if router:
assert ((allow_wildcard_ssl and router.get(
'spec', {}).get('wildcard-ssl-domain') == wildcard_ssl_domain)
or (not allow_wildcard_ssl and not router.get('spec', {}))
), f'invalid router wildcard ssl config: {router}'
else:
# We don't want to create traefik routes if no dns_provider, unless it's minikube
if dns_provider.lower(
) == 'none' and cluster_manager.get_provider_id() != 'minikube':
pass
else:
routers_manager.create(
router_name,
routers_manager.get_traefik_router_spec(
dns_provider=dns_provider,
wildcard_ssl_domain=wildcard_ssl_domain))
from .storage.manager import initialize as ckan_storage_initialize
ckan_storage_initialize(interactive=interactive)
from .deployment.manager import initialize as ckan_deployment_initialize
ckan_deployment_initialize(interactive=interactive)
def initialize(interactive=False):
logs.info('env-id is a single character identifier of the environment')
logs.info('.e.g p for production, s for staging, d for development')
logs.info("For the first cluster it's recommended to use 'p'")
default_dns_provider = {
'aws': 'route53',
}.get(cluster_manager.get_provider_id(), 'cloudflare')
if cluster_manager.get_provider_id() == 'kamatera':
from ckan_cloud_operator.providers.cluster.kamatera.management import manager as kamatera_management_manager
default_root_domain = kamatera_management_manager.get_management_machine_secrets('RootDomainName')
else:
default_root_domain = None
config_manager.interactive_set(
{
'env-id': 'p',
'default-root-domain': default_root_domain,
'dns-provider': default_dns_provider
},
configmap_name='routers-config',
interactive=interactive
)
dns_provider = config_manager.get(key='dns-provider', configmap_name='routers-config')
logs.info(dns_provider=dns_provider)
if dns_provider == 'cloudflare':
if cluster_manager.get_provider_id() == 'kamatera':
cloudflare_api_key = config_manager.get(
'CloudflareApiKey',
secret_name='cco-kamatera-management-server',
namespace='ckan-cloud-operator',
required=True
)
cloudflare_email = config_manager.get(
'CloudflareEmail',
secret_name='cco-kamatera-management-server',
namespace='ckan-cloud-operator',
required=True
)
config_manager.set(values={
'cloudflare-email': cloudflare_email,
'cloudflare-api-key': cloudflare_api_key
}, secret_name='routers-secrets')
else:
config_manager.interactive_set(
{
'cloudflare-email': None,
'cloudflare-api-key': None
},
secret_name='routers-secrets',
interactive=interactive
)
routers_manager.install_crds()
infra_router_name = routers_manager.get_default_infra_router_name()
default_root_domain = config_manager.get('default-root-domain', configmap_name='routers-config', required=True)
router_type = {
'kamatera': 'nginx'
}.get(cluster_manager.get_provider_id(), 'traefik')
logs.info('Creating infra router', infra_router_name=infra_router_name, default_root_domain=default_root_domain, router_type=router_type)
if router_type == 'traefik':
router_spec = routers_manager.get_traefik_router_spec(
default_root_domain,
config_manager.get('cloudflare-email', secret_name='routers-secrets', required=False, default=None),
config_manager.get('cloudflare-api-key', secret_name='routers-secrets', required=False, default=None),
dns_provider=dns_provider
)
else:
router_spec = routers_manager.get_nginx_router_spec(
default_root_domain,
config_manager.get('cloudflare-email', secret_name='routers-secrets', required=False, default=None),
config_manager.get('cloudflare-api-key', secret_name='routers-secrets', required=False, default=None),
dns_provider=dns_provider
)
routers_manager.create(infra_router_name, router_spec)
def update(instance_id_or_name,
override_spec=None,
persist_overrides=False,
wait_ready=False,
skip_deployment=False,
skip_route=False,
force=False,
dry_run=False):
instance_id, instance_type, instance = _get_instance_id_and_type(
instance_id_or_name, required=not dry_run)
if dry_run:
logs.info('update instance',
instance_id=instance_id,
instance_id_or_name=instance_id_or_name,
override_spec=override_spec,
persist_overrides=persist_overrides,
wait_ready=wait_ready,
skip_deployment=skip_deployment,
skip_route=skip_route,
force=force,
dry_run=dry_run)
else:
pre_update_hook_data = deployment_manager.pre_update_hook(
instance_id, instance_type, instance, override_spec, skip_route)
bucket_credentials = instance['spec'].get('ckanStorageBucket', {}).get(
get_storage_provider_id())
use_cloud_storage = bucket_credentials and config_manager.get(
'use-cloud-native-storage', secret_name=CONFIG_NAME)
if use_cloud_storage:
cluster_provider_id = cluster_manager.get_provider_id()
if bucket_credentials:
literal = []
config_manager.set(values=bucket_credentials,
secret_name='bucket-credentials',
namespace=instance_id)
if instance['spec'].get('operatorCopySecrets'):
for target_secret_name, source_secret_config in json.loads(
instance['spec']['operatorCopySecrets']).items():
for k, v in source_secret_config.items():
source_secret_config[k] = v.replace(
"__INSTANCE_NAME__", instance_id_or_name)
kubectl.update_secret(
target_secret_name,
kubectl.decode_secret(
kubectl.get('secret',
source_secret_config["fromName"],
namespace=source_secret_config.get(
"fromNamespace", "ckan-cloud"))),
namespace=instance_id)
if persist_overrides:
logs.info('Persisting overrides')
kubectl.apply(instance)
if not skip_deployment:
deployment_manager.update(instance_id,
instance_type,
instance,
force=force)
if wait_ready:
wait_instance_ready(instance_id_or_name)
if not skip_route and pre_update_hook_data.get('sub-domain'):
root_domain = pre_update_hook_data.get('root-domain')
sub_domain = pre_update_hook_data['sub-domain']
assert root_domain == routers_manager.get_default_root_domain(
), 'invalid domain, must use default root domain'
logs.info(
f'adding instance default route to {sub_domain}.{root_domain}')
routers_manager.create_subdomain_route(
'instances-default', {
'target-type': 'ckan-instance',
'ckan-instance-id': instance_id,
'root-domain': root_domain,
'sub-domain': sub_domain
})
logs.info(f'updating routers_manager wait_ready: {wait_ready}')
routers_manager.update('instances-default', wait_ready)
else:
logs.info('skipping route creation',
skip_route=skip_route,
sub_domain=pre_update_hook_data.get('sub-domain'))
if not instance['spec'].get('skipCreateCkanAdmin', False):
logs.info('creating ckan admin')
ckan_admin_email = instance['spec'].get(
'ckanAdminEmail', pre_update_hook_data.get('ckan-admin-email'))
ckan_admin_password = pre_update_hook_data.get(
'ckan-admin-password')
ckan_admin_name = instance['spec'].get(
'ckanAdminName',
pre_update_hook_data.get('ckan-admin-name', 'admin'))
res = create_ckan_admin_user(instance_id, ckan_admin_name,
ckan_admin_email, ckan_admin_password)
logs.info(**res)
logs.info('Instance is ready',
instance_id=instance_id,
instance_name=(instance_id_or_name if
instance_id_or_name != instance_id else None))
