def _pre_update_hook_admin_user(instance,
sub_domain,
root_domain,
instance_id,
res,
dry_run=False):
ckan_admin_email = instance['spec'].get('ckanAdminEmail')
if not ckan_admin_email:
ckan_admin_email = f'admin@{sub_domain}.{root_domain}'
ckan_admin_password = config_manager.get(key='CKAN_ADMIN_PASSWORD',
secret_name='ckan-admin-password',
namespace=instance_id,
required=False)
if ckan_admin_password:
logs.info('using existing ckan admin user')
res['ckan-admin-password'] = ckan_admin_password
else:
logs.info('Will create new ckan admin user',
ckan_admin_email=ckan_admin_email)
res['ckan-admin-email'] = ckan_admin_email
res['ckan-admin-password'] = ckan_admin_password = binascii.hexlify(
os.urandom(8)).decode()
config_manager.set(key='CKAN_ADMIN_PASSWORD',
value=ckan_admin_password,
secret_name='ckan-admin-password',
namespace=instance_id,
dry_run=dry_run)
def ssh_management_machine(*args, check_output=False, scp_to_remote_file=None):
machine_secrets = get_management_machine_secrets()
id_rsa = machine_secrets['id_rsa']
server_ip = machine_secrets['server_public_ip']
with NamedTemporaryFile('wb') as f:
f.write(id_rsa.encode('ascii'))
f.flush()
ssh_known_hosts = config_manager.get('ssh_known_hosts', secret_name='cco-kamatera-management-server', namespace='ckan-cloud-operator', required=False, default=None)
if ssh_known_hosts:
os.makedirs('/root/.ssh', exist_ok=True)
with open('/root/.ssh/known_hosts', 'w') as known_hosts_file:
known_hosts_file.write(ssh_known_hosts)
if scp_to_remote_file:
with NamedTemporaryFile('w') as tempfile:
tempfile.write("\n".join(args))
tempfile.flush()
cmd = ['scp', '-i', f.name, tempfile.name, 'root@' + server_ip + ':' + scp_to_remote_file]
subprocess.check_call(cmd)
res = 0
else:
cmd = ['ssh', '-i', f.name, 'root@' + server_ip, *args]
if check_output:
res = subprocess.check_output(cmd)
else:
res = subprocess.call(cmd)
if os.path.exists('/root/.ssh/known_hosts'):
with open('/root/.ssh/known_hosts') as f:
if f.read().strip() != ssh_known_hosts.strip():
config_manager.set('ssh_known_hosts', f.read(), secret_name='cco-kamatera-management-server', namespace='ckan-cloud-operator')
return res
def _set_plural_kind_suffix(singular,
plural_suffix,
kind_suffix,
hash_names=False):
hash_names = 'y' if hash_names else 'n'
config_manager.set(f'installed-crd-{singular}',
f'{plural_suffix},{kind_suffix},{hash_names}')
def _init_ckan_infra_secret(instance_id, dry_run=False):
logs.debug('Initializing ckan infra secret', instance_id=instance_id)
ckan_infra = config_manager.get(secret_name='ckan-infra',
namespace=instance_id,
required=False)
if ckan_infra:
logs.info('ckan-infra secret already exists')
else:
admin_user, admin_password, db_name = db_manager.get_admin_db_credentials(
)
db_host, db_port = db_manager.get_internal_unproxied_db_host_port()
assert int(db_port) == 5432
logs.debug('Creating ckan-infra secret',
admin_user=admin_user,
admin_password=admin_password,
db_name=db_name,
db_host=db_host,
db_port=db_port)
config_manager.set(values={
'POSTGRES_HOST': db_host,
'POSTGRES_PASSWORD': admin_password,
'POSTGRES_USER': admin_user
},
secret_name='ckan-infra',
namespace=instance_id,
dry_run=dry_run)
def config_set(submodule, provider_id=None, key=None, value=None, values=None, namespace=None, is_secret=False, suffix=None):
"""store key/values in a secret or configmap"""
resource_name = get_resource_name(submodule, provider_id=provider_id, suffix=suffix)
config_manager.set(
key=key,
value=value,
values=values,
secret_name=resource_name if is_secret else None,
configmap_name=None if is_secret else resource_name,
namespace=namespace
)
def update(instance_id_or_name, override_spec=None, persist_overrides=False, wait_ready=False, skip_deployment=False,
skip_route=False, force=False, dry_run=False, skip_solr=False):
instance_id, instance_type, instance = _get_instance_id_and_type(instance_id_or_name, required=not dry_run)
if dry_run:
logs.info('update instance', instance_id=instance_id, instance_id_or_name=instance_id_or_name,
override_spec=override_spec, persist_overrides=persist_overrides, wait_ready=wait_ready,
skip_deployment=skip_deployment, skip_route=skip_route, force=force, dry_run=dry_run)
else:
pre_update_hook_data = deployment_manager.pre_update_hook(instance_id, instance_type, instance, override_spec,
skip_route)
bucket_credentials = instance['spec'].get('ckanStorageBucket', {}).get(get_storage_provider_id())
use_cloud_storage = bucket_credentials and config_manager.get('use-cloud-native-storage', secret_name=CONFIG_NAME)
if use_cloud_storage and bucket_credentials:
config_manager.set(
values=bucket_credentials,
secret_name='bucket-credentials',
namespace=instance_id
)
if persist_overrides:
logs.info('Persisting overrides')
kubectl.apply(instance)
if not skip_deployment:
deployment_manager.update(instance_id, instance_type, instance, force=force, skip_solr=skip_solr)
if wait_ready:
wait_instance_ready(instance_id_or_name)
if not skip_route and pre_update_hook_data.get('sub-domain'):
root_domain = pre_update_hook_data.get('root-domain')
sub_domain = pre_update_hook_data['sub-domain']
assert root_domain == routers_manager.get_default_root_domain(), 'invalid domain, must use default root domain'
logs.info(f'adding instance default route to {sub_domain}.{root_domain}')
routers_manager.create_subdomain_route('instances-default', {
'target-type': 'ckan-instance',
'ckan-instance-id': instance_id,
'root-domain': root_domain,
'sub-domain': sub_domain
})
logs.info(f'updating routers_manager wait_ready: {wait_ready}')
routers_manager.update('instances-default', wait_ready)
else:
logs.info('skipping route creation', skip_route=skip_route, sub_domain=pre_update_hook_data.get('sub-domain'))
logs.info('creating ckan admin')
# Need to set in values.yaml
if pre_update_hook_data.get('create-sysadmin'):
ckan_admin_email = pre_update_hook_data.get('ckan-admin-email')
ckan_admin_password = pre_update_hook_data.get('ckan-admin-password')
ckan_admin_name = pre_update_hook_data.get('ckan-admin-name', 'admin')
res = create_ckan_admin_user(instance_id, ckan_admin_name, ckan_admin_email, ckan_admin_password)
logs.info(**res)
logs.info('Instance is ready', instance_id=instance_id, instance_name=(instance_id_or_name if instance_id_or_name != instance_id else None))
def initialize(interactive=False, dry_run=False):
if cluster_manager.get_provider_id() == 'minikube':
config_manager.set(
'container-spec-overrides',
'{"resources":{"limits":{"memory":"1Gi"}}}',
configmap_name='ckan-cloud-provider-solr-solrcloud-sc-config')
zk_host_names = initialize_zookeeper(interactive, dry_run=dry_run)
_config_set('zk-host-names',
yaml.dump(zk_host_names, default_flow_style=False))
logs.info(f'Initialized zookeeper: {zk_host_names}')
zoonavigator_deployment_name = _apply_zoonavigator_deployment(
dry_run=dry_run)
logs.info(f'Initialized zoonavigator: {zoonavigator_deployment_name}')
sc_host_names = initialize_solrcloud(zk_host_names,
pause_deployment=False,
interactive=interactive,
dry_run=dry_run)
_config_set('sc-host-names',
yaml.dump(sc_host_names, default_flow_style=False))
logs.info(f'Initialized solrcloud: {sc_host_names}')
solrcloud_host_name = _apply_solrcloud_service(dry_run=dry_run)
_config_set('sc-main-host-name', solrcloud_host_name)
logs.info(f'Initialized solrcloud service: {solrcloud_host_name}')
# TODO - need to check the pod names and ensure these are solrcloud ones
# TODO - actual number is _+ 1_ and not _+ 2_
expected_running = len(sc_host_names) + len(zk_host_names) + 2
RETRIES = 40 # ~20 minutes
for retry in range(RETRIES):
pods = kubectl.get('pods')
running = len(
[x for x in pods['items'] if x['status']['phase'] == 'Running'])
time.sleep(30)
logs.info('Waiting for SolrCloud to start... %d/%d' %
(running, expected_running))
for x in pods['items']:
logs.info(' - %-10s | %s: %s' %
(x['metadata'].get('labels', {}).get('app'),
x['metadata']['name'], x['status']['phase']))
if running == expected_running:
break
assert retry < RETRIES - 1, 'Gave up on waiting for SolrCloud'
_set_provider()
def get_load_balancer_ip(router_name, failfast=False):
load_balancer_ip = config_manager.get(
'load-balancer-ip',
configmap_name=f'nginx-router-{router_name}-deployment',
required=False,
default=None)
if not load_balancer_ip:
load_balancer_ip = config_manager.get(
'server_public_ip',
secret_name='cco-kamatera-management-server',
namespace='ckan-cloud-operator',
required=True)
config_manager.set(
'load-balancer-ip',
load_balancer_ip,
configmap_name=f'nginx-router-{router_name}-deployment')
return load_balancer_ip
def config_set(singular,
name,
key=None,
value=None,
values=None,
namespace=None,
is_secret=False):
"""store key/values in a secret or configmap"""
resource_name = get_resource_name(singular, name)
config_manager.set(key=key,
value=value,
values=values,
secret_name=resource_name if is_secret else None,
configmap_name=None if is_secret else resource_name,
namespace=namespace,
extra_operator_labels=_get_label_suffixes(
singular, name))
def initialize(interactive=False):
config_manager.interactive_set(
{
'env-id': None,
'default-root-domain': None,
},
configmap_name='routers-config',
interactive=interactive)
if cluster_manager.get_provider_id() == 'aws':
dns_provider = 'route53'
else:
dns_provider = 'cloudflare'
logs.info(dns_provider=dns_provider)
config_manager.set(key='dns-provider',
value=dns_provider,
configmap_name='routers-config')
if dns_provider == 'cloudflare':
config_manager.interactive_set(
{
'cloudflare-email': None,
'cloudflare-api-key': None
},
configmap_name='routers-config',
interactive=interactive)
routers_manager.install_crds()
infra_router_name = routers_manager.get_default_infra_router_name()
default_root_domain = config_manager.get('default-root-domain',
configmap_name='routers-config',
required=True)
logs.info('Creating infra router',
infra_router_name=infra_router_name,
default_root_domain=default_root_domain)
routers_manager.create(
infra_router_name,
routers_manager.get_traefik_router_spec(
default_root_domain,
config_manager.get('cloudflare-email',
configmap_name='routers-config',
required=False,
default=None),
config_manager.get('cloudflare-api-key',
configmap_name='routers-config',
required=False,
default=None),
dns_provider=dns_provider))
def _init_ckan_infra_secret(instance_id):
ckan_infra = config_manager.get(secret_name='ckan-infra',
namespace=instance_id,
required=False)
if ckan_infra:
print('ckan-infra secret already exists')
else:
admin_user, admin_password, db_name = db_manager.get_admin_db_credentials(
)
db_host, db_port = db_manager.get_internal_unproxied_db_host_port()
assert int(db_port) == 5432
config_manager.set(values={
'POSTGRES_HOST': db_host,
'POSTGRES_PASSWORD': admin_password,
'POSTGRES_USER': admin_user
},
secret_name='ckan-infra',
namespace=instance_id)
def initialize(log_kwargs=None, interactive=False):
logs.info('setting label-prefix: ckan-cloud', **(log_kwargs or {}))
config_manager.set('label-prefix', 'ckan-cloud')
config_manager.interactive_set({'short-label-prefix': 'cc'},
interactive=interactive)
def set_provider(submodule, provider_id):
config_manager.set(key=get_operator_configmap_key(
submodule, suffix='main-provider-id'),
value=provider_id)
def initialize(log_kwargs=None):
values = {'crd-group': 'stable.viderum.com', 'crd-prefix': 'CkanCloud'}
logs.info(f'Setting default crds module configurations',
**(log_kwargs or {}), **values)
config_manager.set(values=values)
def update(instance_id_or_name,
override_spec=None,
persist_overrides=False,
wait_ready=False,
skip_deployment=False,
skip_route=False,
force=False,
dry_run=False):
instance_id, instance_type, instance = _get_instance_id_and_type(
instance_id_or_name, required=not dry_run)
if dry_run:
logs.info('update instance',
instance_id=instance_id,
instance_id_or_name=instance_id_or_name,
override_spec=override_spec,
persist_overrides=persist_overrides,
wait_ready=wait_ready,
skip_deployment=skip_deployment,
skip_route=skip_route,
force=force,
dry_run=dry_run)
else:
pre_update_hook_data = deployment_manager.pre_update_hook(
instance_id, instance_type, instance, override_spec, skip_route)
bucket_credentials = instance['spec'].get('ckanStorageBucket', {}).get(
get_storage_provider_id())
use_cloud_storage = bucket_credentials and config_manager.get(
'use-cloud-native-storage', secret_name=CONFIG_NAME)
if use_cloud_storage:
cluster_provider_id = cluster_manager.get_provider_id()
if bucket_credentials:
literal = []
config_manager.set(values=bucket_credentials,
secret_name='bucket-credentials',
namespace=instance_id)
if instance['spec'].get('operatorCopySecrets'):
for target_secret_name, source_secret_config in json.loads(
instance['spec']['operatorCopySecrets']).items():
for k, v in source_secret_config.items():
source_secret_config[k] = v.replace(
"__INSTANCE_NAME__", instance_id_or_name)
kubectl.update_secret(
target_secret_name,
kubectl.decode_secret(
kubectl.get('secret',
source_secret_config["fromName"],
namespace=source_secret_config.get(
"fromNamespace", "ckan-cloud"))),
namespace=instance_id)
if persist_overrides:
logs.info('Persisting overrides')
kubectl.apply(instance)
if not skip_deployment:
deployment_manager.update(instance_id,
instance_type,
instance,
force=force)
if wait_ready:
wait_instance_ready(instance_id_or_name)
if not skip_route and pre_update_hook_data.get('sub-domain'):
root_domain = pre_update_hook_data.get('root-domain')
sub_domain = pre_update_hook_data['sub-domain']
assert root_domain == routers_manager.get_default_root_domain(
), 'invalid domain, must use default root domain'
logs.info(
f'adding instance default route to {sub_domain}.{root_domain}')
routers_manager.create_subdomain_route(
'instances-default', {
'target-type': 'ckan-instance',
'ckan-instance-id': instance_id,
'root-domain': root_domain,
'sub-domain': sub_domain
})
logs.info(f'updating routers_manager wait_ready: {wait_ready}')
routers_manager.update('instances-default', wait_ready)
else:
logs.info('skipping route creation',
skip_route=skip_route,
sub_domain=pre_update_hook_data.get('sub-domain'))
if not instance['spec'].get('skipCreateCkanAdmin', False):
logs.info('creating ckan admin')
ckan_admin_email = instance['spec'].get(
'ckanAdminEmail', pre_update_hook_data.get('ckan-admin-email'))
ckan_admin_password = pre_update_hook_data.get(
'ckan-admin-password')
ckan_admin_name = instance['spec'].get(
'ckanAdminName',
pre_update_hook_data.get('ckan-admin-name', 'admin'))
res = create_ckan_admin_user(instance_id, ckan_admin_name,
ckan_admin_email, ckan_admin_password)
logs.info(**res)
logs.info('Instance is ready',
instance_id=instance_id,
instance_name=(instance_id_or_name if
instance_id_or_name != instance_id else None))
def initialize(interactive=False, dry_run=False):
if cluster_manager.get_provider_id() == 'minikube':
config_manager.set(
'container-spec-overrides',
'{"resources":{"limits":{"memory":"1Gi"}}}',
configmap_name='ckan-cloud-provider-solr-solrcloud-sc-config')
# This is for setting solr's configurations and their defaults
# This can be modified from interactive,yaml or with prompt answers on cluster initialize
config_manager.interactive_set(
{
'sc-cpu': '1',
'sc-mem': '1Gi',
'zk-cpu': '0.2',
'zk-mem': '200Mi',
'zn-cpu': '0.01',
'zn-mem': '0.01Gi',
'sc-cpu-limit': '2.5',
'sc-mem-limit': '8Gi',
'sc-image': 'solr:5.5.5',
'sc-init-image': 'alpine',
'zk-cpu-limit': '0.5',
'zk-mem-limit': '200Mi',
'zk-image': 'gcr.io/google_samples/k8szk:v3',
'zn-cpu-limit': '0.5',
'zn-mem-limit': '0.5Gi',
},
secret_name='solr-config',
interactive=interactive)
registry_secrets = config_manager.interactive_set(
{'private-registry': 'n'},
secret_name='ckan-docker-registry',
interactive=interactive)
if registry_secrets.get('private-registry') == 'y':
registry_secrets = config_manager.interactive_set(
{
'docker-server': None,
'docker-username': None,
'docker-password': None,
'docker-email': None,
'docker-image-pull-secret-name': 'container-registry',
},
secret_name='ckan-docker-registry',
interactive=interactive)
helm_manager._create_private_container_registry_secret('ckan-cloud')
zk_host_names = initialize_zookeeper(interactive, dry_run=dry_run)
_config_set('zk-host-names',
yaml.dump(zk_host_names, default_flow_style=False))
logs.info(f'Initialized zookeeper: {zk_host_names}')
zoonavigator_deployment_name = _apply_zoonavigator_deployment(
dry_run=dry_run)
logs.info(f'Initialized zoonavigator: {zoonavigator_deployment_name}')
sc_host_names = initialize_solrcloud(zk_host_names,
pause_deployment=False,
interactive=interactive,
dry_run=dry_run)
_config_set('sc-host-names',
yaml.dump(sc_host_names, default_flow_style=False))
logs.info(f'Initialized solrcloud: {sc_host_names}')
solrcloud_host_name = _apply_solrcloud_service(dry_run=dry_run)
_config_set('sc-main-host-name', solrcloud_host_name)
logs.info(f'Initialized solrcloud service: {solrcloud_host_name}')
expected_running = len(sc_host_names) + len(zk_host_names) + 1
RETRIES = 40 # ~20 minutes
for retry in range(RETRIES):
pods = kubectl.get('pods')
running = len([
x for x in pods['items'] if x['status']['phase'] == 'Running'
and x['metadata']['labels']['app'].startswith(
_get_resource_labels(for_deployment=True)['app'])
])
time.sleep(45)
logs.info('Waiting for SolrCloud to start... %d/%d' %
(running, expected_running))
for x in pods['items']:
logs.info(' - %-10s | %s: %s' %
(x['metadata'].get('labels', {}).get('app'),
x['metadata']['name'], x['status']['phase']))
if running == expected_running:
break
assert retry < RETRIES - 1, 'Gave up on waiting for SolrCloud'
_set_provider()
def initialize(log_kwargs=None):
logs.info('setting label-prefix: ckan-cloud', **(log_kwargs or {}))
config_manager.set('label-prefix', 'ckan-cloud')
def set(**kwargs):
"""Set a configuration value"""
manager.set(**kwargs)
logs.exit_great_success()
def initialize(interactive=False, dry_run=False):
if cluster_manager.get_provider_id() == 'minikube':
config_manager.set(
'container-spec-overrides',
'{"resources":{"limits":{"memory":"1Gi"}}}',
configmap_name='ckan-cloud-provider-solr-solrcloud-sc-config')
config_manager.interactive_set(
{
'sc-cpu': '1',
'sc-mem': '1Gi',
'zk-cpu': '0.2',
'zk-mem': '200Mi',
'zn-cpu': '0.01',
'zn-mem': '0.01Gi',
'sc-cpu-limit': '2.5',
'sc-mem-limit': '8Gi',
'zk-cpu-limit': '0.5',
'zk-mem-limit': '200Mi',
'zn-cpu-limit': '0.5',
'zn-mem-limit': '0.5Gi',
'sc-suffixes': DEFAULT_SC_SUFFIXES,
'zk-suffixes': DEFAULT_ZK_SUFFIXES
},
secret_name='solr-config',
interactive=interactive)
zk_host_names = initialize_zookeeper(interactive, dry_run=dry_run)
_config_set('zk-host-names',
yaml.dump(zk_host_names, default_flow_style=False))
logs.info(f'Initialized zookeeper: {zk_host_names}')
zoonavigator_deployment_name = _apply_zoonavigator_deployment(
dry_run=dry_run)
logs.info(f'Initialized zoonavigator: {zoonavigator_deployment_name}')
sc_host_names = initialize_solrcloud(zk_host_names,
pause_deployment=False,
interactive=interactive,
dry_run=dry_run)
_config_set('sc-host-names',
yaml.dump(sc_host_names, default_flow_style=False))
logs.info(f'Initialized solrcloud: {sc_host_names}')
solrcloud_host_name = _apply_solrcloud_service(dry_run=dry_run)
_config_set('sc-main-host-name', solrcloud_host_name)
logs.info(f'Initialized solrcloud service: {solrcloud_host_name}')
expected_running = len(sc_host_names) + len(zk_host_names) + 1
RETRIES = 40 # ~20 minutes
for retry in range(RETRIES):
pods = kubectl.get('pods')
running = len([
x for x in pods['items'] if x['status']['phase'] == 'Running'
and x['metadata']['labels']['app'].startswith(
_get_resource_labels(for_deployment=True)['app'])
])
time.sleep(45)
logs.info('Waiting for SolrCloud to start... %d/%d' %
(running, expected_running))
for x in pods['items']:
logs.info(' - %-10s | %s: %s' %
(x['metadata'].get('labels', {}).get('app'),
x['metadata']['name'], x['status']['phase']))
if running == expected_running:
break
assert retry < RETRIES - 1, 'Gave up on waiting for SolrCloud'
_set_provider()
def initialize(interactive=False):
logs.info('env-id is a single character identifier of the environment')
logs.info('.e.g p for production, s for staging, d for development')
logs.info("For the first cluster it's recommended to use 'p'")
default_dns_provider = {
'aws': 'route53',
}.get(cluster_manager.get_provider_id(), 'cloudflare')
if cluster_manager.get_provider_id() == 'kamatera':
from ckan_cloud_operator.providers.cluster.kamatera.management import manager as kamatera_management_manager
default_root_domain = kamatera_management_manager.get_management_machine_secrets('RootDomainName')
else:
default_root_domain = None
config_manager.interactive_set(
{
'env-id': 'p',
'default-root-domain': default_root_domain,
'dns-provider': default_dns_provider
},
configmap_name='routers-config',
interactive=interactive
)
dns_provider = config_manager.get(key='dns-provider', configmap_name='routers-config')
logs.info(dns_provider=dns_provider)
if dns_provider == 'cloudflare':
if cluster_manager.get_provider_id() == 'kamatera':
cloudflare_api_key = config_manager.get(
'CloudflareApiKey',
secret_name='cco-kamatera-management-server',
namespace='ckan-cloud-operator',
required=True
)
cloudflare_email = config_manager.get(
'CloudflareEmail',
secret_name='cco-kamatera-management-server',
namespace='ckan-cloud-operator',
required=True
)
config_manager.set(values={
'cloudflare-email': cloudflare_email,
'cloudflare-api-key': cloudflare_api_key
}, secret_name='routers-secrets')
else:
config_manager.interactive_set(
{
'cloudflare-email': None,
'cloudflare-api-key': None
},
secret_name='routers-secrets',
interactive=interactive
)
routers_manager.install_crds()
infra_router_name = routers_manager.get_default_infra_router_name()
default_root_domain = config_manager.get('default-root-domain', configmap_name='routers-config', required=True)
router_type = {
'kamatera': 'nginx'
}.get(cluster_manager.get_provider_id(), 'traefik')
logs.info('Creating infra router', infra_router_name=infra_router_name, default_root_domain=default_root_domain, router_type=router_type)
if router_type == 'traefik':
router_spec = routers_manager.get_traefik_router_spec(
default_root_domain,
config_manager.get('cloudflare-email', secret_name='routers-secrets', required=False, default=None),
config_manager.get('cloudflare-api-key', secret_name='routers-secrets', required=False, default=None),
dns_provider=dns_provider
)
else:
router_spec = routers_manager.get_nginx_router_spec(
default_root_domain,
config_manager.get('cloudflare-email', secret_name='routers-secrets', required=False, default=None),
config_manager.get('cloudflare-api-key', secret_name='routers-secrets', required=False, default=None),
dns_provider=dns_provider
)
routers_manager.create(infra_router_name, router_spec)
