def create_volume(disk_size_gb, labels, use_existing_disk_name=None, zone=0):
if zone != 0:
logs.warning(f'variable zone for create_volume has been deprecated.')
disk_id = use_existing_disk_name or 'cc' + _generate_password(12)
if use_existing_disk_name:
logs.info(f'using existing persistent disk {disk_id}')
else:
logs.info(
f'creating persistent disk {disk_id} with size {disk_size_gb}GB')
labels = ','.join([
'{}={}'.format(k.replace('/', '_'), v.replace('/', '_'))
for k, v in labels.items()
])
kubectl.apply({
"kind": "PersistentVolumeClaim",
"apiVersion": "v1",
"metadata": {
"name": disk_id,
"namespace": "ckan-cloud"
},
"spec": {
"accessModes": ["ReadWriteOnce"],
"resources": {
"requests": {
"storage": f'{disk_size_gb}G'
}
},
"storageClassName": "cca-ckan"
}
})
return {'persistentVolumeClaim': {'claimName': disk_id}}
def _add_letsencrypt(dns_provider,
config,
letsencrypt_cloudflare_email,
domains,
wildcard_ssl_domain=None,
external_domains=False):
logs.info('Adding Letsencrypt acme Traefik configuration',
dns_provider=dns_provider,
letsencrypt_cloudflare_email=letsencrypt_cloudflare_email,
domains=domains,
wildcard_ssl_domain=wildcard_ssl_domain,
external_domains=external_domains)
assert dns_provider in ['route53', 'cloudflare']
config['defaultEntryPoints'].append('https')
config['entryPoints']['https'] = {'address': ':443', 'tls': {}}
config['acme'] = {
'email':
letsencrypt_cloudflare_email,
'storage':
'/traefik-acme/acme.json',
'entryPoint':
'https',
**({
'tlsChallenge': {}
} if external_domains else {
'dnsChallenge': {
'provider': dns_provider
}
}), 'domains':
list(
_get_acme_domains(domains,
wildcard_ssl_domain=wildcard_ssl_domain,
external_domains=external_domains))
}
def _helm_deploy(values,
tiller_namespace_name,
chart_repo,
chart_name,
chart_version,
release_name,
instance_id,
dry_run=False,
chart_repo_name=None):
assert chart_repo_name, 'chart-repo-name is required'
helm_driver.init(tiller_namespace_name)
logs.info(
f'Deploying helm chart {chart_repo_name} {chart_repo} {chart_version} {chart_name} to release {release_name} '
f'(instance_id={instance_id})')
with tempfile.NamedTemporaryFile('w') as f:
yaml.dump(values, f, default_flow_style=False)
f.flush()
helm_driver.deploy(tiller_namespace=tiller_namespace_name,
chart_repo=chart_repo,
chart_name=chart_name,
chart_version=chart_version,
release_name=release_name,
values_filename=f.name,
namespace=instance_id,
dry_run=dry_run,
chart_repo_name=chart_repo_name)
def _init_namespace(instance_id, dry_run=False):
logs.debug('Initializing helm-based instance deployment namespace',
namespace=instance_id)
if kubectl.get('ns', instance_id, required=False):
logs.info(f'instance namespace already exists ({instance_id})')
else:
logs.info(f'creating instance namespace ({instance_id})')
kubectl.apply(kubectl.get_resource('v1', 'Namespace', instance_id, {}),
dry_run=dry_run)
service_account_name = f'ckan-{instance_id}-operator'
logs.debug('Creating service account',
service_account_name=service_account_name)
if not dry_run:
kubectl_rbac_driver.update_service_account(
f'ckan-{instance_id}-operator', {}, namespace=instance_id)
role_name = f'ckan-{instance_id}-operator-role'
logs.debug('Creating role and binding to the service account',
role_name=role_name)
if not dry_run:
kubectl_rbac_driver.update_role(role_name, {}, [{
"apiGroups": ["*"],
"resources":
['secrets', 'pods', 'pods/exec', 'pods/portforward'],
"verbs": ["list", "get", "create"]
}],
namespace=instance_id)
kubectl_rbac_driver.update_role_binding(
name=f'ckan-{instance_id}-operator-rolebinding',
role_name=f'ckan-{instance_id}-operator-role',
namespace=instance_id,
service_account_name=f'ckan-{instance_id}-operator',
labels={})
def _scale_down_scale_up(deployment='ckan', namespace=None, replicas=1):
logs.info('Scaling ckan replicas')
kubectl.call(f'scale deployment {deployment} --replicas=0',
namespace=namespace)
kubectl.call(f'scale deployment {deployment} --replicas={replicas}',
namespace=namespace)
time.sleep(20)
def get_or_create_multi_user_volume_claim(label_suffixes):
assert len(label_suffixes) > 0, 'must provide some labels to identify the volume'
claim_labels = labels_manager.get_resource_labels(label_suffixes=dict(
label_suffixes, **_get_cluster_volume_label_suffixes()
))
pvcs = kubectl.get_items_by_labels('PersistentVolumeClaim', claim_labels, required=False)
if len(pvcs) > 0:
assert len(pvcs) == 1
claim_name = pvcs[0]['metadata']['name']
else:
storage_class_name = get_multi_user_storage_class_name()
claim_name = 'cc' + _generate_password(12)
logs.info(f'Creating persistent volume claim: {claim_name}')
logs.info(f'labels: {claim_labels}')
kubectl.apply(kubectl.get_persistent_volume_claim(
claim_name,
claim_labels,
{
'storageClassName': storage_class_name,
'accessModes': ['ReadWriteMany'],
'resources': {
'requests': {
'storage': '1Mi'
}
}
}
))
return {'persistentVolumeClaim': {'claimName': claim_name}}
def update_dns_record(sub_domain, root_domain, load_balancer_hostname):
logs.info('updating Route53 DNS record', sub_domain=sub_domain, root_domain=root_domain, load_balancer_hostname=load_balancer_hostname)
hosted_zone_id = get_dns_hosted_zone_id(root_domain)
logs.info(hosted_zone_id=hosted_zone_id)
response = get_boto3_client('route53').change_resource_record_sets(
HostedZoneId=hosted_zone_id,
ChangeBatch={
'Comment': 'ckan-cloud-operator',
'Changes': [
{
'Action': 'UPSERT',
'ResourceRecordSet': {
'Name': f'{sub_domain}.{root_domain}.',
'Type': 'CNAME',
'TTL': 300,
'ResourceRecords': [
{
'Value': load_balancer_hostname
},
],
}
},
]
}
)
assert response['ResponseMetadata']['HTTPStatusCode'] == 200
def create(router):
router_name = router['metadata']['name']
router_spec = router['spec']
cloudflare_spec = router_spec.get('cloudflare', {})
cloudflare_email = cloudflare_spec.get('email')
cloudflare_api_key = cloudflare_spec.get('api-key')
default_root_domain = router_spec.get('default-root-domain')
dns_provider = router_spec.get('dns-provider')
from ckan_cloud_operator.providers.cluster import manager as cluster_manager
default_dns_provider = 'route53' if cluster_manager.get_provider_id(
) == 'aws' else 'cloudflare'
logs.info(dns_provider=dns_provider,
default_dns_provider=default_dns_provider)
if not dns_provider:
dns_provider = default_dns_provider
router_spec['dns-provider'] = dns_provider
assert all([default_root_domain,
dns_provider]), f'invalid traefik router spec: {router_spec}'
if dns_provider == 'cloudflare':
assert cloudflare_email and cloudflare_api_key, 'invalid traefik router spec for cloudflare dns provider'
# cloudflare credentials are stored in a secret, not in the spec
if 'cloudflare' in router_spec:
del router_spec['cloudflare']
kubectl.apply(router)
annotations = CkanRoutersAnnotations(router_name, router)
if dns_provider == 'cloudflare':
annotations.update_flag(
'letsencryptCloudflareEnabled',
lambda: annotations.set_secrets(
{
'LETSENCRYPT_CLOUDFLARE_EMAIL': cloudflare_email,
'LETSENCRYPT_CLOUDFLARE_API_KEY': cloudflare_api_key
}),
force_update=True)
return router
def zk_put_configs(configs_dir):
def retry_if_fails(command, max_retries=15):
if max_retries < 0:
return
try:
kubectl.check_output(command)
except:
time.sleep(5)
retry_if_fails(command, max_retries=max_retries - 1)
pod_name = kubectl.get('pods',
'-l',
'app=provider-solr-solrcloud-zk',
required=True)['items'][0]['metadata']['name']
logs.info(f'using pod {pod_name}')
for input_filename in glob.glob(f'{configs_dir}/**/*', recursive=True):
if not os.path.isfile(input_filename): continue
output_filename = '/configs' + input_filename.replace(configs_dir, '')
logs.info(f'{input_filename} --> {output_filename}')
output_filepath = ''
for output_filepart in output_filename.split('/')[:-1]:
output_filepart = output_filepart.strip()
if not output_filepart:
continue
output_filepath += f'/{output_filepart}'
logs.info(f'create {output_filepath} null')
retry_if_fails(
f'exec {pod_name} zkCli.sh create {output_filepath} null')
logs.info(f'copy {output_filename}')
retry_if_fails(f'cp {input_filename} {pod_name}:/tmp/zk_input')
logs.info(f'create {output_filename}')
retry_if_fails(
f"exec {pod_name} -- /bin/bash -c '/usr/bin/zkCli.sh create {output_filename} \"$(cat /tmp/zk_input)\"'"
)
def set_ssh_key(server):
try:
authorized_keys = subprocess.check_output([
'sshpass', '-f', server['passwordfile'], 'ssh',
f'root@{server["public_ip"]}', '-o', 'StrictHostKeyChecking=no',
'--', 'cat', '.ssh/authorized_keys'
]).decode()
except subprocess.CalledProcessError:
authorized_keys = None
if not authorized_keys or server['name'] not in authorized_keys:
logs.info(f"Adding authorized key for server",
server_name=server['name'])
with open(server['keyfile'] + '.pub') as f:
public_key = f.read()
subprocess.check_call([
'sshpass', '-f', server['passwordfile'], 'ssh',
f'root@{server["public_ip"]}', '-o', 'StrictHostKeyChecking=no',
'--', 'mkdir', '-p', ".ssh"
])
subprocess.check_call([
'sshpass', '-f', server['passwordfile'], 'ssh',
f'root@{server["public_ip"]}', '-o', 'StrictHostKeyChecking=no',
f'echo "{public_key}" > .ssh/authorized_keys'
])
helpers.ssh(server, ["echo successfull ssh connection using key"])
def initialize(interactive=False,
provider_id=None,
storage_suffix=None,
use_existing_disk_name=None,
dry_run=False):
if not provider_id and interactive:
config_manager.interactive_set({'use-cloud-native-storage': True},
secret_name=CONFIG_NAME,
interactive=interactive)
if config_manager.get('use-cloud-native-storage',
secret_name=CONFIG_NAME):
provider_id = get_provider_id()
logs.info(f'Storage init: chosen provider_id: {provider_id}')
provider = get_provider(default=minio_provider_id,
provider_id=provider_id).initialize(
interactive=interactive,
storage_suffix=storage_suffix,
use_existing_disk_name=use_existing_disk_name,
dry_run=dry_run)
if provider:
provider.initialize(interactive=interactive,
storage_suffix=storage_suffix,
use_existing_disk_name=use_existing_disk_name,
dry_run=dry_run)
def initialize(interactive=False):
_set_provider()
if interactive:
print('\nEnter the Resource Group name\n')
_config_interactive_set({'azure-rg': None}, is_secret=False)
print(
'\nEnter the location of the Kubernets cluster is hosted on [westus2]\n'
)
_config_interactive_set({'azure-default-location': None},
is_secret=False)
print('\nEnter the name of your cluster\n')
_config_interactive_set({'azure-cluster-name': None}, is_secret=False)
print('\nEnter the Subscribtion ID\n')
_config_interactive_set({'azure-subscribtion-id': None},
is_secret=True)
print('\nEnter the Tenant ID\n')
_config_interactive_set({'azure-tenant-id': None}, is_secret=True)
print('\nEnter the Service Principal ID\n')
_config_interactive_set({'azure-client-id': None}, is_secret=True)
print('\nEnter the Service Principal Secret\n')
_config_interactive_set({'azure-client-secret': None}, is_secret=True)
_create_storage_classes()
else:
logs.info(
'Skipping initial cluster set up as `--interactive` flag was not set'
)
_create_storage_classes()
def create_volume(disk_size_gb, labels, use_existing_disk_name=None, zone=0):
rg = _config_get('azure-rg')
location = zone or _config_get('azure-default-location')
disk_id = use_existing_disk_name or 'cc' + _generate_password(12)
if use_existing_disk_name:
logs.info(f'using existing persistent disk {disk_id}')
else:
logs.info(
f'creating persistent disk {disk_id} with size {disk_size_gb}GB')
_, zone = get_project_zone()
labels = ','.join([
'{}={}'.format(k.replace('/', '_'), v.replace('/', '_'))
for k, v in labels.items()
])
kubectl.apply({
"kind": "PersistentVolumeClaim",
"apiVersion": "v1",
"metadata": {
"name": disk_id,
"namespace": "ckan-cloud"
},
"spec": {
"accessModes": ["ReadWriteOnce"],
"resources": {
"requests": {
"storage": f'{disk_size_gb}G'
}
},
"storageClassName": "cca-ckan"
}
})
return {'persistentVolumeClaim': {'claimName': disk_id}}
def create(instance_type, instance_id=None, instance_name=None, values=None, values_filename=None, exists_ok=False,
dry_run=False, update_=False, wait_ready=False, skip_deployment=False, skip_route=False, force=False):
if not instance_id:
if instance_name:
instance_id = '{}-{}'.format(instance_name, _generate_password(6))
else:
instance_id = _generate_password(12)
if values_filename:
assert values is None
if values_filename != '-':
with open(values_filename) as f:
values = yaml.load(f.read())
else:
values = yaml.load(sys.stdin.read())
if not exists_ok and crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id, required=False):
raise Exception('instance already exists')
values_id = values.get('id')
if values_id and values_id != instance_id:
logs.warning(f'changing instance id in spec from {values_id} to the instance id {instance_id}')
values['id'] = instance_id
logs.info('Creating instance', instance_id=instance_id)
kubectl.apply(crds_manager.get_resource(
INSTANCE_CRD_SINGULAR, instance_id,
extra_label_suffixes={'instance-type': instance_type},
spec=values
), dry_run=dry_run)
if instance_name:
set_name(instance_id, instance_name, dry_run=dry_run)
if update_:
update(instance_id, wait_ready=wait_ready, skip_deployment=skip_deployment, skip_route=skip_route, force=force,
dry_run=dry_run)
return instance_id
def _add_route(config, domains, route, enable_ssl_redirect):
route_name = routes_manager.get_name(route)
logs.info(f'adding route to nginx config: {route_name}')
logs.debug_verbose(config=config,
domains=domains,
route=route,
enable_ssl_redirect=enable_ssl_redirect)
backend_url = routes_manager.get_backend_url(route)
frontend_hostname = routes_manager.get_frontend_hostname(route)
print(f'F/B = {frontend_hostname} {backend_url}')
root_domain, sub_domain = routes_manager.get_domain_parts(route)
domains.setdefault(root_domain, []).append(sub_domain)
# if route['spec'].get('extra-no-dns-subdomains'):
# extra_hostnames = ',' + ','.join([f'{s}.{root_domain}' for s in route['spec']['extra-no-dns-subdomains']])
# else:
extra_hostnames = ''
logs.debug_verbose(route_name=route_name,
backend_url=backend_url,
frontend_hostname=frontend_hostname,
root_domain=root_domain,
sub_domain=sub_domain,
domains=domains,
extra_hostnames=extra_hostnames)
if backend_url:
raise NotImplementedError()
def update(instance_id_or_name, override_spec_json, persist_overrides,
wait_ready, skip_deployment, skip_route, force, override_spec_file,
ckan_cloud_docker_latest_tag):
"""Update an instance to the latest resource spec, optionally applying the given json override to the resource spec
Examples:
ckan-cloud-operator ckan instance update <INSTANCE_ID_OR_NAME> '{"siteUrl": "http://localhost:5000"}' --wait-ready
ckan-cloud-operator ckan instance update <INSTANCE_ID_OR_NAME> '{"replicas": 3}' --persist-overrides
"""
override_spec = json.loads(
override_spec_json) if override_spec_json else None
if override_spec:
assert not override_spec_file, "Can't specify both OVERRIDE_SPEC_JSON and override_spec_file"
elif override_spec_file:
with open(override_spec_file) as f:
override_spec = yaml.safe_load(f)
if ckan_cloud_docker_latest_tag:
assert override_spec.get(
"ckanImage"
), "ckanImage attribute is required in spec for --ckan-cloud-docker-latest-tag arg"
if override_spec["ckanImage"].endswith(":latest"):
override_spec["ckanImage"] = override_spec["ckanImage"].replace(
":latest", ":%s" % ckan_cloud_docker_latest_tag)
logs.info("Replacing ckanImage to latest tag: %s" %
override_spec["ckanImage"])
manager.update(instance_id_or_name,
override_spec=override_spec,
persist_overrides=persist_overrides,
wait_ready=wait_ready,
skip_deployment=skip_deployment,
skip_route=skip_route,
force=force)
logs.exit_great_success()
def get_provider(submodule,
required=True,
supported_provider_ids=None,
default=None,
verbose=False,
provider_id=None):
if default: required = False
if not provider_id:
provider_id = get_provider_id(submodule, required=required)
if not provider_id:
if default:
provider_id = default
else:
return None
if verbose:
logs.info(f'submodule {submodule} using provider_id {provider_id}')
if not supported_provider_ids or provider_id in supported_provider_ids:
provider_manager = _get_submodule_ids_provider_or_provider_ids(
submodule, provider_id)
else:
provider_manager = None
if provider_manager:
return provider_manager
else:
msg = f' (supported provider ids: {supported_provider_ids})' if supported_provider_ids else ''
logs.critical(
f'Invalid submodule / provider: {submodule} / {provider_id}{msg}')
raise Exception('failed to get provider')
def create_all_backups(db_prefix=None, dry_run=False):
logs.info('Fetching all database names')
db_names = get_all_db_names(db_prefix=db_prefix)
logs.info('{} DBs'.format(len(db_names)))
for db in db_names:
if db not in ['postgres'] and not db.startswith('template'):
assert create_backup(db, db_prefix=db_prefix, dry_run=dry_run)
def create_backup(database, connection_string=None, db_prefix=None):
filename = f'{database}_' + datetime.datetime.now().strftime(
'%Y%m%d%H%M') + '.gz'
gs_url = os.path.join(
_credentials_get(db_prefix, key='backups-gs-base-url', required=True),
datetime.datetime.now().strftime('%Y/%m/%d/%H'), filename)
if not connection_string:
from ckan_cloud_operator.providers.db import manager as db_manager
connection_string = db_manager.get_external_admin_connection_string(
db_name=database)
logs.info(f'Dumping DB: {filename}')
subprocess.check_call([
"bash",
"-o",
"pipefail",
"-c",
f"pg_dump -d {connection_string} --format=plain --no-owner --no-acl --schema=public | "
f"sed -E 's/(DROP|CREATE|COMMENT ON) EXTENSION/-- \\1 EXTENSION/g' | "
f"gzip -c > {filename}",
])
subprocess.check_call(f'ls -lah {filename}', shell=True)
logs.info(f'Copying to: {gs_url}')
gcloud_driver.check_call(*_gcloud().get_project_zone(),
f'cp -m ./{filename} {gs_url} && rm {filename}',
gsutil=True)
def _create_down_time_approval_code(instance_id, old_db_prefix, new_db_prefix,
db_name, datastore_name):
jenkins_user_token = ckan_manager.get_jenkins_token(
'ckan-cloud-operator-jenkins-creds')
backups_console_log = jenkins_driver.curl(
*jenkins_user_token,
'https://cc-p-jenkins.ckan.io/job/check%20individual%20DB%20backups/lastBuild/console',
raw=True)
backups_console_log = backups_console_log.split(
'Finished: SUCCESS')[0].split('Updated property [compute/zone].')[1]
backups = yaml.safe_load(backups_console_log)
backups = {
k: v
for k, v in backups.items()
if (k == db_name or k == datastore_name or k == f'{db_name}-total-size'
or k == f'{datastore_name}-total-size')
}
logs.info('last instance backups')
logs.print_yaml_dump(backups)
return scripts_helpers.create_file_based_approval_code({
'instance-id':
instance_id,
'old-db-prefix':
old_db_prefix,
'new-db-prefix':
new_db_prefix,
'db-name':
db_name,
'datastore-name':
datastore_name
})
def _init_ckan_infra_secret(instance_id, dry_run=False):
logs.debug('Initializing ckan infra secret', instance_id=instance_id)
ckan_infra = config_manager.get(secret_name='ckan-infra',
namespace=instance_id,
required=False)
if ckan_infra:
logs.info('ckan-infra secret already exists')
else:
admin_user, admin_password, db_name = db_manager.get_admin_db_credentials(
)
db_host, db_port = db_manager.get_internal_unproxied_db_host_port()
assert int(db_port) == 5432
logs.debug('Creating ckan-infra secret',
admin_user=admin_user,
admin_password=admin_password,
db_name=db_name,
db_host=db_host,
db_port=db_port)
config_manager.set(values={
'POSTGRES_HOST': db_host,
'POSTGRES_PASSWORD': admin_password,
'POSTGRES_USER': admin_user
},
secret_name='ckan-infra',
namespace=instance_id,
dry_run=dry_run)
def _log_container_error(text, pod_name, container_name=None):
heading = f"****** {text} ******"
logs.info()
logs.info(heading)
logs.info(f'Pod: {pod_name} - Container: {container_name}')
logs.info("-" * len(heading))
logs.info()
def _pre_update_hook_admin_user(instance,
sub_domain,
root_domain,
instance_id,
res,
dry_run=False):
ckan_admin_email = instance['spec'].get('ckanAdminEmail')
if not ckan_admin_email:
ckan_admin_email = f'admin@{sub_domain}.{root_domain}'
ckan_admin_password = config_manager.get(key='CKAN_ADMIN_PASSWORD',
secret_name='ckan-admin-password',
namespace=instance_id,
required=False)
if ckan_admin_password:
logs.info('using existing ckan admin user')
res['ckan-admin-password'] = ckan_admin_password
else:
logs.info('Will create new ckan admin user',
ckan_admin_email=ckan_admin_email)
res['ckan-admin-email'] = ckan_admin_email
res['ckan-admin-password'] = ckan_admin_password = binascii.hexlify(
os.urandom(8)).decode()
config_manager.set(key='CKAN_ADMIN_PASSWORD',
value=ckan_admin_password,
secret_name='ckan-admin-password',
namespace=instance_id,
dry_run=dry_run)
def initialize(log_kwargs=None, interactive=False):
log_kwargs = log_kwargs or {}
logs.info(f'Installing crds', **log_kwargs)
crds_manager.install_crd(CRD_SINGULAR,
CRD_PLURAL,
CRD_KIND,
hash_names=True)
ckan_infra = CkanInfra(required=False)
if interactive:
providers_manager.config_interactive_set(
PROVIDER_SUBMODULE,
default_values={
'gcloud-storage-import-bucket':
ckan_infra.GCLOUD_SQL_DEIS_IMPORT_BUCKET
},
suffix='deis-migration')
else:
if not providers_manager.config_get(PROVIDER_SUBMODULE,
key='gcloud-storage-import-bucket',
suffix='deis-migration',
required=False):
providers_manager.config_set(
PROVIDER_SUBMODULE,
values={
'gcloud-storage-import-bucket':
ckan_infra.GCLOUD_SQL_DEIS_IMPORT_BUCKET
},
suffix='deis-migration')
def get(routes,
letsencrypt_cloudflare_email,
enable_access_log=False,
wildcard_ssl_domain=None,
external_domains=False,
dns_provider=None,
force=False):
if not dns_provider:
dns_provider = 'cloudflare'
logs.info('Generating traefik configuration',
routes_len=len(routes) if routes else 0,
letsencrypt_cloudflare_email=letsencrypt_cloudflare_email,
enable_access_log=enable_access_log,
wildcard_ssl_domain=wildcard_ssl_domain,
external_domains=external_domains)
config = _get_base_config(**({
'accessLog': {
"format": "json",
"fields": {
'defaultMode': "keep"
}
},
} if enable_access_log else {}))
domains = {}
if dns_provider == 'cloudflare' and letsencrypt_cloudflare_email:
enable_ssl_redirect = True
elif dns_provider == 'route53':
enable_ssl_redirect = True
else:
enable_ssl_redirect = False
logs.info(enable_ssl_redirect=enable_ssl_redirect)
logs.info('Adding routes')
i = 0
errors = 0
for route in routes:
try:
_add_route(config, domains, route, enable_ssl_redirect)
i += 1
except Exception as e:
if force:
logs.error(traceback.format_exc())
logs.error(str(e))
errors += 1
else:
raise
logs.info(f'Added {i} routes')
if errors > 0:
logs.warning(f'Encountered {errors} errors')
if ((dns_provider == 'cloudflare' and letsencrypt_cloudflare_email)
or (dns_provider == 'route53')):
_add_letsencrypt(dns_provider,
config,
letsencrypt_cloudflare_email,
domains,
wildcard_ssl_domain=wildcard_ssl_domain,
external_domains=external_domains)
else:
logs.info('No valid dns_provider, will not setup SSL',
dns_provider=dns_provider)
return config
def _get_or_create_migration_db_passwords(migration_name,
create_if_not_exists=True,
skip_keys=None):
passwords = []
errors = []
for password_config_key in [
'database-password', 'datastore-password',
'datastore-readonly-password'
]:
if skip_keys is not None and password_config_key not in skip_keys:
passwords.append(None)
continue
password = crds_manager.config_get(CRD_SINGULAR,
migration_name,
key=password_config_key,
is_secret=True,
required=False)
if create_if_not_exists:
if password:
logs.info(f'Password already exists: {password_config_key}')
errors.append(f'password-exists: {password_config_key}')
else:
logs.info(f'Generating new password for {password_config_key}')
password = _generate_password()
crds_manager.config_set(CRD_SINGULAR,
migration_name,
key=password_config_key,
value=password,
is_secret=True)
passwords.append(password)
return [errors] + passwords
def solr_curl(path, required=False, debug=False, max_retries=15):
deployment_name = _get_resource_name(_get_sc_suffixes()[0])
if debug:
kubectl.check_call(
f'exec deployment-pod::{deployment_name} -- curl \'localhost:8983/solr{path}\'',
use_first_pod=True)
else:
exitcode, output = kubectl.getstatusoutput(
f'exec deployment-pod::{deployment_name} -- curl -s -f \'localhost:8983/solr{path}\'',
use_first_pod=True)
if exitcode == 0:
return output
elif required:
if max_retries > 0:
logs.info(
f'Failed to run solr curl: localhost:8983/solr{path} - retring in 30 seconds'
)
time.sleep(30)
solr_curl(path,
required=required,
debug=debug,
max_retries=max_retries - 1)
logs.critical(output)
raise Exception(
f'Failed to run solr curl: localhost:8983/solr{path}')
else:
logs.warning(output)
return False
def _update_db_proxy(db_name, datastore_name, datastore_ro_name, db_password,
datastore_password, datastore_ro_password):
logs.info('Updating db proxy')
db_proxy_manager.update(wait_updated=False)
ok = False
for i in range(5):
try:
for user, password, db in [
(db_name, db_password, db_name),
(datastore_name, datastore_password, datastore_name),
(datastore_ro_name, datastore_ro_password, datastore_name)
]:
if user:
db_manager.check_connection_string(
db_manager.get_external_connection_string(
user, password, db))
ok = True
break
except Exception as e:
logs.warning(str(e))
logs.info(f'Waiting for connection to db proxy...')
# 40 seconds on first iteration - to ensure secret is updated in pgbouncer volume
time.sleep(40 if i == 0 else 1)
db_proxy_manager.reload()
time.sleep(10 if i == 2 else 5)
assert ok, 'failed to get connection to db proxy'
yield {
'step':
'update-db-proxy',
'msg':
f'Updated DB Proxy with the new dbs and roles: {db_name}, {datastore_name}, {datastore_ro_name}'
}
def create_volume(disk_size_gb, labels, use_existing_disk_name=None, zone=None):
disk_id = use_existing_disk_name or 'cc' + _generate_password(12)
if use_existing_disk_name:
logs.info(f'using existing persistent disk {disk_id}')
else:
logs.info(f'creating persistent disk {disk_id} with size {disk_size_gb}')
_, zone = get_project_zone()
labels = ','.join([
'{}={}'.format(k.replace('/', '_'), v.replace('/', '_')) for k, v in labels.items()
])
gcloud_driver.check_call(*get_project_zone(), f'compute disks create {disk_id} --size={disk_size_gb}GB --zone={zone} --labels={labels}')
kubectl.apply({
'apiVersion': 'v1', 'kind': 'PersistentVolume',
'metadata': {'name': disk_id, 'namespace': 'ckan-cloud'},
'spec': {
'storageClassName': '',
'capacity': {'storage': f'{disk_size_gb}G'},
'accessModes': ['ReadWriteOnce'],
'gcePersistentDisk': {'pdName': disk_id}
}
})
kubectl.apply({
'apiVersion': 'v1', 'kind': 'PersistentVolumeClaim',
'metadata': {'name': disk_id, 'namespace': 'ckan-cloud'},
'spec': {
'storageClassName': '',
'volumeName': disk_id,
'accessModes': ['ReadWriteOnce'],
'resources': {'requests': {'storage': f'{disk_size_gb}G'}}
}
})
return {'persistentVolumeClaim': {'claimName': disk_id}}
def create(instance_type, instance_id=None, instance_name=None, values=None, values_filename=None, exists_ok=False, dry_run=False):
if not instance_id:
if instance_name:
instance_id = '{}-{}'.format(instance_name, _generate_password(6))
else:
instance_id = _generate_password(12)
if values_filename:
assert values is None
with open(values_filename) as f:
values = yaml.load(f.read())
if not exists_ok and crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id, required=False):
raise Exception('instance already exists')
values_id = values.get('id')
if values_id:
assert values_id == instance_id, f'instance spec has conflicting instance_id ({values_id} != {instance_id})'
values['id'] = instance_id
logs.info('Creating instance', instance_id=instance_id)
kubectl.apply(crds_manager.get_resource(
INSTANCE_CRD_SINGULAR, instance_id,
extra_label_suffixes={'instance-type': instance_type},
spec=values
), dry_run=dry_run)
if instance_name:
set_name(instance_id, instance_name, dry_run=dry_run)
return instance_id