def gitlab_token(token_name=None):
if token_name:
return config_manager.get(token_name,
secret_name='ckan-gitlab-tokens',
required=True)
else:
return config_manager.get('gitlab-token',
secret_name='ckan-migration-secrets',
required=True)
def _apply_zoonavigator_deployment(dry_run=False):
cpu_req = config_manager.get('zn-cpu', secret_name='solr-config')
mem_req = config_manager.get('zn-mem', secret_name='solr-config')
cpu_lim = config_manager.get('zn-cpu-limit', secret_name='solr-config')
mem_lim = config_manager.get('zn-mem-limit', secret_name='solr-config')
suffix = 'zoonavigator'
deployment_name = _get_resource_name(suffix)
kubectl.apply(kubectl.get_deployment(
deployment_name,
_get_resource_labels(for_deployment=True, suffix=suffix), {
'replicas': 1,
'revisionHistoryLimit': 2,
'selector': {
'matchLabels':
_get_resource_labels(for_deployment=True, suffix=suffix),
},
'template': {
'metadata': {
'labels':
_get_resource_labels(for_deployment=True, suffix=suffix),
'annotations':
_get_resource_annotations()
},
'spec': {
'containers': [{
'env': [{
'name': 'API_HOST',
'value': 'localhost'
}, {
'name': 'API_PORT',
'value': '9000'
}, {
'name': 'WEB_HTTP_PORT',
'value': '8000'
}],
'image':
'elkozmon/zoonavigator:0.7.1',
'name':
'zoonavigator',
'ports': [{
'containerPort': 8000,
'name': '8000tcp02',
'protocol': 'TCP'
}],
'resources': {}
}]
}
}
}),
dry_run=dry_run)
return deployment_name
def _create_private_container_registry_secret(instance_id):
if config_manager.get('private-registry',
secret_name='ckan-docker-registry') == 'y':
docker_server, docker_username, docker_password, docker_email = ckan_manager.get_docker_credentials(
)
image_pull_secret_name = config_manager.get(
'docker-image-pull-secret-name',
secret_name='ckan-docker-registry')
subprocess.call(
f'kubectl -n {instance_id} create secret docker-registry {image_pull_secret_name} '
f'--docker-password={docker_password} '
f'--docker-server={docker_server} '
f'--docker-username={docker_username} '
f'--docker-email={docker_email}',
shell=True)
def _pre_update_hook_admin_user(instance,
sub_domain,
root_domain,
instance_id,
res,
dry_run=False):
ckan_admin_email = instance['spec'].get('ckanAdminEmail')
if not ckan_admin_email:
ckan_admin_email = f'admin@{sub_domain}.{root_domain}'
ckan_admin_password = config_manager.get(key='CKAN_ADMIN_PASSWORD',
secret_name='ckan-admin-password',
namespace=instance_id,
required=False)
if ckan_admin_password:
logs.info('using existing ckan admin user')
res['ckan-admin-password'] = ckan_admin_password
else:
logs.info('Will create new ckan admin user',
ckan_admin_email=ckan_admin_email)
res['ckan-admin-email'] = ckan_admin_email
res['ckan-admin-password'] = ckan_admin_password = binascii.hexlify(
os.urandom(8)).decode()
config_manager.set(key='CKAN_ADMIN_PASSWORD',
value=ckan_admin_password,
secret_name='ckan-admin-password',
namespace=instance_id,
dry_run=dry_run)
def get_provider_id(submodule, required=True, default=None):
if default: required = False
return config_manager.get(
get_operator_configmap_key(submodule, suffix='main-provider-id'),
required=required,
default=default
)
def _init_ckan_infra_secret(instance_id, dry_run=False):
logs.debug('Initializing ckan infra secret', instance_id=instance_id)
ckan_infra = config_manager.get(secret_name='ckan-infra',
namespace=instance_id,
required=False)
if ckan_infra:
logs.info('ckan-infra secret already exists')
else:
admin_user, admin_password, db_name = db_manager.get_admin_db_credentials(
)
db_host, db_port = db_manager.get_internal_unproxied_db_host_port()
assert int(db_port) == 5432
logs.debug('Creating ckan-infra secret',
admin_user=admin_user,
admin_password=admin_password,
db_name=db_name,
db_host=db_host,
db_port=db_port)
config_manager.set(values={
'POSTGRES_HOST': db_host,
'POSTGRES_PASSWORD': admin_password,
'POSTGRES_USER': admin_user
},
secret_name='ckan-infra',
namespace=instance_id,
dry_run=dry_run)
def list_instances(full=False, quick=False, withCredentials=False, name=None):
if quick:
for instance in get_all_instance_id_names():
if name is not None and instance['name'] != name: continue
yield {**instance, 'ready': None}
else:
for instance_data in get_all_instances():
metadata_keys = ('name',)
spec_keys = ('id', 'siteUrl', 'siteTitle', 'domain', 'registerSubdomain')
try:
spec = instance_data['spec']
for k in spec_keys:
instance_data[k] = spec.get(k)
metadata = instance_data['metadata']
for k in metadata_keys:
instance_data[k] = metadata.get(k)
instance_type = instance_data['metadata']['labels'].get('{}/instance-type'.format(labels_manager.get_label_prefix()))
deployment = deployment_manager.get(instance_data['id'], instance_type, instance_data)
instance_data['ready'] = deployment.get('ready')
except Exception as e:
pass
if not full:
instance_data = dict(
(k, v)
for k, v in instance_data.items()
if k in ('ready', *spec_keys, *metadata_keys)
)
if withCredentials:
instance_data['admin_password'] = config_manager.get(
'CKAN_ADMIN_PASSWORD',
secret_name='ckan-admin-password',
namespace=instance_data['id']
)
yield instance_data
def ssh_management_machine(*args, check_output=False, scp_to_remote_file=None):
machine_secrets = get_management_machine_secrets()
id_rsa = machine_secrets['id_rsa']
server_ip = machine_secrets['server_public_ip']
with NamedTemporaryFile('wb') as f:
f.write(id_rsa.encode('ascii'))
f.flush()
ssh_known_hosts = config_manager.get('ssh_known_hosts', secret_name='cco-kamatera-management-server', namespace='ckan-cloud-operator', required=False, default=None)
if ssh_known_hosts:
os.makedirs('/root/.ssh', exist_ok=True)
with open('/root/.ssh/known_hosts', 'w') as known_hosts_file:
known_hosts_file.write(ssh_known_hosts)
if scp_to_remote_file:
with NamedTemporaryFile('w') as tempfile:
tempfile.write("\n".join(args))
tempfile.flush()
cmd = ['scp', '-i', f.name, tempfile.name, 'root@' + server_ip + ':' + scp_to_remote_file]
subprocess.check_call(cmd)
res = 0
else:
cmd = ['ssh', '-i', f.name, 'root@' + server_ip, *args]
if check_output:
res = subprocess.check_output(cmd)
else:
res = subprocess.call(cmd)
if os.path.exists('/root/.ssh/known_hosts'):
with open('/root/.ssh/known_hosts') as f:
if f.read().strip() != ssh_known_hosts.strip():
config_manager.set('ssh_known_hosts', f.read(), secret_name='cco-kamatera-management-server', namespace='ckan-cloud-operator')
return res
def initialize():
install_crds()
datapusher_envvars = {'PORT': '8000'}
router_name = 'datapushers'
if not routers_manager.get(router_name, required=False):
routers_manager.create(router_name,
routers_manager.get_traefik_router_spec())
if config_manager.get('enable-deis-ckan',
configmap_name='global-ckan-config') == 'y':
create(
'datapusher-1',
'registry.gitlab.com/viderum/docker-datapusher:cloud-datapusher-1-v9',
datapusher_envvars, router_name)
create(
'datapusher-de',
'registry.gitlab.com/viderum/docker-datapusher:cloud-de-git-943fc3e0',
datapusher_envvars, router_name)
create(
'datapusher-giga',
'registry.gitlab.com/viderum/docker-datapusher:cloud-giga-git-2b05b22d',
datapusher_envvars, router_name)
create(
'datapusher-increased-max-length',
'registry.gitlab.com/viderum/docker-datapusher:cloud-increased-max-length-git-84e86116',
datapusher_envvars, router_name)
update('datapusher-1')
update('datapusher-de')
update('datapusher-giga')
update('datapusher-increased-max-length')
routers_manager.update(router_name)
def initialize(interactive=False,
provider_id=None,
storage_suffix=None,
use_existing_disk_name=None,
dry_run=False):
if not provider_id and interactive:
config_manager.interactive_set({'use-cloud-native-storage': True},
secret_name=CONFIG_NAME,
interactive=interactive)
if config_manager.get('use-cloud-native-storage',
secret_name=CONFIG_NAME):
provider_id = get_provider_id()
logs.info(f'Storage init: chosen provider_id: {provider_id}')
provider = get_provider(default=minio_provider_id,
provider_id=provider_id).initialize(
interactive=interactive,
storage_suffix=storage_suffix,
use_existing_disk_name=use_existing_disk_name,
dry_run=dry_run)
if provider:
provider.initialize(interactive=interactive,
storage_suffix=storage_suffix,
use_existing_disk_name=use_existing_disk_name,
dry_run=dry_run)
def gsutil_publish(filename, gsurl, duration='7d'):
if gsurl:
if filename:
subprocess.check_call(f'ls -lah {filename}', shell=True)
gcloud_driver.check_call(
*cluster_manager.get_provider().get_project_zone(),
f'cp ./{filename} {gsurl}',
gsutil=True)
with tempfile.NamedTemporaryFile('w') as f:
f.write(
config_manager.get(
key='service-account-json',
secret_name='ckan-cloud-provider-cluster-gcloud'))
f.flush()
output = gcloud_driver.check_output(
*cluster_manager.get_provider().get_project_zone(),
f'signurl -d {duration} {f.name} {gsurl}',
gsutil=True)
signed_gsurls = [
line for line in [
line.strip().split('\t')[-1].strip()
for line in output.decode().splitlines()
] if len(line) > 20
]
assert len(signed_gsurls) == 1
return signed_gsurls[0]
else:
return None
def _get_expected_operator_image_tag():
from ckan_cloud_operator.config import manager as config_manager
expected_image = config_manager.get('ckan-cloud-operator-image')
if not expected_image:
return
assert '@' not in expected_image and ':' in expected_image, f'invalid expected image: {expected_image}'
_, expected_image_tag = expected_image.split(':')
return expected_image_tag
def get_load_balancer_ip(router_name, failfast=False):
load_balancer_ip = config_manager.get(
'load-balancer-ip',
configmap_name=f'nginx-router-{router_name}-deployment',
required=False,
default=None)
if not load_balancer_ip:
load_balancer_ip = config_manager.get(
'server_public_ip',
secret_name='cco-kamatera-management-server',
namespace='ckan-cloud-operator',
required=True)
config_manager.set(
'load-balancer-ip',
load_balancer_ip,
configmap_name=f'nginx-router-{router_name}-deployment')
return load_balancer_ip
def initialize(interactive=False):
config_manager.interactive_set(
{
'env-id':
None,
'default-root-domain':
None,
'dns-provider':
'route53' if cluster_manager.get_provider_id() == 'aws' else None
},
configmap_name='routers-config',
interactive=interactive)
dns_provider = get_dns_provider()
if dns_provider.lower() == 'none':
if cluster_manager.get_provider_id() != 'minikube':
return
logs.info(dns_provider=dns_provider)
if dns_provider == 'cloudflare':
config_manager.interactive_set(
{
'cloudflare-email': None,
'cloudflare-api-key': None
},
configmap_name='routers-config',
interactive=interactive)
routers_manager.install_crds()
infra_router_name = routers_manager.get_default_infra_router_name()
default_root_domain = config_manager.get('default-root-domain',
configmap_name='routers-config',
required=True)
logs.info('Creating infra router',
infra_router_name=infra_router_name,
default_root_domain=default_root_domain)
routers_manager.create(
infra_router_name,
routers_manager.get_traefik_router_spec(
default_root_domain,
config_manager.get('cloudflare-email',
configmap_name='routers-config',
required=False,
default=None),
config_manager.get('cloudflare-api-key',
configmap_name='routers-config',
required=False,
default=None),
dns_provider=dns_provider))
def get_docker_credentials():
return [
config_manager.get(key,
secret_name='ckan-docker-registry',
required=True) for key in [
'docker-server', 'docker-username',
'docker-password', 'docker-email'
]
]
def create_bucket(instance_id, region=None, exists_ok=False, dry_run=False):
if not region:
region = cluster_config_get('azure-default-location')
assert region, 'No default region set for the cluster'
bucket_exists = instance_id in list_azure_buckets()
if not exists_ok and bucket_exists:
raise Exception('Bucket for this instance already exists')
if not dry_run and not bucket_exists:
cred_options = _get_cred_options()
az_check_output(f'storage container create -n {instance_id} {cred_options}')
return {
'BUCKET_NAME': instance_id,
'STORAGE_ACCOUNT_NAME': config_manager.get('storage-account-name', secret_name=CONFIG_NAME),
'STORAGE_ACCOUNT_KEY': config_manager.get('storage-account-key', secret_name=CONFIG_NAME)
}
def initialize(interactive=False):
ckan_db_migration_manager.initialize(interactive=interactive)
if config_manager.get('enable-deis-ckan',
configmap_name='global-ckan-config') == 'y':
ckan_infra = CkanInfra(required=False)
config_manager.interactive_set(
{
'deis-kubeconfig': ckan_infra.DEIS_KUBECONFIG,
},
from_file=True,
secret_name='ckan-migration-secrets',
interactive=interactive)
config_manager.interactive_set(
{
'gitlab-token': ckan_infra.GITLAB_TOKEN_PASSWORD,
},
secret_name='ckan-migration-secrets',
interactive=interactive)
config_manager.interactive_set(
{
'docker-server': ckan_infra.DOCKER_REGISTRY_SERVER,
'docker-username': ckan_infra.DOCKER_REGISTRY_USERNAME,
'docker-password': ckan_infra.DOCKER_REGISTRY_PASSWORD,
'docker-email': ckan_infra.DOCKER_REGISTRY_EMAIL,
},
secret_name='ckan-docker-registry',
interactive=interactive)
crds_manager.install_crd(INSTANCE_CRD_SINGULAR, INSTANCE_CRD_PLURAL_SUFFIX,
INSTANCE_CRD_KIND_SUFFIX)
crds_manager.install_crd(INSTANCE_NAME_CRD_SINGULAR,
INSTANCE_NAME_CRD_PLURAL_SUFFIX,
INSTANCE_NAME_CRD_KIND_SUFFIX)
from ckan_cloud_operator.datapushers import initialize as datapusher_initialize
datapusher_initialize()
from ckan_cloud_operator.routers import manager as routers_manager
router_name = get_default_instances_router_name()
wildcard_ssl_domain = routers_manager.get_default_root_domain()
print(f'wildcard_ssl_domain={wildcard_ssl_domain}')
allow_wildcard_ssl = routers_manager.get_env_id() == 'p'
router = routers_manager.get(router_name, required=False)
if router:
assert ((allow_wildcard_ssl and router.get(
'spec', {}).get('wildcard-ssl-domain') == wildcard_ssl_domain)
or (not allow_wildcard_ssl and not router.get('spec', {}))
), f'invalid router wildcard ssl config: {router}'
else:
routers_manager.create(
router_name,
routers_manager.get_traefik_router_spec(
wildcard_ssl_domain=wildcard_ssl_domain))
from .storage.manager import initialize as ckan_storage_initialize
ckan_storage_initialize(interactive=interactive)
def __init__(self, instance):
config = config_manager.get(secret_name='uptime-statuscake-api')
self.statuscake_api_user = config['user']
self.statuscake_api_key = config['key']
self.statuscake_group = config['group']
self.instance_id = instance.id
self.instance = instance
self.timeout = 10
self.session = requests.Session()
self.session.mount('https://www.statuscake.com', HTTPAdapter(max_retries=5))
def initialize(interactive=False):
ckan_infra = CkanInfra(required=False)
config_manager.interactive_set(
{
'env-id': ckan_infra.ROUTERS_ENV_ID,
'default-root-domain': ckan_infra.ROUTERS_DEFAULT_ROOT_DOMAIN,
'cloudflare-email': ckan_infra.ROUTERS_DEFAULT_CLOUDFLARE_EMAIL,
'cloudflare-api-key': ckan_infra.ROUTERS_DEFAULT_CLOUDFLARE_AUTH_KEY
},
configmap_name='routers-config',
interactive=interactive
)
routers_manager.install_crds()
routers_manager.create(
routers_manager.get_default_infra_router_name(),
routers_manager.get_traefik_router_spec(
config_manager.get('default-root-domain', configmap_name='routers-config'),
config_manager.get('cloudflare-email', configmap_name='routers-config'),
config_manager.get('cloudflare-api-key', configmap_name='routers-config')
)
)
def _apply_deployment(db_prefix=None):
db_host = config_manager.get(
secret_name='ckan-cloud-provider-db-azuresql-credentials'
)['azuresql-host']
resource_label = _get_resource_labels(for_deployment=True,
suffix=db_prefix or '')
kubectl.apply(
kubectl.get_deployment(
_get_resource_name(suffix=db_prefix), resource_label, {
'replicas': 1,
'revisionHistoryLimit': 10,
'strategy': {
'type': 'RollingUpdate',
},
'selector': {
'matchLabels': resource_label
},
'template': {
'metadata': {
'labels':
resource_label,
'annotations':
_get_resource_annotations(suffix=db_prefix or '')
},
'spec': {
'containers': [{
'name':
'proxy',
'image':
'viderum/docker-tcp-proxy:latest',
'env': [{
'name': 'LISTEN',
'value': ':5432'
}, {
'name': 'TALK',
'value': f'{db_host}:5432',
}],
'ports': [{
'containerPort': 5432
}],
'resources': {
'limits': {
'memory': '0.3Gi',
},
'requests': {
'cpu': '0.1',
'memory': '0.2Gi',
}
}
}],
}
}
}))
def config_get(submodule, provider_id=None, key=None, default=None, required=False, namespace=None, is_secret=False,
suffix=None, template=None):
resource_name = get_resource_name(submodule, provider_id=provider_id, suffix=suffix)
return config_manager.get(
key=key,
default=default,
secret_name=resource_name if is_secret else None,
configmap_name=None if is_secret else resource_name,
required=required,
namespace=namespace,
template=template
)
def update(instance_id_or_name, override_spec=None, persist_overrides=False, wait_ready=False, skip_deployment=False,
skip_route=False, force=False, dry_run=False, skip_solr=False):
instance_id, instance_type, instance = _get_instance_id_and_type(instance_id_or_name, required=not dry_run)
if dry_run:
logs.info('update instance', instance_id=instance_id, instance_id_or_name=instance_id_or_name,
override_spec=override_spec, persist_overrides=persist_overrides, wait_ready=wait_ready,
skip_deployment=skip_deployment, skip_route=skip_route, force=force, dry_run=dry_run)
else:
pre_update_hook_data = deployment_manager.pre_update_hook(instance_id, instance_type, instance, override_spec,
skip_route)
bucket_credentials = instance['spec'].get('ckanStorageBucket', {}).get(get_storage_provider_id())
use_cloud_storage = bucket_credentials and config_manager.get('use-cloud-native-storage', secret_name=CONFIG_NAME)
if use_cloud_storage and bucket_credentials:
config_manager.set(
values=bucket_credentials,
secret_name='bucket-credentials',
namespace=instance_id
)
if persist_overrides:
logs.info('Persisting overrides')
kubectl.apply(instance)
if not skip_deployment:
deployment_manager.update(instance_id, instance_type, instance, force=force, skip_solr=skip_solr)
if wait_ready:
wait_instance_ready(instance_id_or_name)
if not skip_route and pre_update_hook_data.get('sub-domain'):
root_domain = pre_update_hook_data.get('root-domain')
sub_domain = pre_update_hook_data['sub-domain']
assert root_domain == routers_manager.get_default_root_domain(), 'invalid domain, must use default root domain'
logs.info(f'adding instance default route to {sub_domain}.{root_domain}')
routers_manager.create_subdomain_route('instances-default', {
'target-type': 'ckan-instance',
'ckan-instance-id': instance_id,
'root-domain': root_domain,
'sub-domain': sub_domain
})
logs.info(f'updating routers_manager wait_ready: {wait_ready}')
routers_manager.update('instances-default', wait_ready)
else:
logs.info('skipping route creation', skip_route=skip_route, sub_domain=pre_update_hook_data.get('sub-domain'))
logs.info('creating ckan admin')
# Need to set in values.yaml
if pre_update_hook_data.get('create-sysadmin'):
ckan_admin_email = pre_update_hook_data.get('ckan-admin-email')
ckan_admin_password = pre_update_hook_data.get('ckan-admin-password')
ckan_admin_name = pre_update_hook_data.get('ckan-admin-name', 'admin')
res = create_ckan_admin_user(instance_id, ckan_admin_name, ckan_admin_email, ckan_admin_password)
logs.info(**res)
logs.info('Instance is ready', instance_id=instance_id, instance_name=(instance_id_or_name if instance_id_or_name != instance_id else None))
def _apply_deployment(db_prefix=None):
rds_host = config_manager.get(
secret_name='ckan-cloud-provider-db-rds-credentials')['rds-host']
kubectl.apply(
kubectl.get_deployment(
_get_resource_name(suffix=db_prefix),
_get_resource_labels(for_deployment=True, suffix=db_prefix or ''),
{
'replicas': 1,
'revisionHistoryLimit': 10,
'strategy': {
'type': 'RollingUpdate',
},
'template': {
'metadata': {
'labels':
_get_resource_labels(for_deployment=True,
suffix=db_prefix or ''),
'annotations':
_get_resource_annotations(suffix=db_prefix or '')
},
'spec': {
'containers': [{
'name':
'proxy',
# https://github.com/OriHoch/docker-tcp-proxy
'image':
'viderum/docker-tcp-proxy:latest',
'env': [{
'name': 'LISTEN',
'value': ':5432'
}, {
'name': 'TALK',
'value': f'{rds_host}:5432',
}],
'ports': [{
'containerPort': 5432
}],
'resources': {
'limits': {
'memory': '0.5Gi',
},
'requests': {
'cpu': '0.1',
'memory': '0.2Gi',
}
}
}]
}
}
}))
def gitlab_search_replace(lines):
values = config_manager.get('gitlab-search-replace', secret_name='ckan-migration-secrets', required=False)
if values: values = json.loads(values)
needs_update = False
_lines = []
for line in lines:
if values:
for k, v in values.items():
if k in line:
logs.info(f'Search-replace gitlab line: {line} / {k} = {v}')
line = line.replace(k, v)
needs_update = True
_lines.append(line)
return needs_update, _lines
def config_get(singular,
name,
key=None,
default=None,
required=False,
namespace=None,
is_secret=False):
resource_name = get_resource_name(singular, name)
return config_manager.get(
key=key,
default=default,
secret_name=resource_name if is_secret else None,
configmap_name=None if is_secret else resource_name,
required=required,
namespace=namespace)
def get(**kwargs):
"""Get configuration values
Examples:
- get key foo from default configmap/namespace: get --key foo
- get key foo from secret in default namespace: get --key foo --secret-name my-secret
- get all keys from secret in given namespace: get --secret-name my-secret --namespace my-namespace
"""
raw = kwargs.pop('raw', False)
data = manager.get(**kwargs)
if raw or kwargs.get('template'):
print(data)
else:
print(yaml.dump(data, default_flow_style=False))
def create(instance_type, instance_id=None, instance_name=None, values=None, values_filename=None, exists_ok=False,
dry_run=False, update_=False, wait_ready=False, skip_deployment=False, skip_route=False, force=False, skip_solr=False):
if not instance_id:
if instance_name:
instance_id = '{}-{}'.format(instance_name, _generate_password(6))
else:
instance_id = _generate_password(12)
if values_filename:
assert values is None
if values_filename != '-':
with open(values_filename) as f:
values = yaml.load(f.read())
else:
values = yaml.load(sys.stdin.read())
instance = crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id, required=False)
if not exists_ok and instance:
raise Exception('instance already exists')
values_id = values.get('id')
if values_id and values_id != instance_id:
logs.warning(f'changing instance id in spec from {values_id} to the instance id {instance_id}')
values['id'] = instance_id
use_cloud_storage = values.get('useCloudStorage') and config_manager.get('use-cloud-native-storage', secret_name=CONFIG_NAME)
values['useCloudStorage'] = use_cloud_storage
if instance:
logs.info('Updating instance', instance_id=instance_id)
instance['spec'] = values
kubectl.apply(instance, dry_run=dry_run)
else:
logs.info('Creating instance', instance_id=instance_id)
kubectl.apply(crds_manager.get_resource(
INSTANCE_CRD_SINGULAR, instance_id,
extra_label_suffixes={'instance-type': instance_type},
spec=values
), dry_run=dry_run)
if instance_name:
set_name(instance_id, instance_name, dry_run=dry_run)
if use_cloud_storage:
set_storage(instance_id, instance_name, dry_run=dry_run)
if update_:
update(instance_id, wait_ready=wait_ready, skip_deployment=skip_deployment, skip_route=skip_route, force=force,
dry_run=dry_run, skip_solr=skip_solr)
return instance_id
def update_deis_instance_envvars(deis_instance, envvars):
smtp_creds = config_manager.get(secret_name='ckan-default-smtp-credentials')
old_froms = smtp_creds.get('old-froms')
old_froms = old_froms.split(',') if old_froms else []
if smtp_creds and envvars.get('CKAN_SMTP_MAIL_FROM', '') in ['', smtp_creds['from']] + old_froms:
logs.info(f'updating smtp credentials for deis instance {deis_instance.id}')
envvars.update(CKAN_SMTP_MAIL_FROM=smtp_creds['from'],
CKAN_SMTP_SERVER=smtp_creds['server'],
CKAN_SMTP_USER=smtp_creds['user'],
CKAN_SMTP_PASSWORD=smtp_creds['password'])
if envvars.get('CKANEXT__ORGPORTALS__SMTP__MAIL__FROM'):
logs.info(f'updating orgportls smtp credentials for deis instance {deis_instance.id}')
envvars.update(CKANEXT__ORGPORTALS__SMTP__MAIL__FROM=smtp_creds['from'],
CKANEXT__ORGPORTALS__SMTP__SERVER=smtp_creds['server'],
CKANEXT__ORGPORTALS__SMTP__USER=smtp_creds['user'],
CKANEXT__ORGPORTALS__SMTP__PASSWORD=smtp_creds['password'])
def get_operator_version(verify=False):
installed_image_tag = None
if os.path.exists('/etc/CKAN_CLOUD_OPERATOR_IMAGE_TAG'):
with open('/etc/CKAN_CLOUD_OPERATOR_IMAGE_TAG') as f:
installed_image_tag = f.read().strip()
if not installed_image_tag or len(installed_image_tag) < 2:
#assert not verify, "Failed operator version verification, no version tag could be found in /etc/CKAN_CLOUD_OPERATOR_IMAGE_TAG"
return None
else:
if verify:
from ckan_cloud_operator.config import manager as config_manager
expected_image = config_manager.get('ckan-cloud-operator-image')
assert '@' not in expected_image and ':' in expected_image, f'invalid expected image: {expected_image}'
expected_image, expected_image_tag = expected_image.split(':')
assert installed_image_tag == expected_image_tag, f'installed tag mismatch (expected={expected_image_tag}, actual={installed_image_tag})'
return installed_image_tag
def _init_ckan_infra_secret(instance_id):
ckan_infra = config_manager.get(secret_name='ckan-infra',
namespace=instance_id,
required=False)
if ckan_infra:
print('ckan-infra secret already exists')
else:
admin_user, admin_password, db_name = db_manager.get_admin_db_credentials(
)
db_host, db_port = db_manager.get_internal_unproxied_db_host_port()
assert int(db_port) == 5432
config_manager.set(values={
'POSTGRES_HOST': db_host,
'POSTGRES_PASSWORD': admin_password,
'POSTGRES_USER': admin_user
},
secret_name='ckan-infra',
namespace=instance_id)
