def get(routes,
letsencrypt_cloudflare_email,
enable_access_log=False,
wildcard_ssl_domain=None,
external_domains=False,
dns_provider=None,
force=False):
if not dns_provider:
dns_provider = 'cloudflare'
logs.info('Generating traefik configuration',
routes_len=len(routes) if routes else 0,
letsencrypt_cloudflare_email=letsencrypt_cloudflare_email,
enable_access_log=enable_access_log,
wildcard_ssl_domain=wildcard_ssl_domain,
external_domains=external_domains)
config = _get_base_config(**({
'accessLog': {
"format": "json",
"fields": {
'defaultMode': "keep"
}
},
} if enable_access_log else {}))
domains = {}
if dns_provider == 'cloudflare' and letsencrypt_cloudflare_email:
enable_ssl_redirect = True
elif dns_provider == 'route53':
enable_ssl_redirect = True
else:
enable_ssl_redirect = False
logs.info(enable_ssl_redirect=enable_ssl_redirect)
logs.info('Adding routes')
i = 0
errors = 0
for route in routes:
try:
_add_route(config, domains, route, enable_ssl_redirect)
i += 1
except Exception as e:
if force:
logs.error(traceback.format_exc())
logs.error(str(e))
errors += 1
else:
raise
logs.info(f'Added {i} routes')
if errors > 0:
logs.warning(f'Encountered {errors} errors')
if ((dns_provider == 'cloudflare' and letsencrypt_cloudflare_email)
or (dns_provider == 'route53')):
_add_letsencrypt(dns_provider,
config,
letsencrypt_cloudflare_email,
domains,
wildcard_ssl_domain=wildcard_ssl_domain,
external_domains=external_domains)
else:
logs.info('No valid dns_provider, will not setup SSL',
dns_provider=dns_provider)
return config
def get_operator_version(verify=False):
installed_image_tag = _get_installed_operator_image_tag()
if verify:
expected_image_tag = _get_expected_operator_image_tag()
if not expected_image_tag:
logs.info('No configmap created yet')
return 'not-configured'
if installed_image_tag and len(installed_image_tag) >= 2:
if installed_image_tag != expected_image_tag:
logs.error(
f'installed tag mismatch (expected={expected_image_tag}, actual={installed_image_tag})'
)
logs.info(
'delete the operator image tag file to skip this check: /etc/CKAN_CLOUD_OPERATOR_IMAGE_TAG'
)
logs.exit_catastrophic_failure()
else:
logs.error(
"No version tag could be found. for local development, "
"make sure you use correct version and run the following "
"to create the file with correct version:\n\n"
f"echo {expected_image_tag} | sudo tee /etc/CKAN_CLOUD_OPERATOR_IMAGE_TAG\n"
)
logs.exit_catastrophic_failure()
return installed_image_tag
def get(routes,
letsencrypt_cloudflare_email,
enable_access_log=False,
wildcard_ssl_domain=None,
external_domains=False,
dns_provider=None,
force=False):
assert dns_provider == 'cloudflare'
logs.info('Generating nginx configuration',
routes_len=len(routes) if routes else 0,
letsencrypt_cloudflare_email=letsencrypt_cloudflare_email,
enable_access_log=enable_access_log,
wildcard_ssl_domain=wildcard_ssl_domain,
external_domains=external_domains)
config = {}
domains = {}
enable_ssl_redirect = True
logs.info('Adding routes')
i = 0
errors = 0
for route in routes:
try:
_add_route(config, domains, route, enable_ssl_redirect)
i += 1
except Exception as e:
if force:
logs.error(traceback.format_exc())
logs.error(str(e))
errors += 1
else:
raise
logs.info(f'Added {i} routes')
if errors > 0:
logs.warning(f'Encountered {errors} errors')
return config
def get_dynamic(routes,
letsencrypt_cloudflare_email,
wildcard_ssl_domain=None,
external_domains=False,
dns_provider=None,
force=False):
if not dns_provider:
dns_provider = 'cloudflare'
logs.info('Generating traefik v2 dynamic configuration',
routes_len=len(routes) if routes else 0,
letsencrypt_cloudflare_email=letsencrypt_cloudflare_email,
wildcard_ssl_domain=wildcard_ssl_domain,
external_domains=external_domains)
dynamic_config = {
'http': {
'routers': {},
'services': {},
'middlewares': {
'SSLRedirect': {
'redirectScheme': {
'scheme': 'https',
'permanent': True
}
}
}
}
}
domains = {}
if dns_provider == 'cloudflare' and letsencrypt_cloudflare_email:
enable_ssl_redirect = True
elif dns_provider == 'route53':
enable_ssl_redirect = True
elif dns_provider == 'azure':
enable_ssl_redirect = True
else:
enable_ssl_redirect = False
logs.info(enable_ssl_redirect=enable_ssl_redirect)
logs.info('Adding routes')
i = 0
errors = 0
for route in routes:
try:
_add_route(dynamic_config, domains, route, enable_ssl_redirect,
external_domains, wildcard_ssl_domain)
i += 1
except Exception as e:
if force:
logs.error(traceback.format_exc())
logs.error(str(e))
errors += 1
else:
raise
logs.info(f'Added {i} routes')
if errors > 0:
logs.warning(f'Encountered {errors} errors')
return dynamic_config
def initialize(server, role):
assert role == 'rke-node'
try:
ssh(server, [
"""
mkdir -p /usr/local/lib/cco &&\
ls -lah /usr/local/lib/cco/rancher_install_docker.sh
"""
])
except Exception:
scp_to_server(server, '/usr/local/lib/cco/rancher_install_docker.sh')
lsmod = ssh(server, ["lsmod"], method="check_output").decode()
lsmods = [
line.split(' ')[0] for line in lsmod.split("\n") if len(line) > 0
]
missing_mods = []
for mod in "br_netfilter ip6_udp_tunnel ip_set ip_set_hash_ip ip_set_hash_net iptable_filter iptable_nat iptable_mangle iptable_raw nf_conntrack_netlink nf_conntrack nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat nf_nat_ipv4 nf_nat_masquerade_ipv4 nfnetlink udp_tunnel veth vxlan x_tables xt_addrtype xt_conntrack xt_comment xt_mark xt_multiport xt_nat xt_recent xt_set xt_statistic xt_tcpudp".split(
" "):
if len(mod) < 1: continue
if mod not in lsmods:
missing_mods.append(mod)
for missing_mod in missing_mods:
logs.info("installing missing mod", missing_mod=missing_mod)
ssh(server,
[f"echo {missing_mod} >> /etc/modules && modprobe {missing_mod}"])
lsmod = ssh(server, ["lsmod"], method="check_output").decode()
lsmods = [
line.split(' ')[0] for line in lsmod.split("\n") if len(line) > 0
]
missing_mods = []
for mod in "br_netfilter ip6_udp_tunnel ip_set ip_set_hash_ip ip_set_hash_net iptable_filter iptable_nat iptable_mangle iptable_raw nf_conntrack_netlink nf_conntrack nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat nf_nat_ipv4 nf_nat_masquerade_ipv4 nfnetlink udp_tunnel veth vxlan x_tables xt_addrtype xt_conntrack xt_comment xt_mark xt_multiport xt_nat xt_recent xt_set xt_statistic xt_tcpudp".split(
" "):
if len(mod) < 1: continue
if mod not in lsmods:
missing_mods.append(mod)
if missing_mods:
logs.error(missing_mods=missing_mods)
exit(1)
sysctlconf = ssh(server, ["cat", "/etc/sysctl.conf"],
method="check_output").decode()
if 'net.bridge.bridge-nf-call-iptables = 1' not in sysctlconf:
ssh(server, [
"echo net.bridge.bridge-nf-call-iptables = 1 >> /etc/sysctl.conf"
])
ssh(server, ["sysctl", "-p"])
sshdconfig = ssh(server, ["cat", "/etc/ssh/sshd_config"],
method="check_output").decode()
if 'cco-provider-kamatera managed config' not in sshdconfig:
ssh(server, [f'echo "{SSHD_CONFIG}" > /etc/ssh/sshd_config'])
ssh(server, [
"chmod +x /usr/local/lib/cco/rancher_install_docker.sh &&"
" if ! which docker; then /usr/local/lib/cco/rancher_install_docker.sh; fi"
])
def get_operator_version(verify=False):
installed_image_tag = _get_installed_operator_image_tag()
if verify:
expected_image_tag = _get_expected_operator_image_tag()
if installed_image_tag and len(installed_image_tag) >= 2:
assert installed_image_tag == expected_image_tag, \
f'installed tag mismatch (expected={expected_image_tag}, actual={installed_image_tag})'
else:
logs.error("No version tag could be found. for local development, "
"make sure you use correct version and run the following "
"to create the file with correct version:\n\n"
f"echo {expected_image_tag} | sudo tee /etc/CKAN_CLOUD_OPERATOR_IMAGE_TAG\n")
logs.exit_catastrophic_failure()
return installed_image_tag
def delete_instances(instance_ids_or_names=None,
dry_run=True,
instance_ids=None):
if instance_ids:
assert not instance_ids_or_names
use_instance_ids = True
else:
assert not instance_ids
use_instance_ids = False
logs.info(use_instance_ids=use_instance_ids,
instance_ids=instance_ids,
instance_ids_or_names=instance_ids_or_names)
for instance_id_or_name in (instance_ids if use_instance_ids else
instance_ids_or_names):
logs.info(instance_id_or_name=instance_id_or_name)
if use_instance_ids:
instance_id = instance_id_or_name
else:
instance_id, _ = _get_instance(instance_id_or_name, required=False)
instance_name = instance_id_or_name if instance_id != instance_id_or_name else None
yield {
'id': instance_id,
**({
'name': instance_name
} if instance_name else {})
}
errors = []
if instance_id:
if not dry_run:
try:
delete(instance_id=instance_id)
except Exception as e:
errors.append(str(e))
logs.error(
'exception raised during instance deletion, '
'this does not always indicate deletion was not successful',
exception=str(e))
else:
errors.append(
f'Failed to get instance_id for instance_id_or_name {instance_id_or_name}'
)
if instance_name:
try:
delete_name(instance_name)
except Exception as e:
errors.append(str(e))
logs.error('exception raised during instance name deletion.',
exception=str(e))
yield {'errors': errors}
def create_storage_class():
if kubectl.get("storageclass/cca-ckan", required=False) is None:
if _config_get("deploy-nfs-client-provisioner") == "yes":
deploy_nfs_client_provisioner()
else:
logs.error("Missing cca-ckan storage class")
logs.exit_catastrophic_failure()
kubectl.apply({
'apiVersion': 'storage.k8s.io/v1', 'kind': 'StorageClass',
'metadata': {
'name': 'cca-storage',
},
'provisioner': 'kubernetes.io/gce-pd',
'reclaimPolicy': 'Delete',
'volumeBindingMode': 'Immediate',
'parameters': {
'type': 'pd-standard',
}
})
def initialize(cluster_id, skip_server_init=False, dry_run=False):
cluster = get_cluster(cluster_id)
cluster['servers'] = server_manager.create_cluster_servers(
cluster, skip_server_init)
save_cluster(cluster)
has_rke_nodes = False
for server in cluster['servers']:
if server.get('disabled'):
continue
if 'rke-node' in server['roles']:
has_rke_nodes = True
if has_rke_nodes:
rke_cluster_config = get_rke_cluster_config(cluster)
rke_config_filename = f'{get_cluster_path(cluster_id)}/rke-cluster.yml'
with open(rke_config_filename, 'w') as f:
f.write(logs.yaml_dump(rke_cluster_config))
if dry_run:
logs.info('skipping rke command',
rke_config_filename=rke_config_filename)
else:
i = 0
while True:
i += 1
logs.info('rke up', i=i)
err = None
try:
subprocess.check_call(
['rke', 'up', '--config', rke_config_filename])
except subprocess.CalledProcessError as e:
logs.error(f'{e}')
err = e
if err:
if i > 4:
raise Exception('Too many iterations')
logs.info('sleeping 60 seconds before retrying...')
time.sleep(60)
else:
break
subprocess.check_call([
'kubectl', '--kubeconfig',
f'{get_cluster_path(cluster_id)}/kube_config_rke-cluster.yml',
'get', 'nodes'
])
def get_preset_answer(namespace, configmap_name, secret_name, key):
interactive_file = os.environ.get('CCO_INTERACTIVE_CI')
if not interactive_file:
return
answers = yaml.load(open(interactive_file))
section = '?'
subsection = '?'
namespace = namespace or 'default'
try:
if configmap_name:
section = 'config'
subsection = configmap_name
else:
section = 'secrets'
subsection = secret_name
return answers[namespace][section][subsection][key]
except:
logs.error(f'Failed to find in interactive file value for {namespace}.{section}.{subsection}.{key}')
raise
def create(name, spec, force=False, exists_ok=False, delete_dbs=False):
migration = get(name, required=False)
if migration:
if force or delete_dbs:
delete(name, delete_dbs=delete_dbs)
elif exists_ok:
return migration
else:
logs.error('migration name conflict: ' +
str(migration.get('spec')))
raise Exception(
f'migration already exists: {name}, Run with --force / --rerun / --recreate-dbs'
)
logs.info(f'Creating migration name {name} (spec={spec})')
migration = crds_manager.get_resource(CRD_SINGULAR,
name,
spec=_get_spec(name, spec),
extra_label_suffixes=_get_labels(
name, spec))
kubectl.apply(migration)
return migration
def print_event_exit_on_complete(event, details_msg, soft_exit=False):
if not details_msg: details_msg = ''
msg = event.pop('msg')
step = event.pop('step')
logs.info(f'{step}: {msg}')
if len(event) > 0:
logs.info(f'event metadata: {event}')
event_completed_successfully = get_event_migration_complete_status(event)
if event_completed_successfully:
logs.info(f'Migration completed successfully ({details_msg})')
if soft_exit:
return True
else:
logs.exit_great_success()
elif event_completed_successfully is False:
logs.error(f'Migration failed ({details_msg})')
if soft_exit:
return False
else:
logs.exit_catastrophic_failure()
else:
return None
def delete(instance_id):
try:
instance_id, instance = _get_instance(instance_id=instance_id)
except Exception:
logs.error(traceback.format_exc())
instance, instance_type = None, None
try:
deployment_manager.delete(instance_id, instance)
except Exception as e:
logs.error('error during deployment delete', error=str(e))
try:
crds_manager.delete(APP_CRD_SINGULAR, instance_id)
except Exception as e:
logs.error('error during crd delete', error=str(e))
