def get(routes, letsencrypt_cloudflare_email, enable_access_log=False, wildcard_ssl_domain=None, external_domains=False, dns_provider=None, force=False): if not dns_provider: dns_provider = 'cloudflare' logs.info('Generating traefik configuration', routes_len=len(routes) if routes else 0, letsencrypt_cloudflare_email=letsencrypt_cloudflare_email, enable_access_log=enable_access_log, wildcard_ssl_domain=wildcard_ssl_domain, external_domains=external_domains) config = _get_base_config(**({ 'accessLog': { "format": "json", "fields": { 'defaultMode': "keep" } }, } if enable_access_log else {})) domains = {} if dns_provider == 'cloudflare' and letsencrypt_cloudflare_email: enable_ssl_redirect = True elif dns_provider == 'route53': enable_ssl_redirect = True else: enable_ssl_redirect = False logs.info(enable_ssl_redirect=enable_ssl_redirect) logs.info('Adding routes') i = 0 errors = 0 for route in routes: try: _add_route(config, domains, route, enable_ssl_redirect) i += 1 except Exception as e: if force: logs.error(traceback.format_exc()) logs.error(str(e)) errors += 1 else: raise logs.info(f'Added {i} routes') if errors > 0: logs.warning(f'Encountered {errors} errors') if ((dns_provider == 'cloudflare' and letsencrypt_cloudflare_email) or (dns_provider == 'route53')): _add_letsencrypt(dns_provider, config, letsencrypt_cloudflare_email, domains, wildcard_ssl_domain=wildcard_ssl_domain, external_domains=external_domains) else: logs.info('No valid dns_provider, will not setup SSL', dns_provider=dns_provider) return config
def get_operator_version(verify=False): installed_image_tag = _get_installed_operator_image_tag() if verify: expected_image_tag = _get_expected_operator_image_tag() if not expected_image_tag: logs.info('No configmap created yet') return 'not-configured' if installed_image_tag and len(installed_image_tag) >= 2: if installed_image_tag != expected_image_tag: logs.error( f'installed tag mismatch (expected={expected_image_tag}, actual={installed_image_tag})' ) logs.info( 'delete the operator image tag file to skip this check: /etc/CKAN_CLOUD_OPERATOR_IMAGE_TAG' ) logs.exit_catastrophic_failure() else: logs.error( "No version tag could be found. for local development, " "make sure you use correct version and run the following " "to create the file with correct version:\n\n" f"echo {expected_image_tag} | sudo tee /etc/CKAN_CLOUD_OPERATOR_IMAGE_TAG\n" ) logs.exit_catastrophic_failure() return installed_image_tag
def get(routes, letsencrypt_cloudflare_email, enable_access_log=False, wildcard_ssl_domain=None, external_domains=False, dns_provider=None, force=False): assert dns_provider == 'cloudflare' logs.info('Generating nginx configuration', routes_len=len(routes) if routes else 0, letsencrypt_cloudflare_email=letsencrypt_cloudflare_email, enable_access_log=enable_access_log, wildcard_ssl_domain=wildcard_ssl_domain, external_domains=external_domains) config = {} domains = {} enable_ssl_redirect = True logs.info('Adding routes') i = 0 errors = 0 for route in routes: try: _add_route(config, domains, route, enable_ssl_redirect) i += 1 except Exception as e: if force: logs.error(traceback.format_exc()) logs.error(str(e)) errors += 1 else: raise logs.info(f'Added {i} routes') if errors > 0: logs.warning(f'Encountered {errors} errors') return config
def get_dynamic(routes, letsencrypt_cloudflare_email, wildcard_ssl_domain=None, external_domains=False, dns_provider=None, force=False): if not dns_provider: dns_provider = 'cloudflare' logs.info('Generating traefik v2 dynamic configuration', routes_len=len(routes) if routes else 0, letsencrypt_cloudflare_email=letsencrypt_cloudflare_email, wildcard_ssl_domain=wildcard_ssl_domain, external_domains=external_domains) dynamic_config = { 'http': { 'routers': {}, 'services': {}, 'middlewares': { 'SSLRedirect': { 'redirectScheme': { 'scheme': 'https', 'permanent': True } } } } } domains = {} if dns_provider == 'cloudflare' and letsencrypt_cloudflare_email: enable_ssl_redirect = True elif dns_provider == 'route53': enable_ssl_redirect = True elif dns_provider == 'azure': enable_ssl_redirect = True else: enable_ssl_redirect = False logs.info(enable_ssl_redirect=enable_ssl_redirect) logs.info('Adding routes') i = 0 errors = 0 for route in routes: try: _add_route(dynamic_config, domains, route, enable_ssl_redirect, external_domains, wildcard_ssl_domain) i += 1 except Exception as e: if force: logs.error(traceback.format_exc()) logs.error(str(e)) errors += 1 else: raise logs.info(f'Added {i} routes') if errors > 0: logs.warning(f'Encountered {errors} errors') return dynamic_config
def initialize(server, role): assert role == 'rke-node' try: ssh(server, [ """ mkdir -p /usr/local/lib/cco &&\ ls -lah /usr/local/lib/cco/rancher_install_docker.sh """ ]) except Exception: scp_to_server(server, '/usr/local/lib/cco/rancher_install_docker.sh') lsmod = ssh(server, ["lsmod"], method="check_output").decode() lsmods = [ line.split(' ')[0] for line in lsmod.split("\n") if len(line) > 0 ] missing_mods = [] for mod in "br_netfilter ip6_udp_tunnel ip_set ip_set_hash_ip ip_set_hash_net iptable_filter iptable_nat iptable_mangle iptable_raw nf_conntrack_netlink nf_conntrack nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat nf_nat_ipv4 nf_nat_masquerade_ipv4 nfnetlink udp_tunnel veth vxlan x_tables xt_addrtype xt_conntrack xt_comment xt_mark xt_multiport xt_nat xt_recent xt_set xt_statistic xt_tcpudp".split( " "): if len(mod) < 1: continue if mod not in lsmods: missing_mods.append(mod) for missing_mod in missing_mods: logs.info("installing missing mod", missing_mod=missing_mod) ssh(server, [f"echo {missing_mod} >> /etc/modules && modprobe {missing_mod}"]) lsmod = ssh(server, ["lsmod"], method="check_output").decode() lsmods = [ line.split(' ')[0] for line in lsmod.split("\n") if len(line) > 0 ] missing_mods = [] for mod in "br_netfilter ip6_udp_tunnel ip_set ip_set_hash_ip ip_set_hash_net iptable_filter iptable_nat iptable_mangle iptable_raw nf_conntrack_netlink nf_conntrack nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat nf_nat_ipv4 nf_nat_masquerade_ipv4 nfnetlink udp_tunnel veth vxlan x_tables xt_addrtype xt_conntrack xt_comment xt_mark xt_multiport xt_nat xt_recent xt_set xt_statistic xt_tcpudp".split( " "): if len(mod) < 1: continue if mod not in lsmods: missing_mods.append(mod) if missing_mods: logs.error(missing_mods=missing_mods) exit(1) sysctlconf = ssh(server, ["cat", "/etc/sysctl.conf"], method="check_output").decode() if 'net.bridge.bridge-nf-call-iptables = 1' not in sysctlconf: ssh(server, [ "echo net.bridge.bridge-nf-call-iptables = 1 >> /etc/sysctl.conf" ]) ssh(server, ["sysctl", "-p"]) sshdconfig = ssh(server, ["cat", "/etc/ssh/sshd_config"], method="check_output").decode() if 'cco-provider-kamatera managed config' not in sshdconfig: ssh(server, [f'echo "{SSHD_CONFIG}" > /etc/ssh/sshd_config']) ssh(server, [ "chmod +x /usr/local/lib/cco/rancher_install_docker.sh &&" " if ! which docker; then /usr/local/lib/cco/rancher_install_docker.sh; fi" ])
def get_operator_version(verify=False): installed_image_tag = _get_installed_operator_image_tag() if verify: expected_image_tag = _get_expected_operator_image_tag() if installed_image_tag and len(installed_image_tag) >= 2: assert installed_image_tag == expected_image_tag, \ f'installed tag mismatch (expected={expected_image_tag}, actual={installed_image_tag})' else: logs.error("No version tag could be found. for local development, " "make sure you use correct version and run the following " "to create the file with correct version:\n\n" f"echo {expected_image_tag} | sudo tee /etc/CKAN_CLOUD_OPERATOR_IMAGE_TAG\n") logs.exit_catastrophic_failure() return installed_image_tag
def delete_instances(instance_ids_or_names=None, dry_run=True, instance_ids=None): if instance_ids: assert not instance_ids_or_names use_instance_ids = True else: assert not instance_ids use_instance_ids = False logs.info(use_instance_ids=use_instance_ids, instance_ids=instance_ids, instance_ids_or_names=instance_ids_or_names) for instance_id_or_name in (instance_ids if use_instance_ids else instance_ids_or_names): logs.info(instance_id_or_name=instance_id_or_name) if use_instance_ids: instance_id = instance_id_or_name else: instance_id, _ = _get_instance(instance_id_or_name, required=False) instance_name = instance_id_or_name if instance_id != instance_id_or_name else None yield { 'id': instance_id, **({ 'name': instance_name } if instance_name else {}) } errors = [] if instance_id: if not dry_run: try: delete(instance_id=instance_id) except Exception as e: errors.append(str(e)) logs.error( 'exception raised during instance deletion, ' 'this does not always indicate deletion was not successful', exception=str(e)) else: errors.append( f'Failed to get instance_id for instance_id_or_name {instance_id_or_name}' ) if instance_name: try: delete_name(instance_name) except Exception as e: errors.append(str(e)) logs.error('exception raised during instance name deletion.', exception=str(e)) yield {'errors': errors}
def create_storage_class(): if kubectl.get("storageclass/cca-ckan", required=False) is None: if _config_get("deploy-nfs-client-provisioner") == "yes": deploy_nfs_client_provisioner() else: logs.error("Missing cca-ckan storage class") logs.exit_catastrophic_failure() kubectl.apply({ 'apiVersion': 'storage.k8s.io/v1', 'kind': 'StorageClass', 'metadata': { 'name': 'cca-storage', }, 'provisioner': 'kubernetes.io/gce-pd', 'reclaimPolicy': 'Delete', 'volumeBindingMode': 'Immediate', 'parameters': { 'type': 'pd-standard', } })
def initialize(cluster_id, skip_server_init=False, dry_run=False): cluster = get_cluster(cluster_id) cluster['servers'] = server_manager.create_cluster_servers( cluster, skip_server_init) save_cluster(cluster) has_rke_nodes = False for server in cluster['servers']: if server.get('disabled'): continue if 'rke-node' in server['roles']: has_rke_nodes = True if has_rke_nodes: rke_cluster_config = get_rke_cluster_config(cluster) rke_config_filename = f'{get_cluster_path(cluster_id)}/rke-cluster.yml' with open(rke_config_filename, 'w') as f: f.write(logs.yaml_dump(rke_cluster_config)) if dry_run: logs.info('skipping rke command', rke_config_filename=rke_config_filename) else: i = 0 while True: i += 1 logs.info('rke up', i=i) err = None try: subprocess.check_call( ['rke', 'up', '--config', rke_config_filename]) except subprocess.CalledProcessError as e: logs.error(f'{e}') err = e if err: if i > 4: raise Exception('Too many iterations') logs.info('sleeping 60 seconds before retrying...') time.sleep(60) else: break subprocess.check_call([ 'kubectl', '--kubeconfig', f'{get_cluster_path(cluster_id)}/kube_config_rke-cluster.yml', 'get', 'nodes' ])
def get_preset_answer(namespace, configmap_name, secret_name, key): interactive_file = os.environ.get('CCO_INTERACTIVE_CI') if not interactive_file: return answers = yaml.load(open(interactive_file)) section = '?' subsection = '?' namespace = namespace or 'default' try: if configmap_name: section = 'config' subsection = configmap_name else: section = 'secrets' subsection = secret_name return answers[namespace][section][subsection][key] except: logs.error(f'Failed to find in interactive file value for {namespace}.{section}.{subsection}.{key}') raise
def create(name, spec, force=False, exists_ok=False, delete_dbs=False): migration = get(name, required=False) if migration: if force or delete_dbs: delete(name, delete_dbs=delete_dbs) elif exists_ok: return migration else: logs.error('migration name conflict: ' + str(migration.get('spec'))) raise Exception( f'migration already exists: {name}, Run with --force / --rerun / --recreate-dbs' ) logs.info(f'Creating migration name {name} (spec={spec})') migration = crds_manager.get_resource(CRD_SINGULAR, name, spec=_get_spec(name, spec), extra_label_suffixes=_get_labels( name, spec)) kubectl.apply(migration) return migration
def print_event_exit_on_complete(event, details_msg, soft_exit=False): if not details_msg: details_msg = '' msg = event.pop('msg') step = event.pop('step') logs.info(f'{step}: {msg}') if len(event) > 0: logs.info(f'event metadata: {event}') event_completed_successfully = get_event_migration_complete_status(event) if event_completed_successfully: logs.info(f'Migration completed successfully ({details_msg})') if soft_exit: return True else: logs.exit_great_success() elif event_completed_successfully is False: logs.error(f'Migration failed ({details_msg})') if soft_exit: return False else: logs.exit_catastrophic_failure() else: return None
def delete(instance_id): try: instance_id, instance = _get_instance(instance_id=instance_id) except Exception: logs.error(traceback.format_exc()) instance, instance_type = None, None try: deployment_manager.delete(instance_id, instance) except Exception as e: logs.error('error during deployment delete', error=str(e)) try: crds_manager.delete(APP_CRD_SINGULAR, instance_id) except Exception as e: logs.error('error during crd delete', error=str(e))