コード例 #1
0
def create_volume(disk_size_gb, labels, use_existing_disk_name=None, zone=0):
    if zone != 0:
        logs.warning(f'variable zone for create_volume has been deprecated.')

    disk_id = use_existing_disk_name or 'cc' + _generate_password(12)
    if use_existing_disk_name:
        logs.info(f'using existing persistent disk {disk_id}')
    else:
        logs.info(
            f'creating persistent disk {disk_id} with size {disk_size_gb}GB')
        labels = ','.join([
            '{}={}'.format(k.replace('/', '_'), v.replace('/', '_'))
            for k, v in labels.items()
        ])

    kubectl.apply({
        "kind": "PersistentVolumeClaim",
        "apiVersion": "v1",
        "metadata": {
            "name": disk_id,
            "namespace": "ckan-cloud"
        },
        "spec": {
            "accessModes": ["ReadWriteOnce"],
            "resources": {
                "requests": {
                    "storage": f'{disk_size_gb}G'
                }
            },
            "storageClassName": "cca-ckan"
        }
    })
    return {'persistentVolumeClaim': {'claimName': disk_id}}
コード例 #2
0
def _add_letsencrypt(dns_provider,
                     config,
                     letsencrypt_cloudflare_email,
                     domains,
                     wildcard_ssl_domain=None,
                     external_domains=False):
    logs.info('Adding Letsencrypt acme Traefik configuration',
              dns_provider=dns_provider,
              letsencrypt_cloudflare_email=letsencrypt_cloudflare_email,
              domains=domains,
              wildcard_ssl_domain=wildcard_ssl_domain,
              external_domains=external_domains)
    assert dns_provider in ['route53', 'cloudflare']
    config['defaultEntryPoints'].append('https')
    config['entryPoints']['https'] = {'address': ':443', 'tls': {}}
    config['acme'] = {
        'email':
        letsencrypt_cloudflare_email,
        'storage':
        '/traefik-acme/acme.json',
        'entryPoint':
        'https',
        **({
            'tlsChallenge': {}
        } if external_domains else {
               'dnsChallenge': {
                   'provider': dns_provider
               }
           }), 'domains':
        list(
            _get_acme_domains(domains,
                              wildcard_ssl_domain=wildcard_ssl_domain,
                              external_domains=external_domains))
    }
コード例 #3
0
ファイル: manager.py プロジェクト: zelima/ckan-cloud-operator
def _helm_deploy(values,
                 tiller_namespace_name,
                 chart_repo,
                 chart_name,
                 chart_version,
                 release_name,
                 instance_id,
                 dry_run=False,
                 chart_repo_name=None):
    assert chart_repo_name, 'chart-repo-name is required'
    helm_driver.init(tiller_namespace_name)
    logs.info(
        f'Deploying helm chart {chart_repo_name} {chart_repo} {chart_version} {chart_name} to release {release_name} '
        f'(instance_id={instance_id})')
    with tempfile.NamedTemporaryFile('w') as f:
        yaml.dump(values, f, default_flow_style=False)
        f.flush()
        helm_driver.deploy(tiller_namespace=tiller_namespace_name,
                           chart_repo=chart_repo,
                           chart_name=chart_name,
                           chart_version=chart_version,
                           release_name=release_name,
                           values_filename=f.name,
                           namespace=instance_id,
                           dry_run=dry_run,
                           chart_repo_name=chart_repo_name)
コード例 #4
0
def _init_namespace(instance_id, dry_run=False):
    logs.debug('Initializing helm-based instance deployment namespace',
               namespace=instance_id)
    if kubectl.get('ns', instance_id, required=False):
        logs.info(f'instance namespace already exists ({instance_id})')
    else:
        logs.info(f'creating instance namespace ({instance_id})')
        kubectl.apply(kubectl.get_resource('v1', 'Namespace', instance_id, {}),
                      dry_run=dry_run)
        service_account_name = f'ckan-{instance_id}-operator'
        logs.debug('Creating service account',
                   service_account_name=service_account_name)
        if not dry_run:
            kubectl_rbac_driver.update_service_account(
                f'ckan-{instance_id}-operator', {}, namespace=instance_id)
        role_name = f'ckan-{instance_id}-operator-role'
        logs.debug('Creating role and binding to the service account',
                   role_name=role_name)
        if not dry_run:
            kubectl_rbac_driver.update_role(role_name, {}, [{
                "apiGroups": ["*"],
                "resources":
                ['secrets', 'pods', 'pods/exec', 'pods/portforward'],
                "verbs": ["list", "get", "create"]
            }],
                                            namespace=instance_id)
            kubectl_rbac_driver.update_role_binding(
                name=f'ckan-{instance_id}-operator-rolebinding',
                role_name=f'ckan-{instance_id}-operator-role',
                namespace=instance_id,
                service_account_name=f'ckan-{instance_id}-operator',
                labels={})
コード例 #5
0
def _scale_down_scale_up(deployment='ckan', namespace=None, replicas=1):
    logs.info('Scaling ckan replicas')
    kubectl.call(f'scale deployment {deployment} --replicas=0',
                 namespace=namespace)
    kubectl.call(f'scale deployment {deployment} --replicas={replicas}',
                 namespace=namespace)
    time.sleep(20)
コード例 #6
0
def get_or_create_multi_user_volume_claim(label_suffixes):
    assert len(label_suffixes) > 0, 'must provide some labels to identify the volume'
    claim_labels = labels_manager.get_resource_labels(label_suffixes=dict(
        label_suffixes, **_get_cluster_volume_label_suffixes()
    ))
    pvcs = kubectl.get_items_by_labels('PersistentVolumeClaim', claim_labels, required=False)
    if len(pvcs) > 0:
        assert len(pvcs) == 1
        claim_name = pvcs[0]['metadata']['name']
    else:
        storage_class_name = get_multi_user_storage_class_name()
        claim_name = 'cc' + _generate_password(12)
        logs.info(f'Creating persistent volume claim: {claim_name}')
        logs.info(f'labels: {claim_labels}')
        kubectl.apply(kubectl.get_persistent_volume_claim(
            claim_name,
            claim_labels,
            {
                'storageClassName': storage_class_name,
                'accessModes': ['ReadWriteMany'],
                'resources': {
                    'requests': {
                        'storage': '1Mi'
                    }
                }
            }
        ))
    return {'persistentVolumeClaim': {'claimName': claim_name}}
コード例 #7
0
def update_dns_record(sub_domain, root_domain, load_balancer_hostname):
    logs.info('updating Route53 DNS record', sub_domain=sub_domain, root_domain=root_domain, load_balancer_hostname=load_balancer_hostname)
    hosted_zone_id = get_dns_hosted_zone_id(root_domain)
    logs.info(hosted_zone_id=hosted_zone_id)
    response = get_boto3_client('route53').change_resource_record_sets(
        HostedZoneId=hosted_zone_id,
        ChangeBatch={
            'Comment': 'ckan-cloud-operator',
            'Changes': [
                {
                    'Action': 'UPSERT',
                    'ResourceRecordSet': {
                        'Name': f'{sub_domain}.{root_domain}.',
                        'Type': 'CNAME',
                        'TTL': 300,
                        'ResourceRecords': [
                            {
                                'Value': load_balancer_hostname
                            },
                        ],
                    }
                },
            ]
        }
    )
    assert response['ResponseMetadata']['HTTPStatusCode'] == 200
コード例 #8
0
ファイル: manager.py プロジェクト: zelima/ckan-cloud-operator
def create(router):
    router_name = router['metadata']['name']
    router_spec = router['spec']
    cloudflare_spec = router_spec.get('cloudflare', {})
    cloudflare_email = cloudflare_spec.get('email')
    cloudflare_api_key = cloudflare_spec.get('api-key')
    default_root_domain = router_spec.get('default-root-domain')
    dns_provider = router_spec.get('dns-provider')
    from ckan_cloud_operator.providers.cluster import manager as cluster_manager
    default_dns_provider = 'route53' if cluster_manager.get_provider_id(
    ) == 'aws' else 'cloudflare'
    logs.info(dns_provider=dns_provider,
              default_dns_provider=default_dns_provider)
    if not dns_provider:
        dns_provider = default_dns_provider
    router_spec['dns-provider'] = dns_provider
    assert all([default_root_domain,
                dns_provider]), f'invalid traefik router spec: {router_spec}'
    if dns_provider == 'cloudflare':
        assert cloudflare_email and cloudflare_api_key, 'invalid traefik router spec for cloudflare dns provider'
    # cloudflare credentials are stored in a secret, not in the spec
    if 'cloudflare' in router_spec:
        del router_spec['cloudflare']
    kubectl.apply(router)
    annotations = CkanRoutersAnnotations(router_name, router)
    if dns_provider == 'cloudflare':
        annotations.update_flag(
            'letsencryptCloudflareEnabled',
            lambda: annotations.set_secrets(
                {
                    'LETSENCRYPT_CLOUDFLARE_EMAIL': cloudflare_email,
                    'LETSENCRYPT_CLOUDFLARE_API_KEY': cloudflare_api_key
                }),
            force_update=True)
    return router
コード例 #9
0
def zk_put_configs(configs_dir):
    def retry_if_fails(command, max_retries=15):
        if max_retries < 0:
            return
        try:
            kubectl.check_output(command)
        except:
            time.sleep(5)
            retry_if_fails(command, max_retries=max_retries - 1)

    pod_name = kubectl.get('pods',
                           '-l',
                           'app=provider-solr-solrcloud-zk',
                           required=True)['items'][0]['metadata']['name']
    logs.info(f'using pod {pod_name}')
    for input_filename in glob.glob(f'{configs_dir}/**/*', recursive=True):
        if not os.path.isfile(input_filename): continue
        output_filename = '/configs' + input_filename.replace(configs_dir, '')
        logs.info(f'{input_filename} --> {output_filename}')
        output_filepath = ''
        for output_filepart in output_filename.split('/')[:-1]:
            output_filepart = output_filepart.strip()
            if not output_filepart:
                continue
            output_filepath += f'/{output_filepart}'
            logs.info(f'create {output_filepath} null')
            retry_if_fails(
                f'exec {pod_name} zkCli.sh create {output_filepath} null')
        logs.info(f'copy {output_filename}')
        retry_if_fails(f'cp {input_filename} {pod_name}:/tmp/zk_input')
        logs.info(f'create {output_filename}')
        retry_if_fails(
            f"exec {pod_name} -- /bin/bash -c '/usr/bin/zkCli.sh create {output_filename} \"$(cat /tmp/zk_input)\"'"
        )
コード例 #10
0
def set_ssh_key(server):
    try:
        authorized_keys = subprocess.check_output([
            'sshpass', '-f', server['passwordfile'], 'ssh',
            f'root@{server["public_ip"]}', '-o', 'StrictHostKeyChecking=no',
            '--', 'cat', '.ssh/authorized_keys'
        ]).decode()
    except subprocess.CalledProcessError:
        authorized_keys = None
    if not authorized_keys or server['name'] not in authorized_keys:
        logs.info(f"Adding authorized key for server",
                  server_name=server['name'])
        with open(server['keyfile'] + '.pub') as f:
            public_key = f.read()
        subprocess.check_call([
            'sshpass', '-f', server['passwordfile'], 'ssh',
            f'root@{server["public_ip"]}', '-o', 'StrictHostKeyChecking=no',
            '--', 'mkdir', '-p', ".ssh"
        ])
        subprocess.check_call([
            'sshpass', '-f', server['passwordfile'], 'ssh',
            f'root@{server["public_ip"]}', '-o', 'StrictHostKeyChecking=no',
            f'echo "{public_key}" > .ssh/authorized_keys'
        ])
        helpers.ssh(server, ["echo successfull ssh connection using key"])
コード例 #11
0
def initialize(interactive=False,
               provider_id=None,
               storage_suffix=None,
               use_existing_disk_name=None,
               dry_run=False):
    if not provider_id and interactive:
        config_manager.interactive_set({'use-cloud-native-storage': True},
                                       secret_name=CONFIG_NAME,
                                       interactive=interactive)
        if config_manager.get('use-cloud-native-storage',
                              secret_name=CONFIG_NAME):
            provider_id = get_provider_id()

    logs.info(f'Storage init: chosen provider_id: {provider_id}')
    provider = get_provider(default=minio_provider_id,
                            provider_id=provider_id).initialize(
                                interactive=interactive,
                                storage_suffix=storage_suffix,
                                use_existing_disk_name=use_existing_disk_name,
                                dry_run=dry_run)
    if provider:
        provider.initialize(interactive=interactive,
                            storage_suffix=storage_suffix,
                            use_existing_disk_name=use_existing_disk_name,
                            dry_run=dry_run)
コード例 #12
0
def initialize(interactive=False):
    _set_provider()
    if interactive:
        print('\nEnter the Resource Group name\n')
        _config_interactive_set({'azure-rg': None}, is_secret=False)
        print(
            '\nEnter the location of the Kubernets cluster is hosted on [westus2]\n'
        )
        _config_interactive_set({'azure-default-location': None},
                                is_secret=False)
        print('\nEnter the name of your cluster\n')
        _config_interactive_set({'azure-cluster-name': None}, is_secret=False)
        print('\nEnter the Subscribtion ID\n')
        _config_interactive_set({'azure-subscribtion-id': None},
                                is_secret=True)
        print('\nEnter the Tenant ID\n')
        _config_interactive_set({'azure-tenant-id': None}, is_secret=True)
        print('\nEnter the Service Principal ID\n')
        _config_interactive_set({'azure-client-id': None}, is_secret=True)
        print('\nEnter the Service Principal Secret\n')
        _config_interactive_set({'azure-client-secret': None}, is_secret=True)

        _create_storage_classes()
    else:
        logs.info(
            'Skipping initial cluster set up as `--interactive` flag was not set'
        )
    _create_storage_classes()
コード例 #13
0
def create_volume(disk_size_gb, labels, use_existing_disk_name=None, zone=0):
    rg = _config_get('azure-rg')

    location = zone or _config_get('azure-default-location')

    disk_id = use_existing_disk_name or 'cc' + _generate_password(12)
    if use_existing_disk_name:
        logs.info(f'using existing persistent disk {disk_id}')
    else:
        logs.info(
            f'creating persistent disk {disk_id} with size {disk_size_gb}GB')
        _, zone = get_project_zone()
        labels = ','.join([
            '{}={}'.format(k.replace('/', '_'), v.replace('/', '_'))
            for k, v in labels.items()
        ])

    kubectl.apply({
        "kind": "PersistentVolumeClaim",
        "apiVersion": "v1",
        "metadata": {
            "name": disk_id,
            "namespace": "ckan-cloud"
        },
        "spec": {
            "accessModes": ["ReadWriteOnce"],
            "resources": {
                "requests": {
                    "storage": f'{disk_size_gb}G'
                }
            },
            "storageClassName": "cca-ckan"
        }
    })
    return {'persistentVolumeClaim': {'claimName': disk_id}}
コード例 #14
0
def create(instance_type, instance_id=None, instance_name=None, values=None, values_filename=None, exists_ok=False,
           dry_run=False, update_=False, wait_ready=False, skip_deployment=False, skip_route=False, force=False):
    if not instance_id:
        if instance_name:
            instance_id = '{}-{}'.format(instance_name, _generate_password(6))
        else:
            instance_id = _generate_password(12)
    if values_filename:
        assert values is None
        if values_filename != '-':
            with open(values_filename) as f:
                values = yaml.load(f.read())
        else:
            values = yaml.load(sys.stdin.read())
    if not exists_ok and crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id, required=False):
        raise Exception('instance already exists')
    values_id = values.get('id')
    if values_id and values_id != instance_id:
        logs.warning(f'changing instance id in spec from {values_id} to the instance id {instance_id}')
    values['id'] = instance_id
    logs.info('Creating instance', instance_id=instance_id)
    kubectl.apply(crds_manager.get_resource(
        INSTANCE_CRD_SINGULAR, instance_id,
        extra_label_suffixes={'instance-type': instance_type},
        spec=values
    ), dry_run=dry_run)
    if instance_name:
        set_name(instance_id, instance_name, dry_run=dry_run)
    if update_:
        update(instance_id, wait_ready=wait_ready, skip_deployment=skip_deployment, skip_route=skip_route, force=force,
               dry_run=dry_run)

    return instance_id
コード例 #15
0
def _add_route(config, domains, route, enable_ssl_redirect):
    route_name = routes_manager.get_name(route)
    logs.info(f'adding route to nginx config: {route_name}')
    logs.debug_verbose(config=config,
                       domains=domains,
                       route=route,
                       enable_ssl_redirect=enable_ssl_redirect)
    backend_url = routes_manager.get_backend_url(route)
    frontend_hostname = routes_manager.get_frontend_hostname(route)
    print(f'F/B = {frontend_hostname} {backend_url}')
    root_domain, sub_domain = routes_manager.get_domain_parts(route)
    domains.setdefault(root_domain, []).append(sub_domain)
    # if route['spec'].get('extra-no-dns-subdomains'):
    #     extra_hostnames = ',' + ','.join([f'{s}.{root_domain}' for s in route['spec']['extra-no-dns-subdomains']])
    # else:
    extra_hostnames = ''
    logs.debug_verbose(route_name=route_name,
                       backend_url=backend_url,
                       frontend_hostname=frontend_hostname,
                       root_domain=root_domain,
                       sub_domain=sub_domain,
                       domains=domains,
                       extra_hostnames=extra_hostnames)
    if backend_url:
        raise NotImplementedError()
コード例 #16
0
def update(instance_id_or_name, override_spec_json, persist_overrides,
           wait_ready, skip_deployment, skip_route, force, override_spec_file,
           ckan_cloud_docker_latest_tag):
    """Update an instance to the latest resource spec, optionally applying the given json override to the resource spec

    Examples:

    ckan-cloud-operator ckan instance update <INSTANCE_ID_OR_NAME> '{"siteUrl": "http://localhost:5000"}' --wait-ready

    ckan-cloud-operator ckan instance update <INSTANCE_ID_OR_NAME> '{"replicas": 3}' --persist-overrides
    """
    override_spec = json.loads(
        override_spec_json) if override_spec_json else None
    if override_spec:
        assert not override_spec_file, "Can't specify both OVERRIDE_SPEC_JSON and override_spec_file"
    elif override_spec_file:
        with open(override_spec_file) as f:
            override_spec = yaml.safe_load(f)
    if ckan_cloud_docker_latest_tag:
        assert override_spec.get(
            "ckanImage"
        ), "ckanImage attribute is required in spec for --ckan-cloud-docker-latest-tag arg"
        if override_spec["ckanImage"].endswith(":latest"):
            override_spec["ckanImage"] = override_spec["ckanImage"].replace(
                ":latest", ":%s" % ckan_cloud_docker_latest_tag)
            logs.info("Replacing ckanImage to latest tag: %s" %
                      override_spec["ckanImage"])
    manager.update(instance_id_or_name,
                   override_spec=override_spec,
                   persist_overrides=persist_overrides,
                   wait_ready=wait_ready,
                   skip_deployment=skip_deployment,
                   skip_route=skip_route,
                   force=force)
    logs.exit_great_success()
コード例 #17
0
def get_provider(submodule,
                 required=True,
                 supported_provider_ids=None,
                 default=None,
                 verbose=False,
                 provider_id=None):
    if default: required = False
    if not provider_id:
        provider_id = get_provider_id(submodule, required=required)
        if not provider_id:
            if default:
                provider_id = default
            else:
                return None
        if verbose:
            logs.info(f'submodule {submodule} using provider_id {provider_id}')
    if not supported_provider_ids or provider_id in supported_provider_ids:
        provider_manager = _get_submodule_ids_provider_or_provider_ids(
            submodule, provider_id)
    else:
        provider_manager = None
    if provider_manager:
        return provider_manager
    else:
        msg = f' (supported provider ids: {supported_provider_ids})' if supported_provider_ids else ''
        logs.critical(
            f'Invalid submodule / provider: {submodule} / {provider_id}{msg}')
        raise Exception('failed to get provider')
コード例 #18
0
def create_all_backups(db_prefix=None, dry_run=False):
    logs.info('Fetching all database names')
    db_names = get_all_db_names(db_prefix=db_prefix)
    logs.info('{} DBs'.format(len(db_names)))
    for db in db_names:
        if db not in ['postgres'] and not db.startswith('template'):
            assert create_backup(db, db_prefix=db_prefix, dry_run=dry_run)
コード例 #19
0
def create_backup(database, connection_string=None, db_prefix=None):
    filename = f'{database}_' + datetime.datetime.now().strftime(
        '%Y%m%d%H%M') + '.gz'
    gs_url = os.path.join(
        _credentials_get(db_prefix, key='backups-gs-base-url', required=True),
        datetime.datetime.now().strftime('%Y/%m/%d/%H'), filename)
    if not connection_string:
        from ckan_cloud_operator.providers.db import manager as db_manager
        connection_string = db_manager.get_external_admin_connection_string(
            db_name=database)
    logs.info(f'Dumping DB: {filename}')
    subprocess.check_call([
        "bash",
        "-o",
        "pipefail",
        "-c",
        f"pg_dump -d {connection_string} --format=plain --no-owner --no-acl --schema=public | "
        f"sed -E 's/(DROP|CREATE|COMMENT ON) EXTENSION/-- \\1 EXTENSION/g' | "
        f"gzip -c > {filename}",
    ])
    subprocess.check_call(f'ls -lah {filename}', shell=True)
    logs.info(f'Copying to: {gs_url}')
    gcloud_driver.check_call(*_gcloud().get_project_zone(),
                             f'cp -m ./{filename} {gs_url} && rm {filename}',
                             gsutil=True)
コード例 #20
0
def _create_down_time_approval_code(instance_id, old_db_prefix, new_db_prefix,
                                    db_name, datastore_name):
    jenkins_user_token = ckan_manager.get_jenkins_token(
        'ckan-cloud-operator-jenkins-creds')
    backups_console_log = jenkins_driver.curl(
        *jenkins_user_token,
        'https://cc-p-jenkins.ckan.io/job/check%20individual%20DB%20backups/lastBuild/console',
        raw=True)
    backups_console_log = backups_console_log.split(
        'Finished: SUCCESS')[0].split('Updated property [compute/zone].')[1]
    backups = yaml.safe_load(backups_console_log)
    backups = {
        k: v
        for k, v in backups.items()
        if (k == db_name or k == datastore_name or k == f'{db_name}-total-size'
            or k == f'{datastore_name}-total-size')
    }
    logs.info('last instance backups')
    logs.print_yaml_dump(backups)
    return scripts_helpers.create_file_based_approval_code({
        'instance-id':
        instance_id,
        'old-db-prefix':
        old_db_prefix,
        'new-db-prefix':
        new_db_prefix,
        'db-name':
        db_name,
        'datastore-name':
        datastore_name
    })
コード例 #21
0
def _init_ckan_infra_secret(instance_id, dry_run=False):
    logs.debug('Initializing ckan infra secret', instance_id=instance_id)
    ckan_infra = config_manager.get(secret_name='ckan-infra',
                                    namespace=instance_id,
                                    required=False)
    if ckan_infra:
        logs.info('ckan-infra secret already exists')
    else:
        admin_user, admin_password, db_name = db_manager.get_admin_db_credentials(
        )
        db_host, db_port = db_manager.get_internal_unproxied_db_host_port()
        assert int(db_port) == 5432
        logs.debug('Creating ckan-infra secret',
                   admin_user=admin_user,
                   admin_password=admin_password,
                   db_name=db_name,
                   db_host=db_host,
                   db_port=db_port)
        config_manager.set(values={
            'POSTGRES_HOST': db_host,
            'POSTGRES_PASSWORD': admin_password,
            'POSTGRES_USER': admin_user
        },
                           secret_name='ckan-infra',
                           namespace=instance_id,
                           dry_run=dry_run)
コード例 #22
0
def _log_container_error(text, pod_name, container_name=None):
    heading = f"****** {text} ******"
    logs.info()
    logs.info(heading)
    logs.info(f'Pod: {pod_name} - Container: {container_name}')
    logs.info("-" * len(heading))
    logs.info()
コード例 #23
0
def _pre_update_hook_admin_user(instance,
                                sub_domain,
                                root_domain,
                                instance_id,
                                res,
                                dry_run=False):
    ckan_admin_email = instance['spec'].get('ckanAdminEmail')
    if not ckan_admin_email:
        ckan_admin_email = f'admin@{sub_domain}.{root_domain}'
    ckan_admin_password = config_manager.get(key='CKAN_ADMIN_PASSWORD',
                                             secret_name='ckan-admin-password',
                                             namespace=instance_id,
                                             required=False)
    if ckan_admin_password:
        logs.info('using existing ckan admin user')
        res['ckan-admin-password'] = ckan_admin_password
    else:
        logs.info('Will create new ckan admin user',
                  ckan_admin_email=ckan_admin_email)
        res['ckan-admin-email'] = ckan_admin_email
        res['ckan-admin-password'] = ckan_admin_password = binascii.hexlify(
            os.urandom(8)).decode()
        config_manager.set(key='CKAN_ADMIN_PASSWORD',
                           value=ckan_admin_password,
                           secret_name='ckan-admin-password',
                           namespace=instance_id,
                           dry_run=dry_run)
コード例 #24
0
def initialize(log_kwargs=None, interactive=False):
    log_kwargs = log_kwargs or {}
    logs.info(f'Installing crds', **log_kwargs)
    crds_manager.install_crd(CRD_SINGULAR,
                             CRD_PLURAL,
                             CRD_KIND,
                             hash_names=True)
    ckan_infra = CkanInfra(required=False)
    if interactive:
        providers_manager.config_interactive_set(
            PROVIDER_SUBMODULE,
            default_values={
                'gcloud-storage-import-bucket':
                ckan_infra.GCLOUD_SQL_DEIS_IMPORT_BUCKET
            },
            suffix='deis-migration')
    else:
        if not providers_manager.config_get(PROVIDER_SUBMODULE,
                                            key='gcloud-storage-import-bucket',
                                            suffix='deis-migration',
                                            required=False):
            providers_manager.config_set(
                PROVIDER_SUBMODULE,
                values={
                    'gcloud-storage-import-bucket':
                    ckan_infra.GCLOUD_SQL_DEIS_IMPORT_BUCKET
                },
                suffix='deis-migration')
コード例 #25
0
def get(routes,
        letsencrypt_cloudflare_email,
        enable_access_log=False,
        wildcard_ssl_domain=None,
        external_domains=False,
        dns_provider=None,
        force=False):
    if not dns_provider:
        dns_provider = 'cloudflare'
    logs.info('Generating traefik configuration',
              routes_len=len(routes) if routes else 0,
              letsencrypt_cloudflare_email=letsencrypt_cloudflare_email,
              enable_access_log=enable_access_log,
              wildcard_ssl_domain=wildcard_ssl_domain,
              external_domains=external_domains)
    config = _get_base_config(**({
        'accessLog': {
            "format": "json",
            "fields": {
                'defaultMode': "keep"
            }
        },
    } if enable_access_log else {}))
    domains = {}
    if dns_provider == 'cloudflare' and letsencrypt_cloudflare_email:
        enable_ssl_redirect = True
    elif dns_provider == 'route53':
        enable_ssl_redirect = True
    else:
        enable_ssl_redirect = False
    logs.info(enable_ssl_redirect=enable_ssl_redirect)
    logs.info('Adding routes')
    i = 0
    errors = 0
    for route in routes:
        try:
            _add_route(config, domains, route, enable_ssl_redirect)
            i += 1
        except Exception as e:
            if force:
                logs.error(traceback.format_exc())
                logs.error(str(e))
                errors += 1
            else:
                raise
    logs.info(f'Added {i} routes')
    if errors > 0:
        logs.warning(f'Encountered {errors} errors')
    if ((dns_provider == 'cloudflare' and letsencrypt_cloudflare_email)
            or (dns_provider == 'route53')):
        _add_letsencrypt(dns_provider,
                         config,
                         letsencrypt_cloudflare_email,
                         domains,
                         wildcard_ssl_domain=wildcard_ssl_domain,
                         external_domains=external_domains)
    else:
        logs.info('No valid dns_provider, will not setup SSL',
                  dns_provider=dns_provider)
    return config
コード例 #26
0
def _get_or_create_migration_db_passwords(migration_name,
                                          create_if_not_exists=True,
                                          skip_keys=None):
    passwords = []
    errors = []
    for password_config_key in [
            'database-password', 'datastore-password',
            'datastore-readonly-password'
    ]:
        if skip_keys is not None and password_config_key not in skip_keys:
            passwords.append(None)
            continue
        password = crds_manager.config_get(CRD_SINGULAR,
                                           migration_name,
                                           key=password_config_key,
                                           is_secret=True,
                                           required=False)
        if create_if_not_exists:
            if password:
                logs.info(f'Password already exists: {password_config_key}')
                errors.append(f'password-exists: {password_config_key}')
            else:
                logs.info(f'Generating new password for {password_config_key}')
                password = _generate_password()
                crds_manager.config_set(CRD_SINGULAR,
                                        migration_name,
                                        key=password_config_key,
                                        value=password,
                                        is_secret=True)
        passwords.append(password)
    return [errors] + passwords
コード例 #27
0
def solr_curl(path, required=False, debug=False, max_retries=15):
    deployment_name = _get_resource_name(_get_sc_suffixes()[0])
    if debug:
        kubectl.check_call(
            f'exec deployment-pod::{deployment_name} -- curl \'localhost:8983/solr{path}\'',
            use_first_pod=True)
    else:
        exitcode, output = kubectl.getstatusoutput(
            f'exec deployment-pod::{deployment_name} -- curl -s -f \'localhost:8983/solr{path}\'',
            use_first_pod=True)
        if exitcode == 0:
            return output
        elif required:
            if max_retries > 0:
                logs.info(
                    f'Failed to run solr curl: localhost:8983/solr{path} - retring in 30 seconds'
                )
                time.sleep(30)
                solr_curl(path,
                          required=required,
                          debug=debug,
                          max_retries=max_retries - 1)
            logs.critical(output)
            raise Exception(
                f'Failed to run solr curl: localhost:8983/solr{path}')
        else:
            logs.warning(output)
            return False
コード例 #28
0
def _update_db_proxy(db_name, datastore_name, datastore_ro_name, db_password,
                     datastore_password, datastore_ro_password):
    logs.info('Updating db proxy')
    db_proxy_manager.update(wait_updated=False)
    ok = False
    for i in range(5):
        try:
            for user, password, db in [
                (db_name, db_password, db_name),
                (datastore_name, datastore_password, datastore_name),
                (datastore_ro_name, datastore_ro_password, datastore_name)
            ]:
                if user:
                    db_manager.check_connection_string(
                        db_manager.get_external_connection_string(
                            user, password, db))
            ok = True
            break
        except Exception as e:
            logs.warning(str(e))
        logs.info(f'Waiting for connection to db proxy...')
        # 40 seconds on first iteration - to ensure secret is updated in pgbouncer volume
        time.sleep(40 if i == 0 else 1)
        db_proxy_manager.reload()
        time.sleep(10 if i == 2 else 5)
    assert ok, 'failed to get connection to db proxy'
    yield {
        'step':
        'update-db-proxy',
        'msg':
        f'Updated DB Proxy with the new dbs and roles: {db_name}, {datastore_name}, {datastore_ro_name}'
    }
コード例 #29
0
def create_volume(disk_size_gb, labels, use_existing_disk_name=None, zone=None):
    disk_id = use_existing_disk_name or 'cc' + _generate_password(12)
    if use_existing_disk_name:
        logs.info(f'using existing persistent disk {disk_id}')
    else:
        logs.info(f'creating persistent disk {disk_id} with size {disk_size_gb}')
        _, zone = get_project_zone()
        labels = ','.join([
            '{}={}'.format(k.replace('/', '_'), v.replace('/', '_')) for k, v in labels.items()
        ])
        gcloud_driver.check_call(*get_project_zone(), f'compute disks create {disk_id} --size={disk_size_gb}GB --zone={zone} --labels={labels}')
    kubectl.apply({
        'apiVersion': 'v1', 'kind': 'PersistentVolume',
        'metadata': {'name': disk_id, 'namespace': 'ckan-cloud'},
        'spec': {
            'storageClassName': '',
            'capacity': {'storage': f'{disk_size_gb}G'},
            'accessModes': ['ReadWriteOnce'],
            'gcePersistentDisk': {'pdName': disk_id}
        }
    })
    kubectl.apply({
        'apiVersion': 'v1', 'kind': 'PersistentVolumeClaim',
        'metadata': {'name': disk_id, 'namespace': 'ckan-cloud'},
        'spec': {
            'storageClassName': '',
            'volumeName': disk_id,
            'accessModes': ['ReadWriteOnce'],
            'resources': {'requests': {'storage': f'{disk_size_gb}G'}}
        }
    })
    return {'persistentVolumeClaim': {'claimName': disk_id}}
コード例 #30
0
def create(instance_type, instance_id=None, instance_name=None, values=None, values_filename=None, exists_ok=False, dry_run=False):
    if not instance_id:
        if instance_name:
            instance_id = '{}-{}'.format(instance_name, _generate_password(6))
        else:
            instance_id = _generate_password(12)
    if values_filename:
        assert values is None
        with open(values_filename) as f:
            values = yaml.load(f.read())
    if not exists_ok and crds_manager.get(INSTANCE_CRD_SINGULAR, name=instance_id, required=False):
        raise Exception('instance already exists')
    values_id = values.get('id')
    if values_id:
        assert values_id == instance_id, f'instance spec has conflicting instance_id ({values_id} != {instance_id})'
    values['id'] = instance_id
    logs.info('Creating instance', instance_id=instance_id)
    kubectl.apply(crds_manager.get_resource(
        INSTANCE_CRD_SINGULAR, instance_id,
        extra_label_suffixes={'instance-type': instance_type},
        spec=values
    ), dry_run=dry_run)
    if instance_name:
        set_name(instance_id, instance_name, dry_run=dry_run)
    return instance_id