def upgrade():
    """An upgrade has been triggered."""
    pki_directory = os.path.join(easyrsa_directory, "pki")
    if os.path.isdir(pki_directory):
        # specific handling if the upgrade is from a previous version
        # where certificate_authority_serial is not set at install
        serial_file = "serial"
        with chdir(pki_directory):
            # if the ca and ca_key are set and serial is not
            # set this to serial in the pki directory
            if (os.path.isfile(serial_file)
                    and leader_get("certificate_authority")
                    and leader_get("certificate_authority_key")
                    and not leader_get("certificate_authority_serial")):
                with open(serial_file, "r") as stream:
                    ca_serial = stream.read()
                # set the previously unset certificate authority serial
                leader_set({"certificate_authority_serial": ca_serial})

        charm_pki_directory = os.path.join(charm_directory, "pki")
        # When the charm pki directory exists, it is stale, remove it.
        if os.path.isdir(charm_pki_directory):
            shutil.rmtree(charm_pki_directory)
        # Copy the EasyRSA/pki to the charm pki directory.
        shutil.copytree(pki_directory, charm_pki_directory, symlinks=True)
    clear_flag("easyrsa.installed")
    clear_flag("easyrsa.configured")
示例#2
0
def prepare_end_user_package():
    """ Prepare the tarball package for clients to use to connet to the
        swarm cluster using the default client credentials. """

    # If we are a follower, we dont have keys and need to fetch them
    # from leader-data, which triggered `leadership.set.client_cert`
    # So it better be there!
    if not path.exists('swarm_credentials'):
        makedirs('swarm_credentials')
        with open('swarm_credentials/key.pem', 'w+') as fp:
            fp.write(leader_get('client_key'))
        with open('swarm_credentials/cert.pem', 'w+') as fp:
            fp.write(leader_get('client_cert'))
        with open('swarm_credentials/ca.pem', 'w+') as fp:
            fp.write(leader_get('certificate_authority'))

    # Render the client package script
    template_vars = {'public_address': unit_get('public-address')}
    render('enable.sh', './swarm_credentials/enable.sh', template_vars)

    # clear out any stale credentials package
    if path.exists('swarm_credentials.tar'):
        remove('swarm_credentials.tar')

    cmd = 'tar cvfz swarm_credentials.tar.gz swarm_credentials'
    subprocess.check_call(split(cmd))
    copyfile('swarm_credentials.tar.gz',
             '/home/ubuntu/swarm_credentials.tar.gz')
    set_state('client.credentials.placed')
示例#3
0
def prepare_end_user_package():
    """ Prepare the tarball package for clients to use to connet to the
        swarm cluster using the default client credentials. """

    # If we are a follower, we dont have keys and need to fetch them
    # from leader-data, which triggered `leadership.set.client_cert`
    # So it better be there!
    if not path.exists("swarm_credentials"):
        makedirs("swarm_credentials")
        with open("swarm_credentials/key.pem", "w+") as fp:
            fp.write(leader_get("client_key"))
        with open("swarm_credentials/cert.pem", "w+") as fp:
            fp.write(leader_get("client_cert"))
        with open("swarm_credentials/ca.pem", "w+") as fp:
            fp.write(leader_get("certificate_authority"))

    # Render the client package script
    template_vars = {"public_address": unit_get("public-address")}
    render("enable.sh", "./swarm_credentials/enable.sh", template_vars)

    # clear out any stale credentials package
    if path.exists("swarm_credentials.tar"):
        remove("swarm_credentials.tar")

    cmd = "tar cvfz swarm_credentials.tar.gz swarm_credentials"
    subprocess.check_call(split(cmd))
    copyfile("swarm_credentials.tar.gz", "/home/ubuntu/swarm_credentials.tar.gz")
    set_state("client.credentials.placed")
示例#4
0
def write_cert_secret():
    ''' Write returned certificate into a secret for the webhook.
    This data is also shared across the leadership data to other
    units. '''
    hookenv.status_set('maintenance', 'Writing certificates')

    cert = leader_get('cert').encode('utf-8')
    key = leader_get('key').encode('utf-8')

    context = {}
    context['namespace'] = namespace
    context['cert'] = base64.b64encode(cert).decode('utf-8')
    context['key'] = base64.b64encode(key).decode('utf-8')

    render('certs.yaml', secret_yaml, context)
    hookenv.log('Updating AWS-IAM secret.')
    try:
        _kubectl('apply', '-f', secret_yaml)
    except CalledProcessError as e:
        hookenv.log(e)
        hookenv.log(
            'Failed to create AWS_IAM secret. Will attempt again next update.'
        )  # noqa
        return

    set_flag('charm.aws-iam.certificate-written')
示例#5
0
文件: pdi.py 项目: tdeleeuw/layer-pdi
def render_slave_config():
    render('carte-config/slave.xml.j2', '/home/etl/carte-config.xml', {
        'carteslaveport': leader_get('port'),
        'carteslavehostname': hookenv.unit_private_ip(),
        'cartemasterhostname': leader_get('hostname'),
        'carteslavepassword': leader_get('password'),
        'cartemasterpassword': leader_get('password'),
        'cartemasterport': leader_get('port')
    })
示例#6
0
def send_ca(tls):
    '''The client relationship has been established, read the CA and client
    certificate from leadership data to set them on the relationship object.'''
    certificate_authority = leader_get('certificate_authority')
    tls.set_ca(certificate_authority)
    # The client cert and key should be same for all connections.
    client_cert = leader_get('client_certificate')
    client_key = leader_get('client_key')
    # Set the client certificate and key on the relationship object.
    tls.set_client_cert(client_cert, client_key)
示例#7
0
def send_ca(tls):
    '''The client relationship has been established, read the CA and client
    certificate from leadership data to set them on the relationship object.'''
    certificate_authority = leader_get('certificate_authority')
    tls.set_ca(certificate_authority)
    # The client cert and key should be same for all connections.
    client_cert = leader_get('client_certificate')
    client_key = leader_get('client_key')
    # Set the client certificate and key on the relationship object.
    tls.set_client_cert(client_cert, client_key)
示例#8
0
def write_webhook_yaml():
    ''' Write out the webhook yaml file for the api server to use.
    Everyone, including the leader, does this with leadership data
    set by the leader. '''
    hookenv.status_set('maintenance',
                       'Writing apiserver webhook configuration')
    context = {}
    cert = leader_get('cert').encode('utf-8')
    context['cert'] = base64.b64encode(cert).decode('utf-8')
    context['service_ip'] = leader_get('service_ip')
    render('webhook.yaml', webhook_path, context)
    aws_iam = endpoint_from_flag('endpoint.aws-iam.available')
    aws_iam.set_webhook_status(True)
    set_flag('charm.aws-iam.written-webhook')
示例#9
0
def setSlaveProperties():
    cid = leader_get('cluster')
    ats = leader_get('authtoken')
    directory = '/var/lib/dcos'
    if not os.path.exists(directory):
        os.makedirs(directory)
    f = open('/var/lib/dcos/cluster-id', 'w')
    f.write(cid)
    f.close()
    f = open('/var/lib/dcos/auth-token-secret', 'w')
    f.write(ats)
    f.close()
    startDCOS()
    set_state('dcos-master.running')
    status_set('active', 'DC/OS started')
示例#10
0
 def new_munge_consumer(self):
     remote_unit = hookenv.remote_unit()
     if remote_unit:
         mk = leadership.leader_get('munge_key')
         hookenv.log(
             'new_munge_consumer(): join event from %s, publishing key: %s'
             % (remote_unit, mk))
def process_snapd_timer():
    """
    Set the snapd refresh timer on the leader so all cluster members
    (present and future) will refresh near the same time.

    :return: None
    """
    # Get the current snapd refresh timer; we know layer-snap has set this
    # when the 'snap.refresh.set' flag is present.
    timer = snap.get(snapname="core",
                     key="refresh.timer").decode("utf-8").strip()
    if not timer:
        # The core snap timer is empty. This likely means a subordinate timer
        # reset ours. Try to set it back to a previously leader-set value,
        # falling back to config if needed. Luckily, this should only happen
        # during subordinate install, so this should remain stable afterward.
        timer = leader_get("snapd_refresh") or hookenv.config("snapd_refresh")
        snap.set_refresh_timer(timer)

        # Ensure we have the timer known by snapd (it may differ from config).
        timer = snap.get(snapname="core",
                         key="refresh.timer").decode("utf-8").strip()

    # The first time through, data_changed will be true. Subsequent calls
    # should only update leader data if something changed.
    if data_changed("snapd_refresh", timer):
        hookenv.log("setting leader snapd_refresh timer to: {}".format(timer))
        leader_set({"snapd_refresh": timer})
示例#12
0
def write_config_file():
    cc = hookenv.config()
    lc = leadership.leader_get()
    config = {
        "api-macaroon-timeout": cc["api-macaroon-timeout"],
        "discharge-macaroon-timeout": cc["discharge-macaroon-timeout"],
        "discharge-token-timeout": cc["discharge-token-timeout"],
        "enable-email-login": cc["enable-email-login"],
        "logging-config": cc["logging-config"],
        "private-addr": hookenv.unit_private_ip(),
        "rendezvous-timeout": cc["rendezvous-timeout"],
        "skip-location-for-cookie-paths": cc["skip-location-for-cookie-paths"],
    }
    if cc["admin-agent-public-key"]:
        config["admin-agent-public-key"] = cc["admin-agent-public-key"]
    if cc["http-proxy"]:
        config["http-proxy"] = cc["http-proxy"]
        # extend no-proxy to include all candid units.
        no_proxy = [cc["no-proxy"]]
        if not no_proxy[0]:
            no_proxy = no_proxy[1:]
        ep = endpoint_from_flag('candid.connected')
        if ep:
            no_proxy.extend(ep.addresses)
        config["no-proxy"] = ",".join(no_proxy)
    if cc["identity-providers"]:
        try:
            config["identity-providers"] = \
                candid.parse_identity_providers(cc["identity-providers"])
        except candid.IdentityProvidersParseError as e:
            hookenv.log("invalid identity providers: {}".format(e),
                        level="error")
    if cc["location"]:
        config["location"] = cc["location"]
    if cc["private-key"]:
        config["private-key"] = cc["private-key"]
    elif lc.get("private-key"):
        config["private-key"] = lc["private-key"]
    if cc["public-key"]:
        config["public-key"] = cc["public-key"]
    elif lc.get("public-key"):
        config["public-key"] = lc["public-key"]
    if cc["redirect-login-trusted-urls"]:
        config["redirect-login-trusted-urls"] = \
            _parse_list(cc["redirect-login-trusted-urls"])
    if cc["redirect-login-trusted-domains"]:
        config["redirect-login-trusted-domains"] = \
            _parse_list(cc["redirect-login-trusted-domains"])
    pg = endpoint_from_flag('postgres.master.available')
    if pg:
        config["storage"] = {
            "type": "postgres",
            "connection-string": str(pg.master),
        }
    else:
        config["storage"] = {"type": "memory"}

    candid.update_config(CONFIG_FILE, config)
    set_flag('candid.configured')
    set_flag('candid.restart')
示例#13
0
def enable_client_tls():
    """
    Copy the TLS certificates in place and generate mount points for the swarm
    manager to mount the certs. This enables client-side TLS security on the
    TCP service.
    """
    if not path.exists("/etc/docker"):
        makedirs("/etc/docker")

    kv = unitdata.kv()
    cert = kv.get("tls.server.certificate")
    with open("/etc/docker/server.pem", "w+") as f:
        f.write(cert)
    with open("/etc/docker/ca.pem", "w+") as f:
        f.write(leader_get("certificate_authority"))

    # schenanigans
    keypath = "easy-rsa/easyrsa3/pki/private/{}.key"
    server = getenv("JUJU_UNIT_NAME").replace("/", "_")
    if path.exists(keypath.format(server)):
        copyfile(keypath.format(server), "/etc/docker/server-key.pem")
    else:
        copyfile(keypath.format(unit_get("public-address")), "/etc/docker/server-key.pem")

    opts = DockerOpts()
    config_dir = "/etc/docker"
    cert_path = "{}/server.pem".format(config_dir)
    ca_path = "{}/ca.pem".format(config_dir)
    key_path = "{}/server-key.pem".format(config_dir)
    opts.add("tlscert", cert_path)
    opts.add("tlscacert", ca_path)
    opts.add("tlskey", key_path)
    opts.add("tlsverify", None)
    render("docker.defaults", "/etc/default/docker", {"opts": opts.to_s()})
示例#14
0
def controller_ip_port_list(cls):
    """List of ip:port pairs for Amphorae instances health reporting.

    The list is built based on information from individual Octavia units
    coordinated, stored and shared among all units trhough leader storage.

    :param cls: charms_openstack.adapters.ConfigurationAdapter derived class
                instance.  Charm class instance is at cls.charm_instance.
    :type: cls: charms_openstack.adapters.ConfiguartionAdapter
    :returns: Comma separated list of ip:port pairs.
    :rtype: str
    """

    # -> contrail addition
    return ch_net_ip.get_host_ip(ch_core.hookenv.network_get("public")["ingress-addresses"][0]) + ':' + OCTAVIA_HEALTH_LISTEN_PORT
    # <- contrail addition

    try:
        ip_list = json.loads(
            leadership.leader_get('controller-ip-port-list'))
    except TypeError:
        return
    if ip_list:
        port_suffix = ':' + OCTAVIA_HEALTH_LISTEN_PORT
        return (port_suffix + ', ').join(sorted(ip_list)) + port_suffix
示例#15
0
def enable_client_tls():
    """
    Copy the TLS certificates in place and generate mount points for the swarm
    manager to mount the certs. This enables client-side TLS security on the
    TCP service.
    """
    if not path.exists('/etc/docker'):
        makedirs('/etc/docker')

    kv = unitdata.kv()
    cert = kv.get('tls.server.certificate')
    with open('/etc/docker/server.pem', 'w+') as f:
        f.write(cert)
    with open('/etc/docker/ca.pem', 'w+') as f:
        f.write(leader_get('certificate_authority'))

    # schenanigans
    keypath = 'easy-rsa/easyrsa3/pki/private/{}.key'
    server = getenv('JUJU_UNIT_NAME').replace('/', '_')
    if path.exists(keypath.format(server)):
        copyfile(keypath.format(server), '/etc/docker/server-key.pem')
    else:
        copyfile(keypath.format(unit_get('public-address')),
                 '/etc/docker/server-key.pem')

    opts = DockerOpts()
    config_dir = '/etc/docker'
    cert_path = '{}/server.pem'.format(config_dir)
    ca_path = '{}/ca.pem'.format(config_dir)
    key_path = '{}/server-key.pem'.format(config_dir)
    opts.add('tlscert', cert_path)
    opts.add('tlscacert', ca_path)
    opts.add('tlskey', key_path)
    opts.add('tlsverify', None)
    render('docker.defaults', '/etc/default/docker', {'opts': opts.to_s()})
示例#16
0
def return_secrets(secrets=None):
    """Return secrets dict
    """

    conf = config()
    if secrets:
        secrets_mod = secrets
    else:
        secrets_mod = {}

    secrets_mod['redis_host'] = kv.get('redis_host')
    secrets_mod['redis_port'] = kv.get('redis_port')
    secrets_mod['postgresql_host'] = kv.get('postgresql_host')
    secrets_mod['postgresql_port'] = kv.get('postgresql_port')
    secrets_mod['postgresql_user'] = kv.get('postgresql_user')
    secrets_mod['postgresql_password'] = kv.get('postgresql_password')
    secrets_mod['postgresql_dbname'] = kv.get('postgresql_dbname')
    secrets_mod['system_secret_key'] = leader_get('system_secret_key')

    if conf.get('aws-key'):
        secrets_mod['AWS_KEY'] = config('aws-key')
    if conf.get('aws-secret'):
        secrets_mod['AWS_SECRET'] = config('aws-secret')
    if conf.get('aws-region'):
        secrets_mod['AWS_REGION'] = config('aws-region')

    if conf.get('secrets', ''):
        secrets_from_config = config('secrets').strip().split(",")
        for secret in secrets_from_config:
            s = secret.split("=")
            secrets_mod[s[0]] = s[1]
    return secrets_mod
示例#17
0
def get_seed_ips():
    '''Return the set of seed ip addresses.

    We use ip addresses rather than unit names, as we may need to use
    external seed ips at some point.
    '''
    return set((leadership.leader_get('seeds') or '').split(','))
def db_monitor_respond():
    """Response to db-monitor relation changed."""
    ch_core.hookenv.log("db-monitor connected", ch_core.hookenv.DEBUG)
    db_monitor = reactive.endpoint_from_flag("db-monitor.connected")

    # get related application name = user
    username = related_app = ch_core.hookenv.remote_service_name()

    # get or create db-monitor user password
    db_monitor_stored_passwd_key = "db-monitor.{}.passwd".format(related_app)
    password = leadership.leader_get(db_monitor_stored_passwd_key)
    if not password:
        password = ch_core.host.pwgen()
        leadership.leader_set({db_monitor_stored_passwd_key: password})

    # provide relation data
    with charm.provide_charm_instance() as instance:
        # NOTE (rgildein): Create a custom user with administrator privileges,
        # but read-only access.
        if not instance.create_cluster_user(db_monitor.relation_ip, username,
                                            password, True):
            ch_core.hookenv.log("db-monitor user was not created.",
                                ch_core.hookenv.WARNING)
            return

        db_monitor.provide_access(
            port=instance.cluster_port,
            user=username,
            password=password,
        )

        instance.assess_status()
def get_keys_from_leader(keys, overwrite_local=False):
    """
    Gets the broadcasted keys from the leader and stores them in
    the corresponding files.

    Args:
        keys: list of keys. Keys are actually files on the FS.

    Returns: True if all key were fetched, False if not.

    """
    # This races with other codepaths, and seems to require being created first
    # This block may be extracted later, but for now seems to work as intended
    os.makedirs('/root/cdk', exist_ok=True)

    for k in keys:
        # If the path does not exist, assume we need it
        if not os.path.exists(k) or overwrite_local:
            # Fetch data from leadership broadcast
            contents = leader_get(k)
            # Default to logging the warning and wait for leader data to be set
            if contents is None:
                msg = "Waiting on leaders crypto keys."
                hookenv.status_set('waiting', msg)
                hookenv.log('Missing content for file {}'.format(k))
                return False
            # Write out the file and move on to the next item
            with open(k, 'w+') as fp:
                fp.write(contents)
                fp.write('\n')

    return True
示例#20
0
def reinstall_spark():
    spark_master_host = leadership.leader_get('master-fqdn')
    peers = []
    zks = []
    if is_state('zookeeper.ready'):
        # if ZK is availuable we are in HA. We do not want reconfigurations if a leader fails
        # HA takes care of this
        spark_master_host = ''
        zk = RelationBase.from_state('zookeeper.ready')
        zks = zk.zookeepers()
        # We need reconfigure Spark when in HA and peers change ignore otherwise
        peers = get_spark_peers()

    deployment_matrix = {
        'spark_master': spark_master_host,
        'yarn_ready': is_state('hadoop.yarn.ready'),
        'hdfs_ready': is_state('hadoop.hdfs.ready'),
        'zookeepers': zks,
        'peers': peers,
    }

    if not data_changed('deployment_matrix', deployment_matrix):
        return

    hookenv.status_set('maintenance', 'configuring spark')
    hadoop = (RelationBase.from_state('hadoop.yarn.ready') or
              RelationBase.from_state('hadoop.hdfs.ready'))
    if install_spark(hadoop, zks):
        if is_state('hadoop.yarn.ready'):
            set_deployment_mode_state('spark.yarn.installed')
        else:
            set_deployment_mode_state('spark.standalone.installed')

        report_status()
示例#21
0
def install_spark_yarn():
    """
    Called in 'yarn-*' mode after Juju has elected a leader. The
    'hadoop.yarn.ready' state must be set.
    """
    hosts = {
        'spark-master': leadership.leader_get('master-fqdn'),
    }
    hadoop = (RelationBase.from_state('hadoop.yarn.ready')
              or RelationBase.from_state('hadoop.hdfs.ready'))
    rms = hadoop.resourcemanagers()
    hosts['resourcemanager'] = rms[0]

    # Probably don't need to check this since yarn.ready implies hdfs.ready
    # for us, but it doesn't hurt.
    if is_state('hadoop.hdfs.ready'):
        nns = hadoop.namenodes()
        hosts['namenode'] = nns[0]

    # Always include native hadoop libs in yarn mode; add cuda libs if present.
    extra_libs = ["/usr/lib/hadoop/lib/native"]
    if is_state('cuda.installed'):
        extra_libs.append("/usr/local/cuda/lib64")

    spark = Spark()
    spark.configure(hosts, zk_units=None, peers=None, extra_libs=extra_libs)
    set_deployment_mode_state('spark.yarn.installed')
示例#22
0
def end_admin_party(config_path='/etc/couchdb'):
    """
    Couch starts out in 'admin party' mode, which means that anyone
    can create and edit databases. This routine secures couch, and
    flags us to restart.

    @param str config_path: The location of the config files in the system.

    """
    log("Ending the admin party.", DEBUG)
    _maybe_generate_passwords()

    passwords = json.loads(leader_get('passwords'))

    entries = [
        {'section': 'admins', 'key': 'admin', 'value': passwords['admin_pass']},
        {'section': 'admins', 'key': 'replication', 'value': passwords['repl_pass']},
        {'section': 'couch_httpd_auth', 'key': 'require_valid_user', 'value': 'true'},
    ]
    if config("human-auditable-creds"):
        entries += [
            {'section': 'juju_notes', 'key': 'admin_pass', 'value': passwords['admin_pass']},
            {'section': 'juju_notes', 'key': 'repl_pass', 'value': passwords['repl_pass']},
        ]
    _write_config(config_path, 'local', entries)

    remove_state('couchdb.admin_party')
示例#23
0
def install_spark_standalone(zks, peers):
    """
    Called in local/standalone mode after Juju has elected a leader.
    """
    hosts = {
        'spark-master': leadership.leader_get('master-fqdn'),
    }

    # If zks have changed and we are not handling a departed spark peer,
    # give the ensemble time to settle. Otherwise we might try to start
    # spark master with data from the wrong zk leader. Doing so will cause
    # spark-master to shutdown:
    #  https://issues.apache.org/jira/browse/SPARK-15544
    if (zks and data_changed('zks', zks)
            and not is_state('sparkpeers.departed')):
        hookenv.status_set('maintenance',
                           'waiting for zookeeper ensemble to settle')
        hookenv.log(
            "Waiting 2m to ensure zk ensemble has settled: {}".format(zks))
        time.sleep(120)

    # Let spark know if we have cuda libs installed.
    # NB: spark packages prereq hadoop (boo), so even in standalone mode, we'll
    # have hadoop libs installed. May as well include them in our lib path.
    extra_libs = ["/usr/lib/hadoop/lib/native"]
    if is_state('cuda.installed'):
        extra_libs.append("/usr/local/cuda/lib64")

    spark = Spark()
    spark.configure(hosts, zk_units=zks, peers=peers, extra_libs=extra_libs)
    set_deployment_mode_state('spark.standalone.installed')
示例#24
0
文件: spark.py 项目: apache/bigtop
def install_spark_standalone(zks, peers):
    """
    Called in local/standalone mode after Juju has elected a leader.
    """
    hosts = {
        'spark-master': leadership.leader_get('master-fqdn'),
    }

    # If zks have changed and we are not handling a departed spark peer,
    # give the ensemble time to settle. Otherwise we might try to start
    # spark master with data from the wrong zk leader. Doing so will cause
    # spark-master to shutdown:
    #  https://issues.apache.org/jira/browse/SPARK-15544
    if (zks and data_changed('zks', zks) and not is_state('sparkpeers.departed')):
        hookenv.status_set('maintenance',
                           'waiting for zookeeper ensemble to settle')
        hookenv.log("Waiting 2m to ensure zk ensemble has settled: {}".format(zks))
        time.sleep(120)

    # Let spark know if we have cuda libs installed.
    # NB: spark packages prereq hadoop (boo), so even in standalone mode, we'll
    # have hadoop libs installed. May as well include them in our lib path.
    extra_libs = ["/usr/lib/hadoop/lib/native"]
    if is_state('cuda.installed'):
        extra_libs.append("/usr/local/cuda/lib64")

    spark = Spark()
    spark.configure(hosts, zk_units=zks, peers=peers, extra_libs=extra_libs)
    set_deployment_mode_state('spark.standalone.installed')
示例#25
0
文件: spark.py 项目: apache/bigtop
def install_spark_yarn():
    """
    Called in 'yarn-*' mode after Juju has elected a leader. The
    'hadoop.yarn.ready' state must be set.
    """
    hosts = {
        'spark-master': leadership.leader_get('master-fqdn'),
    }
    hadoop = (RelationBase.from_state('hadoop.yarn.ready') or
              RelationBase.from_state('hadoop.hdfs.ready'))
    rms = hadoop.resourcemanagers()
    hosts['resourcemanager'] = rms[0]

    # Probably don't need to check this since yarn.ready implies hdfs.ready
    # for us, but it doesn't hurt.
    if is_state('hadoop.hdfs.ready'):
        nns = hadoop.namenodes()
        hosts['namenode'] = nns[0]

    # Always include native hadoop libs in yarn mode; add cuda libs if present.
    extra_libs = ["/usr/lib/hadoop/lib/native"]
    if is_state('cuda.installed'):
        extra_libs.append("/usr/local/cuda/lib64")

    spark = Spark()
    spark.configure(hosts, zk_units=None, peers=None, extra_libs=extra_libs)
    set_deployment_mode_state('spark.yarn.installed')
示例#26
0
def return_secrets(secrets=None):
    """Return secrets dict
    """

    if secrets:
        secrets_mod = secrets
    else:
        secrets_mod = {}

    for k in ('redis_host',
              'redis_port',
              'postgresql_host',
              'postgresql_port',
              'postgresql_user',
              'postgresql_password',
              'postgresql_dbname',
              'github_app_id',
              'github_api_secret',
              'github_extended_permissions',
              'social_auth_redirect_is_https',
              'email_server_host',
              'email_server_port',
              'email_server_username',
              'email_server_password',
              'email_server_tls',
              'email_from',
              'enable_statsd',
              'beacon'):
        secrets_mod[k] = kv.get(k)

    secrets_mod['system_secret_key'] = leader_get('system_secret_key')

    return secrets_mod
示例#27
0
def get_keys_from_leader(keys, overwrite_local=False):
    """
    Gets the broadcasted keys from the leader and stores them in
    the corresponding files.

    Args:
        keys: list of keys. Keys are actually files on the FS.

    Returns: True if all key were fetched, False if not.

    """
    # This races with other codepaths, and seems to require being created first
    # This block may be extracted later, but for now seems to work as intended
    os.makedirs('/root/cdk', exist_ok=True)

    for k in keys:
        # If the path does not exist, assume we need it
        if not os.path.exists(k) or overwrite_local:
            # Fetch data from leadership broadcast
            contents = leader_get(k)
            # Default to logging the warning and wait for leader data to be set
            if contents is None:
                msg = "Waiting on leaders crypto keys."
                hookenv.status_set('waiting', msg)
                hookenv.log('Missing content for file {}'.format(k))
                return False
            # Write out the file and move on to the next item
            with open(k, 'w+') as fp:
                fp.write(contents)
                fp.write('\n')

    return True
示例#28
0
def provide_munge_key_to_interface(munge_provider):
    '''Provide munge key if any consumers are related and if '''
    munge_key = leadership.leader_get('munge_key')
    hookenv.log('provide_munge_key_to_interface(): exposing munge key: %s' %
                munge_key)
    munge_provider.expose_munge_key(munge_key)
    munge_provider.provide_munge_key()
    flags.set_flag('munge.exposed')
示例#29
0
def configure_munge_key():
    munge_key = leadership.leader_get('munge_key')
    munge.render_munge_key(context={'munge_key': munge_key})
    hookenv.log(
        'configure_munge_key(): leadership detected new munge key, rendered new file'
    )
    # set a flag confirming that munge key is rendered
    flags.set_flag('munge.configured')
示例#30
0
def _maybe_generate_passwords():
    """
    If the leader hasn't generated passwords yet, generate them.

    """
    if not leader_get('passwords'):
        admin_pass = subprocess.check_output(['pwgen', '-N1']).strip().decode('utf-8')
        repl_pass = subprocess.check_output(['pwgen', '-N1']).strip().decode('utf-8')
        leader_set(passwords=json.dumps({'admin_pass': admin_pass, 'repl_pass': repl_pass}))
示例#31
0
def update_recovery_conf(follow):
    assert follow != hookenv.local_unit()

    peer_rel = helpers.get_peer_relation()
    follow_relinfo = peer_rel.get(follow)
    assert follow_relinfo is not None, "Invalid upstream {}".format(follow)

    current_follow = get_following()

    if follow != current_follow:
        status_set("maintenance", "Following new unit {}".format(follow))
        set_following(follow)
        # Setting the state to defer publication until after restart.
        reactive.set_state("postgresql.replication.publish_following")

    else:
        # Even though the master is unchanged, we still regenerate
        # recovery.conf in case connection details such as IP addresses
        # have changed.
        hookenv.log("Continuing to follow {}".format(follow))

    pg12 = postgresql.has_version("12")
    if pg12:
        path = postgresql.hot_standby_conf_path()
        template = "hot_standby.conf.tmpl"
    else:
        path = postgresql.recovery_conf_path()
        template = "recovery.conf.tmpl"

    config = hookenv.config()

    data = dict(
        streaming_replication=config["streaming_replication"],
        host=follow_relinfo["host"],
        port=follow_relinfo["port"],
        user=replication_username(),
        password=leader_get("replication_password"),
    )

    if reactive.helpers.is_state("postgresql.wal_e.enabled"):
        data["restore_command"] = wal_e.wal_e_restore_command()

    templating.render(template,
                      path,
                      data,
                      owner="postgres",
                      group="postgres",
                      perms=0o600)

    if pg12:
        touch(postgresql.hot_standby_signal_path())

    # Use @when_file_changed for this when Issue #44 is resolved.
    if reactive.helpers.any_file_changed([path]):
        reactive.set_state("postgresql.cluster.needs_restart")
        if reactive.is_state("postgresql.replication.cloned"):
            reactive.set_state("postgresql.replication.check_following")
示例#32
0
    def inform_restart(self):
        '''
        Inform our peers that we have restarted, usually as part of a
        rolling restart.

        '''
        for conv in self.conversations():
            nonce = leader_get('restart_nonce')
            conv.set_remote('restarted.{}'.format(nonce), json.dumps(True))
def set_snapd_timer():
    ''' Set the snapd refresh.timer on non-leader cluster members. '''
    # NB: This method should only be run when 'snap.refresh.set' is present.
    # Layer-snap will always set a core refresh.timer, which may not be the
    # same as our leader. Gating with 'snap.refresh.set' ensures layer-snap
    # has finished and we are free to set our config to the leader's timer.
    timer = leadership.leader_get('snapd_refresh') or ''  # None will error
    hookenv.log('setting snapd_refresh timer to: {}'.format(timer))
    snap.set_refresh_timer(timer)
示例#34
0
def create_replication_user():
    username = replication_username()
    hookenv.log("Creating replication user {}".format(username))
    con = postgresql.connect()
    postgresql.ensure_user(con,
                           username,
                           leader_get("replication_password"),
                           replication=True)
    con.commit()
    reactive.set_state("postgresql.replication.replication_user_created")
示例#35
0
    def restarted_nodes(self):
        nodes = []
        nonce = leader_get('restart_nonce')
        if not nonce:
            return nodes  # We're not restarting if no nonce is set.
        for conv in self.conversations():
            if conv.get_remote('restarted.{}'.format(nonce)):
                nodes.append((conv.scope, conv.get_remote('private-address')))

        return nodes
示例#36
0
    def states_to_check(self, required_relations=None):
        """Custom state check function for charm specific state check needs.

        Interface used for ``neutron_openvswitch`` subordinate lacks a
        ``available`` state.

        The ``Octavia`` service will not operate normally until Nova and
        Neutron resources have been created, this needs to be tracked in
        workload status.
        """
        states_to_check = super().states_to_check(required_relations)
        override_relation = 'neutron-openvswitch'
        if override_relation in states_to_check:
            states_to_check[override_relation] = [
                ("{}.connected".format(override_relation), "blocked",
                 "'{}' missing".format(override_relation))
            ]
        if not leadership.leader_get('amp-boot-network-list'):
            if not reactive.is_flag_set('config.default.create-mgmt-network'):
                # we are configured to not create required resources and they
                # are not present, prompt end-user to create them.
                states_to_check['crud'] = [(
                    'crud.available',  # imaginate ``crud`` relation
                    'blocked',
                    'Awaiting end-user to create required resources and '
                    'execute `configure-resources` action')]
            else:
                if reactive.is_flag_set('leadership.is_leader'):
                    who = 'end-user execution of `configure-resources` action'
                else:
                    who = 'leader'
                states_to_check['crud'] = [(
                    'crud.available',  # imaginate ``crud`` relation
                    'blocked',
                    'Awaiting {} to create required resources'.format(who))]
        # if these configuration options are at default value it means they are
        # not set by end-user, they are required for successfull creation of
        # load balancer instances.
        if (reactive.is_flag_set('config.default.lb-mgmt-issuing-cacert')
                or reactive.is_flag_set(
                    'config.default.lb-mgmt-issuing-ca-private-key')
                or reactive.is_flag_set(
                    'config.default.lb-mgmt-issuing-ca-key-passphrase')
                or reactive.is_flag_set(
                    'config.default.lb-mgmt-controller-cacert')
                or reactive.is_flag_set(
                    'config.default.lb-mgmt-controller-cert')):
            # set workload status to prompt end-user attention
            states_to_check['config'] = [(
                'config._required_certs',  # imaginate flag
                'blocked',
                'Missing required certificate configuration, please '
                'examine documentation')]
        return states_to_check
示例#37
0
def write_cert_to_leadership_data():
    cert_ep = endpoint_from_flag('certificates.certs.available')
    my_cert = cert_ep.server_certs_map[_get_cert_common_name()]

    leader_set({'cert': my_cert.cert, 'key': my_cert.key})
    # we also use this time to generate the cluster id
    if not leader_get('cluster_id'):
        cluster_id = ''.join(
            random.choice(string.ascii_letters + string.digits)
            for i in range(24))
        leader_set({'cluster_id': cluster_id})
示例#38
0
def create_global_client_cert():
    """
    This is for backwards compatibility with older tls-certificate clients
    only.  Obviously, it's not good security / design to have clients sharing
    a certificate, but it seems that there are clients that depend on this
    (though some, like etcd, only block on the flag that it triggers but don't
    actually use the cert), so we have to set it for now.
    """
    client_cert = leader_get("client_certificate")
    client_key = leader_get("client_key")
    if not client_cert or not client_key:
        hookenv.log("Unable to find global client cert on "
                    "leadership data, generating...")
        client_cert, client_key = create_client_certificate()
        # Set the client certificate and key on leadership data.
        leader_set({"client_certificate": client_cert})
        leader_set({"client_key": client_key})
    else:
        hookenv.log("found global client cert on leadership "
                    "data, not generating...")
    set_flag("easyrsa.global-client-cert.created")
示例#39
0
def upgrade_charm():
    remove_state('calico.binaries.installed')
    remove_state('calico.service.installed')
    remove_state('calico.pool.configured')
    remove_state('calico.image.pulled')
    remove_state('calico.npc.deployed')
    if is_leader() and not leader_get('calico-v3-data-ready'):
        leader_set({
            'calico-v3-data-migration-needed': True,
            'calico-v3-npc-cleanup-needed': True,
            'calico-v3-completion-needed': True
        })
示例#40
0
def mirror_credentials(rel):
    pub = rel.to_publish_raw
    config = cassandra.config()
    if config['authenticator'].lower() == 'allowallauthenticator':
        if 'username' in pub:
            del pub['username']
            del pub['password']
        return
    source_unit = leadership.leader_get('client_rel_source')
    source_data = hookenv.relation_get(unit=source_unit, rid=rel.relation_id)
    if source_data and 'username' in source_data:
        hookenv.log("Mirroring credentials for {} ({}) from {}".format(rel.application_name,
                                                                       rel.relation_id, source_unit))
        pub['username'] = source_data['username']
        pub['password'] = source_data['password']
示例#41
0
def update_restart_queue(zkpeer):
    '''
    If a Zookeeper node has restarted as part of a rolling restart,
    pop it off of the queue.

    '''
    queue = json.loads(leader_get('restart_queue') or '[]')
    if not queue:
        return

    restarted_nodes = _ip_list(zkpeer.restarted_nodes())
    new_queue = [node for node in queue if node not in restarted_nodes]

    if new_queue != queue:
        hookenv.log('Leader updating restart queue: {}'.format(queue))
        leader_set(restart_queue=json.dumps(new_queue))
示例#42
0
def db_relation_joined():
    """
    Hook to run when somebody connects to us.

    """

    passwords = json.loads(leader_get('passwords'))  # TODO: Exception handling.

    relation_set(
        host=config('couchdb-host'),
        ip=unit_public_ip(),
        port=config('couchdb-port'),
        admin_pass=passwords['admin_pass'],
        repl_pass=passwords['repl_pass']
    )

    log("{} joined".format(os.getenv('ENSEMBLE_REMOTE_UNIT')), INFO)
def install_spark(hadoop=None):
    spark_master_host = leadership.leader_get('master-fqdn')
    hosts = {
        'spark-master': spark_master_host,
    }

    if is_state('hadoop.yarn.ready'):
        rms = hadoop.resourcemanagers()
        hosts['resourcemanager'] = rms[0]

    if is_state('hadoop.hdfs.ready'):
        nns = hadoop.namenodes()
        hosts['namenode'] = nns[0]

    dist = get_dist_config()
    spark = Spark(dist)
    spark.configure(hosts)
示例#44
0
文件: pdi.py 项目: OSBI/layer-pdi
def start():
    currentenv = dict(os.environ)
    port = hookenv.config('carte_port')
    javaopts = hookenv.config('java_opts')

    if javaopts:
        currentenv['JAVA_OPTS'] = javaopts

    try:
        check_call(['pgrep', '-f', 'org.pentaho.di.www.Carte'])
    except CalledProcessError:
        check_call(['su', 'etl', '-c',
                    '/opt/data-integration/carte.sh '
                    '/home/etl/carte-config.xml &'],
                   env=currentenv, cwd="/opt/data-integration")

    hookenv.open_port(port)
    status_set('active',
               'Carte is ready! Master is:' + leader_get('public_ip'))
示例#45
0
def etcd_data_change(etcd):
    ''' Etcd scale events block master reconfiguration due to the
        kubernetes-master.components.started state. We need a way to
        handle these events consistenly only when the number of etcd
        units has actually changed '''

    # key off of the connection string
    connection_string = etcd.get_connection_string()

    # If the connection string changes, remove the started state to trigger
    # handling of the master components
    if data_changed('etcd-connect', connection_string):
        remove_state('kubernetes-master.components.started')

    # We are the leader and the etcd_version is not set meaning
    # this is the first time we connect to etcd.
    if is_state('leadership.is_leader') and not leader_get('etcd_version'):
        if etcd.get_version().startswith('3.'):
            leader_set(etcd_version='etcd3')
        else:
            leader_set(etcd_version='etcd2')
示例#46
0
文件: spark.py 项目: Guavus/bigtop
def install_spark(hadoop=None, zks=None):
    spark_master_host = leadership.leader_get('master-fqdn')
    if not spark_master_host:
        hookenv.status_set('waiting', 'master not elected yet')
        return False

    hosts = {
        'spark-master': spark_master_host,
    }

    if is_state('hadoop.yarn.ready'):
        rms = hadoop.resourcemanagers()
        hosts['resourcemanager'] = rms[0]

    if is_state('hadoop.hdfs.ready'):
        nns = hadoop.namenodes()
        hosts['namenode'] = nns[0]

    spark = Spark()
    spark.configure(hosts, zks, get_spark_peers())
    return True
def reinstall_spark():
    spark_master_host = leadership.leader_get('master-fqdn')
    deployment_matrix = {
        'spark_master': spark_master_host,
        'yarn_ready': is_state('hadoop.yarn.ready'),
        'hdfs_ready': is_state('hadoop.hdfs.ready'),
    }

    if not data_changed('deployment_matrix', deployment_matrix):
        return

    hookenv.status_set('maintenance', 'Configuring Spark')
    hadoop = (RelationBase.from_state('hadoop.yarn.ready') or
              RelationBase.from_state('hadoop.hdfs.ready'))
    install_spark(hadoop)
    if is_state('hadoop.yarn.ready'):
        set_deployment_mode_state('spark.yarn.installed')
    else:
        set_deployment_mode_state('spark.standalone.installed')

    report_status()
示例#48
0
def start_master(etcd):
    '''Run the Kubernetes master components.'''
    hookenv.status_set('maintenance',
                       'Configuring the Kubernetes master services.')
    freeze_service_cidr()
    if not etcd.get_connection_string():
        # etcd is not returning a connection string. This happens when
        # the master unit disconnects from etcd and is ready to terminate.
        # No point in trying to start master services and fail. Just return.
        return

    # TODO: Make sure below relation is handled on change
    # https://github.com/kubernetes/kubernetes/issues/43461
    handle_etcd_relation(etcd)

    # Add CLI options to all components
    leader_etcd_version = leader_get('etcd_version')
    configure_apiserver(etcd.get_connection_string(), leader_etcd_version)
    configure_controller_manager()
    configure_scheduler()
    set_state('kubernetes-master.components.started')
    hookenv.open_port(6443)
示例#49
0
def restart_for_quorum(zkpeer):
    '''
    If we're the next node in the restart queue, restart, and then
    inform the leader that we've restarted. (If we are the leader,
    remove ourselves from the queue, and update the leadership data.)

    '''
    private_address = hookenv.unit_get('private-address')
    queue = json.loads(leader_get('restart_queue') or '[]')

    if not queue:
        # Everything has restarted.
        return

    if private_address == queue[0]:
        # It's our turn to restart.
        _restart_zookeeper('rolling restart for quorum update')
        if is_state('leadership.is_leader'):
            queue = queue[1:]
            hookenv.log('Leader updating restart queue: {}'.format(queue))
            leader_set(restart_queue=json.dumps(queue))
        else:
            zkpeer.inform_restart()
示例#50
0
def check_for_upgrade_needed():
    '''An upgrade charm event was triggered by Juju, react to that here.'''
    hookenv.status_set('maintenance', 'Checking resources')

    migrate_from_pre_snaps()
    add_rbac_roles()
    set_state('reconfigure.authentication.setup')
    remove_state('authentication.setup')
    changed = snap_resources_changed()
    if changed == 'yes':
        set_upgrade_needed()
    elif changed == 'unknown':
        # We are here on an upgrade from non-rolling master
        # Since this upgrade might also include resource updates eg
        # juju upgrade-charm kubernetes-master --resource kube-any=my.snap
        # we take no risk and forcibly upgrade the snaps.
        # Forcibly means we do not prompt the user to call the upgrade action.
        set_upgrade_needed(forced=True)

    # Set the auto storage backend to etcd2.
    auto_storage_backend = leader_get('auto_storage_backend')
    is_leader = is_state('leadership.is_leader')
    if not auto_storage_backend and is_leader:
        leader_set(auto_storage_backend='etcd2')
示例#51
0
def upgrade_for_etcd():
    # we are upgrading the charm.
    # If this is an old deployment etcd_version is not set
    # so if we are the leader we need to set it to v2
    if not leader_get('etcd_version') and is_state('leadership.is_leader'):
        leader_set(etcd_version='etcd2')
def get_cluster_nodes():
    return json.loads(leadership.leader_get('cluster-nodes') or '[]')
示例#53
0
def on_config_api_extra_args_change(etcd):
    configure_apiserver(etcd.get_connection_string(),
                        leader_get('etcd_version'))
示例#54
0
def getStorageBackend():
    storage_backend = hookenv.config('storage-backend')
    if storage_backend == 'auto':
        storage_backend = leader_get('auto_storage_backend')
    return storage_backend
def send_ssh_key(datanode):
    datanode.send_ssh_key(leadership.leader_get('ssh-key-pub'))
示例#56
0
文件: pdi.py 项目: OSBI/layer-pdi
def render_master_config():
    render('carte-config/master.xml.j2', '/home/etl/carte-config.xml', {
        'carteport': leader_get('port'),
        'cartehostname': hookenv.unit_private_ip()
    })