Пример #1
0
def provision(config):
    try:
        # Open a connection to CM and get a CM object
        api = util.get_api_handle(config)

        # Get dictionary of hostname to host ids
        host_id_map = util.host_id_map(config, api)

        # Update CM config
        cm.update_config(config, api)

        # Update CM license
        cm.update_license(config, api)

        # Update all hosts configuration
        cm.update_hosts_config(config, api)

        # Create Cloudera Management Services
        mgmt.add_or_update(config, api, host_id_map)

        # Add cluster and services
        cluster.add_or_update(config, api, host_id_map)

    except ApiException, e:
        logging.error("Error: %s" % e)
Пример #2
0
def enable_rmha(conf):

    ha_hosts_map = {}
    standby_rm_key = "StandbyResourceManager"

    try:
        # Open a connection to CM and get a CM object
        api = util.get_api_handle(conf)
        cm = api.get_cloudera_manager()
        cl = None
        if 'cluster' in conf and 'name' in conf['cluster']:
            cl = api.get_cluster(conf['cluster']['name'])
        else:
            raise ProvisionatorException("No cluster specified")


        host_id_map = util.host_id_map(config, api)

        #todo: check if enabled already
        yarn_svc = config.get_cluster_service_by_type(conf, 'YARN')
        yarn = cl.get_service(yarn_svc['name'])
        standby_rm = yarn_svc['haconfig']['standby_rm']
        ha_hosts_map[standby_rm_key] = standby_rm

        LOG.info("host_id_map: " + str(host_id_map))
        LOG.info("ha_host_map: " + str(ha_hosts_map))
            for ha_host in ha_hosts_map:
            LOG.info ("ha_hosts_map[ha_host]: " + ha_hosts_map[ha_host])
            try:
                LOG.info("Host ID for " + ha_host + ": "+ host_id_map[ha_hosts_map[ha_host]])
            except KeyError:
                LOG.error("Cannot find host " + ha_hosts_map[ha_host] + " for role \""
                    + ha_host + "\" in list of hosts (" + str(list(host_id_map)) + ")")
                return

        zk_svc = config.get_cluster_service_by_type(conf, 'ZOOKEEPER')
        zk_svc_name = zk_svc['name']

        LOG.info("Running YARN HA configuration now...")
        # new_rm_host_id - ID of host where Standby Resource Manager will be created.
        # zk_service_name - Name of the ZooKeeper service to use for auto-failover.
        cmd = yarn.enable_rm_ha(
            host_id_map[standby_rm],
            zk_service_name=zk_svc_name
        )

        util.wait_for_command(cmd, True)
        #TODO: in some cases time.sleep(x) is needed to avoid race conditions
        try:
            mgmt_svc = cm.get_service()
            LOG.info("Restarting management services")
            mgmt.restart(cm)
        except ApiException:
	        pass

        LOG.info("Restarting cluster")
        cluster.restart(cl)
    except ApiException, e:
        raise ProvisionatorException(e)
Пример #3
0
def enable_kerberos(conf):
    try:
        # Open a connection to CM and get a CM object
        api = util.get_api_handle(conf)
        cm = api.get_cloudera_manager()
        cl = None
        if 'cluster' in conf and 'name' in conf['cluster']:
            cl = api.get_cluster(conf['cluster']['name'])
        else:
            raise ProvisionatorException("No cluster specified")

        # Check the HDFS service to see if Kerberos is already enabled
        hdfs_svc = config.get_cluster_service_by_type(conf, 'HDFS')
        hdfs = cl.get_service(hdfs_svc['name'])
        hdfs_cfg, hdfs_roletype_cfg = hdfs.get_config(view='full')

        if not hdfs_cfg['hadoop_security_authentication'].value == 'kerberos':
            # Kerberos has not been enabled - add the KDC creds
            if 'kdc_user' in conf['cm'] and 'kdc_pass' in conf['cm']:
                check_creds(conf['cm']['kdc_user'], conf['cm']['kdc_pass'])
                LOG.info("Importing credentials for administrative account")
                cmd = cm.import_admin_credentials(conf['cm']['kdc_user'],
                                                  conf['cm']['kdc_pass'])
                util.wait_for_command(cmd, True)

            # OK let's do this
            datanode_transceiver_port = DATANODE_TRANSCEIVER_PORT
            datanode_web_port = DATANODE_WEB_PORT

            cm_cfg = cm.get_config()
            if 'SINGLE_USER_ENABLED' in cm_cfg:
                # TODO: don't hardcode this
                datanode_transceiver_port = 4004
                datanode_web_port = 4006

            LOG.info("Running Kerberos configuration now...")
            cmd = cl.configure_for_kerberos(
                datanode_transceiver_port=datanode_transceiver_port,
                datanode_web_port=datanode_web_port)

            util.wait_for_command(cmd, True)
            #TODO: in some cases time.sleep(x) is needed to avoid race conditions
            try:
                mgmt_svc = cm.get_service()
                LOG.info("Restarting management services")
                mgmt.restart(cm)
            except ApiException:
                pass
            LOG.info("Restarting cluster")
            cluster.restart(cl)
        else:
            LOG.info("Kerberos already enabled")
    except ApiException, e:
        raise ProvisionatorException(e)
Пример #4
0
def enable_kerberos(conf):
    try:
        # Open a connection to CM and get a CM object
        api = util.get_api_handle(conf)
        cm = api.get_cloudera_manager()
        cl = None
        if 'cluster' in conf and 'name' in conf['cluster']:
            cl = api.get_cluster(conf['cluster']['name'])
        else:
            raise ProvisionatorException("No cluster specified")

        # Check the HDFS service to see if Kerberos is already enabled
        hdfs_svc = config.get_cluster_service_by_type(conf, 'HDFS')
        hdfs = cl.get_service(hdfs_svc['name'])
        hdfs_cfg, hdfs_roletype_cfg = hdfs.get_config(view='full')

        if not hdfs_cfg['hadoop_security_authentication'].value == 'kerberos':
            # Kerberos has not been enabled - add the KDC creds
            if 'kdc_user' in conf['cm'] and 'kdc_pass' in conf['cm']:
                check_creds(conf['cm']['kdc_user'], conf['cm']['kdc_pass'])
                LOG.info("Importing credentials for administrative account")
                cmd = cm.import_admin_credentials(conf['cm']['kdc_user'], conf['cm']['kdc_pass'])
                util.wait_for_command(cmd, True)

            # OK let's do this
            datanode_transceiver_port = DATANODE_TRANSCEIVER_PORT
            datanode_web_port = DATANODE_WEB_PORT

            cm_cfg = cm.get_config()
            if 'SINGLE_USER_ENABLED' in cm_cfg:
                # TODO: don't hardcode this
                datanode_transceiver_port=4004
                datanode_web_port=4006

            LOG.info("Running Kerberos configuration now...")
            cmd = cl.configure_for_kerberos(datanode_transceiver_port=datanode_transceiver_port,
                                            datanode_web_port=datanode_web_port)

            util.wait_for_command(cmd, True)
            #TODO: in some cases time.sleep(x) is needed to avoid race conditions
            try:
                mgmt_svc = cm.get_service()
                LOG.info("Restarting management services")
                mgmt.restart(cm)
            except ApiException:
                pass
            LOG.info("Restarting cluster")
            cluster.restart(cl)
        else:
            LOG.info("Kerberos already enabled")
    except ApiException, e:
        raise ProvisionatorException(e)
Пример #5
0
def enable_hdfsha(conf):
    ha_hosts_map = {}
    active_namenode_key = "ActiveNamenode"
    standby_namenode_key = "StandbyNamenode"
    journalnode1_key = "JournalNode1"
    journalnode2_key = "JournalNode2"
    journalnode3_key = "JournalNode3"

    try:
        # Open a connection to CM and get a CM object
        api = util.get_api_handle(conf)
        cm = api.get_cloudera_manager()
        cl = None
        if 'cluster' in conf and 'name' in conf['cluster']:
            cl = api.get_cluster(conf['cluster']['name'])
        else:
            raise ProvisionatorException("No cluster specified")

        #todo: check if enabled already
        hdfs_svc = config.get_cluster_service_by_type(conf, 'HDFS')
        hdfs = cl.get_service(hdfs_svc['name'])
        hdfs_cfg, hdfs_roletype_cfg = hdfs.get_config(view='full')
        host_id_map = util.host_id_map(config, api)
        active_namenode = hdfs_svc['haconfig']['active_namenode']
        standby_namenode = hdfs_svc['haconfig']['standby_namenode']
        journalnode1 = hdfs_svc['haconfig']['journalnode1']
        journalnode2 = hdfs_svc['haconfig']['journalnode2']
        journalnode3 = hdfs_svc['haconfig']['journalnode3']
        ns = hdfs_svc['haconfig']['ns']
        jnEditsDir = hdfs_svc['haconfig']['jnEditsDir']

        ha_hosts_map[active_namenode_key] = active_namenode
        ha_hosts_map[standby_namenode_key] = standby_namenode
        ha_hosts_map[journalnode1_key] = journalnode1
        ha_hosts_map[journalnode2_key] = journalnode2
        ha_hosts_map[journalnode3_key] = journalnode3

        LOG.info("host_id_map: " + str(host_id_map))
        LOG.info("ha_host_map: " + str(ha_hosts_map))
    	for ha_host in ha_hosts_map:
		LOG.info ("ha_hosts_map[ha_host]: " + ha_hosts_map[ha_host])
		try:
			LOG.info("Host ID for " + ha_host + ": "+ host_id_map[ha_hosts_map[ha_host]])
		except KeyError:
			LOG.error("Cannot find host " + ha_hosts_map[ha_host] + " for role \""
				+ ha_host + "\" in list of hosts (" + str(list(host_id_map)) + ")")
			return


        jns = []
        jns.append({'jnHostId': host_id_map[journalnode1], 'jnName': journalnode1_key, 'jnEditsDir': jnEditsDir})
        jns.append({'jnHostId': host_id_map[journalnode2], 'jnName': journalnode2_key, 'jnEditsDir': jnEditsDir})
        jns.append({'jnHostId': host_id_map[journalnode3], 'jnName': journalnode3_key, 'jnEditsDir': jnEditsDir})

        zk_svc = config.get_cluster_service_by_type(conf, 'ZOOKEEPER')
        zk_svc_name = zk_svc['name']

        nn = hdfs.get_roles_by_type("NAMENODE")

        if not nn:
            raise ProvisionatorException("Could not find the active Namenode for current configuration")



        if len([instance.name for instance in nn]) > 1:
            LOG.warn("Found more than one Namnode in current configuration")
        nn_name = ([instance.name for instance in nn])[0]
        LOG.info("Active Namenode name in current config: " + nn_name)

        LOG.info("Running HDFS HA configuration now...")
        # standby_host_id - ID of host where Standby NameNode will be created.
        # nameservice - Nameservice to be used while enabling HA
        # jns - List of Journal Nodes. Each element must be a dict containing the following keys:
        # jnHostId: ID of the host where the new JournalNode will be created.
        # jnName: Name of the JournalNode role (optional)
        # jnEditsDir: Edits dir of the JournalNode. Can be omitted if the config is already set at RCG level.
        # zk_service_name - Name of the ZooKeeper service to use for auto-failover.
        cmd = hdfs.enable_nn_ha(
            nn_name,
            host_id_map[standby_namenode],
            ns,
            jns,
            standby_name_dir_list=None,
            qj_name=None,
            standby_name=None,
            active_fc_name=None,
            standby_fc_name=None,
            zk_service_name=zk_svc_name,
            force_init_znode=True,
            clear_existing_standby_name_dirs=True,
            clear_existing_jn_edits_dir=True
        )

        util.wait_for_command(cmd, True)
        #TODO: in some cases time.sleep(x) is needed to avoid race conditions
        try:
	    mgmt_svc = cm.get_service()
	    LOG.info("Restarting management services")
	    mgmt.restart(cm)
        except ApiException:
	    pass
        LOG.info("Restarting cluster")
        cluster.restart(cl)
    except ApiException, e:
        raise ProvisionatorException(e)
Пример #6
0
import logging
import cm_api
import config, mgmt, util, cluster, service
from cm_api.api_client import ApiException

if __name__ == "__main__":
  logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
  if len(sys.argv) < 2:
    logging.fatal("No configuration file specified")
    sys.exit(-1)

  config = config.read_config(sys.argv[1])

  try:
    # Open a connection to CM and get a CM object
    api = util.get_api_handle(config)
    host_id_map = util.host_id_map(config, api)
    cm = api.get_cloudera_manager()

    # Update CM config
    if 'config' in config['cm']:
      cm.update_config(config['cm']['config'])

    # Create Cloudera Management Services
    mgmt.create(config, cm, host_id_map)
    mgmt.start(cm)

    # Add cluster
    cl = cluster.add(config, api, host_id_map)

    # Add services