Exemplo n.º 1
0
def main():
    """
    Kerberizes a cluster.

    @rtype:   number
    @returns: A number representing the status of success.
    """
    settings = retrieve_args()

    api = ApiResource(settings.host, settings.port, settings.username,
                      settings.password, settings.use_tls, 8)

    cloudera_manager = api.get_cloudera_manager()
    cluster = api.get_cluster(settings.cluster)
    mgmt_service = cloudera_manager.get_service()

    if verify_cloudera_manager_has_kerberos_principal(cloudera_manager):
        wait_for_command('Stopping the cluster', cluster.stop())
        wait_for_command('Stopping MGMT services', mgmt_service.stop())
        configure_services(cluster)
        wait_for_generate_credentials(cloudera_manager)
        wait_for_command('Deploying client configs.', cluster.deploy_client_config())
        wait_for_command('Deploying cluster client configs', cluster.deploy_cluster_client_config())
        wait_for_command('Starting MGMT services', mgmt_service.start())
        wait_for_command('Starting the cluster', cluster.start())
    else:
        print "Cluster does not have Kerberos admin credentials.  Exiting!"

    return 0
Exemplo n.º 2
0
def main():
    module = AnsibleModule(argument_spec=MODULE_ARGUMENTS)
    host_a = module.params.get('host', None)
    pass_a = module.params.get('pass', None)
    user_a = module.params.get('user', None)

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    result = sock.connect_ex((host_a, 7183))

    if result == 0:
        # Port is open then use https port
        api = ApiResource(server_host=host_a,
                          username=user_a,
                          password=pass_a,
                          use_tls=True,
                          version=9)
    else:
        # Port is not open then use http port
        api = ApiResource(server_host=host_a,
                          username=user_a,
                          password=pass_a,
                          version=9)

    cdh_config = CdhConfiguration(api, CLUSTER_NAME)
    module.exit_json(changed=True,
                     msg='Collected',
                     config=cdh_config.get_config_for_cluster(CLUSTER_NAME))
Exemplo n.º 3
0
def init_cluster():
    # wait for all cloudera agent processes to come up
    setup_logger.info("Creating Clutser.")
    BDVLIB_ServiceWait([["services", "cloudera_scm_agent", NODE_GROUP_ID]])
    # make sure cloudera manager has received registration
    # for all new agents
    all_cloudera_hosts = get_hosts_for_service(
        ["services", "cloudera_scm_agent"])
    api = ApiResource(CM_HOST, username=ADMIN_USER, password=ADMIN_PASS)
    while True:
        current_all_hosts = map(lambda x: x.hostname, api.get_all_hosts())
        setup_logger.info("Currently registered hosts with CM " +
                          str(current_all_hosts))
        if all(x in current_all_hosts for x in all_cloudera_hosts):
            break
        setup_logger.info(
            "waiting for new nodes to register with cloudera manager")
        time.sleep(10)
    manager = api.get_cloudera_manager()
    manager.update_config(CM_CONFIG)
    cluster = api.create_cluster(CLUSTER_NAME, CDH_MAJOR_VERSION,
                                 CDH_FULL_VERSION)
    cluster.add_hosts(ALL_HOSTS)

    # turn off host swap alerting
    hosts_swap_alert_off(api)

    setup_logger.info("Setting Up SPARK2 Repo....")
    add_spark2_repo(api)
    ##Set java home
    setup_logger.info("Setting Up Java Path....")
    hosts_set_javahome(api)

    return (cluster, manager)
Exemplo n.º 4
0
def main():
    """
    Add peer to the cluster.
    @rtype:   number
    @returns: A number representing the status of success.
    """
    settings = parse_args()
    if len(sys.argv) == 1 or len(sys.argv) > 17:
        print_usage_message()
        quit(1)

    api_target = ApiResource(settings.server, settings.port, settings.username,
                             settings.password, settings.use_tls, 14)
    cloudera_manager = api_target.get_cloudera_manager()
    try:
        cloudera_manager.create_peer(settings.peer_name,
                                     settings.source_cm_url,
                                     settings.source_user,
                                     settings.source_password)
        print "Peer Successfully Added"
    except ApiException as error:
        if 'already exists' in str(error):
            print 'Peer Already exists'
        else:
            raise error

    return 0
Exemplo n.º 5
0
def connect(cm_api, cm_username, cm_password, use_proxy=False):
    '''
    Wait for ten minutes for CM to come up
    '''

    for _ in xrange(120):
        try:
            logging.info("Checking CM availability....")
            # change name of proxy if necessary
            proxy = urllib2.ProxyHandler({'http': 'proxy'})

            api = ApiResource(cm_api,
                              username=cm_username,
                              password=cm_password,
                              version=14)

            if use_proxy:
                # pylint: disable=W0212
                api._client._opener.add_handler(proxy)

            cloudera_manager = api.get_cloudera_manager()
            api.get_user(cm_username)

            return api, cloudera_manager
        except Exception:
            logging.warning("CM is not up")
            time.sleep(5)
    logging.error("CM did not come UP")
    sys.exit(-1)
Exemplo n.º 6
0
class ImpalaCluster(object):
  def __init__(self, cm_host, cm_cluster_name, username, password):
    self.cm_api = ApiResource(cm_host, username=username, password=password)
    self.hosts = dict()
    self.services = list()
    self.cluster = self.cm_api.get_cluster(cm_cluster_name)
    if self.cluster is None:
      raise RuntimeError, 'Cluster name "%s" not found' % cm_cluster_name

    self.__load_hosts()
    self.__impala_service = ImpalaService(self)

  def _get_all_services(self):
    return self.cluster.get_all_services()

  def get_impala_service(self):
    return self.__impala_service

  def __load_hosts(self):
    self.hosts = dict()
    # Search for all hosts that are in the target cluster.
    # There is no API that provides the list of host in a given cluster, so to find them
    # we must loop through all the hosts and check the cluster name matches.
    for host_info in self.cm_api.get_all_hosts():
      # host_info doesn't include a link to the roleRef so need to do another lookup
      # based on the hostId.
      host = self.cm_api.get_host(host_info.hostId)
      for roleRef.get('clusterName') == self.cluster_name:
        self.hosts[host_info.hostId] = Host(host)
          break
Exemplo n.º 7
0
    def __init__(self,service,role,host,list):
        self.service = service.lower()
        self.role = role.lower()
        self.host = host.lower()
	self.list = list.lower()
        cm_host = '10.7.177.234'
        self.api = ApiResource(cm_host, username="******", password="******")
Exemplo n.º 8
0
def main():
    """
    Kerberizes a cluster.

    @rtype:   number
    @returns: A number representing the status of success.
    """
    settings = retrieve_args()

    api = ApiResource(settings.host, settings.port, settings.username,
                      settings.password, settings.use_tls, 8)

    cloudera_manager = api.get_cloudera_manager()
    cluster = api.get_cluster(settings.cluster)
    mgmt_service = cloudera_manager.get_service()

    if verify_cloudera_manager_has_kerberos_principal(cloudera_manager):
        wait_for_command('Stopping the cluster', cluster.stop())
        wait_for_command('Stopping MGMT services', mgmt_service.stop())
        configure_services(cluster)
        wait_for_generate_credentials(cloudera_manager)
        wait_for_command('Deploying client configs.',
                         cluster.deploy_client_config())
        wait_for_command('Deploying cluster client configs',
                         cluster.deploy_cluster_client_config())
        wait_for_command('Starting MGMT services', mgmt_service.start())
        wait_for_command('Starting the cluster', cluster.start())
    else:
        print "Cluster does not have Kerberos admin credentials.  Exiting!"

    return 0
Exemplo n.º 9
0
 def get_hosts(self):
     hosts = {}
     from cm_api.api_client import ApiResource
     api = ApiResource(self.host, self.port, self.username, self.password)
     for h in api.get_all_hosts():
         hosts[h.hostId] = h.ipAddress
     return hosts
Exemplo n.º 10
0
def do_call(host, port, user, password, cluster_name, service_role_name,
            random_index):
    api = ApiResource(host, port, user, password, False, MAN_API_VERSION)
    for cluster in api.get_all_clusters():
        if cluster_name is None:
            break
        elif cluster_name == cluster.name:
            break
    if cluster_name is not None and cluster_name != cluster.name:
        print >> sys.stderr, "Cloud not find cluster: " + cluster_name
        return -2
    do_print_header()
    for service in cluster.get_all_services():
        do_print_line_item(api, service, service_role_name, random_index,
                           'HDFS', 'NAMENODE', 'namenode_port', [], [])
        do_print_line_item(api, service, service_role_name, random_index,
                           'KUDU', 'KUDU_MASTER', 'webserver_port', [], [])
        do_print_line_item(api, service, service_role_name, random_index,
                           'HUE', 'HUE_SERVER', 'hue_http_port', [], [])
        do_print_line_item(api, service, service_role_name, random_index,
                           'HIVE', 'HIVESERVER2', 'hs2_thrift_address_port',
                           [], [])
        do_print_line_item(api, service, service_role_name, random_index,
                           'IMPALA', 'IMPALAD', 'beeswax_port', [], [])
        do_print_line_item(api, service, service_role_name, random_index,
                           'FLUME', 'AGENT', 'agent_http_port', [], [])
        do_print_line_item(api, service, service_role_name, random_index,
                           'KAFKA', 'KAFKA_BROKER', 'port', [], [])
        do_print_line_item(api, service, service_role_name, random_index,
                           'ZOOKEEPER', 'SERVER', 'clientPort', [], [])
    do_print_footer()
Exemplo n.º 11
0
    def setup(self,
              p_cm_host,
              p_cm_user,
              p_cm_pass,
              p_cm_version,
              p_cluster,
              p_cm_port=None,
              p_use_tls=False):
        self.cm_api = ApiResource(p_cm_host,
                                  server_port=p_cm_port,
                                  version=p_cm_version,
                                  username=p_cm_user,
                                  password=p_cm_pass,
                                  use_tls=p_use_tls)
        handler_cm_api.cluster_hosts = self.cm_api.get_all_hosts()
        if p_cluster:
            self.cluster = filter(lambda x: x.displayName == p_cluster,
                                  self.cm_api.get_all_clusters())[0]
            if not self.cluster:
                print("Error: That cluster is not valid.")
                return
            else:
                self.services = self.cluster.get_all_services()
                self.name = self.cluster.displayName

        tmp_topology = self.cluster.list_hosts()
        self.topology = {}

        for i in range(len(tmp_topology)):
            tmp_host = filter(lambda x: x.hostId == tmp_topology[i].hostId,
                              handler_cm_api.cluster_hosts)[0]
            self.topology[tmp_topology[i].hostId] = tmp_host.hostname
def main():
    module = build_module()
    choice_map = {'present': present, 'distributed': distributed, 'activated': activated, 'absent': absent, 'infos': infos}
    params = module.params
    has_changed = False

    api = ApiResource(params["cm_host"], username=params["cm_login"], password=params["cm_login"], version=params["api_version"])

    try:
        cluster = api.get_cluster(params["cluster_name"])
    except ApiException as e:
        module.fail_json(msg="Cluster error : {0}".format(e))

    if params["product"] and params["version"]:
        parcel = get_parcel(cluster, params["product"], params["version"])
        if params["state"] != "infos":
            error, has_changed, result, meta = choice_map.get(params['state'])(cluster, parcel)

            if error:
                module.fail_json(msg=result)
            module.exit_json(changed=has_changed, msg=result, meta=meta)
        else:
            meta = {
                "product": parcel.product,
                "version": parcel.version,
                "stage": parcel.stage
            }
            module.exit_json(changed=False, msg="Parcel informations gathered", meta=meta)
    elif not params["product"] and not params["version"] and params["state"] == "infos":
        module.exit_json(changed=has_changed, msg="Parcel informations gathered", meta=infos(cluster))
Exemplo n.º 13
0
def init_cluster():
    # wait for all cloudera agent processes to come up
    BDVLIB_ServiceWait(
        [["services", "cloudera_scm_agent", NODE_GROUP_ID, "kts"]])
    # make sure cloudera manager has received registration
    # for all new agents
    all_cloudera_hosts = get_hosts_for_service(
        ["services", "cloudera_scm_agent"])
    api = ApiResource(CM_HOST, username="******", password="******")
    while True:
        current_all_hosts = map(lambda x: x.hostname, api.get_all_hosts())
        setup_logger.info("Currently registered hosts with CM " +
                          str(current_all_hosts))
        if all(x in current_all_hosts for x in all_cloudera_hosts):
            break
        setup_logger.info(
            "waiting for new nodes to register with cloudera manager")
        time.sleep(10)
    manager = api.get_cloudera_manager()
    manager.update_config(CM_CONFIG)
    cluster = api.create_cluster(CLUSTER_NAME, CDH_MAJOR_VERSION,
                                 CDH_FULL_VERSION)
    KTS_HOSTS = ConfigMeta.getWithTokens(
        ['nodegroups', NODE_GROUP_ID, 'roles', 'kts', 'fqdns'])
    cluster.add_hosts(KTS_HOSTS)

    return (cluster, manager)
Exemplo n.º 14
0
def main():
    #print sys.argv[0]
    #for i in range(1, len(sys.argv)):
    #    print "param ", i, sys.argv[i]

    # get a handle on the instance of CM that we have running
    api = ApiResource(cm_host, cm_port, cm_username, cm_password, version=13)

    # get the CM instancepython2.7 setuptools
    cm = ClouderaManager(api)

    cluster = api.get_cluster(cluster_name)

    # distribution_parcels(api, cluster)

    cmd = cluster.first_run()

    while cmd.success == None:
        cmd = cmd.fetch()

    if cmd.success != True:
        print "The first run command failed: " + cmd.resultMessage()
        exit(0)

    print "First run successfully executed. Your cluster has been set up!"
Exemplo n.º 15
0
def adjust_yarn_memory_limits(region, stack_name, restart=True):
    ec2_conn = create_ec2_connection(region)
    manager_instance = get_manager_instance(ec2_conn, stack_name)
    with cm_tunnel_ctx(manager_instance) as local_port:
        cm_api = ApiResource('localhost', username='******', password='******',
                             server_port=local_port, version=9)
        cluster = list(cm_api.get_all_clusters())[0]
        host = list(cm_api.get_all_hosts())[0]  # all hosts same instance type
        yarn = filter(lambda x: x.type == 'YARN',
                      list(cluster.get_all_services()))[0]
        rm_cg = filter(lambda x: x.roleType == 'RESOURCEMANAGER',
                       list(yarn.get_all_role_config_groups()))[0]
        nm_cg = filter(lambda x: x.roleType == 'NODEMANAGER',
                       list(yarn.get_all_role_config_groups()))[0]
        rm_cg.update_config({
            'yarn_scheduler_maximum_allocation_mb': (
                int(host.totalPhysMemBytes / 1024. / 1024.)),
            'yarn_scheduler_maximum_allocation_vcores': host.numCores})
        nm_cg.update_config({
            'yarn_nodemanager_resource_memory_mb': (
                int(host.totalPhysMemBytes / 1024. / 1024.)),
            'yarn_nodemanager_resource_cpu_vcores': host.numCores})
        cluster.deploy_client_config().wait()
        if restart:
            cluster.restart().wait()
Exemplo n.º 16
0
def create_cluster():
    CM_HOST = (load_cfg(ansible_path+"/group_vars/all")).get("cm_host")
    USERNAME = (load_cfg(ansible_path+"/group_vars/all")).get("cm_username")
    PASSWORD = (load_cfg(ansible_path+"/group_vars/all")).get("cm_password")
    api = ApiResource(CM_HOST, version=API_VERSION, username=USERNAME, password=PASSWORD)
    cluster = api.get_cluster(CLUSTER_NAME)
    return cluster
Exemplo n.º 17
0
def connect(cm_api, cm_username, cm_password, use_proxy=False):
    '''
    Wait for ten minutes for CM to come up
    '''

    for _ in xrange(120):
        try:
            logging.info("Checking CM availability....")
            # change name of proxy if necessary
            proxy = urllib2.ProxyHandler({'http': 'proxy'})

            api = ApiResource(cm_api, username=cm_username, password=cm_password, version=14)

            if use_proxy:
            # pylint: disable=W0212
                api._client._opener.add_handler(proxy)

            cloudera_manager = api.get_cloudera_manager()
            api.get_user(cm_username)

            return api, cloudera_manager
        except Exception:
            logging.warning("CM is not up")
            time.sleep(5)
    logging.error("CM did not come UP")
    sys.exit(-1)
Exemplo n.º 18
0
    def __init__(self,
                 hostname=None,
                 port='7180',
                 username='******',
                 password='******'):
        '''
        Creates a Resource object that provides API endpoints.

        @param hostname: Hostname of the Cloudera Manager server.
        @param port:     Port of the server. Defaults to 7180 (http).
        @param username: Login name.
        @param password: Login password.
        '''
        config = Config()
        salt   = Salt()

        if hostname is None:
            hostname = salt.function('roles.dict', 'cloudera-cm4-server')['cloudera-cm4-server'][0]

        try:
            logger.debug('Acquiring Cloudera Manager API resource')
            ApiResource.__init__(self, hostname, port, username, password)
        except ApiException:
            logger.error('Problem acquiring Cloudera Manager API resource')
#           sys.exit(1)
        except URLError:
            logger.error('Problem connecting to %s', hostname)
            sys.exit(1)
        else:
            self.config = config
            self.salt   = salt

            cluster = ClouderaManagerCluster(self)
            host    = ClouderaManagerHost(self)
Exemplo n.º 19
0
    def setup_api_resources(self):
        self.api = ApiResource(server_host=self.cm_server_address, server_port=self.cm_server_port,
                               username=self.username, password=self.password,
                               version=self._get_api_version())

        self.cm = self.api.get_cloudera_manager()
        self.cluster = self.api.get_cluster('Cluster 1 (clusterdock)')
Exemplo n.º 20
0
def getActiveCMConfig(totalconfig):
    cmConfig = {}
    for cm in totalconfig['cmfqdn']:
        api = ApiResource(cm, totalconfig[cm]['port'], totalconfig[cm]['user'],
                          totalconfig[cm]['passwd'], totalconfig[cm]['tls'],
                          totalconfig[cm]['apiv'])
        clusters = api.get_all_clusters()
        cmConfig[cm] = {}
        for cluster in clusters:
            cmConfig[cm][cluster.displayName] = {}
            services = cluster.get_all_services()
            for service in services:
                cmConfig[cm][cluster.displayName][service.name] = {}
                cmConfig[cm][cluster.displayName][service.name]['Service'] = {}
                for name, config in service.get_config(view='full')[0].items():
                    cmConfig[cm][cluster.displayName][
                        service.name]['Service'][name] = {
                            'value': config.value,
                            'default': config.default
                        }
                for roleGroup in service.get_all_role_config_groups():
                    cmConfig[cm][cluster.displayName][service.name][
                        roleGroup.roleType] = {}
                    for name, config in roleGroup.get_config(
                            view='full').items():
                        cmConfig[cm][cluster.displayName][service.name][
                            roleGroup.roleType][name] = {
                                'value': config.value,
                                'default': config.default
                            }
                    print(roleGroup.roleType)
    #print(json.dumps(cmConfig, indent=4))
    return cmConfig
Exemplo n.º 21
0
def main():
    """
    Add peer to the cluster.
    @rtype:   number
    @returns: A number representing the status of success.
    """
    settings = parse_args()
    if len(sys.argv) == 1 or len(sys.argv) > 17:
        print_usage_message()
        quit(1)

    api_target = ApiResource(settings.server, settings.port, settings.username,
                             settings.password, settings.use_tls, 14)
    type_name = 'AWS_ACCESS_KEY_AUTH'
    account_configs = {
        'aws_access_key': settings.aws_access_key,
        'aws_secret_key': settings.aws_secret_key
    }
    try:
        api_target.create_external_account(settings.account_name,
                                           settings.account_name,
                                           type_name,
                                           account_configs=account_configs)
        print "S3 Account Successfully Added"
    except ApiException as error:
        if 'already exists' in str(error):
            print 'Peer Already exists'
        else:
            raise error

    return 0
Exemplo n.º 22
0
    def __init__(self, hostname=None, port="7180", username="******", password="******"):
        """
        Creates a Resource object that provides API endpoints.

        @param hostname: Hostname of the Cloudera Manager server.
        @param port:     Port of the server. Defaults to 7180 (http).
        @param username: Login name.
        @param password: Login password.
        """
        config = Config()
        salt = Salt()

        if hostname is None:
            hostname = salt.function("roles.dict", "cloudera-cm5-server")["cloudera-cm5-server"][0]

        try:
            logger.debug("Acquiring Cloudera Manager API resource")
            ApiResource.__init__(self, hostname, port, username, password)
        except ApiException:
            logger.error("Problem acquiring Cloudera Manager API resource")
        #           sys.exit(1)
        except URLError:
            logger.error("Problem connecting to %s", hostname)
            sys.exit(1)
        else:
            self.config = config
            self.salt = salt

            cluster = ClouderaManagerCluster(self)
            host = ClouderaManagerHost(self)
Exemplo n.º 23
0
def main():
    # connect cm api
    api = ApiResource(CM_HOST,
                      7180,
                      username=CM_USERNAME,
                      password=CM_PASSWORD)
    manager = api.get_cloudera_manager()
    # no need to update cm config
    #manager.update_config(cm_host)
    print("[INFO] Connected to CM host on " + CM_HOST)

    # create cluster object
    try:
        cluster = api.get_cluster(name=CLUSTER_NAME)
    except:
        cluster = init_cluster(api, CLUSTER_NAME, CLUSTER_VERSION,
                               CLUSTER_NODE_COUNT)
    print("[INFO] Initialized cluster " + CLUSTER_NAME +
          " which uses CDH version " + CLUSTER_VERSION)

    #
    mgmt_servicename = "MGMT"
    amon_role_name = "ACTIVITYMONITOR"
    apub_role_name = "ALERTPUBLISHER"
    eserv_role_name = "EVENTSERVER"
    hmon_role_name = "HOSTMONITOR"
    smon_role_name = "SERVICEMONITOR"
    nav_role_name = "NAVIGATOR"
    navms_role_name = "NAVIGATORMETADATASERVER"
    rman_role_name = "REPORTMANAGER"
    deploy_management(manager, mgmt_servicename, amon_role_name,
                      apub_role_name, eserv_role_name, hmon_role_name,
                      smon_role_name, nav_role_name, navms_role_name,
                      rman_role_name)
    print("[INFO] Deployed CM management service " + mgmt_servicename +
          " to run on " + CM_HOST)

    #
    assign_roles(api, cluster)
    print("[INFO] all roles have assigned.")

    #
    # Custom role config groups cannot be automatically configured: Gateway Group 1 (error 400)
    try:
        cluster.auto_configure()
    except:
        pass
    update_custom_config(api, cluster)
    print("[INFO] all servies and roles have configured.")
    #
    cmd = cluster.first_run()
    while cmd.success == None:
        cmd = cmd.fetch()
    if not cmd.success:
        print("[ERROR] The first run command failed: " + cmd.resultMessage())
    else:
        print(
            "[INFO] First run successfully executed. Your cluster has been set up!"
        )
Exemplo n.º 24
0
def list_hosts(host, username, password, cafile):
  context = ssl.create_default_context(cafile=cafile)

  api = ApiResource(host, username=username, password=password, use_tls=True,
                    ssl_context=context)

  for h in api.get_all_hosts():
    print h.hostname
Exemplo n.º 25
0
def main():
    cmhost = os.environ['DEPLOYMENT_HOST_PORT'].split(":")[0]
    api = ApiResource(cmhost, username='******', password='******')
    all_clusters = api.get_all_clusters()
    for cluster in all_clusters:
        if (cluster.name == os.environ['CLUSTER_NAME']):
            break

    template = cluster.create_host_template("cdsw-gateway")
Exemplo n.º 26
0
def main():
   API = ApiResource(CM_HOST, version=5, username=ADMIN_USER, password=ADMIN_PASS)
   print "Connected to CM host on " + CM_HOST

   CLUSTER = API.get_cluster(CLUSTER_NAME)

   print "About to restart cluster."
   CLUSTER.restart().wait()
   print "Done restarting cluster."
Exemplo n.º 27
0
def main():
   API = ApiResource(CM_HOST, version=5, username=ADMIN_USER, password=ADMIN_PASS)
   print "Connected to CM host on " + CM_HOST

   CLUSTER = API.get_cluster(CLUSTER_NAME)

   print "About to stop cluster."
   CLUSTER.stop().wait()
   print "Done stopping cluster."
Exemplo n.º 28
0
def get_cluster_specs():
    cm_api = ApiResource(os.environ['MANAGER_HOST'], username='******',
                         password='******', server_port=7180, version=9)
    host = list(cm_api.get_all_hosts())[0]  # all hosts same instance type
    cluster = list(cm_api.get_all_clusters())[0]
    yarn = filter(lambda x: x.type == 'YARN',
                  list(cluster.get_all_services()))[0]
    return {'num_worker_nodes': len(yarn.get_roles_by_type('NODEMANAGER')),
            'num_cores': host.numCores, 'node_memory': host.totalPhysMemBytes}
Exemplo n.º 29
0
def main(cm_host, user, password):
    api = ApiResource(cm_host, username=user, password=password)
    cm = api.get_cloudera_manager()
    cm.update_all_hosts_config(
        {"java_home": "/usr/java/jdk1.8.0_121-cloudera"})
    print("restarting CM service - this will take a minute or so")
    cm.get_service().restart().wait()
    print("restarting cluster - this will take 2-5 minutes")
    api.get_all_clusters()[0].restart(restart_only_stale_services=True,
                                      redeploy_client_configuration=True).wait()
Exemplo n.º 30
0
def start(host, user, passw):
    cm_host = str(host)
    api = ApiResource(cm_host,
                      7180,
                      username=str(username),
                      password=str(password),
                      version=9)
    mgmt = api.get_cloudera_manager().get_service()
    mgmt.restart()
    print("Services successfully started")
Exemplo n.º 31
0
 def getClusterInformation(self):
     api = ApiResource(self.cm_host, username=self.user, password=self.passwd)
     logger.info('Received; user -> %s, password -> %s, host -> %s', self.user, self.passwd, self.cm_host)
     for c in api.get_all_clusters():
         clusterInf = "Cluster name %s and version %s" %(c.name, c.version)
         #print "Cluster name %s and version %s" %(c.name, c.version)
         logger.info("Cluster name %s and version %s", c.name, c.version)
         if c.version == "CDH5":
             cdh5 = c
     return cdh5, clusterInf
Exemplo n.º 32
0
def main():
  parser = cm_args_parser()
  args = parser.parse_args()
  print "connecting to host:" + args.cm_host + "..."
  api = ApiResource(args.cm_host, username=args.cm_user, password=args.cm_password)
  print "host connected, getting cloudera manager "
  MANAGER = api.get_cloudera_manager()
  print "have cloudera manager object" 
  deploy_management(MANAGER, MGMT_SERVICENAME, MGMT_SERVICE_CONFIG, MGMT_ROLE_CONFIG, AMON_ROLENAME, AMON_ROLE_CONFIG, APUB_ROLENAME, APUB_ROLE_CONFIG, ESERV_ROLENAME, ESERV_ROLE_CONFIG, HMON_ROLENAME, HMON_ROLE_CONFIG, SMON_ROLENAME, SMON_ROLE_CONFIG, RMAN_ROLENAME, RMAN_ROLE_CONFIG)
  print "Deployed CM management service " + MGMT_SERVICENAME + " to run on " + CM_HOST
Exemplo n.º 33
0
  def __init__(self, cm_host, cm_cluster_name, username, password):
    self.cm_api = ApiResource(cm_host, username=username, password=password)
    self.hosts = dict()
    self.services = list()
    self.cluster = self.cm_api.get_cluster(cm_cluster_name)
    if self.cluster is None:
      raise RuntimeError, 'Cluster name "%s" not found' % cm_cluster_name

    self.__load_hosts()
    self.__impala_service = ImpalaService(self)
Exemplo n.º 34
0
 def __init__(self, module, config, trial=False, license_txt=None):
     self.api = ApiResource(config['cm']['host'],
                            username=config['cm']['username'],
                            password=config['cm']['password'])
     self.manager = self.api.get_cloudera_manager()
     self.config = config
     self.module = module
     self.trial = trial
     self.license_txt = license_txt
     self.cluster = None
def main():
    """
    Add peer to the cluster.
    @rtype:   number
    @returns: A number representing the status of success.
    """
    settings = parse_args()
    if len(sys.argv) == 1 or len(sys.argv) > 17:
        print_usage_message()
        quit(1)

    api = ApiResource(settings.server, settings.port, settings.username,
                      settings.password, settings.use_tls, 14)

    yarn_service = get_service_name('YARN', api, settings.cluster_name)
    hdfs_name = get_service_name('HDFS', api, settings.cluster_name)

    hdfs = api.get_cluster(settings.cluster_name).get_service(hdfs_name)

    hdfs_cloud_args = ApiHdfsCloudReplicationArguments(None)
    hdfs_cloud_args.sourceService = ApiServiceRef(None,
                                                  peerName=None,
                                                  clusterName=settings.cluster_name,
                                                  serviceName=hdfs_name)
    hdfs_cloud_args.sourcePath = settings.source_path
    hdfs_cloud_args.destinationPath = settings.target_path
    hdfs_cloud_args.destinationAccount = settings.account_name
    hdfs_cloud_args.mapreduceServiceName = yarn_service

    # creating a schedule with daily frequency
    start = datetime.datetime.now()
    # The time at which the scheduled activity is triggered for the first time.
    end = start + datetime.timedelta(days=365)
    # The time after which the scheduled activity will no longer be triggered.

    schedule = hdfs.create_replication_schedule(start, end, "DAY", 1, True, hdfs_cloud_args)

    ## Updating the Schedule's properties
    # schedule.hdfsArguments.removeMissingFiles = False
    schedule.alertOnFail = True
    schedule = hdfs.update_replication_schedule(schedule.id, schedule)

    print "Schedule created with Schdule ID: " + str(schedule.id)
    # print schedule.alertOnFail
    # print schedule.hdfsArguments.removeMissingFiles
    # print schedule.hdfsArguments.sourcePath
    # print schedule.hdfsArguments.preserveXAttrs
    # print schedule.hdfsArguments.exclusionFilters
    # print schedule.hdfsArguments.replicationStrategy
    # print schedule.hdfsArguments.numMaps
    # print schedule.hdfsArguments.userName
    # print schedule.hdfsArguments.schedulerPoolName
    # print type(schedule)

    return 0
Exemplo n.º 36
0
def connect_cm(cm_host, cm_username, cm_password):
    """
    Connects to Cloudera Manager API Resource instance to retrieve Endpoint details
    :param cm_host: Cloudera Manager host
    :param cm_username: Username for authentication
    :param cm_password: Password for authentication
    :return:
    """
    api = ApiResource(cm_host, version=6, username=cm_username, password=cm_password)
    cm_manager = api.get_cloudera_manager()
    return api, cm_manager
Exemplo n.º 37
0
def connect_cm(cm_host, cm_username, cm_password):
    """
    Connects to Cloudera Manager API Resource instance to retrieve Endpoint details
    :param cm_host: Cloudera Manager host
    :param cm_username: Username for authentication
    :param cm_password: Password for authentication
    :return:
    """
    api = ApiResource(cm_host, version=6, username=cm_username, password=cm_password)
    cm_manager = api.get_cloudera_manager()
    return api, cm_manager
Exemplo n.º 38
0
def get_cluster(cm_host, user, pwd, cluster_name):

    global api
    api = ApiResource(cm_host, username=user, password=pwd, version=12)
    for c in api.get_all_clusters():
        if cluster_name in c.name:
            # print("Cluster, Version : " + c.name + ", " + c.version)
            return True, c
        else:
            print("[ERR] : Cluster \"" + cluster_name + "\" not found at \"" +
                  cm_host + "\"")
            return False, ""
Exemplo n.º 39
0
def find_impala_in_cm(cm_host, cm_user, cm_password, cm_cluster_name):
  """Finds the Impala service in CM and returns an Impala instance."""
  cm = ApiResource(cm_host, username=cm_user, password=cm_password)
  cm_impalas = [service for cluster in cm.get_all_clusters()
                if cm_cluster_name is None or cm_cluster_name == cluster.name
                for service in cluster.get_all_services() if service.type == "IMPALA"]
  if len(cm_impalas) > 1:
    raise Exception("Found %s Impala services in CM;" % len(cm_impalas) +
        " use --cm-cluster-name option to specify which one to use.")
  if len(cm_impalas) == 0:
    raise Exception("No Impala services found in CM")
  return Impala(cm_impalas[0])
Exemplo n.º 40
0
 def cm_api_resource(self):
     ar = None
     try:
         ar = ApiResource(self.cm_host, self.cm_port,
                          self.cm_username, self.cm_password)
         ar.echo('Authenticated')  # Issue a sample request to test the conn
     except ApiException, aexc:
         if aexc.code == 401:
             log.debug("Changing default API username to {0}".format(self.cm_username))
             self.cm_username = self.host_username
             self.cm_password = self.host_password
             ar = ApiResource(self.cm_host, self.cm_port,
                              self.cm_username, self.cm_password)
         else:
             log.error("Api Exception connecting to ClouderaManager: {0}".format(aexc))
Exemplo n.º 41
0
def main():
    # connect cm api
    api = ApiResource(CM_HOST, 7180, username=CM_USERNAME, password=CM_PASSWORD)
    manager = api.get_cloudera_manager()
    # no need to update cm config
    #manager.update_config(cm_host)
    print("[INFO] Connected to CM host on " + CM_HOST)

    # create cluster object
    try:
        cluster = api.get_cluster(name=CLUSTER_NAME)
    except:
        cluster = init_cluster(api, CLUSTER_NAME, CLUSTER_VERSION, CLUSTER_NODE_COUNT)
    print("[INFO] Initialized cluster " + CLUSTER_NAME + " which uses CDH version " + CLUSTER_VERSION)

    #
    mgmt_servicename = "MGMT"
    amon_role_name = "ACTIVITYMONITOR"
    apub_role_name = "ALERTPUBLISHER"
    eserv_role_name = "EVENTSERVER"
    hmon_role_name = "HOSTMONITOR"
    smon_role_name = "SERVICEMONITOR"
    nav_role_name = "NAVIGATOR"
    navms_role_name = "NAVIGATORMETADATASERVER"
    rman_role_name = "REPORTMANAGER"
    deploy_management(manager, mgmt_servicename, amon_role_name, apub_role_name, eserv_role_name, hmon_role_name, smon_role_name, nav_role_name, navms_role_name, rman_role_name)
    print("[INFO] Deployed CM management service " + mgmt_servicename + " to run on " + CM_HOST)

    #
    assign_roles(api, cluster)
    print("[INFO] all roles have assigned.")

    #
    # Custom role config groups cannot be automatically configured: Gateway Group 1 (error 400)
    try:
        cluster.auto_configure()
    except:
        pass
    update_custom_config(api, cluster)
    print("[INFO] all servies and roles have configured.")
    #
    cmd = cluster.first_run()
    while cmd.success == None:
        cmd = cmd.fetch()
    if not cmd.success:
        print("[ERROR] The first run command failed: " + cmd.resultMessage())
    else:
        print("[INFO] First run successfully executed. Your cluster has been set up!")
Exemplo n.º 42
0
def main():
    s,a = arg_handle()
    for i in range(0,15):
        while True:
            try:
                cm_host = "127.0.0.1"
                api = ApiResource(cm_host, username="******", password="******")
                cdh=api.get_all_clusters()[0]
            except:
                print "Failed to connect to Cloudera Manager."
                print "Attempting to connect to Cloudera Manager..."
                time.sleep(15)
                continue
            break
    srv=cdh.get_service(s)
    actions[a](srv,s)
Exemplo n.º 43
0
def main():
    resource = ApiResource("localhost", 7180, "cloudera", "cloudera", version=19)
    cluster = resource.get_cluster("Cloudera Quickstart")

    cm_manager = resource.get_cloudera_manager()
    cm_manager.update_config({'REMOTE_PARCEL_REPO_URLS': PARCEL_REPO})
    cm_manager.update_all_hosts_config(JDK_CONFIG)
    time.sleep(5)

    for parcel in PARCELS:
        ParcelInstaller(parcel['name'], parcel['version']).install(cluster)

    print "Restarting cluster"
    cluster.stop().wait()
    cluster.start().wait()
    print "Done restarting cluster"
Exemplo n.º 44
0
def main():

    AMON_ROLE_CONFIG = {
   'firehose_heapsize': '1173741824',
}

    API = ApiResource("ec2-52-24-151-222.us-west-2.compute.amazonaws.com", version=5, username="******", password="******")
    MANAGER = API.get_cloudera_manager()
    mgmt = MANAGER.get_service()
    #cf = mgmt.get_config()

    for group in mgmt.get_all_role_config_groups():
       if group.roleType == "ACTIVITYMONITOR":
           group.update_config(AMON_ROLE_CONFIG)

    pass
Exemplo n.º 45
0
def update_cm(cm_host, cm_port, username, password):
    """Update config using the CM API (note: will restart service)"""
    elts = generate_xml_elements()
    cm_api = ApiResource(cm_host, username=username, password=password,
                         server_port=cm_port, version=9)
    cluster = list(cm_api.get_all_clusters())[0]
    hdfs = filter(lambda x: x.type == 'HDFS',
                  list(cluster.get_all_services()))[0]
    print("Updating HFDS core-site.xml safety valve...")
    _ = hdfs.update_config({
        'core_site_safety_valve': '\n'.join(tostring(e) for e in elts)})
    print("Deploying client config across the cluster...")
    cluster.deploy_client_config().wait()
    print("Restarting necessary services...")
    cluster.restart().wait()
    print("Done!")
Exemplo n.º 46
0
  def __init__(self, host_name, port=7180, user="******", password="******",
      cluster_name=None, ssh_user=None, ssh_port=None, ssh_key_file=None):
    # Initialize strptime() to workaround https://bugs.python.org/issue7980. Apparently
    # something in the CM API uses strptime().
    strptime("2015", "%Y")

    Cluster.__init__(self)
    self.cm = CmApiResource(host_name, server_port=port, username=user, password=password)
    clusters = self.cm.get_all_clusters()
    if not clusters:
      raise Exception("No clusters found in CM at %s" % host_name)
    if cluster_name:
      clusters_by_name = dict((c.name, c) for c in clusters)
      if cluster_name not in clusters_by_name:
        raise Exception(("No clusters named %s found in CM at %s."
            "Available clusters are %s.")
            % (cluster_name, host_name, ", ".join(sorted(clusters_by_name.keys()))))
      self.cm_cluster = clusters_by_name[cluster_name]
    else:
      if len(clusters) > 1:
        raise Exception(("Too many clusters found in CM at %s;"
            " a cluster name must be provided")
            % host_name)
      self.cm_cluster = clusters[-1]

    self.ssh_user = ssh_user
    self.ssh_port = ssh_port
    self.ssh_key_file = ssh_key_file
    self._ssh_client_lock = Lock()
    self._ssh_clients_by_host_name = defaultdict(list)
Exemplo n.º 47
0
class TimeSeriesQuery(object):
  """
  """
  def __init__(self):
    self._api = ApiResource(CM_HOST, username=CM_USER, password=CM_PASSWD, use_tls=CM_USE_TLS)

  def query(self, query, from_time, to_time):
    return self._api.query_timeseries(query, from_time, to_time)
def main():
    """
    This is a script to export a current Cloudera Manager cluster configuration into an Hadrian supported format.
    You can then use these configuration files as the basis for your new cluster configs.
    """

    parser = argparse.ArgumentParser(description='Export Cloudera Manager configs in an Hadrian friendly format.')
    parser.add_argument('-H', '--host', '--hostname', action='store', dest='hostname', required=True, help='CM Server Name')
    parser.add_argument('-p', '--port', action='store', dest='port', type=int, default=7180, help='CM Port')
    parser.add_argument('-u', '--user', '--username', action='store', dest='username', required=True, help='CM username')
    args = parser.parse_args()

    password = getpass.getpass('Please enter your Cloudera Manager passsword: ')
    api = ApiResource(args.hostname, args.port, args.username, password, version=4)

    for cluster in api.get_all_clusters():
        conf_dir = './confs/' + cluster.name
        if not os.path.exists(conf_dir):
            os.makedirs(conf_dir)

            for service in cluster.get_all_services():
                with open(conf_dir + '/' + service.name + '.ini', 'w') as f:
                    print 'Dumping Service config for ' + service.name
                    rcg = list()

                    for i in service.get_all_role_config_groups():
                        rcg.append(i.name)
                        f.write('[' + service.type + ']\n')
                        f.write('config_groups=' + ','.join(rcg))
                        f.write('\n\n')
                        f.write('[' + service.name + '-svc-config]\n')

                    for item in service.get_config():
                        for k,v in item.iteritems():
                            f.write(k + '=' + str(v) + '\n')

                    for i in service.get_all_role_config_groups():
                        f.write('\n')
                        f.write('[' + i.name + ']\n')
                        for k,v in i.get_config('full').iteritems():
                            if v.value is not None:
                                f.write(k + '=' + str(v.value) + '\n')
                    f.close()
        else:
            print 'Cluster config dir already exists.  Please rename or remove existing config dir: ' + conf_dir
Exemplo n.º 49
0
 def __init__(self, module, config, trial=False, license_txt=None):
     self.api = ApiResource(config['cm']['host'], username=config['cm']['username'],
                            password=config['cm']['password'])
     self.manager = self.api.get_cloudera_manager()
     self.config = config
     self.module = module
     self.trial = trial
     self.license_txt = license_txt
     self.cluster = None
Exemplo n.º 50
0
def do_call(host, port, user, password, cluster_name, service_role_name, random_index):
    api = ApiResource(host, port, user, password, False, MAN_API_VERSION);
    for cluster in api.get_all_clusters():
        if cluster_name is None:
            break
        elif cluster_name == cluster.name:
            break
    if cluster_name is not None and cluster_name != cluster.name:
        print >> sys.stderr, "Cloud not find cluster: " + cluster_name
        return -2;
    do_print_header()
    for service in cluster.get_all_services():
        do_print_line_item(api, service, service_role_name, random_index, 'HDFS', 'NAMENODE', 'namenode_port', [], [])
        do_print_line_item(api, service, service_role_name, random_index, 'KUDU', 'KUDU_MASTER', 'webserver_port', [], [])
        do_print_line_item(api, service, service_role_name, random_index, 'HUE', 'HUE_SERVER', 'hue_http_port', [], [])
        do_print_line_item(api, service, service_role_name, random_index, 'HIVE', 'HIVESERVER2', 'hs2_thrift_address_port', [], [])
        do_print_line_item(api, service, service_role_name, random_index, 'IMPALA', 'IMPALAD', 'beeswax_port', [], [])
        do_print_line_item(api, service, service_role_name, random_index, 'FLUME', 'AGENT', 'agent_http_port', [], [])
        do_print_line_item(api, service, service_role_name, random_index, 'KAFKA', 'KAFKA_BROKER', 'port', [], [])
        do_print_line_item(api, service, service_role_name, random_index, 'ZOOKEEPER', 'SERVER', 'clientPort', [], [])
    do_print_footer()
Exemplo n.º 51
0
def reset_cm(cm_host, cm_port, username, password):
    """Elim S3 config from CM API safety valve (service restart necessary)"""
    s3_props = set(get_s3_properties())
    cm_api = ApiResource(cm_host, username=username, password=password,
                         server_port=cm_port, version=9)
    cluster = list(cm_api.get_all_clusters())[0]
    hdfs = filter(lambda x: x.type == 'HDFS',
                  list(cluster.get_all_services()))[0]
    print("Getting current safety valve config")
    current_config = hdfs.get_config('full')[0]['core_site_safety_valve'].value
    # need the "<foo>...</foo>" to make it valid XML (bc it requires root elt)
    elts = list(fromstring('<foo>' + current_config + '</foo>'))
    new_elts = filter(lambda x: x.find('name').text not in s3_props, elts)
    print("Updating safety valve and deleting S3 config")
    _ = hdfs.update_config({
        'core_site_safety_valve': '\n'.join(tostring(e) for e in new_elts)})
    print("Deploying client config across the cluster...")
    cluster.deploy_client_config().wait()
    print("Restarting necessary services...")
    cluster.restart().wait()
    print("Done!")
Exemplo n.º 52
0
def main():
    """
    Enables HDFS HA on a cluster.

    @rtype:   number
    @returns: A number representing the status of success.
    """
    settings = retrieve_args()

    api = ApiResource(settings.host, settings.port, settings.username, settings.password,
                      version=6)

    if not validate_cluster(api, settings.cluster):
        write_to_stdout("Cluster does not satisfy preconditions for enabling HDFS HA. Exiting!")
        return 1

    if settings.wait_for_good_health:
        write_to_stdout("Waiting for GOOD health... ")
        if not wait_for_good_health(api, settings.cluster):
            write_to_stdout("Cluster health is not GOOD.  Exiting!\n")
            return 1
    else:
        write_to_stdout("Checking cluster health... ")
        if not check_health(api, settings.cluster):
            write_to_stdout("Cluster health is not GOOD.  Exiting!\n")

    write_to_stdout("Cluster health is GOOD!\n")

    cluster = api.get_cluster(settings.cluster)

    invoke_hdfs_enable_nn_ha(cluster, settings.nameservice)
    update_hive_for_ha_hdfs(cluster)

    # Restarting the MGMT services to make sure the HDFS file browser functions
    # as expected.
    cloudera_manager = api.get_cloudera_manager()
    mgmt_service = cloudera_manager.get_service()
    wait_for_command('Restarting MGMT services', mgmt_service.restart())

    return 0
Exemplo n.º 53
0
def main():
    
    """
    TODO: This probably needs some work.  You get the idea though.  
    An example of how to do a bulk config update to Cloudera Manager.  This is helpful if you have a bunch of changes
    That you want to make but don't want to use the GUI.  
    """
    
    parser = argparse.ArgumentParser(description='Cloudera Manager Bulk Config Update Script')
    parser.add_argument('-H', '--host', '--hostname', action='store', dest='hostname', required=True, help='CM server host')
    parser.add_argument('-p', '--port', action='store', dest='port', type=int, default=7180, help='example: 7180')
    parser.add_argument('-u', '--user', '--username', action='store', dest='username', required=True, help='example: admin')
    parser.add_argument('-c', '--cluster', action='store', dest='cluster', required=True, help='example: hadrian-cluster')
    args = parser.parse_args() 
    password = getpass.getpass('Please enter your Cloudera Manager passsword: ')
    
    # read configuration files:
    for i in os.listdir('./conf/' + args.cluster):
        config.read('./conf/' + args.cluster + '/' + i)
    
    api = ApiResource(args.hostname, args.port, args.username, password)
    cluster = api.get_cluster(args.cluster)
    services = cluster.get_all_services()
   
    # update services based with configuration file parameters   
    for service in services:
        if config_grabber.has_section(service.type):
            service.update_config(svc_config=config_grabber(service.name + '-svc-config'))
            config_groups = config_grabber(service.name)['config_groups']
            for config_group in config_groups.split(','):
                print section
                temp_config_group = service.get_role_config_group(section)
                temp_config_group.update_config(config_grabber(section))
        else:
            print 'unknown service: ' + service.name

    print 'Starting final client configuration deployment for all services.'
    cmd = cluster.deploy_client_config()
    if not cmd.wait(CMD_TIMEOUT).success:
        print 'Failed to deploy client configuration.'
Exemplo n.º 54
0
def get_cluster_info(manager_host, server_port=7180, username='******',
                     password='******'):
    cm_api = ApiResource(manager_host, username=username, password=password,
                         server_port=server_port, version=9)
    host = list(cm_api.get_all_hosts())[0]  # all hosts same instance type
    cluster = list(cm_api.get_all_clusters())[0]
    yarn = filter(lambda x: x.type == 'YARN',
                  list(cluster.get_all_services()))[0]
    hive = filter(lambda x: x.type == 'HIVE',
                  list(cluster.get_all_services()))[0]
    impala = filter(lambda x: x.type == 'IMPALA',
                    list(cluster.get_all_services()))[0]
    hive_hs2 = hive.get_roles_by_type('HIVESERVER2')[0]
    hive_host = cm_api.get_host(hive_hs2.hostRef.hostId).hostname
    hive_port = int(
        hive_hs2.get_config('full')['hs2_thrift_address_port'].default)
    impala_hs2 = impala.get_roles_by_type('IMPALAD')[0]
    impala_host = cm_api.get_host(impala_hs2.hostRef.hostId).hostname
    impala_port = int(impala_hs2.get_config('full')['hs2_port'].default)
    return {'num_worker_nodes': len(yarn.get_roles_by_type('NODEMANAGER')),
            'node_cores': host.numCores, 'node_memory': host.totalPhysMemBytes,
            'hive_host': hive_host, 'hive_port': hive_port,
            'impala_host': impala_host, 'impala_port': impala_port}