def main(): """ Kerberizes a cluster. @rtype: number @returns: A number representing the status of success. """ settings = retrieve_args() api = ApiResource(settings.host, settings.port, settings.username, settings.password, settings.use_tls, 8) cloudera_manager = api.get_cloudera_manager() cluster = api.get_cluster(settings.cluster) mgmt_service = cloudera_manager.get_service() if verify_cloudera_manager_has_kerberos_principal(cloudera_manager): wait_for_command('Stopping the cluster', cluster.stop()) wait_for_command('Stopping MGMT services', mgmt_service.stop()) configure_services(cluster) wait_for_generate_credentials(cloudera_manager) wait_for_command('Deploying client configs.', cluster.deploy_client_config()) wait_for_command('Deploying cluster client configs', cluster.deploy_cluster_client_config()) wait_for_command('Starting MGMT services', mgmt_service.start()) wait_for_command('Starting the cluster', cluster.start()) else: print "Cluster does not have Kerberos admin credentials. Exiting!" return 0
def main(): module = AnsibleModule(argument_spec=MODULE_ARGUMENTS) host_a = module.params.get('host', None) pass_a = module.params.get('pass', None) user_a = module.params.get('user', None) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) result = sock.connect_ex((host_a, 7183)) if result == 0: # Port is open then use https port api = ApiResource(server_host=host_a, username=user_a, password=pass_a, use_tls=True, version=9) else: # Port is not open then use http port api = ApiResource(server_host=host_a, username=user_a, password=pass_a, version=9) cdh_config = CdhConfiguration(api, CLUSTER_NAME) module.exit_json(changed=True, msg='Collected', config=cdh_config.get_config_for_cluster(CLUSTER_NAME))
def init_cluster(): # wait for all cloudera agent processes to come up setup_logger.info("Creating Clutser.") BDVLIB_ServiceWait([["services", "cloudera_scm_agent", NODE_GROUP_ID]]) # make sure cloudera manager has received registration # for all new agents all_cloudera_hosts = get_hosts_for_service( ["services", "cloudera_scm_agent"]) api = ApiResource(CM_HOST, username=ADMIN_USER, password=ADMIN_PASS) while True: current_all_hosts = map(lambda x: x.hostname, api.get_all_hosts()) setup_logger.info("Currently registered hosts with CM " + str(current_all_hosts)) if all(x in current_all_hosts for x in all_cloudera_hosts): break setup_logger.info( "waiting for new nodes to register with cloudera manager") time.sleep(10) manager = api.get_cloudera_manager() manager.update_config(CM_CONFIG) cluster = api.create_cluster(CLUSTER_NAME, CDH_MAJOR_VERSION, CDH_FULL_VERSION) cluster.add_hosts(ALL_HOSTS) # turn off host swap alerting hosts_swap_alert_off(api) setup_logger.info("Setting Up SPARK2 Repo....") add_spark2_repo(api) ##Set java home setup_logger.info("Setting Up Java Path....") hosts_set_javahome(api) return (cluster, manager)
def main(): """ Add peer to the cluster. @rtype: number @returns: A number representing the status of success. """ settings = parse_args() if len(sys.argv) == 1 or len(sys.argv) > 17: print_usage_message() quit(1) api_target = ApiResource(settings.server, settings.port, settings.username, settings.password, settings.use_tls, 14) cloudera_manager = api_target.get_cloudera_manager() try: cloudera_manager.create_peer(settings.peer_name, settings.source_cm_url, settings.source_user, settings.source_password) print "Peer Successfully Added" except ApiException as error: if 'already exists' in str(error): print 'Peer Already exists' else: raise error return 0
def connect(cm_api, cm_username, cm_password, use_proxy=False): ''' Wait for ten minutes for CM to come up ''' for _ in xrange(120): try: logging.info("Checking CM availability....") # change name of proxy if necessary proxy = urllib2.ProxyHandler({'http': 'proxy'}) api = ApiResource(cm_api, username=cm_username, password=cm_password, version=14) if use_proxy: # pylint: disable=W0212 api._client._opener.add_handler(proxy) cloudera_manager = api.get_cloudera_manager() api.get_user(cm_username) return api, cloudera_manager except Exception: logging.warning("CM is not up") time.sleep(5) logging.error("CM did not come UP") sys.exit(-1)
class ImpalaCluster(object): def __init__(self, cm_host, cm_cluster_name, username, password): self.cm_api = ApiResource(cm_host, username=username, password=password) self.hosts = dict() self.services = list() self.cluster = self.cm_api.get_cluster(cm_cluster_name) if self.cluster is None: raise RuntimeError, 'Cluster name "%s" not found' % cm_cluster_name self.__load_hosts() self.__impala_service = ImpalaService(self) def _get_all_services(self): return self.cluster.get_all_services() def get_impala_service(self): return self.__impala_service def __load_hosts(self): self.hosts = dict() # Search for all hosts that are in the target cluster. # There is no API that provides the list of host in a given cluster, so to find them # we must loop through all the hosts and check the cluster name matches. for host_info in self.cm_api.get_all_hosts(): # host_info doesn't include a link to the roleRef so need to do another lookup # based on the hostId. host = self.cm_api.get_host(host_info.hostId) for roleRef.get('clusterName') == self.cluster_name: self.hosts[host_info.hostId] = Host(host) break
def __init__(self,service,role,host,list): self.service = service.lower() self.role = role.lower() self.host = host.lower() self.list = list.lower() cm_host = '10.7.177.234' self.api = ApiResource(cm_host, username="******", password="******")
def main(): """ Kerberizes a cluster. @rtype: number @returns: A number representing the status of success. """ settings = retrieve_args() api = ApiResource(settings.host, settings.port, settings.username, settings.password, settings.use_tls, 8) cloudera_manager = api.get_cloudera_manager() cluster = api.get_cluster(settings.cluster) mgmt_service = cloudera_manager.get_service() if verify_cloudera_manager_has_kerberos_principal(cloudera_manager): wait_for_command('Stopping the cluster', cluster.stop()) wait_for_command('Stopping MGMT services', mgmt_service.stop()) configure_services(cluster) wait_for_generate_credentials(cloudera_manager) wait_for_command('Deploying client configs.', cluster.deploy_client_config()) wait_for_command('Deploying cluster client configs', cluster.deploy_cluster_client_config()) wait_for_command('Starting MGMT services', mgmt_service.start()) wait_for_command('Starting the cluster', cluster.start()) else: print "Cluster does not have Kerberos admin credentials. Exiting!" return 0
def get_hosts(self): hosts = {} from cm_api.api_client import ApiResource api = ApiResource(self.host, self.port, self.username, self.password) for h in api.get_all_hosts(): hosts[h.hostId] = h.ipAddress return hosts
def do_call(host, port, user, password, cluster_name, service_role_name, random_index): api = ApiResource(host, port, user, password, False, MAN_API_VERSION) for cluster in api.get_all_clusters(): if cluster_name is None: break elif cluster_name == cluster.name: break if cluster_name is not None and cluster_name != cluster.name: print >> sys.stderr, "Cloud not find cluster: " + cluster_name return -2 do_print_header() for service in cluster.get_all_services(): do_print_line_item(api, service, service_role_name, random_index, 'HDFS', 'NAMENODE', 'namenode_port', [], []) do_print_line_item(api, service, service_role_name, random_index, 'KUDU', 'KUDU_MASTER', 'webserver_port', [], []) do_print_line_item(api, service, service_role_name, random_index, 'HUE', 'HUE_SERVER', 'hue_http_port', [], []) do_print_line_item(api, service, service_role_name, random_index, 'HIVE', 'HIVESERVER2', 'hs2_thrift_address_port', [], []) do_print_line_item(api, service, service_role_name, random_index, 'IMPALA', 'IMPALAD', 'beeswax_port', [], []) do_print_line_item(api, service, service_role_name, random_index, 'FLUME', 'AGENT', 'agent_http_port', [], []) do_print_line_item(api, service, service_role_name, random_index, 'KAFKA', 'KAFKA_BROKER', 'port', [], []) do_print_line_item(api, service, service_role_name, random_index, 'ZOOKEEPER', 'SERVER', 'clientPort', [], []) do_print_footer()
def setup(self, p_cm_host, p_cm_user, p_cm_pass, p_cm_version, p_cluster, p_cm_port=None, p_use_tls=False): self.cm_api = ApiResource(p_cm_host, server_port=p_cm_port, version=p_cm_version, username=p_cm_user, password=p_cm_pass, use_tls=p_use_tls) handler_cm_api.cluster_hosts = self.cm_api.get_all_hosts() if p_cluster: self.cluster = filter(lambda x: x.displayName == p_cluster, self.cm_api.get_all_clusters())[0] if not self.cluster: print("Error: That cluster is not valid.") return else: self.services = self.cluster.get_all_services() self.name = self.cluster.displayName tmp_topology = self.cluster.list_hosts() self.topology = {} for i in range(len(tmp_topology)): tmp_host = filter(lambda x: x.hostId == tmp_topology[i].hostId, handler_cm_api.cluster_hosts)[0] self.topology[tmp_topology[i].hostId] = tmp_host.hostname
def main(): module = build_module() choice_map = {'present': present, 'distributed': distributed, 'activated': activated, 'absent': absent, 'infos': infos} params = module.params has_changed = False api = ApiResource(params["cm_host"], username=params["cm_login"], password=params["cm_login"], version=params["api_version"]) try: cluster = api.get_cluster(params["cluster_name"]) except ApiException as e: module.fail_json(msg="Cluster error : {0}".format(e)) if params["product"] and params["version"]: parcel = get_parcel(cluster, params["product"], params["version"]) if params["state"] != "infos": error, has_changed, result, meta = choice_map.get(params['state'])(cluster, parcel) if error: module.fail_json(msg=result) module.exit_json(changed=has_changed, msg=result, meta=meta) else: meta = { "product": parcel.product, "version": parcel.version, "stage": parcel.stage } module.exit_json(changed=False, msg="Parcel informations gathered", meta=meta) elif not params["product"] and not params["version"] and params["state"] == "infos": module.exit_json(changed=has_changed, msg="Parcel informations gathered", meta=infos(cluster))
def init_cluster(): # wait for all cloudera agent processes to come up BDVLIB_ServiceWait( [["services", "cloudera_scm_agent", NODE_GROUP_ID, "kts"]]) # make sure cloudera manager has received registration # for all new agents all_cloudera_hosts = get_hosts_for_service( ["services", "cloudera_scm_agent"]) api = ApiResource(CM_HOST, username="******", password="******") while True: current_all_hosts = map(lambda x: x.hostname, api.get_all_hosts()) setup_logger.info("Currently registered hosts with CM " + str(current_all_hosts)) if all(x in current_all_hosts for x in all_cloudera_hosts): break setup_logger.info( "waiting for new nodes to register with cloudera manager") time.sleep(10) manager = api.get_cloudera_manager() manager.update_config(CM_CONFIG) cluster = api.create_cluster(CLUSTER_NAME, CDH_MAJOR_VERSION, CDH_FULL_VERSION) KTS_HOSTS = ConfigMeta.getWithTokens( ['nodegroups', NODE_GROUP_ID, 'roles', 'kts', 'fqdns']) cluster.add_hosts(KTS_HOSTS) return (cluster, manager)
def main(): #print sys.argv[0] #for i in range(1, len(sys.argv)): # print "param ", i, sys.argv[i] # get a handle on the instance of CM that we have running api = ApiResource(cm_host, cm_port, cm_username, cm_password, version=13) # get the CM instancepython2.7 setuptools cm = ClouderaManager(api) cluster = api.get_cluster(cluster_name) # distribution_parcels(api, cluster) cmd = cluster.first_run() while cmd.success == None: cmd = cmd.fetch() if cmd.success != True: print "The first run command failed: " + cmd.resultMessage() exit(0) print "First run successfully executed. Your cluster has been set up!"
def adjust_yarn_memory_limits(region, stack_name, restart=True): ec2_conn = create_ec2_connection(region) manager_instance = get_manager_instance(ec2_conn, stack_name) with cm_tunnel_ctx(manager_instance) as local_port: cm_api = ApiResource('localhost', username='******', password='******', server_port=local_port, version=9) cluster = list(cm_api.get_all_clusters())[0] host = list(cm_api.get_all_hosts())[0] # all hosts same instance type yarn = filter(lambda x: x.type == 'YARN', list(cluster.get_all_services()))[0] rm_cg = filter(lambda x: x.roleType == 'RESOURCEMANAGER', list(yarn.get_all_role_config_groups()))[0] nm_cg = filter(lambda x: x.roleType == 'NODEMANAGER', list(yarn.get_all_role_config_groups()))[0] rm_cg.update_config({ 'yarn_scheduler_maximum_allocation_mb': ( int(host.totalPhysMemBytes / 1024. / 1024.)), 'yarn_scheduler_maximum_allocation_vcores': host.numCores}) nm_cg.update_config({ 'yarn_nodemanager_resource_memory_mb': ( int(host.totalPhysMemBytes / 1024. / 1024.)), 'yarn_nodemanager_resource_cpu_vcores': host.numCores}) cluster.deploy_client_config().wait() if restart: cluster.restart().wait()
def create_cluster(): CM_HOST = (load_cfg(ansible_path+"/group_vars/all")).get("cm_host") USERNAME = (load_cfg(ansible_path+"/group_vars/all")).get("cm_username") PASSWORD = (load_cfg(ansible_path+"/group_vars/all")).get("cm_password") api = ApiResource(CM_HOST, version=API_VERSION, username=USERNAME, password=PASSWORD) cluster = api.get_cluster(CLUSTER_NAME) return cluster
def connect(cm_api, cm_username, cm_password, use_proxy=False): ''' Wait for ten minutes for CM to come up ''' for _ in xrange(120): try: logging.info("Checking CM availability....") # change name of proxy if necessary proxy = urllib2.ProxyHandler({'http': 'proxy'}) api = ApiResource(cm_api, username=cm_username, password=cm_password, version=14) if use_proxy: # pylint: disable=W0212 api._client._opener.add_handler(proxy) cloudera_manager = api.get_cloudera_manager() api.get_user(cm_username) return api, cloudera_manager except Exception: logging.warning("CM is not up") time.sleep(5) logging.error("CM did not come UP") sys.exit(-1)
def __init__(self, hostname=None, port='7180', username='******', password='******'): ''' Creates a Resource object that provides API endpoints. @param hostname: Hostname of the Cloudera Manager server. @param port: Port of the server. Defaults to 7180 (http). @param username: Login name. @param password: Login password. ''' config = Config() salt = Salt() if hostname is None: hostname = salt.function('roles.dict', 'cloudera-cm4-server')['cloudera-cm4-server'][0] try: logger.debug('Acquiring Cloudera Manager API resource') ApiResource.__init__(self, hostname, port, username, password) except ApiException: logger.error('Problem acquiring Cloudera Manager API resource') # sys.exit(1) except URLError: logger.error('Problem connecting to %s', hostname) sys.exit(1) else: self.config = config self.salt = salt cluster = ClouderaManagerCluster(self) host = ClouderaManagerHost(self)
def setup_api_resources(self): self.api = ApiResource(server_host=self.cm_server_address, server_port=self.cm_server_port, username=self.username, password=self.password, version=self._get_api_version()) self.cm = self.api.get_cloudera_manager() self.cluster = self.api.get_cluster('Cluster 1 (clusterdock)')
def getActiveCMConfig(totalconfig): cmConfig = {} for cm in totalconfig['cmfqdn']: api = ApiResource(cm, totalconfig[cm]['port'], totalconfig[cm]['user'], totalconfig[cm]['passwd'], totalconfig[cm]['tls'], totalconfig[cm]['apiv']) clusters = api.get_all_clusters() cmConfig[cm] = {} for cluster in clusters: cmConfig[cm][cluster.displayName] = {} services = cluster.get_all_services() for service in services: cmConfig[cm][cluster.displayName][service.name] = {} cmConfig[cm][cluster.displayName][service.name]['Service'] = {} for name, config in service.get_config(view='full')[0].items(): cmConfig[cm][cluster.displayName][ service.name]['Service'][name] = { 'value': config.value, 'default': config.default } for roleGroup in service.get_all_role_config_groups(): cmConfig[cm][cluster.displayName][service.name][ roleGroup.roleType] = {} for name, config in roleGroup.get_config( view='full').items(): cmConfig[cm][cluster.displayName][service.name][ roleGroup.roleType][name] = { 'value': config.value, 'default': config.default } print(roleGroup.roleType) #print(json.dumps(cmConfig, indent=4)) return cmConfig
def main(): """ Add peer to the cluster. @rtype: number @returns: A number representing the status of success. """ settings = parse_args() if len(sys.argv) == 1 or len(sys.argv) > 17: print_usage_message() quit(1) api_target = ApiResource(settings.server, settings.port, settings.username, settings.password, settings.use_tls, 14) type_name = 'AWS_ACCESS_KEY_AUTH' account_configs = { 'aws_access_key': settings.aws_access_key, 'aws_secret_key': settings.aws_secret_key } try: api_target.create_external_account(settings.account_name, settings.account_name, type_name, account_configs=account_configs) print "S3 Account Successfully Added" except ApiException as error: if 'already exists' in str(error): print 'Peer Already exists' else: raise error return 0
def __init__(self, hostname=None, port="7180", username="******", password="******"): """ Creates a Resource object that provides API endpoints. @param hostname: Hostname of the Cloudera Manager server. @param port: Port of the server. Defaults to 7180 (http). @param username: Login name. @param password: Login password. """ config = Config() salt = Salt() if hostname is None: hostname = salt.function("roles.dict", "cloudera-cm5-server")["cloudera-cm5-server"][0] try: logger.debug("Acquiring Cloudera Manager API resource") ApiResource.__init__(self, hostname, port, username, password) except ApiException: logger.error("Problem acquiring Cloudera Manager API resource") # sys.exit(1) except URLError: logger.error("Problem connecting to %s", hostname) sys.exit(1) else: self.config = config self.salt = salt cluster = ClouderaManagerCluster(self) host = ClouderaManagerHost(self)
def main(): # connect cm api api = ApiResource(CM_HOST, 7180, username=CM_USERNAME, password=CM_PASSWORD) manager = api.get_cloudera_manager() # no need to update cm config #manager.update_config(cm_host) print("[INFO] Connected to CM host on " + CM_HOST) # create cluster object try: cluster = api.get_cluster(name=CLUSTER_NAME) except: cluster = init_cluster(api, CLUSTER_NAME, CLUSTER_VERSION, CLUSTER_NODE_COUNT) print("[INFO] Initialized cluster " + CLUSTER_NAME + " which uses CDH version " + CLUSTER_VERSION) # mgmt_servicename = "MGMT" amon_role_name = "ACTIVITYMONITOR" apub_role_name = "ALERTPUBLISHER" eserv_role_name = "EVENTSERVER" hmon_role_name = "HOSTMONITOR" smon_role_name = "SERVICEMONITOR" nav_role_name = "NAVIGATOR" navms_role_name = "NAVIGATORMETADATASERVER" rman_role_name = "REPORTMANAGER" deploy_management(manager, mgmt_servicename, amon_role_name, apub_role_name, eserv_role_name, hmon_role_name, smon_role_name, nav_role_name, navms_role_name, rman_role_name) print("[INFO] Deployed CM management service " + mgmt_servicename + " to run on " + CM_HOST) # assign_roles(api, cluster) print("[INFO] all roles have assigned.") # # Custom role config groups cannot be automatically configured: Gateway Group 1 (error 400) try: cluster.auto_configure() except: pass update_custom_config(api, cluster) print("[INFO] all servies and roles have configured.") # cmd = cluster.first_run() while cmd.success == None: cmd = cmd.fetch() if not cmd.success: print("[ERROR] The first run command failed: " + cmd.resultMessage()) else: print( "[INFO] First run successfully executed. Your cluster has been set up!" )
def list_hosts(host, username, password, cafile): context = ssl.create_default_context(cafile=cafile) api = ApiResource(host, username=username, password=password, use_tls=True, ssl_context=context) for h in api.get_all_hosts(): print h.hostname
def main(): cmhost = os.environ['DEPLOYMENT_HOST_PORT'].split(":")[0] api = ApiResource(cmhost, username='******', password='******') all_clusters = api.get_all_clusters() for cluster in all_clusters: if (cluster.name == os.environ['CLUSTER_NAME']): break template = cluster.create_host_template("cdsw-gateway")
def main(): API = ApiResource(CM_HOST, version=5, username=ADMIN_USER, password=ADMIN_PASS) print "Connected to CM host on " + CM_HOST CLUSTER = API.get_cluster(CLUSTER_NAME) print "About to restart cluster." CLUSTER.restart().wait() print "Done restarting cluster."
def main(): API = ApiResource(CM_HOST, version=5, username=ADMIN_USER, password=ADMIN_PASS) print "Connected to CM host on " + CM_HOST CLUSTER = API.get_cluster(CLUSTER_NAME) print "About to stop cluster." CLUSTER.stop().wait() print "Done stopping cluster."
def get_cluster_specs(): cm_api = ApiResource(os.environ['MANAGER_HOST'], username='******', password='******', server_port=7180, version=9) host = list(cm_api.get_all_hosts())[0] # all hosts same instance type cluster = list(cm_api.get_all_clusters())[0] yarn = filter(lambda x: x.type == 'YARN', list(cluster.get_all_services()))[0] return {'num_worker_nodes': len(yarn.get_roles_by_type('NODEMANAGER')), 'num_cores': host.numCores, 'node_memory': host.totalPhysMemBytes}
def main(cm_host, user, password): api = ApiResource(cm_host, username=user, password=password) cm = api.get_cloudera_manager() cm.update_all_hosts_config( {"java_home": "/usr/java/jdk1.8.0_121-cloudera"}) print("restarting CM service - this will take a minute or so") cm.get_service().restart().wait() print("restarting cluster - this will take 2-5 minutes") api.get_all_clusters()[0].restart(restart_only_stale_services=True, redeploy_client_configuration=True).wait()
def start(host, user, passw): cm_host = str(host) api = ApiResource(cm_host, 7180, username=str(username), password=str(password), version=9) mgmt = api.get_cloudera_manager().get_service() mgmt.restart() print("Services successfully started")
def getClusterInformation(self): api = ApiResource(self.cm_host, username=self.user, password=self.passwd) logger.info('Received; user -> %s, password -> %s, host -> %s', self.user, self.passwd, self.cm_host) for c in api.get_all_clusters(): clusterInf = "Cluster name %s and version %s" %(c.name, c.version) #print "Cluster name %s and version %s" %(c.name, c.version) logger.info("Cluster name %s and version %s", c.name, c.version) if c.version == "CDH5": cdh5 = c return cdh5, clusterInf
def main(): parser = cm_args_parser() args = parser.parse_args() print "connecting to host:" + args.cm_host + "..." api = ApiResource(args.cm_host, username=args.cm_user, password=args.cm_password) print "host connected, getting cloudera manager " MANAGER = api.get_cloudera_manager() print "have cloudera manager object" deploy_management(MANAGER, MGMT_SERVICENAME, MGMT_SERVICE_CONFIG, MGMT_ROLE_CONFIG, AMON_ROLENAME, AMON_ROLE_CONFIG, APUB_ROLENAME, APUB_ROLE_CONFIG, ESERV_ROLENAME, ESERV_ROLE_CONFIG, HMON_ROLENAME, HMON_ROLE_CONFIG, SMON_ROLENAME, SMON_ROLE_CONFIG, RMAN_ROLENAME, RMAN_ROLE_CONFIG) print "Deployed CM management service " + MGMT_SERVICENAME + " to run on " + CM_HOST
def __init__(self, cm_host, cm_cluster_name, username, password): self.cm_api = ApiResource(cm_host, username=username, password=password) self.hosts = dict() self.services = list() self.cluster = self.cm_api.get_cluster(cm_cluster_name) if self.cluster is None: raise RuntimeError, 'Cluster name "%s" not found' % cm_cluster_name self.__load_hosts() self.__impala_service = ImpalaService(self)
def __init__(self, module, config, trial=False, license_txt=None): self.api = ApiResource(config['cm']['host'], username=config['cm']['username'], password=config['cm']['password']) self.manager = self.api.get_cloudera_manager() self.config = config self.module = module self.trial = trial self.license_txt = license_txt self.cluster = None
def main(): """ Add peer to the cluster. @rtype: number @returns: A number representing the status of success. """ settings = parse_args() if len(sys.argv) == 1 or len(sys.argv) > 17: print_usage_message() quit(1) api = ApiResource(settings.server, settings.port, settings.username, settings.password, settings.use_tls, 14) yarn_service = get_service_name('YARN', api, settings.cluster_name) hdfs_name = get_service_name('HDFS', api, settings.cluster_name) hdfs = api.get_cluster(settings.cluster_name).get_service(hdfs_name) hdfs_cloud_args = ApiHdfsCloudReplicationArguments(None) hdfs_cloud_args.sourceService = ApiServiceRef(None, peerName=None, clusterName=settings.cluster_name, serviceName=hdfs_name) hdfs_cloud_args.sourcePath = settings.source_path hdfs_cloud_args.destinationPath = settings.target_path hdfs_cloud_args.destinationAccount = settings.account_name hdfs_cloud_args.mapreduceServiceName = yarn_service # creating a schedule with daily frequency start = datetime.datetime.now() # The time at which the scheduled activity is triggered for the first time. end = start + datetime.timedelta(days=365) # The time after which the scheduled activity will no longer be triggered. schedule = hdfs.create_replication_schedule(start, end, "DAY", 1, True, hdfs_cloud_args) ## Updating the Schedule's properties # schedule.hdfsArguments.removeMissingFiles = False schedule.alertOnFail = True schedule = hdfs.update_replication_schedule(schedule.id, schedule) print "Schedule created with Schdule ID: " + str(schedule.id) # print schedule.alertOnFail # print schedule.hdfsArguments.removeMissingFiles # print schedule.hdfsArguments.sourcePath # print schedule.hdfsArguments.preserveXAttrs # print schedule.hdfsArguments.exclusionFilters # print schedule.hdfsArguments.replicationStrategy # print schedule.hdfsArguments.numMaps # print schedule.hdfsArguments.userName # print schedule.hdfsArguments.schedulerPoolName # print type(schedule) return 0
def connect_cm(cm_host, cm_username, cm_password): """ Connects to Cloudera Manager API Resource instance to retrieve Endpoint details :param cm_host: Cloudera Manager host :param cm_username: Username for authentication :param cm_password: Password for authentication :return: """ api = ApiResource(cm_host, version=6, username=cm_username, password=cm_password) cm_manager = api.get_cloudera_manager() return api, cm_manager
def connect_cm(cm_host, cm_username, cm_password): """ Connects to Cloudera Manager API Resource instance to retrieve Endpoint details :param cm_host: Cloudera Manager host :param cm_username: Username for authentication :param cm_password: Password for authentication :return: """ api = ApiResource(cm_host, version=6, username=cm_username, password=cm_password) cm_manager = api.get_cloudera_manager() return api, cm_manager
def get_cluster(cm_host, user, pwd, cluster_name): global api api = ApiResource(cm_host, username=user, password=pwd, version=12) for c in api.get_all_clusters(): if cluster_name in c.name: # print("Cluster, Version : " + c.name + ", " + c.version) return True, c else: print("[ERR] : Cluster \"" + cluster_name + "\" not found at \"" + cm_host + "\"") return False, ""
def find_impala_in_cm(cm_host, cm_user, cm_password, cm_cluster_name): """Finds the Impala service in CM and returns an Impala instance.""" cm = ApiResource(cm_host, username=cm_user, password=cm_password) cm_impalas = [service for cluster in cm.get_all_clusters() if cm_cluster_name is None or cm_cluster_name == cluster.name for service in cluster.get_all_services() if service.type == "IMPALA"] if len(cm_impalas) > 1: raise Exception("Found %s Impala services in CM;" % len(cm_impalas) + " use --cm-cluster-name option to specify which one to use.") if len(cm_impalas) == 0: raise Exception("No Impala services found in CM") return Impala(cm_impalas[0])
def cm_api_resource(self): ar = None try: ar = ApiResource(self.cm_host, self.cm_port, self.cm_username, self.cm_password) ar.echo('Authenticated') # Issue a sample request to test the conn except ApiException, aexc: if aexc.code == 401: log.debug("Changing default API username to {0}".format(self.cm_username)) self.cm_username = self.host_username self.cm_password = self.host_password ar = ApiResource(self.cm_host, self.cm_port, self.cm_username, self.cm_password) else: log.error("Api Exception connecting to ClouderaManager: {0}".format(aexc))
def main(): # connect cm api api = ApiResource(CM_HOST, 7180, username=CM_USERNAME, password=CM_PASSWORD) manager = api.get_cloudera_manager() # no need to update cm config #manager.update_config(cm_host) print("[INFO] Connected to CM host on " + CM_HOST) # create cluster object try: cluster = api.get_cluster(name=CLUSTER_NAME) except: cluster = init_cluster(api, CLUSTER_NAME, CLUSTER_VERSION, CLUSTER_NODE_COUNT) print("[INFO] Initialized cluster " + CLUSTER_NAME + " which uses CDH version " + CLUSTER_VERSION) # mgmt_servicename = "MGMT" amon_role_name = "ACTIVITYMONITOR" apub_role_name = "ALERTPUBLISHER" eserv_role_name = "EVENTSERVER" hmon_role_name = "HOSTMONITOR" smon_role_name = "SERVICEMONITOR" nav_role_name = "NAVIGATOR" navms_role_name = "NAVIGATORMETADATASERVER" rman_role_name = "REPORTMANAGER" deploy_management(manager, mgmt_servicename, amon_role_name, apub_role_name, eserv_role_name, hmon_role_name, smon_role_name, nav_role_name, navms_role_name, rman_role_name) print("[INFO] Deployed CM management service " + mgmt_servicename + " to run on " + CM_HOST) # assign_roles(api, cluster) print("[INFO] all roles have assigned.") # # Custom role config groups cannot be automatically configured: Gateway Group 1 (error 400) try: cluster.auto_configure() except: pass update_custom_config(api, cluster) print("[INFO] all servies and roles have configured.") # cmd = cluster.first_run() while cmd.success == None: cmd = cmd.fetch() if not cmd.success: print("[ERROR] The first run command failed: " + cmd.resultMessage()) else: print("[INFO] First run successfully executed. Your cluster has been set up!")
def main(): s,a = arg_handle() for i in range(0,15): while True: try: cm_host = "127.0.0.1" api = ApiResource(cm_host, username="******", password="******") cdh=api.get_all_clusters()[0] except: print "Failed to connect to Cloudera Manager." print "Attempting to connect to Cloudera Manager..." time.sleep(15) continue break srv=cdh.get_service(s) actions[a](srv,s)
def main(): resource = ApiResource("localhost", 7180, "cloudera", "cloudera", version=19) cluster = resource.get_cluster("Cloudera Quickstart") cm_manager = resource.get_cloudera_manager() cm_manager.update_config({'REMOTE_PARCEL_REPO_URLS': PARCEL_REPO}) cm_manager.update_all_hosts_config(JDK_CONFIG) time.sleep(5) for parcel in PARCELS: ParcelInstaller(parcel['name'], parcel['version']).install(cluster) print "Restarting cluster" cluster.stop().wait() cluster.start().wait() print "Done restarting cluster"
def main(): AMON_ROLE_CONFIG = { 'firehose_heapsize': '1173741824', } API = ApiResource("ec2-52-24-151-222.us-west-2.compute.amazonaws.com", version=5, username="******", password="******") MANAGER = API.get_cloudera_manager() mgmt = MANAGER.get_service() #cf = mgmt.get_config() for group in mgmt.get_all_role_config_groups(): if group.roleType == "ACTIVITYMONITOR": group.update_config(AMON_ROLE_CONFIG) pass
def update_cm(cm_host, cm_port, username, password): """Update config using the CM API (note: will restart service)""" elts = generate_xml_elements() cm_api = ApiResource(cm_host, username=username, password=password, server_port=cm_port, version=9) cluster = list(cm_api.get_all_clusters())[0] hdfs = filter(lambda x: x.type == 'HDFS', list(cluster.get_all_services()))[0] print("Updating HFDS core-site.xml safety valve...") _ = hdfs.update_config({ 'core_site_safety_valve': '\n'.join(tostring(e) for e in elts)}) print("Deploying client config across the cluster...") cluster.deploy_client_config().wait() print("Restarting necessary services...") cluster.restart().wait() print("Done!")
def __init__(self, host_name, port=7180, user="******", password="******", cluster_name=None, ssh_user=None, ssh_port=None, ssh_key_file=None): # Initialize strptime() to workaround https://bugs.python.org/issue7980. Apparently # something in the CM API uses strptime(). strptime("2015", "%Y") Cluster.__init__(self) self.cm = CmApiResource(host_name, server_port=port, username=user, password=password) clusters = self.cm.get_all_clusters() if not clusters: raise Exception("No clusters found in CM at %s" % host_name) if cluster_name: clusters_by_name = dict((c.name, c) for c in clusters) if cluster_name not in clusters_by_name: raise Exception(("No clusters named %s found in CM at %s." "Available clusters are %s.") % (cluster_name, host_name, ", ".join(sorted(clusters_by_name.keys())))) self.cm_cluster = clusters_by_name[cluster_name] else: if len(clusters) > 1: raise Exception(("Too many clusters found in CM at %s;" " a cluster name must be provided") % host_name) self.cm_cluster = clusters[-1] self.ssh_user = ssh_user self.ssh_port = ssh_port self.ssh_key_file = ssh_key_file self._ssh_client_lock = Lock() self._ssh_clients_by_host_name = defaultdict(list)
class TimeSeriesQuery(object): """ """ def __init__(self): self._api = ApiResource(CM_HOST, username=CM_USER, password=CM_PASSWD, use_tls=CM_USE_TLS) def query(self, query, from_time, to_time): return self._api.query_timeseries(query, from_time, to_time)
def main(): """ This is a script to export a current Cloudera Manager cluster configuration into an Hadrian supported format. You can then use these configuration files as the basis for your new cluster configs. """ parser = argparse.ArgumentParser(description='Export Cloudera Manager configs in an Hadrian friendly format.') parser.add_argument('-H', '--host', '--hostname', action='store', dest='hostname', required=True, help='CM Server Name') parser.add_argument('-p', '--port', action='store', dest='port', type=int, default=7180, help='CM Port') parser.add_argument('-u', '--user', '--username', action='store', dest='username', required=True, help='CM username') args = parser.parse_args() password = getpass.getpass('Please enter your Cloudera Manager passsword: ') api = ApiResource(args.hostname, args.port, args.username, password, version=4) for cluster in api.get_all_clusters(): conf_dir = './confs/' + cluster.name if not os.path.exists(conf_dir): os.makedirs(conf_dir) for service in cluster.get_all_services(): with open(conf_dir + '/' + service.name + '.ini', 'w') as f: print 'Dumping Service config for ' + service.name rcg = list() for i in service.get_all_role_config_groups(): rcg.append(i.name) f.write('[' + service.type + ']\n') f.write('config_groups=' + ','.join(rcg)) f.write('\n\n') f.write('[' + service.name + '-svc-config]\n') for item in service.get_config(): for k,v in item.iteritems(): f.write(k + '=' + str(v) + '\n') for i in service.get_all_role_config_groups(): f.write('\n') f.write('[' + i.name + ']\n') for k,v in i.get_config('full').iteritems(): if v.value is not None: f.write(k + '=' + str(v.value) + '\n') f.close() else: print 'Cluster config dir already exists. Please rename or remove existing config dir: ' + conf_dir
def __init__(self, module, config, trial=False, license_txt=None): self.api = ApiResource(config['cm']['host'], username=config['cm']['username'], password=config['cm']['password']) self.manager = self.api.get_cloudera_manager() self.config = config self.module = module self.trial = trial self.license_txt = license_txt self.cluster = None
def do_call(host, port, user, password, cluster_name, service_role_name, random_index): api = ApiResource(host, port, user, password, False, MAN_API_VERSION); for cluster in api.get_all_clusters(): if cluster_name is None: break elif cluster_name == cluster.name: break if cluster_name is not None and cluster_name != cluster.name: print >> sys.stderr, "Cloud not find cluster: " + cluster_name return -2; do_print_header() for service in cluster.get_all_services(): do_print_line_item(api, service, service_role_name, random_index, 'HDFS', 'NAMENODE', 'namenode_port', [], []) do_print_line_item(api, service, service_role_name, random_index, 'KUDU', 'KUDU_MASTER', 'webserver_port', [], []) do_print_line_item(api, service, service_role_name, random_index, 'HUE', 'HUE_SERVER', 'hue_http_port', [], []) do_print_line_item(api, service, service_role_name, random_index, 'HIVE', 'HIVESERVER2', 'hs2_thrift_address_port', [], []) do_print_line_item(api, service, service_role_name, random_index, 'IMPALA', 'IMPALAD', 'beeswax_port', [], []) do_print_line_item(api, service, service_role_name, random_index, 'FLUME', 'AGENT', 'agent_http_port', [], []) do_print_line_item(api, service, service_role_name, random_index, 'KAFKA', 'KAFKA_BROKER', 'port', [], []) do_print_line_item(api, service, service_role_name, random_index, 'ZOOKEEPER', 'SERVER', 'clientPort', [], []) do_print_footer()
def reset_cm(cm_host, cm_port, username, password): """Elim S3 config from CM API safety valve (service restart necessary)""" s3_props = set(get_s3_properties()) cm_api = ApiResource(cm_host, username=username, password=password, server_port=cm_port, version=9) cluster = list(cm_api.get_all_clusters())[0] hdfs = filter(lambda x: x.type == 'HDFS', list(cluster.get_all_services()))[0] print("Getting current safety valve config") current_config = hdfs.get_config('full')[0]['core_site_safety_valve'].value # need the "<foo>...</foo>" to make it valid XML (bc it requires root elt) elts = list(fromstring('<foo>' + current_config + '</foo>')) new_elts = filter(lambda x: x.find('name').text not in s3_props, elts) print("Updating safety valve and deleting S3 config") _ = hdfs.update_config({ 'core_site_safety_valve': '\n'.join(tostring(e) for e in new_elts)}) print("Deploying client config across the cluster...") cluster.deploy_client_config().wait() print("Restarting necessary services...") cluster.restart().wait() print("Done!")
def main(): """ Enables HDFS HA on a cluster. @rtype: number @returns: A number representing the status of success. """ settings = retrieve_args() api = ApiResource(settings.host, settings.port, settings.username, settings.password, version=6) if not validate_cluster(api, settings.cluster): write_to_stdout("Cluster does not satisfy preconditions for enabling HDFS HA. Exiting!") return 1 if settings.wait_for_good_health: write_to_stdout("Waiting for GOOD health... ") if not wait_for_good_health(api, settings.cluster): write_to_stdout("Cluster health is not GOOD. Exiting!\n") return 1 else: write_to_stdout("Checking cluster health... ") if not check_health(api, settings.cluster): write_to_stdout("Cluster health is not GOOD. Exiting!\n") write_to_stdout("Cluster health is GOOD!\n") cluster = api.get_cluster(settings.cluster) invoke_hdfs_enable_nn_ha(cluster, settings.nameservice) update_hive_for_ha_hdfs(cluster) # Restarting the MGMT services to make sure the HDFS file browser functions # as expected. cloudera_manager = api.get_cloudera_manager() mgmt_service = cloudera_manager.get_service() wait_for_command('Restarting MGMT services', mgmt_service.restart()) return 0
def main(): """ TODO: This probably needs some work. You get the idea though. An example of how to do a bulk config update to Cloudera Manager. This is helpful if you have a bunch of changes That you want to make but don't want to use the GUI. """ parser = argparse.ArgumentParser(description='Cloudera Manager Bulk Config Update Script') parser.add_argument('-H', '--host', '--hostname', action='store', dest='hostname', required=True, help='CM server host') parser.add_argument('-p', '--port', action='store', dest='port', type=int, default=7180, help='example: 7180') parser.add_argument('-u', '--user', '--username', action='store', dest='username', required=True, help='example: admin') parser.add_argument('-c', '--cluster', action='store', dest='cluster', required=True, help='example: hadrian-cluster') args = parser.parse_args() password = getpass.getpass('Please enter your Cloudera Manager passsword: ') # read configuration files: for i in os.listdir('./conf/' + args.cluster): config.read('./conf/' + args.cluster + '/' + i) api = ApiResource(args.hostname, args.port, args.username, password) cluster = api.get_cluster(args.cluster) services = cluster.get_all_services() # update services based with configuration file parameters for service in services: if config_grabber.has_section(service.type): service.update_config(svc_config=config_grabber(service.name + '-svc-config')) config_groups = config_grabber(service.name)['config_groups'] for config_group in config_groups.split(','): print section temp_config_group = service.get_role_config_group(section) temp_config_group.update_config(config_grabber(section)) else: print 'unknown service: ' + service.name print 'Starting final client configuration deployment for all services.' cmd = cluster.deploy_client_config() if not cmd.wait(CMD_TIMEOUT).success: print 'Failed to deploy client configuration.'
def get_cluster_info(manager_host, server_port=7180, username='******', password='******'): cm_api = ApiResource(manager_host, username=username, password=password, server_port=server_port, version=9) host = list(cm_api.get_all_hosts())[0] # all hosts same instance type cluster = list(cm_api.get_all_clusters())[0] yarn = filter(lambda x: x.type == 'YARN', list(cluster.get_all_services()))[0] hive = filter(lambda x: x.type == 'HIVE', list(cluster.get_all_services()))[0] impala = filter(lambda x: x.type == 'IMPALA', list(cluster.get_all_services()))[0] hive_hs2 = hive.get_roles_by_type('HIVESERVER2')[0] hive_host = cm_api.get_host(hive_hs2.hostRef.hostId).hostname hive_port = int( hive_hs2.get_config('full')['hs2_thrift_address_port'].default) impala_hs2 = impala.get_roles_by_type('IMPALAD')[0] impala_host = cm_api.get_host(impala_hs2.hostRef.hostId).hostname impala_port = int(impala_hs2.get_config('full')['hs2_port'].default) return {'num_worker_nodes': len(yarn.get_roles_by_type('NODEMANAGER')), 'node_cores': host.numCores, 'node_memory': host.totalPhysMemBytes, 'hive_host': hive_host, 'hive_port': hive_port, 'impala_host': impala_host, 'impala_port': impala_port}