def __init__(self, config): self.config = config self.logger = logging.getLogger('ClouderaManagerMetricsCollector') self._setup_logging(config) self.logger.info("Load configuration: " + str(config)) self.cluster = None self.services = [] self.availableMetrics = {} self.serviceMetricsMapFromConfig = {} self._create_service_metrics_map() self._setup_termination_handler() self._setup_metrics_processor(config) self._setup_health_check_metrics_processor(config) cm_client.configuration.username = config["clouderaManager"]["user"] cm_client.configuration.password = config["clouderaManager"][ "password"] cm_client.configuration.verify_ssl = False self.cluster_type = config["clouderaManager"]["clusterType"] if socket.gethostname().find('.') >= 0: api_host = socket.gethostname() else: api_host = socket.gethostbyaddr(socket.gethostname())[0] protocol = config["clouderaManager"]["protocol"] port = config["clouderaManager"]["port"] api_version = config["clouderaManager"]["apiVersion"] api_url = protocol + "://" + api_host + ':' + port + '/api/' + api_version self.api_client = cm_client.ApiClient(api_url) self.logger.info("Cloudera Manager Api URL: " + api_url) metrics_api_url = protocol + "://" + api_host + ':' + port + '/api/v5' self.metrics_api_client = cm_client.ApiClient(metrics_api_url) self.logger.info("Cloudera Manager Metrics Api URL: " + metrics_api_url) self.globalFields = config["globalFields"]
def __init__(self): import cm_client as cm cm.configuration.username = "******" cm.configuration.password = "******" self.api_client = cm.ApiClient(self.api_url) self.services_api_health = cm.ServicesResourceApi(self.api_client) self.services_api_IO = cm.TimeSeriesResourceApi(self.api_client)
def api_client(self): if self._api_client is None: if cm_client.configuration.ssl_ca_cert: api_url = "https://" + self.host + ":7183/api" else: api_url = "http://" + self.host + ":7180/api" self._api_client = cm_client.ApiClient(api_url + '/' + self._get_api_version()) return self._api_client
def cluster_stats(): """Gets the total YARN meory & VCores using the Cloudera anager API""" #req = requests.get("http://nightly6x-unsecure-1.nightly6x-unsecure.root.hwx.site:7180/api/v40/tools/echo?message=hello") #print req.content cm_client.configuration.username = '******' cm_client.configuration.password = '******' # Create an instance of the API class api_host = 'http://nightly6x-unsecure-1.nightly6x-unsecure.root.hwx.site' port = '7180' api_version = 'v33' # Construct base URL for API # http://cmhost:7180/api/v30 api_url = api_host + ':' + port + '/api/' + api_version print api_url api_client = cm_client.ApiClient(api_url) cluster_api_instance = cm_client.ClustersResourceApi(api_client) api_response = cluster_api_instance.read_clusters(view='SUMMARY') for cluster in api_response.items: print cluster.name, "-", cluster.full_version services_api_instance = cm_client.ServicesResourceApi(api_client) services = services_api_instance.read_services(cluster.name, view='FULL') for service in services.items: if service.type=="YARN": yarn = service print yarn.name api_url_v5 = api_host + '/api/' + 'v5' api_client_v5 = cm_client.ApiClient(api_url_v5) print api_url_v5 services_api_instance_v5 = cm_client.ServicesResourceApi(api_client_v5) #print services_api_instance_v5.get_metrics(cluster.name, hdfs.name) metrics = services_api_instance_v5.get_metrics(cluster.name, yarn.name) for m in metrics.items: print "%s (%s)" % (m.name, m.unit)
def init_conn(self, host, port, version): """初始化连接,构建cm的连接实例 """ url = host + ':' + port + '/api/' + version client = cm_client.ApiClient(url) # 生成资源API self._cluster_api_instance = cm_client.ClustersResourceApi(client) # 为了获取namemode的资源API self._services_api_instance = cm_client.ServicesResourceApi(client) self._roles_api_instance = cm_client.RolesResourceApi(client) self._host_api_instance = cm_client.HostsResourceApi(client)
def main(): cm_client.configuration.username = ADMIN_USER cm_client.configuration.password = ADMIN_PASS # Create an instance of the API class api_url = "http://{}:{}/api/{}".format(CM_HOST, CM_PORT, API_VERSION) api_client = cm_client.ApiClient(api_url) api_instance = cm_client.ClouderaManagerResourceApi(api_client) try: setup_logger.info("Activating Trial License...") api_instance.begin_trial() setup_logger.info("License Activated...") except: setup_logger.error("Failed to Activate License...")
def cm_kafkaXX1_host_command(update: Update, context: CallbackContext) -> None: chat_ids = update.message.chat_id api_client = cm_client.ApiClient(api_url_dc) api_instance = cm_client.ClustersResourceApi(api_client) api_response = api_instance.list_hosts('kafkaXX1') print(api_response) mo = ('Cluster: <b>kafkaXX1</b>:\n') for cluster in api_response.items: mo = mo + ('Host: ' + cluster.hostname + '\n') context.bot.send_message(chat_id=chat_ids, parse_mode=ParseMode.HTML, text=(mo))
def cm_drc_cluster_command(update: Update, context: CallbackContext) -> None: chat_ids = update.message.chat_id api_client = cm_client.ApiClient(api_url_drc) cluster_api_instance = cm_client.ClustersResourceApi(api_client) api_response = cluster_api_instance.read_clusters(view='full') for cluster in api_response.items: mo = ('Site: <b>DRC</b>:\n') mo = mo + ('Name: <b>' + cluster.name + '</b>\n') mo = mo + ('Version: ' + cluster.full_version + '\n') mo = mo + ('Status: ' + cluster.entity_status + '\n') mo = mo + ('Maintenance: ' + str(cluster.maintenance_mode) + '\n') mo = mo + ('URL: ' + cluster.cluster_url + '\n') context.bot.send_message(chat_id=chat_ids, parse_mode=ParseMode.HTML, text=(mo))
def main(): module = build_module() choice_map = { 'present': 'downloaded', 'distributed': 'distributed', 'activated': 'activated', 'absent': 'available_remotely', 'infos': 'infos' } params = module.params api_url = f"http://{params['cm_host']}:{params['cm_port']}/api/v{params['api_version']}" cm_client.configuration.username = params['cm_login'] cm_client.configuration.password = params['cm_password'] cm_client.configuration.host = api_url api_client = cm_client.ApiClient() if params["product"] and params["version"]: parcel = Parcel(params["product"], params["version"], params["cluster_name"], api_client) try: getattr(parcel, choice_map.get(params["state"]))() except ApiException as e: module.fail_json(msg=f"Cluster error : {e}") module.exit_json(changed=parcel.changed, msg="Parcel informations gathered", meta=parcel.meta()) else: if params["state"] == "infos": api_client_instance = cm_client.ParcelsResourceApi(api_client) parcels = [] try: for parcel in api_client_instance.read_parcels( params["cluster_name"]).items: if params["product"]: if params["product"] != parcel.product: continue parcels.append( Parcel(name=parcel.product, version=parcel.version, cluster_name=parcel.cluster_ref.cluster_name, api_client=api_client).meta()) except ApiException as e: module.fail_json(msg=f"Cluster error : {e}") module.fail_json( changed=False, msg="No valid parameters combination was used, exiting", meta=parcels)
def cm_clusterZ1_services_command(update: Update, context: CallbackContext) -> None: chat_ids = update.message.chat_id api_client = cm_client.ApiClient(api_url_drc) api_instance = cm_client.ServicesResourceApi(api_client) api_response = api_instance.read_services('clusterZ1') mo = ('Cluster: <b>clusterZ1</b>:\n') for cluster in api_response.items: if (cluster.health_summary == 'GOOD'): mo = (mo + '\U00002705') mo = mo + ('[<b>' + cluster.display_name + '</b>] => ' + cluster.health_summary + '=> ' + cluster.service_state + '\n') context.bot.send_message(chat_id=chat_ids, parse_mode=ParseMode.HTML, text=(mo))
def __init__(self, cm_host, env, send_alert): cm_client.configuration.username = '******' cm_client.configuration.password = '******' self.env = env api_host = 'http://' + cm_host port = '7180' api_version = 'v19' api_url = api_host + ':' + port + '/api/' + api_version self.send_alert_email = send_alert print(api_url) self.api_client = cm_client.ApiClient(api_url) self.cluster_api_instance = cm_client.ClustersResourceApi( self.api_client) self.services_api_instance = cm_client.ServicesResourceApi( self.api_client) self.roles_api_instance = cm_client.RolesResourceApi(self.api_client) self.host_api_instance = cm_client.HostsResourceApi(self.api_client) self.host_details = self.get_hostname_by_id() self.bad_alert_count = 0 self.msg_body = ""
def __init__(self, instance): self.log = logging.getLogger(__name__) self.url, user, password, api_version, verify_ssl = get_config( instance) if not user: raise Exception('Cloudera Manager user name is required.') if not password: raise Exception('Cloudera Manager user password is required.') # Configure HTTP basic authorization: basic cm_client.configuration.username = user cm_client.configuration.password = password cm_client.configuration.verify_ssl = verify_ssl # Construct base URL for API api_url = '{0}/api/{1}'.format(self.url, api_version) self.api_client = cm_client.ApiClient(api_url)
def main(): cm_client.configuration.username = ADMIN_USER cm_client.configuration.password = ADMIN_PASS # Create an instance of the API class api_host = 'http://'+ CM_HOST port = '7180' api_version = 'v18' api_url = api_host + ':' + port + '/api/' + api_version api_client = cm_client.ApiClient(api_url) host_api_instance = cm_client.HostsResourceApi(api_client) service_api_instance = cm_client.ServicesResourceApi(api_client) sentry_host_id=fetch_hostid(HostsApi=host_api_instance, hostname=SENTRY_HA_HOST) cluster_name = CLUSTER_NAME service_name = "Sentry" body = cm_client.ApiEnableSentryHaArgs(new_sentry_host_id=sentry_host_id, new_sentry_role_name= "Sentry-Server-2", zk_service_name="ZOOKEEPER") setup_logger.info("Enabling Sentry HA.....") time.sleep(60) res=service_api_instance.enable_sentry_ha_command(cluster_name, service_name, body=body) time.sleep(60) setup_logger.info("Sentry HA Enabled...")
def main(): module = build_module() params = module.params api_url = f"http://{params['cm_host']}:{params['cm_port']}/api/v{params['api_version']}" cm_client.configuration.username = params['cm_login'] cm_client.configuration.password = params['cm_password'] cm_client.configuration.host = api_url api_client = cm_client.ApiClient() cm_config = CM(name=params["cm_host"], api_client=api_client) try: cm_config.set_prop(name=params["name"], state=params["action"], value=params["value"]) except ApiException as e: module.fail_json(msg=f"Cluster error : {e}") module.exit_json(changed=cm_config.changed, msg=f'{params["name"]} is in desired state', meta=cm_config.meta())
def main(argv=None): if argv is None: argv = sys.argv parser = argparse.ArgumentParser(description='Cloudera Cluster and Management Service stop commands') parser.add_argument('configfile', nargs='?', default='config.yaml', help='CM API access information in YAML (default config.yaml)') args = parser.parse_args() try: with open(args.configfile, 'r') as f: config = yaml.safe_load(f) def config_value(key, default): return config[key] if key in config else default # Configure HTTP basic authentication: basic cm_client.configuration.username = os.getenv('CM_ADMIN_USERNAME', config_value('username', 'admin')) cm_client.configuration.password = os.getenv('CM_ADMIN_PASSWORD', config_value('password', 'admin')) # Create an instance of the API class protocol = config_value('protocol', 'http') api_host = config_value('api_host', 'cloudera-manager.example.com') api_port = config_value('api_port', '7180') api_version = config_value('api_version', 'v32') api_uri = f'{protocol}://{api_host}:{api_port}/api/{api_version}' logger.debug(f'CM API URI: {api_uri}') api_client = cm_client.ApiClient(api_uri) stop_all_clusters(api_client, api_host) stop_management_service(api_client, api_host) return 0 except IOError as err: logger.error(f'IOError: [Errno {err.errno}] {err.strerror}: {args.configfile}') return -1
def get_api_handle(cmserver, port='7180', user='******', passwd='admin', api_version='v30'): """ Get a handle to the CM API client :param cmserver: Hostname of the Cloudera Manager Server (CMS) :param port: Port of the server :param user: SCM username :param passwd: SCM password :param tls: Whether to use TLS :return: Resource object referring to the root """ cm_client.configuration.username = user cm_client.configuration.password = passwd api_url = 'http://' + cmserver + ':' + port + '/api/' + api_version api_client = cm_client.ApiClient(api_url) api = None try: api = cm_client.HostsResourceApi(api_client) except ApiException: pass return api
import cm_client from cm_client.rest import ApiException from collections import namedtuple from pprint import pprint import json # Configure HTTP basic authorization: basic cm_client.configuration.username = '******' cm_client.configuration.password = '******' api_url = "http://10.209.239.13:7180/api/v17" api_client = cm_client.ApiClient(api_url) # create an instance of the API class cluster_name = 'Cluster 1' # str | clusters_api_instance = cm_client.ClustersResourceApi(api_client) template = clusters_api_instance.export(cluster_name) # Following step allows python fields with under_score to map # to matching camelCase name in the API model before writing to json file. json_dict = api_client.sanitize_for_serialization(template) with open('/home/tpc/hs/tpcxBB-pipeline/13sep-2019-cluster_template.json', 'w') as out_file: json.dump(json_dict, out_file, indent=4, sort_keys=True)
def main(passed_username, passed_password, passed_database): PEER_NAME = 'PRODUCTION' # Previously TARGET_CLUSTER_NAME = 'DEV' # defined SOURCE_CLUSTER_NAME = 'cluster' # at Experian cm_host = 'br1andvhmn11.passporthealth.com' cm_client.configuration.username = passed_username # Ensure that password is quoted cm_client.configuration.password = "******" + passed_password + "'" sourceDatabase = passed_database # Setup authentication for SSL cm_client.configuration.verify_ssl = True cm_client.configuration.ssl_ca_cert = '/opt/cloudera/security/pki/x509/truststore.pem' # Create an instance of the API class api_host = 'https://br1andvhmn11.passporthealth.com' port = '7183' api_version = 'v30' impala_host = 'br1anprhsn02.passporthealth.com' # Construct base URL for API # http://cmhost:7180/api/v30 api_url = api_host + ':' + port + '/api/' + api_version api_client = cm_client.ApiClient(api_url) cluster_api_instance = cm_client.ClustersResourceApi(api_client) # Lists all known clusters. api_response = cluster_api_instance.read_clusters(view='SUMMARY') for cluster in api_response.items: print cluster.name, "-", cluster.full_version services_api_instance = cm_client.ServicesResourceApi(api_client) services = services_api_instance.read_services(cluster.name, view='FULL') for service in services.items: # print service.display_name, "-", service.type if service.type == 'HIVE': targetHive = service targetCluster = cluster print targetHive.name, targetHive.service_state, targetHive.health_summary for health_check in targetHive.health_checks: print health_check.name, "---", health_check.summary # print "Source database = " + sourceDatabase ###show_statement = "'show tables in " + sourceDatabase +"'" ###streamOperand = "impala-shell -i " + impala_host + " -d default -k --ssl --ca_cert=/opt/cloudera/security/pki/x509/truststore.pem -q " + show_statement ###stream=os.popen(streamOperand) ### ###output=stream.readlines() ###lineno =0 ###numtables = 0 ###tablenames = [] ###for line in output: ### if lineno <= 2: # skip heading lines ### pass ### elif line[0:3] == "+--": # skip last line ### pass ### else: # strip out tablename ### name = line[2:] ### blank = name.index(' ') ### tablenames.append(name[0:blank]) ### numtables +=1 ### lineno +=1 ###print str(numtables) + " tables in database " + sourceDatabase ###for table in tablenames: ### print table tablenames = [] tablenames.append("test") tablenames.append("test2") api_root = ApiResource(cm_host, username=passed_username, password=passed_password, use_tls=True) PEER_NAME = 'PRODUCTION' SOURCE_HDFS_NAME = 'hdfs' TARGET_HDFS_NAME = 'hdfs' SOURCE_HIVE_NAME = 'hive' TARGET_HIVE_NAME = 'hive' SOURCE_CLUSTER_NAME = 'cluster' TARGET_CLUSTER_NAME = 'DEV' TARGET_YARN_SERVICE = 'yarn' # Setup for Hive replication hive = api_root.get_cluster(TARGET_CLUSTER_NAME).get_service(TARGET_HIVE_NAME) hive_args = ApiHiveReplicationArguments(None) hdfs_args = ApiHdfsReplicationArguments(None) # Needed for replicating table data stored in HDFS hive_args.sourceService = ApiServiceRef(None, peerName=PEER_NAME, clusterName=SOURCE_CLUSTER_NAME, serviceName=SOURCE_HIVE_NAME) # Define tables to replicate table_filters = [] table = ApiHiveTable(None) for tab in tablenames: table.database = (passed_database) table.tableName = (tab) table_filters = [] table_filters.append(table) print "Replicating " + passed_database + "." + tab hive_args.tableFilters = table_filters hive_args.force = True # Overwrite existing tables hive_args.replicateData = True # Replicate table data stored in HDFS hdfs_args.skipChecksumChecks = True hdfs_args.skipListingChecksumChecks = True hdfs_args.preserveBlockSize = True hdfs_args.preserveReplicationCount = True hdfs_args.preservePermissions = True # Define HDFS portion of the Hive replication as needed hdfs_args.destinationPath = '/user/bob.marshall/repltest' # Argument? Path relative to servicename? hdfs_args.mapreduceServiceName = TARGET_YARN_SERVICE hdfs_args.userName = passed_username hdfs_args.sourceUser = passed_username hive_args.hdfsArguments = hdfs_args start = datetime.datetime.now() end = start + datetime.timedelta(days=1) interval = "DAY" numinterval = 1 pause = True print "Creating Hive Replication Schedule" schedule = hive.create_replication_schedule(start, end, interval, numinterval, pause, hive_args) print "Starting Hive Replication" cmd = hive.trigger_replication_schedule(schedule.id) print "Waiting for completion" cmd = cmd.wait() print "Getting result" result = hive.get_replication_schedule(schedule.id).history[0].hiveResult print result print "Cleanup... Remove Hive replication schedule" sch = hive.delete_replication_schedule(schedule.id) print sch exit(0) #scheds = hive.get_replication_schedules() #sch = hive.delete_replication_schedule(162) # Setup for HDFS replication hdfs = api_root.get_cluster(TARGET_CLUSTER_NAME).get_service(TARGET_HDFS_NAME) hdfs_args = ApiHdfsReplicationArguments(None) hdfs_args.sourceService = ApiServiceRef(None, peerName=PEER_NAME, clusterName=SOURCE_CLUSTER_NAME, serviceName=SOURCE_HDFS_NAME) hdfs_args.sourcePath = '/user/bob.marshall/repltest' hdfs_args.destinationPath = '/user/bob.marshall/repltest' hdfs_args.mapreduceServiceName = TARGET_YARN_SERVICE hdfs_args.userName = args.username hdfs_args.sourceUser = args.username hdfs_args.preserveBlockSize = True hdfs_args.preserveReplicationCount = True hdfs_args.preservePermissions = True hdfs_args.skipChecksumChecks = True hdfs_args.skipListingChecksumChecks = True start = datetime.datetime.now() end = start + datetime.timedelta(days=1) interval = "DAY" numinterval = 1 pause = True #schedule = hdfs.create_replication_schedule(start, end, interval, interval, pause, hdfs_args) print "Creating HDFS Replication Schedule" schedule = hdfs.create_replication_schedule(start, end, "DAY", 1, True, hdfs_args) print "Starting HDFS Replication" cmd = hdfs.trigger_replication_schedule(schedule.id) print "Waiting for completion" cmd = cmd.wait() print "Getting result" result = hdfs.get_replication_schedule(schedule.id).history[0].hdfsResult print result print "Cleanup... Remove HDFS replication schedule" sch = hdfs.delete_replication_schedule(schedule.id) print sch #scheds = hdfs.get_replication_schedules() #sch = hdfs.delete_replication_schedule(27) if __name__ == "__main__": parser = argparse.ArgumentParser(description='Perform BDR jobs while getting around BDR limitations.') parser.add_argument("username") parser.add_argument("password") parser.add_argument("database") args = parser.parse_args() main(args.username, args.password, args.database)
now = time.time() if deadline < now: return cmd else: time.sleep(min(SLEEP_SECS, deadline - now)) else: time.sleep(SLEEP_SECS) except ApiException as e: print( "Exception when calling ClouderaManagerResourceApi->import_cluster_template: %s\n" % e) cm_client.configuration.username = '******' cm_client.configuration.password = '******' api_client = cm_client.ApiClient("http://localhost:7180/api/v32") cm_api = cm_client.ClouderaManagerResourceApi(api_client) # accept trial licence cm_api.begin_trial() # Install CM Agent on host with open("/root/myRSAkey", "r") as f: key = f.read() instargs = cm_client.ApiHostInstallArguments( host_names=['YourHostname'], user_name='root', private_key=key, cm_repo_url='https://archive.cloudera.com/cm6/6.3.0',
def api_client(self): if self._api_client is None: self._api_client = cm_client.ApiClient( "http://localhost:7180/api/v32") return self._api_client
def main(passed_username, passed_password, passed_database): PEER_NAME = 'PRODUCTION' # Previously TARGET_CLUSTER_NAME = 'DEV' # defined SOURCE_CLUSTER_NAME = 'cluster' # at Experian cm_host = 'br1andvhmn11.passporthealth.com' cm_client.configuration.username = passed_username cm_client.configuration.password = passed_password sourceDatabase = passed_database # Setup authentication for SSL cm_client.configuration.verify_ssl = True cm_client.configuration.ssl_ca_cert = '/opt/cloudera/security/pki/x509/truststore.pem' # Create an instance of the API class api_host = 'https://br1andvhmn11.passporthealth.com' port = '7183' api_version = 'v30' #For testing, until firewall opened, pull tablenames from target cluster. We are therefore simulating an overwrite. #For real, change impalahost to pull tablenames from source cluster. # impala_host = 'br1anprhsn02.passporthealth.com' impala_host = 'br1andvhmn02.passporthealth.com' # Construct base URL for API # http://cmhost:7180/api/v30 api_url = api_host + ':' + port + '/api/' + api_version api_client = cm_client.ApiClient(api_url) cluster_api_instance = cm_client.ClustersResourceApi(api_client) # Lists all known clusters. api_response = cluster_api_instance.read_clusters(view='SUMMARY') for cluster in api_response.items: print cluster.name, "-", cluster.full_version services_api_instance = cm_client.ServicesResourceApi(api_client) services = services_api_instance.read_services(cluster.name, view='FULL') for service in services.items: # print service.display_name, "-", service.type if service.type == 'HIVE': targetHive = service targetCluster = cluster print targetHive.name, targetHive.service_state, targetHive.health_summary for health_check in targetHive.health_checks: print health_check.name, "---", health_check.summary # print "Source database = " + sourceDatabase ###show_statement = "'show tables in " + sourceDatabase +"'" ###streamOperand = "impala-shell -i " + impala_host + " -d default -k --ssl --ca_cert=/opt/cloudera/security/pki/x509/truststore.pem -q " + show_statement ###stream=os.popen(streamOperand) ### ###output=stream.readlines() ###lineno =0 ###numtables = 0 ###tablenames = [] ###for line in output: ### if lineno <= 2: # skip heading lines ### pass ### elif line[0:3] == "+--": # skip last line ### pass ### else: # strip out tablename ### name = line[2:] ### blank = name.index(' ') ### tablenames.append(name[0:blank]) ### numtables +=1 ### lineno +=1 ###print str(numtables) + " tables in database " + sourceDatabase ###for table in tablenames: ### print table tablenames = [] tablenames.append('b05006') tablenames.append('b05007_bgpr_cape_rename') tablenames.append('b07001_bgpr_cape') tablenames.append('b13008_bgpr_cape') tablenames.append('c16001_bgpr_cape') tablenames.append('c24070_bgpr_cape') tablenames.append('cde2018_bg_a1') tablenames.append('cde2018_bg_a2') tablenames.append('cde2018_bg_a3') tablenames.append('cde2018_bg_a4') tablenames.append('cde2018_bg_b') tablenames.append('cde2018_bg_c1') tablenames.append('cde2018_bg_c2') tablenames.append('cde2018_bg_c3') tablenames.append('cde2018_bg_d') tablenames.append('cons_exp_annual_cye2018_bg') tablenames.append('cons_exp_annual_fyp2018_bg') tablenames.append('cons_exp_avgannual_cye2018_bg') tablenames.append('cons_exp_avgannual_fyp2018_bg') tablenames.append('cons_exp_gifts_cye2018_bg') tablenames.append('cons_exp_gifts_fyp2018_bg') tablenames.append('cye2018_bg_a1') tablenames.append('cye2018_bg_a2') tablenames.append('cye2018_bg_a3') tablenames.append('cye2018_bg_a4') tablenames.append('cye2018_bg_b') tablenames.append('cye2018_bg_c1') tablenames.append('cye2018_bg_c2') tablenames.append('cye2018_bg_c3') tablenames.append('cye2018_bg_d') tablenames.append('fyp2018_bg_a1') tablenames.append('fyp2018_bg_a2') tablenames.append('fyp2018_bg_a3') tablenames.append('fyp2018_bg_a4') tablenames.append('fyp2018_bg_b') tablenames.append('fyp2018_bg_c1') tablenames.append('fyp2018_bg_c2') tablenames.append('fyp2018_bg_c3') tablenames.append('fyp2018_bg_d') api_root = ApiResource(cm_host, username=passed_username, password=passed_password, use_tls=True) PEER_NAME = 'PRODUCTION' SOURCE_HDFS_NAME = 'hdfs' TARGET_HDFS_NAME = 'hdfs' SOURCE_HIVE_NAME = 'hive' TARGET_HIVE_NAME = 'hive' SOURCE_CLUSTER_NAME = 'cluster' TARGET_CLUSTER_NAME = 'DEV' TARGET_YARN_SERVICE = 'yarn' # Setup for Hive replication hive = api_root.get_cluster(TARGET_CLUSTER_NAME).get_service(TARGET_HIVE_NAME) hive_args = ApiHiveReplicationArguments(None) hdfs_args = ApiHdfsReplicationArguments(None) # Needed for replicating table data stored in HDFS hive_args.sourceService = ApiServiceRef(None, peerName=PEER_NAME, clusterName=SOURCE_CLUSTER_NAME, serviceName=SOURCE_HIVE_NAME) # Define tables to replicate table_filters = [] table = ApiHiveTable(None) for tab in tablenames: # table.database = (passed_database) # # Hardwire database name for 2020-11-24 replication # table.database = 'lake_consumerview' table.tableName = (tab) table_filters = [] table_filters.append(table) print "Replicating " + passed_database + "." + tab hive_args.tableFilters = table_filters hive_args.force = True # Overwrite existing tables hive_args.replicateData = True # Replicate table data stored in HDFS hdfs_args.skipChecksumChecks = True hdfs_args.skipListingChecksumChecks = True hdfs_args.preserveBlockSize = True hdfs_args.preserveReplicationCount = True hdfs_args.preservePermissions = True # Define HDFS portion of the Hive replication as needed # hdfs_args.destinationPath = '/' hdfs_args.mapreduceServiceName = TARGET_YARN_SERVICE hdfs_args.userName = passed_username hdfs_args.sourceUser = passed_username hive_args.hdfsArguments = hdfs_args start = datetime.datetime.now() end = start + datetime.timedelta(days=1) interval = "DAY" numinterval = 1 pause = True print "Creating Hive Replication Schedule" schedule = hive.create_replication_schedule(start, end, interval, numinterval, pause, hive_args) print "Starting Hive Replication" cmd = hive.trigger_replication_schedule(schedule.id) print "Waiting for completion" cmd = cmd.wait() print "Getting result" result = hive.get_replication_schedule(schedule.id).history[0].hiveResult print result print "Cleanup... Remove Hive replication schedule" sch = hive.delete_replication_schedule(schedule.id) print sch exit(0) #scheds = hive.get_replication_schedules() #sch = hive.delete_replication_schedule(162) # Setup for HDFS replication hdfs = api_root.get_cluster(TARGET_CLUSTER_NAME).get_service(TARGET_HDFS_NAME) hdfs_args = ApiHdfsReplicationArguments(None) hdfs_args.sourceService = ApiServiceRef(None, peerName=PEER_NAME, clusterName=SOURCE_CLUSTER_NAME, serviceName=SOURCE_HDFS_NAME) hdfs_args.sourcePath = '/user/bob.marshall/repltest' hdfs_args.destinationPath = '/user/bob.marshall/repltest' hdfs_args.mapreduceServiceName = TARGET_YARN_SERVICE hdfs_args.userName = args.username hdfs_args.sourceUser = args.username hdfs_args.preserveBlockSize = True hdfs_args.preserveReplicationCount = True hdfs_args.preservePermissions = True hdfs_args.skipChecksumChecks = True hdfs_args.skipListingChecksumChecks = True start = datetime.datetime.now() end = start + datetime.timedelta(days=1) interval = "DAY" numinterval = 1 pause = True #schedule = hdfs.create_replication_schedule(start, end, interval, interval, pause, hdfs_args) print "Creating HDFS Replication Schedule" schedule = hdfs.create_replication_schedule(start, end, "DAY", 1, True, hdfs_args) print "Starting HDFS Replication" cmd = hdfs.trigger_replication_schedule(schedule.id) print "Waiting for completion" cmd = cmd.wait() print "Getting result" result = hdfs.get_replication_schedule(schedule.id).history[0].hdfsResult print result print "Cleanup... Remove HDFS replication schedule" sch = hdfs.delete_replication_schedule(schedule.id) print sch
if deadline is not None: now = time.time() if deadline < now: return cmd else: time.sleep(min(SLEEP_SECS, deadline - now)) else: time.sleep(SLEEP_SECS) except ApiException as e: print("Exception when calling ClouderaManagerResourceApi->import_cluster_template: %s\n" % e) cm_client.configuration.username = '******' cm_client.configuration.password = '******' api_client = cm_client.ApiClient("http://cloudera:7180/api/v40") cm_api = cm_client.ClouderaManagerResourceApi(api_client) # accept trial licence # m_api.begin_trial() # Install CM Agent on host with open ("/root/myRSAkey", "r") as f: key = f.read() instargs = cm_client.ApiHostInstallArguments( host_names=['YourHostname'], user_name='root', private_key=key, cm_repo_url='https://archive.cloudera.com/cm7/7.1.3/',
if sys.argv[2] != 'KILL': print("the only valid second argument is \"KILL\"") printUsageMessage() quit(1) else: kill = True # Lists all known clusters. # Construct base URL for API # http://cmhost:7180/api/v17 api_url = api_host + ':' + port + '/api/' + api_version ## Connect to CM print("\nConnecting to Cloudera Manager at " + api_host + ":" + port) ## Get the IMPALA service api_client = cm_client.ApiClient(api_url) services_api_instance = cm_client.ServicesResourceApi(api_client) services = services_api_instance.read_services(cluster_name, view='FULL') for service in services.items: #print(service.name, "-", service.type) if service.type == "IMPALA": impala_service = service print("Located Impala Service: " + service.name) break if impala_service is None: print("Error: Could not locate Impala Service") quit(1) ## A window of one day assumes queries have not been running more than 24 hours now = datetime.utcnow() start = now - timedelta(days=1)
cm_client_username = args.user cm_client_password = args.password api_protocol = args.protocol api_host = args.host port = args.port api_version = args.apiver cluster = args.cluster service = SERVICE cm_client.configuration.username = cm_client_username cm_client.configuration.password = cm_client_password url = api_protocol + "://" + api_host + ":" + port + "/api/" + api_version api_client = cm_client.ApiClient(url) role_config_group_resource_api = cm_client.RoleConfigGroupsResourceApi( api_client) config_by_roles = role_config_group_resource_api.read_role_config_groups( cluster_name=cluster, service_name=service) config_by_group = {} role_config_group_name_displayname_mapping = {} for item in config_by_roles.items: role_group_name = item.name _.set(role_config_group_name_displayname_mapping, "" + item.display_name, role_group_name) for subitem in item.config.items: _.set(