def __init__(self, logger_arg, log_file, debug_mode=False): """Constructor for KafkaUtils that sets up the logging module and AmbariHelper. """ self.debug_mode = debug_mode self.logger = logger_arg self.logger.setLevel(logging.DEBUG) logging_format = '%(asctime)s - %(filename)s - %(name)s - %(levelname)s - %(message)s' # create console handler and set level to info handler = logging.StreamHandler() handler.setLevel(logging.INFO) formatter = logging.Formatter(logging_format) handler.setFormatter(formatter) self.logger.addHandler(handler) # create file handler and set level to debug handler = logging.FileHandler(log_file, 'a') handler.setLevel(logging.DEBUG) formatter = logging.Formatter(logging_format) handler.setFormatter(formatter) self.logger.addHandler(handler) self.logger.info('Log file: {0}'.format(log_file)) self.ambari_helper = AmbariHelper() self.cluster_manifest = self.ambari_helper.get_cluster_manifest() self.cluster_name = self.cluster_manifest.deployment.cluster_name
def getBrokerInformation(self): ambari_helper = AmbariHelper() cluster_manifest = ambari_helper.get_cluster_manifest() cluster_name = cluster_manifest.deployment.cluster_name hosts_result = ambari_helper.query_url( 'clusters/{0}/hosts'.format(cluster_name)) zookeepers = reduce( lambda r1, r2: r1 + ',' + r2, map( lambda m: m['Hosts']['host_name'] + ':2181', filter( lambda h: h['Hosts']['host_name'].startswith( cluster_manifest.settings[ Constants.ZOOKEEPER_VM_NAME_PREFIX_SETTING_KEY]), hosts_result['items']))) self.logger.info("zookeepers: {0}\n".format(zookeepers)) broker_hosts = map( lambda m: m['Hosts']['host_name'], filter( lambda h: h['Hosts']['host_name'].startswith( cluster_manifest.settings[ Constants.WORKERNODE_VM_NAME_PREFIX_SETTING_KEY]), hosts_result['items'])) self.logger.info("broker_hosts: {0}\n".format(broker_hosts)) brokers = reduce(lambda r1, r2: r1 + ',' + r2, map(lambda m: m + ':9092', broker_hosts)) self.logger.info("brokers: {0}\n".format(brokers)) return zookeepers, broker_hosts, brokers
def get_kafka_log_dirs(): ah = AmbariHelper() json_output = ah.query_url( "clusters/" + ah.cluster_name() + "/configurations/service_config_versions?service_name.in(KAFKA)&is_current=true" ) kafka_brokers_configs = [ element for element in json_output["items"][0]["configurations"] if element["type"] == KAFKA_BROKER ][0] return kafka_brokers_configs["properties"]["log.dirs"].split(',')
def get_zookeeper_connect_string(): ah = AmbariHelper() hosts = ah.get_host_components() zkHosts = "" for item in hosts["items"]: if item["HostRoles"]["component_name"] == "ZOOKEEPER_SERVER": zkHosts += item["HostRoles"]["host_name"] zkHosts += ZOOKEEPER_PORT zkHosts += "," if len(zkHosts) > 2: return zkHosts[:-1] else: raise Exception("Failed to get Zookeeper information from Ambari!")
def get_cluster_topology_json(): ambariHelper = AmbariHelper() cluster_manifest = ambariHelper.get_cluster_manifest() settings = cluster_manifest.settings if "cluster_topology_json_url" in settings: json_url = settings["cluster_topology_json_url"] logger.info("Retrieved Cluster Topology JSON document.") logger.info("URL: %s", json_url) r = requests.get(json_url) topology_info = r.text logger.debug("Cluster Topology: %s", topology_info) return topology_info else: raise Exception( "Failed to get cluster_topology_json_url from cluster manifest")
from hdinsight_common.AmbariHelper import AmbariHelper import requests, json, re import subprocess, sys import pty if(len(sys.argv) < 2): print "Error: Usage: restartdeadregionserverswithlocalhostissue.py <ssh-username>" sys.exit(1) a=AmbariHelper() result=a.query_url("clusters/"+a.cluster_name()+"/hosts") for i in result['items']: if(i['Hosts']['host_name'].startswith("zk")): result_hbase_metrics=a.query_url("clusters/"+a.cluster_name()+"/hosts/"+i['Hosts']['host_name']+"/host_components/HBASE_MASTER") if(result_hbase_metrics['metrics']['hbase']['master']['IsActiveMaster'] == 'true'): url="http://"+i['Hosts']['host_name']+":16010/jmx?qry=Hadoop:service=HBase,name=Master,sub=Server" req=requests.get(url) res=req.json() deadregionservers = res['beans'][0]['tag.deadRegionServers'].split(';') BASH_PATH="/bin/bash" HBASE_DAEMON_PATH="/usr/hdp/current/hbase-regionserver/bin/hbase-daemon.sh" HBASE_CONFIG_PATH="/usr/hdp/current/hbase-regionserver/conf" COMMANDLOGTAIL="sudo tail -n100 /var/log/hbase/hbase-hbase-regionserver-wn*.log" COMMANDSTOP="sudo -u hbase " + BASH_PATH + " " + HBASE_DAEMON_PATH + " --config " + HBASE_CONFIG_PATH + " stop regionserver" COMMANDSTART="sudo -u hbase " + BASH_PATH + " " + HBASE_DAEMON_PATH + " --config " + HBASE_CONFIG_PATH + " start regionserver" LOCALHOSTISSUESTRING = 'WARN \[regionserver/localhost/127.0.0.1:16020\] regionserver.HRegionServer: error telling master we are up' SSHUSER = sys.argv[1] for deadregionserver in filter(None,deadregionservers):
raise_error( "Failed to get cluster_topology_json_url from cluster manifest") def parse_topo_info(cluster_topology_json, fqdn_suffix): print cluster_topology_json workernode_info = json.loads( cluster_topology_json)["hostGroups"]["workernode"] host_info = [] for node in workernode_info: host = { UPDATE_DOMAIN: str(node[UPDATE_DOMAIN]), FQDN: node[FQDN] + "." + str(fqdn_suffix), RACK: "/rack" + str(node[UPDATE_DOMAIN]) } host_info.append(host) return host_info ambariHelper = AmbariHelper() cluster_topology_json = get_cluster_topology_json( ambariHelper.get_cluster_manifest()) host_info = parse_topo_info(cluster_topology_json, ambariHelper.get_fqdn().split('.', 1)[1]) cluster_name = ambariHelper.cluster_name() for node in host_info: ambariHelper.request_url( "clusters/" + cluster_name + "/hosts/" + str(node[FQDN]), "PUT", "{\"Hosts\":{\"rack_info\":\"" + str(node[RACK]) + "\"}}") ambariHelper.restart_all_stale_services()