def dirchange(self, dirn='/home/lin'): os.chdir(dirn) self.host = hostname.hostname() self.username = os.getlogin() self.path = os.getcwd() self.start = self.username + '@' + \ self.host + ':'+ self.path + '$' + ' '
def perform_prestart_checks(expected_hostname): # Check if current hostname is equal to expected one (got from the server # during bootstrap. global config if expected_hostname is not None: current_hostname = hostname.hostname(config) if current_hostname != expected_hostname: print("Determined hostname does not match expected. Please check agent " "log for details") msg = "Ambari agent machine hostname ({0}) does not match expected ambari " \ "server hostname ({1}). Aborting registration. Please check hostname, " \ "hostname -f and /etc/hosts file to confirm your " \ "hostname is setup correctly".format(current_hostname, expected_hostname) logger.error(msg) sys.exit(1) # Check if there is another instance running if os.path.isfile(agent_pidfile) and not OSCheck.get_os_family() == OSConst.WINSRV_FAMILY: print("%s already exists, exiting" % agent_pidfile) sys.exit(1) # check if ambari prefix exists elif config.has_option('agent', 'prefix') and not os.path.isdir(os.path.abspath(config.get('agent', 'prefix'))): msg = "Ambari prefix dir %s does not exists, can't continue" \ % config.get("agent", "prefix") logger.error(msg) print(msg) sys.exit(1) elif not config.has_option('agent', 'prefix'): msg = "Ambari prefix dir %s not configured, can't continue" logger.error(msg) print(msg) sys.exit(1) check_sudo()
def dirchange(self, dirn = '/home/lin'): os.chdir(dirn) self.host = hostname.hostname() self.username = os.getlogin() self.path = os.getcwd() self.start = self.username + '@' + \ self.host + ':'+ self.path + '$' + ' '
def reqSignCrt(self): sign_crt_req_url = self.server_url + '/certs/' + hostname.hostname() agent_crt_req_f = open(self.getAgentCrtReqName()) agent_crt_req_content = agent_crt_req_f.read() passphrase_env_var = self.config.get('security', 'passphrase_env_var_name') passphrase = os.environ[passphrase_env_var] register_data = {'csr' : agent_crt_req_content, 'passphrase' : passphrase} data = json.dumps(register_data) req = urllib2.Request(sign_crt_req_url, data, {'Content-Type': 'application/json'}) f = urllib2.urlopen(req) response = f.read() f.close() data = json.loads(response) logger.debug("Sign response from Server: \n" + pprint.pformat(data)) result=data['result'] if result == 'OK': agentCrtContent=data['signedCa'] agentCrtF = open(self.getAgentCrtName(), "w") agentCrtF.write(agentCrtContent) else: # Possible exception is catched higher at Controller logger.error('Certificate signing failed.' '\nIn order to receive a new agent' ' certificate, remove existing certificate file from keys ' 'directory. As a workaround you can turn off two-way SSL ' 'authentication in server configuration(ambari.properties) ' '\nExiting..') raise ssl.SSLError
def reqSignCrt(self): sign_crt_req_url = self.server_url + '/ws/v1/slider/agents/certs/' + \ hostname.hostname() agent_crt_req_f = open(self.getAgentCrtReqName()) agent_crt_req_content = agent_crt_req_f.read() passphrase_env_var = self.config.get('security', 'passphrase_env_var_name') passphrase = os.environ[passphrase_env_var] register_data = {'csr' : agent_crt_req_content, 'passphrase' : passphrase} data = json.dumps(register_data) req = urllib2.Request(sign_crt_req_url, data, {'Content-Type': 'application/json'}) f = urllib2.urlopen(req) response = f.read() f.close() data = json.loads(response) logger.debug("Sign response from Server: \n" + pprint.pformat(data)) result=data['result'] if result == 'OK': agentCrtContent=data['signedCa'] agentCrtF = open(self.getAgentCrtName(), "w") agentCrtF.write(agentCrtContent) else: # Possible exception is catched higher at Controller logger.error('Certificate signing failed.' '\nIn order to receive a new agent' ' certificate, remove existing certificate file from keys ' 'directory. As a workaround you can turn off two-way SSL ' 'authentication in server configuration.' '\nExiting..') raise ssl.SSLError
def build(self, id='-1', state_interval=-1): global clusterId, clusterDefinitionRevision, firstContact timestamp = int(time.time() * 1000) queueResult = self.actionQueue.result() nodeStatus = {"status": "HEALTHY", "cause": "NONE"} heartbeat = { 'responseId': int(id), 'timestamp': timestamp, 'hostname': hostname.hostname(), 'nodeStatus': nodeStatus } if len(queueResult) != 0: heartbeat['reports'] = queueResult['reports'] heartbeat['componentStatus'] = queueResult['componentStatus'] pass logger.info("Sending heartbeat with response id: " + str(id) + " and " "timestamp: " + str(timestamp)) logger.debug("Heartbeat : " + pformat(heartbeat)) if (int(id) >= 0) and state_interval > 0 and (int(id) % state_interval) == 0: hostInfo = HostInfo() nodeInfo = {} # for now, just do the same work as registration hostInfo.register(nodeInfo) heartbeat['agentEnv'] = nodeInfo logger.debug("agentEnv : " + str(nodeInfo)) return heartbeat
def perform_prestart_checks(expected_hostname): # Check if current hostname is equal to expected one (got from the server # during bootstrap. if expected_hostname is not None: current_hostname = hostname.hostname() if current_hostname != expected_hostname: print("Determined hostname does not match expected. Please check agent " "log for details") msg = "Ambari agent machine hostname ({0}) does not match expected ambari " \ "server hostname ({1}). Aborting registration. Please check hostname, " \ "hostname -f and /etc/hosts file to confirm your " \ "hostname is setup correctly".format(current_hostname, expected_hostname) logger.error(msg) sys.exit(1) # Check if there is another instance running if os.path.isfile(ProcessHelper.pidfile): print("%s already exists, exiting" % ProcessHelper.pidfile) sys.exit(1) # check if ambari prefix exists elif not os.path.isdir(config.get("agent", "prefix")): msg = "Ambari prefix dir %s does not exists, can't continue" \ % config.get("agent", "prefix") logger.error(msg) print(msg) sys.exit(1)
def build(self, id='-1', state_interval=-1, componentsMapped=False): global clusterId, clusterDefinitionRevision, firstContact timestamp = int(time.time() * 1000) queueResult = self.actionQueue.result() nodeStatus = {"status": "HEALTHY", "cause": "NONE"} nodeStatus["alerts"] = [] heartbeat = { 'responseId': int(id), 'timestamp': timestamp, 'hostname': hostname.hostname(self.config), 'nodeStatus': nodeStatus } commandsInProgress = False if not self.actionQueue.commandQueue.empty(): commandsInProgress = True if len(queueResult) != 0: heartbeat['reports'] = queueResult['reports'] heartbeat['componentStatus'] = queueResult['componentStatus'] if len(heartbeat['reports']) > 0: # There may be IN_PROGRESS tasks commandsInProgress = True pass # For first request/heartbeat assume no components are mapped if int(id) == 0: componentsMapped = False logger.info( "Building Heartbeat: {responseId = %s, timestamp = %s, commandsInProgress = %s, componentsMapped = %s}", str(id), str(timestamp), repr(commandsInProgress), repr(componentsMapped)) if logger.isEnabledFor(logging.DEBUG): logger.debug("Heartbeat: %s", pformat(heartbeat)) hostInfo = HostInfo(self.config) if (int(id) >= 0) and state_interval > 0 and (int(id) % state_interval) == 0: nodeInfo = {} # for now, just do the same work as registration # this must be the last step before returning heartbeat hostInfo.register(nodeInfo, componentsMapped, commandsInProgress) heartbeat['agentEnv'] = nodeInfo mounts = Hardware.osdisks() heartbeat['mounts'] = mounts if logger.isEnabledFor(logging.DEBUG): logger.debug("agentEnv: %s", str(nodeInfo)) logger.debug("mounts: %s", str(mounts)) nodeStatus["alerts"] = hostInfo.createAlerts(nodeStatus["alerts"]) if self.collector is not None: heartbeat['alerts'] = self.collector.alerts() return heartbeat
def __init__(self, config, range=30): threading.Thread.__init__(self) logger.debug('Initializing Controller RPC thread.') self.lock = threading.Lock() self.safeMode = True self.credential = None self.config = config self.hostname = hostname.hostname() server_secured_url = 'https://' + config.get( 'server', 'hostname') + ':' + config.get('server', 'secured_url_port') self.registerUrl = server_secured_url + '/agent/v1/register/' + self.hostname self.unregisterUrl = server_secured_url + '/agent/v1/unregister/' + self.hostname self.heartbeatUrl = server_secured_url + '/agent/v1/heartbeat/' + self.hostname self.netutil = NetUtil() self.responseId = -1 self.repeatRegistration = False self.cachedconnect = None self.range = range self.hasMappedComponents = True self.actionQueue = ActionQueue(self.config) self.actionQueue.start() self.register = Register(self.config) self.unregister = Unregister(self.config) self.heartbeat = Heartbeat(self.actionQueue, self.config)
def createAlerts(self, alerts): existingUsers = [] self.checkUsers(self.DEFAULT_USERS, existingUsers) dirs = [] self.checkFolders(self.DEFAULT_DIRS, self.DEFAULT_PROJECT_NAMES, existingUsers, dirs) alert = { 'name': 'host_alert', 'instance': None, 'service': 'AMBARI', 'component': 'host', 'host': hostname.hostname(self.config), 'state': 'OK', 'label': 'Disk space', 'text': 'Used disk space less than 80%' } message = "" mountinfoSet = [] for dir in dirs: if dir["type"] == 'directory': mountinfo = self.osdiskAvailableSpace(dir['name']) if int(mountinfo["percent"].strip('%')) >= 80: if not mountinfo in mountinfoSet: mountinfoSet.append(mountinfo) message += str(dir['name']) + ";\n" if message != "": message = "These discs have low space:\n" + str( mountinfoSet ) + "\n They include following critical directories:\n" + message alert['state'] = 'WARNING' alert['text'] = message alerts.append(alert) return alerts
def perform_prestart_checks(expected_hostname): # Check if current hostname is equal to expected one (got from the server # during bootstrap. if expected_hostname is not None: current_hostname = hostname.hostname() if current_hostname != expected_hostname: print( "Determined hostname does not match expected. Please check agent " "log for details") msg = "Ambari agent machine hostname ({0}) does not match expected ambari " \ "server hostname ({1}). Aborting registration. Please check hostname, " \ "hostname -f and /etc/hosts file to confirm your " \ "hostname is setup correctly".format(current_hostname, expected_hostname) logger.error(msg) sys.exit(1) # Check if there is another instance running if os.path.isfile(ProcessHelper.pidfile): print("%s already exists, exiting" % ProcessHelper.pidfile) sys.exit(1) # check if ambari prefix exists elif not os.path.isdir(config.get("agent", "prefix")): msg = "Ambari prefix dir %s does not exists, can't continue" \ % config.get("agent", "prefix") logger.error(msg) print(msg) sys.exit(1)
def __init__(self): os.chdir('/home/lin') self.host = hostname.hostname() self.username = os.getlogin() self.path = os.getcwd() self.start = self.username + '@' + \ self.host + ':'+ self.path + '$' + ' ' self.conn_ssh()
def build(self, id='-1', state_interval=-1, componentsMapped=False): global clusterId, clusterDefinitionRevision, firstContact timestamp = int(time.time()*1000) queueResult = self.actionQueue.result() nodeStatus = { "status" : "HEALTHY", "cause" : "NONE" } nodeStatus["alerts"] = [] heartbeat = { 'responseId' : int(id), 'timestamp' : timestamp, 'hostname' : hostname.hostname(self.config), 'nodeStatus' : nodeStatus } commandsInProgress = False if not self.actionQueue.commandQueue.empty(): commandsInProgress = True if len(queueResult) != 0: heartbeat['reports'] = queueResult['reports'] heartbeat['componentStatus'] = queueResult['componentStatus'] if len(heartbeat['reports']) > 0: # There may be IN_PROGRESS tasks commandsInProgress = True pass # For first request/heartbeat assume no components are mapped if int(id) == 0: componentsMapped = False logger.info("Building Heartbeat: {responseId = %s, timestamp = %s, commandsInProgress = %s, componentsMapped = %s}", str(id), str(timestamp), repr(commandsInProgress), repr(componentsMapped)) if logger.isEnabledFor(logging.DEBUG): logger.debug("Heartbeat: %s", pformat(heartbeat)) hostInfo = HostInfo(self.config) if (int(id) >= 0) and state_interval > 0 and (int(id) % state_interval) == 0: nodeInfo = { } # for now, just do the same work as registration # this must be the last step before returning heartbeat hostInfo.register(nodeInfo, componentsMapped, commandsInProgress) heartbeat['agentEnv'] = nodeInfo mounts = Hardware.osdisks() heartbeat['mounts'] = mounts if logger.isEnabledFor(logging.DEBUG): logger.debug("agentEnv: %s", str(nodeInfo)) logger.debug("mounts: %s", str(mounts)) nodeStatus["alerts"] = hostInfo.createAlerts(nodeStatus["alerts"]) return heartbeat
def genAgentCrtReq(self): generate_script = GEN_AGENT_KEY % { 'hostname': hostname.hostname(), 'keysdir': self.config.get('security', 'keysdir') } logger.info(generate_script) p = subprocess.Popen([generate_script], shell=True, stdout=subprocess.PIPE) p.communicate()
def build(self, id='-1'): timestamp = int(time.time()*1000) version = self.agentInfo.version() register = { 'responseId' : int(id), 'timestamp' : timestamp, 'hostname' : hostname.hostname(), 'agentVersion' : version } return register
def genAgentCrtReq(self): generate_script = GEN_AGENT_KEY % {'hostname': hostname.hostname(self.config), 'keysdir' : os.path.abspath(self.config.get('security', 'keysdir'))} logger.info(generate_script) if platform.system() == 'Windows': p = subprocess.Popen(generate_script, stdout=subprocess.PIPE) p.communicate() else: p = subprocess.Popen([generate_script], shell=True, stdout=subprocess.PIPE) p.communicate()
def build(self, id='-1'): timestamp = int(time.time() * 1000) version = self.agentInfo.version() register = { 'responseId': int(id), 'timestamp': timestamp, 'hostname': hostname.hostname(), 'agentVersion': version } return register
def genAgentCrtReq(self): generate_script = GEN_AGENT_KEY % { 'hostname': hostname.hostname(self.config), 'keysdir': os.path.abspath(self.config.get('security', 'keysdir'))} logger.info(generate_script) if platform.system() == 'Windows': p = subprocess.Popen(generate_script, stdout=subprocess.PIPE) p.communicate() else: p = subprocess.Popen([generate_script], shell=True, stdout=subprocess.PIPE) p.communicate()
def build(self, id='-1', state_interval=-1, componentsMapped=False): global clusterId, clusterDefinitionRevision, firstContact timestamp = int(time.time()*1000) queueResult = self.actionQueue.result() nodeStatus = { "status" : "HEALTHY", "cause" : "NONE"} heartbeat = { 'responseId' : int(id), 'timestamp' : timestamp, 'hostname' : hostname.hostname(), 'nodeStatus' : nodeStatus } commandsInProgress = False if not self.actionQueue.commandQueue.empty(): commandsInProgress = True if len(queueResult) != 0: heartbeat['reports'] = queueResult['reports'] heartbeat['componentStatus'] = queueResult['componentStatus'] if len(heartbeat['reports']) > 0: # There may be IN_PROGRESS tasks commandsInProgress = True pass # For first request/heartbeat assume no components are mapped if int(id) == 0: componentsMapped = False logger.info("Sending heartbeat with response id: " + str(id) + " and " "timestamp: " + str(timestamp) + ". Command(s) in progress: " + repr(commandsInProgress) + ". Components mapped: " + repr(componentsMapped)) logger.debug("Heartbeat : " + pformat(heartbeat)) if (int(id) >= 0) and state_interval > 0 and (int(id) % state_interval) == 0: hostInfo = HostInfo(self.config) nodeInfo = { } # for now, just do the same work as registration # this must be the last step before returning heartbeat hostInfo.register(nodeInfo, componentsMapped, commandsInProgress) heartbeat['agentEnv'] = nodeInfo logger.debug("agentEnv : " + str(nodeInfo)) mounts = Hardware.osdisks() heartbeat['mounts'] = mounts logger.debug("mounts : " + str(mounts)) return heartbeat
def genAgentCrtReq(self, keyname): keysdir = os.path.abspath(self.config.get('security', 'keysdir')) generate_script = GEN_AGENT_KEY % { 'hostname': hostname.hostname(self.config), 'keysdir': keysdir} logger.info(generate_script) if platform.system() == 'Windows': p = subprocess.Popen(generate_script, stdout=subprocess.PIPE) p.communicate() else: p = subprocess.Popen([generate_script], shell=True, stdout=subprocess.PIPE) p.communicate() # this is required to be 600 for security concerns. os.chmod(keyname, 0600)
def genAgentCrtReq(self, keyname): keysdir = os.path.abspath(self.config.get('security', 'keysdir')) generate_script = GEN_AGENT_KEY % { 'hostname': hostname.hostname(self.config), 'keysdir': keysdir} logger.info(generate_script) if platform.system() == 'Windows': p = subprocess32.Popen(generate_script, stdout=subprocess32.PIPE) p.communicate() else: p = subprocess32.Popen([generate_script], shell=True, stdout=subprocess32.PIPE) p.communicate() # this is required to be 600 for security concerns. os.chmod(keyname, 0600)
def build(self, id='-1'): global clusterId, clusterDefinitionRevision, firstContact timestamp = int(time.time()*1000) hostInfo = HostInfo() agentEnv = { } hostInfo.register(agentEnv) version = self.read_agent_version() register = { 'responseId' : int(id), 'timestamp' : timestamp, 'hostname' : hostname.hostname(), 'publicHostname' : hostname.public_hostname(), 'hardwareProfile' : self.hardware.get(), 'agentEnv' : agentEnv, 'agentVersion' : version } return register
def build(self, version, id='-1'): global clusterId, clusterDefinitionRevision, firstContact timestamp = int(time.time()*1000) hostInfo = HostInfo(self.config) agentEnv = { } hostInfo.register(agentEnv, False, False) current_ping_port = self.config.get('agent','current_ping_port') register = { 'responseId' : int(id), 'timestamp' : timestamp, 'hostname' : hostname.hostname(self.config), 'currentPingPort' : int(current_ping_port), 'publicHostname' : hostname.public_hostname(self.config), 'hardwareProfile' : self.hardware.get(), 'agentEnv' : agentEnv, 'agentVersion' : version, 'prefix' : self.config.get('agent', 'prefix') } return register
def build(self, id='-1'): global clusterId, clusterDefinitionRevision, firstContact timestamp = int(time.time()*1000) hostInfo = HostInfo(self.config) agentEnv = { } hostInfo.register(agentEnv, False, False) version = self.agentInfo.version() current_ping_port = self.config.get('agent','current_ping_port') register = { 'responseId' : int(id), 'timestamp' : timestamp, 'hostname' : hostname.hostname(), 'currentPingPort' : int(current_ping_port), 'publicHostname' : hostname.public_hostname(), 'hardwareProfile' : self.hardware.get(), 'agentEnv' : agentEnv, 'agentVersion' : version } return register
def reqSignCrt(self): sign_crt_req_url = self.server_url + '/certs/' + hostname.hostname() agent_crt_req_f = open(self.getAgentCrtReqName()) agent_crt_req_content = agent_crt_req_f.read() passphrase_env_var = self.config.get('security', 'passphrase_env_var_name') passphrase = os.environ[passphrase_env_var] register_data = {'csr' : agent_crt_req_content, 'passphrase' : passphrase} data = json.dumps(register_data) req = urllib2.Request(sign_crt_req_url, data, {'Content-Type': 'application/json'}) f = urllib2.urlopen(req) response = f.read() f.close() data = json.loads(response) logger.debug("Sign response from Server: \n" + pprint.pformat(data)) result=data['result'] if result == 'OK': agentCrtContent=data['signedCa'] agentCrtF = open(self.getAgentCrtName(), "w") agentCrtF.write(agentCrtContent) else: # Possible exception is catched higher at Controller logger.error("Certificate signing failed")
def __init__(self, config, range=30): threading.Thread.__init__(self) logger.debug('Initializing Controller RPC thread.') self.lock = threading.Lock() self.safeMode = True self.credential = None self.config = config self.hostname = hostname.hostname() server_secured_url = 'https://' + config.get('server', 'hostname') + ':' + config.get('server', 'secured_url_port') self.registerUrl = server_secured_url + '/agent/v1/register/' + self.hostname self.unregisterUrl = server_secured_url + '/agent/v1/unregister/' + self.hostname self.heartbeatUrl = server_secured_url + '/agent/v1/heartbeat/' + self.hostname self.netutil = NetUtil() self.responseId = -1 self.repeatRegistration = False self.cachedconnect = None self.range = range self.hasMappedComponents = True self.actionQueue = ActionQueue(self.config) self.actionQueue.start() self.register = Register(self.config) self.unregister = Unregister(self.config) self.heartbeat = Heartbeat(self.actionQueue, self.config)
def createAlerts(self, alerts): existingUsers = [] self.checkUsers(self.DEFAULT_USERS, existingUsers) dirs = [] self.checkFolders(self.DEFAULT_DIRS, self.DEFAULT_PROJECT_NAMES, existingUsers, dirs) alert = { "name": "host_alert", "instance": None, "service": "AMBARI", "component": "host", "host": hostname.hostname(self.config), "state": "OK", "label": "Disk space", "text": "Used disk space less than 80%", } message = "" mountinfoSet = [] for dir in dirs: if dir["type"] == "directory": mountinfo = self.osdiskAvailableSpace(dir["name"]) if int(mountinfo["percent"].strip("%")) >= 80: if not mountinfo in mountinfoSet: mountinfoSet.append(mountinfo) message += str(dir["name"]) + ";\n" if message != "": message = ( "These discs have low space:\n" + str(mountinfoSet) + "\n They include following critical directories:\n" + message ) alert["state"] = "WARNING" alert["text"] = message alerts.append(alert) return alerts
def getAgentKeyName(self): keysdir = self.config.get('security', 'keysdir') return keysdir + os.sep + hostname.hostname(self.config) + ".key"
def getFqdn(self): return hostname.hostname(self.config)
def getAgentCrtName(self): keysdir = self.config.get('security', 'keysdir') return keysdir + os.sep + hostname.hostname() + ".crt"
def getHostname(self): return hostname.hostname()
def getAgentCrtReqName(self): keysdir = os.path.abspath(self.config.get('security', 'keysdir')) return keysdir + os.sep + hostname.hostname(self.config) + ".csr"
def genAgentCrtReq(self): generate_script = GEN_AGENT_KEY % {'hostname': hostname.hostname(), 'keysdir' : self.config.get('security', 'keysdir')} logger.info(generate_script) p = subprocess.Popen([generate_script], shell=True, stdout=subprocess.PIPE) p.communicate()
def writeHostnames(outputFile): fqdn = hostname.hostname() public_fqdn = hostname.public_hostname() outputFile.write('$myhostname' + " = '" + fqdn + "'" + os.linesep) outputFile.write('$public_hostname' + " = '" + public_fqdn + "'" + os.linesep)