def __init__(self, cluster, service, component, globalConfig, config): self.cluster = cluster self.service = service self.component = component self.globalConfig = globalConfig versionsFileDir = config.getResolvedPath(AgentConfig.APP_TASK_DIR) self.versionsHandler = StackVersionsFileHandler(versionsFileDir)
def __init__(self, pythonExecutor, puppetExecutor, config): self.pythonExecutor = pythonExecutor self.puppetExecutor = puppetExecutor self.stacksDir = config.get('stack', 'upgradeScriptsDir') self.config = config versionsFileDir = config.get('agent', 'prefix') self.versionsHandler = StackVersionsFileHandler(versionsFileDir)
def __init__(self, cluster, service, component, globalConfig, config): self.cluster = cluster self.service = service self.component = component self.globalConfig = globalConfig versionsFileDir = config.get('agent', 'prefix') self.versionsHandler = StackVersionsFileHandler(versionsFileDir) self.actualConfigHandler = ActualConfigHandler(config)
class LiveStatus: SERVICES = [ "HDFS", "MAPREDUCE", "GANGLIA", "HBASE", "NAGIOS", "ZOOKEEPER", "OOZIE", "HCATALOG", "KERBEROS", "TEMPLETON", "HIVE", "WEBHCAT", "YARN", "MAPREDUCE2", "FLUME" ] CLIENT_COMPONENTS = [{ "serviceName": "HBASE", "componentName": "HBASE_CLIENT" }, { "serviceName": "HDFS", "componentName": "HDFS_CLIENT" }, { "serviceName": "MAPREDUCE", "componentName": "MAPREDUCE_CLIENT" }, { "serviceName": "ZOOKEEPER", "componentName": "ZOOKEEPER_CLIENT" }, { "serviceName": "OOZIE", "componentName": "OOZIE_CLIENT" }, { "serviceName": "HCATALOG", "componentName": "HCAT" }, { "serviceName": "HIVE", "componentName": "HIVE_CLIENT" }, { "serviceName": "YARN", "componentName": "YARN_CLIENT" }, { "serviceName": "MAPREDUCE2", "componentName": "MAPREDUCE2_CLIENT" }, { "serviceName": "PIG", "componentName": "PIG" }, { "serviceName": "SQOOP", "componentName": "SQOOP" }] COMPONENTS = [ { "serviceName": "HDFS", "componentName": "DATANODE" }, { "serviceName": "HDFS", "componentName": "NAMENODE" }, { "serviceName": "HDFS", "componentName": "SECONDARY_NAMENODE" }, { "serviceName": "HDFS", "componentName": "JOURNALNODE" }, { "serviceName": "HDFS", "componentName": "ZKFC" }, { "serviceName": "MAPREDUCE", "componentName": "JOBTRACKER" }, { "serviceName": "MAPREDUCE", "componentName": "TASKTRACKER" }, { "serviceName": "GANGLIA", "componentName": "GANGLIA_SERVER" }, { "serviceName": "GANGLIA", "componentName": "GANGLIA_MONITOR" }, { "serviceName": "HBASE", "componentName": "HBASE_MASTER" }, { "serviceName": "HBASE", "componentName": "HBASE_REGIONSERVER" }, { "serviceName": "NAGIOS", "componentName": "NAGIOS_SERVER" }, { "serviceName": "FLUME", "componentName": "FLUME_SERVER" }, { "serviceName": "ZOOKEEPER", "componentName": "ZOOKEEPER_SERVER" }, { "serviceName": "OOZIE", "componentName": "OOZIE_SERVER" }, { "serviceName": "HCATALOG", "componentName": "HCATALOG_SERVER" }, { "serviceName": "KERBEROS", "componentName": "KERBEROS_SERVER" }, { "serviceName": "HIVE", "componentName": "HIVE_SERVER" }, { "serviceName": "HIVE", "componentName": "HIVE_METASTORE" }, { "serviceName": "HIVE", "componentName": "MYSQL_SERVER" }, { "serviceName": "WEBHCAT", "componentName": "WEBHCAT_SERVER" }, { "serviceName": "YARN", "componentName": "RESOURCEMANAGER" }, { "serviceName": "YARN", "componentName": "NODEMANAGER" }, { "serviceName": "MAPREDUCE2", "componentName": "HISTORYSERVER" }, ] LIVE_STATUS = "STARTED" DEAD_STATUS = "INSTALLED" def __init__(self, cluster, service, component, globalConfig, config): self.cluster = cluster self.service = service self.component = component self.globalConfig = globalConfig versionsFileDir = config.get('agent', 'prefix') self.versionsHandler = StackVersionsFileHandler(versionsFileDir) self.actualConfigHandler = ActualConfigHandler(config) def belongsToService(self, component): #TODO: Should also check belonging of server to cluster return component['serviceName'] == self.service # Live status was stripped from heartbeat after revision e1718dd def build(self): global SERVICES, CLIENT_COMPONENTS, COMPONENTS, LIVE_STATUS, DEAD_STATUS statusCheck = StatusCheck(AmbariConfig.servicesToPidNames, AmbariConfig.pidPathesVars, self.globalConfig, AmbariConfig.servicesToLinuxUser) livestatus = None component = { "serviceName": self.service, "componentName": self.component } if component in self.COMPONENTS + self.CLIENT_COMPONENTS: # CLIENT components can't have status STARTED if component in self.CLIENT_COMPONENTS: status = self.DEAD_STATUS else: serviceStatus = statusCheck.getStatus(self.component) if serviceStatus is None: logger.warn("There is no service to pid mapping for " + self.component) status = self.LIVE_STATUS if serviceStatus else self.DEAD_STATUS livestatus = { "componentName": self.component, "msg": "", "status": status, "clusterName": self.cluster, "serviceName": self.service, "stackVersion": self.versionsHandler.read_stack_version(self.component) } active_config = self.actualConfigHandler.read_actual_component( self.component) if not active_config is None: livestatus['configurationTags'] = active_config logger.debug("The live status for component " + str(self.component) +\ " of service " + str(self.service) + " is " + str(livestatus)) return livestatus
class UpgradeExecutor: """ Class that performs the StackVersion stack upgrade""" SCRIPT_DIRS = [ 'pre-upgrade.d', 'upgrade.d', 'post-upgrade.d' ] NAME_PARSING_FAILED_CODE = 999 def __init__(self, pythonExecutor, puppetExecutor, config): self.pythonExecutor = pythonExecutor self.puppetExecutor = puppetExecutor self.stacksDir = config.get('stack', 'upgradeScriptsDir') self.config = config versionsFileDir = config.get('agent', 'prefix') self.versionsHandler = StackVersionsFileHandler(versionsFileDir) def perform_stack_upgrade(self, command, tmpout, tmperr): logger.info("Performing stack upgrade") params = command['commandParams'] srcStack = params['source_stack_version'] tgtStack = params['target_stack_version'] component = command['role'] srcStackTuple = self.split_stack_version(srcStack) tgtStackTuple = self.split_stack_version(tgtStack) if srcStackTuple is None or tgtStackTuple is None: errorstr = "Source (%s) or target (%s) version does not match pattern \ <Name>-<Version>" % (srcStack, tgtStack) logger.info(errorstr) result = { 'exitcode' : 1, 'stdout' : 'None', 'stderr' : errorstr } elif srcStack != tgtStack: paramTuple = sum((srcStackTuple, tgtStackTuple), ()) upgradeId = "%s-%s.%s_%s-%s.%s" % paramTuple # Check stack version (do we need upgrade?) basedir = os.path.join(self.stacksDir, upgradeId, component) if not os.path.isdir(basedir): errorstr = "Upgrade %s is not supported (dir %s does not exist)" \ % (upgradeId, basedir) logger.error(errorstr) result = { 'exitcode' : 1, 'stdout' : errorstr, 'stderr' : errorstr } else: result = { 'exitcode' : 0, 'stdout' : '', 'stderr' : '' } # Request repos update (will be executed once before running any pp file) self.puppetExecutor.discardInstalledRepos() for dir in self.SCRIPT_DIRS: if result['exitcode'] != 0: break tmpRes = self.execute_dir(command, basedir, dir, tmpout, tmperr) result = { 'exitcode' : result['exitcode'] or tmpRes['exitcode'], 'stdout' : "%s\n%s" % (result['stdout'], tmpRes['stdout']), 'stderr' : "%s\n%s" % (result['stderr'], tmpRes['stderr']), } if result['exitcode'] == 0: logger.info("Upgrade %s successfully finished" % upgradeId) self.versionsHandler.write_stack_version(component, tgtStack) else: infostr = "target_stack_version (%s) matches current stack version" \ " for component %s, nothing to do" % (tgtStack, component) logger.info(infostr) result = { 'exitcode' : 0, 'stdout' : infostr, 'stderr' : 'None' } result = { 'exitcode' : result['exitcode'], 'stdout' : grep.tail(result['stdout'], grep.OUTPUT_LAST_LINES), 'stderr' : grep.tail(result['stderr'], grep.OUTPUT_LAST_LINES) } return result def get_key_func(self, name): """ Returns a number from filenames like 70-foobar.* or 999 for not matching filenames """ parts = name.split('-', 1) if not parts or not parts[0].isdigit(): logger.warn("Can't parse script filename number %s" % name) return self.NAME_PARSING_FAILED_CODE # unknown element will be placed to the end of list return int(parts[0]) def split_stack_version(self, verstr): verdict = json.loads(verstr) stack_name = verdict["stackName"].strip() matchObj = re.match( r'(\d+).(\d+)', verdict["stackVersion"].strip(), re.M|re.I) if matchObj: stack_major_ver = matchObj.group(1) stack_minor_ver = matchObj.group(2) return stack_name, stack_major_ver, stack_minor_ver else: return None def execute_dir(self, command, basedir, dir, tmpout, tmperr): """ Executes *.py and *.pp files located in a given directory. Files a executed in a numeric sorting order. """ dirpath = os.path.join(basedir, dir) logger.info("Executing %s" % dirpath) if not os.path.isdir(dirpath): warnstr = "Script directory %s does not exist, skipping" % dirpath logger.warn(warnstr) result = { 'exitcode' : 0, 'stdout' : warnstr, 'stderr' : 'None' } return result fileList=os.listdir(dirpath) fileList.sort(key = self.get_key_func) formattedResult = { 'exitcode' : 0, 'stdout' : '', 'stderr' : '' } for filename in fileList: prevcode = formattedResult['exitcode'] if prevcode != 0 or self.get_key_func(filename) == self.NAME_PARSING_FAILED_CODE: break filepath = os.path.join(dirpath, filename) if filename.endswith(".pp"): logger.info("Running puppet file %s" % filepath) result = self.puppetExecutor.run_manifest(command, filepath, tmpout, tmperr) elif filename.endswith(".py"): logger.info("Running python file %s" % filepath) result = self.pythonExecutor.run_file(command, filepath, tmpout, tmperr) elif filename.endswith(".pyc"): pass # skipping compiled files else: warnstr = "Unrecognized file type, skipping: %s" % filepath logger.warn(warnstr) result = { 'exitcode' : 0, 'stdout' : warnstr, 'stderr' : 'None' } formattedResult = { 'exitcode' : prevcode or result['exitcode'], 'stdout' : "%s\n%s" % (formattedResult['stdout'], result['stdout']), 'stderr' : "%s\n%s" % (formattedResult['stderr'], result['stderr']), } logger.debug("Result of %s: \n %s" % (dirpath, pprint.pformat(formattedResult))) return formattedResult
class LiveStatus: SERVICES = [ "HDFS", "MAPREDUCE", "GANGLIA", "HBASE", "NAGIOS", "ZOOKEEPER", "OOZIE", "HCATALOG", "KERBEROS", "TEMPLETON", "HIVE", "WEBHCAT", "YARN", "MAPREDUCE2", "FLUME", "TEZ", "FALCON", "STORM" ] CLIENT_COMPONENTS = [{ "serviceName": "HBASE", "componentName": "HBASE_CLIENT" }, { "serviceName": "HDFS", "componentName": "HDFS_CLIENT" }, { "serviceName": "MAPREDUCE", "componentName": "MAPREDUCE_CLIENT" }, { "serviceName": "ZOOKEEPER", "componentName": "ZOOKEEPER_CLIENT" }, { "serviceName": "OOZIE", "componentName": "OOZIE_CLIENT" }, { "serviceName": "HCATALOG", "componentName": "HCAT" }, { "serviceName": "HIVE", "componentName": "HIVE_CLIENT" }, { "serviceName": "YARN", "componentName": "YARN_CLIENT" }, { "serviceName": "MAPREDUCE2", "componentName": "MAPREDUCE2_CLIENT" }, { "serviceName": "PIG", "componentName": "PIG" }, { "serviceName": "SQOOP", "componentName": "SQOOP" }, { "serviceName": "TEZ", "componentName": "TEZ_CLIENT" }, { "serviceName": "FALCON", "componentName": "FALCON_CLIENT" }] COMPONENTS = [{ "serviceName": "HDFS", "componentName": "DATANODE" }, { "serviceName": "HDFS", "componentName": "NAMENODE" }, { "serviceName": "HDFS", "componentName": "SECONDARY_NAMENODE" }, { "serviceName": "HDFS", "componentName": "JOURNALNODE" }, { "serviceName": "HDFS", "componentName": "ZKFC" }, { "serviceName": "MAPREDUCE", "componentName": "JOBTRACKER" }, { "serviceName": "MAPREDUCE", "componentName": "TASKTRACKER" }, { "serviceName": "GANGLIA", "componentName": "GANGLIA_SERVER" }, { "serviceName": "GANGLIA", "componentName": "GANGLIA_MONITOR" }, { "serviceName": "HBASE", "componentName": "HBASE_MASTER" }, { "serviceName": "HBASE", "componentName": "HBASE_REGIONSERVER" }, { "serviceName": "NAGIOS", "componentName": "NAGIOS_SERVER" }, { "serviceName": "FLUME", "componentName": "FLUME_SERVER" }, { "serviceName": "ZOOKEEPER", "componentName": "ZOOKEEPER_SERVER" }, { "serviceName": "OOZIE", "componentName": "OOZIE_SERVER" }, { "serviceName": "HCATALOG", "componentName": "HCATALOG_SERVER" }, { "serviceName": "KERBEROS", "componentName": "KERBEROS_SERVER" }, { "serviceName": "HIVE", "componentName": "HIVE_SERVER" }, { "serviceName": "HIVE", "componentName": "HIVE_METASTORE" }, { "serviceName": "HIVE", "componentName": "MYSQL_SERVER" }, { "serviceName": "WEBHCAT", "componentName": "WEBHCAT_SERVER" }, { "serviceName": "YARN", "componentName": "RESOURCEMANAGER" }, { "serviceName": "YARN", "componentName": "NODEMANAGER" }, { "serviceName": "YARN", "componentName": "APP_TIMELINE_SERVER" }, { "serviceName": "MAPREDUCE2", "componentName": "HISTORYSERVER" }, { "serviceName": "FALCON", "componentName": "FALCON_SERVER" }, { "serviceName": "STORM", "componentName": "NIMBUS" }, { "serviceName": "STORM", "componentName": "STORM_REST_API" }, { "serviceName": "STORM", "componentName": "SUPERVISOR" }, { "serviceName": "STORM", "componentName": "STORM_UI_SERVER" }, { "serviceName": "STORM", "componentName": "DRPC_SERVER" }, { "serviceName": "STORM", "componentName": "LOGVIEWER_SERVER" }] LIVE_STATUS = "STARTED" DEAD_STATUS = "INSTALLED" def __init__(self, cluster, service, component, globalConfig, config): self.cluster = cluster self.service = service self.component = component self.globalConfig = globalConfig versionsFileDir = config.getResolvedPath(AgentConfig.APP_TASK_DIR) self.versionsHandler = StackVersionsFileHandler(versionsFileDir) def belongsToService(self, component): #TODO: Should also check belonging of server to cluster return component['serviceName'] == self.service def build(self, forsed_component_status=None): """ If forsed_component_status is explicitly defined, than StatusCheck methods are not used. This feature has been added to support custom (ver 2.0) services. """ global SERVICES, CLIENT_COMPONENTS, COMPONENTS, LIVE_STATUS, DEAD_STATUS livestatus = None component = { "serviceName": self.service, "componentName": self.component } if forsed_component_status: # If already determined status = forsed_component_status # Nothing to do elif component in self.CLIENT_COMPONENTS: status = self.DEAD_STATUS # CLIENT components can't have status STARTED elif component in self.COMPONENTS: statusCheck = StatusCheck(AgentConfig.servicesToPidNames, AgentConfig.pidPathesVars, self.globalConfig, AgentConfig.servicesToLinuxUser) serviceStatus = statusCheck.getStatus(self.component) if serviceStatus is None: logger.warn("There is no service to pid mapping for " + self.component) status = self.LIVE_STATUS if serviceStatus else self.DEAD_STATUS livestatus = { "componentName": self.component, "msg": "", "status": status, "clusterName": self.cluster, "serviceName": self.service, "stackVersion": self.versionsHandler.read_stack_version(self.component) } logger.debug("The live status for component " + str(self.component) +\ " of service " + str(self.service) + " is " + str(livestatus)) return livestatus
class LiveStatus: SERVICES = [] CLIENT_COMPONENTS = [] COMPONENTS = [] LIVE_STATUS = "STARTED" DEAD_STATUS = "INSTALLED" def __init__(self, cluster, service, component, globalConfig, config, configTags): self.cluster = cluster self.service = service self.component = component self.globalConfig = globalConfig versionsFileDir = config.get('agent', 'prefix') self.versionsHandler = StackVersionsFileHandler(versionsFileDir) self.configTags = configTags self.actualConfigHandler = ActualConfigHandler(config, configTags) def belongsToService(self, component): #TODO: Should also check belonging of server to cluster return component['serviceName'] == self.service def build(self, forsed_component_status=None): """ If forsed_component_status is explicitly defined, than StatusCheck methods are not used. This feature has been added to support custom (ver 2.0) services. """ global SERVICES, CLIENT_COMPONENTS, COMPONENTS, LIVE_STATUS, DEAD_STATUS component = { "serviceName": self.service, "componentName": self.component } if forsed_component_status: # If already determined status = forsed_component_status # Nothing to do elif component in self.CLIENT_COMPONENTS: status = self.DEAD_STATUS # CLIENT components can't have status STARTED elif component in self.COMPONENTS: statusCheck = StatusCheck(AmbariConfig.servicesToPidNames, AmbariConfig.pidPathesVars, self.globalConfig, AmbariConfig.servicesToLinuxUser) serviceStatus = statusCheck.getStatus(self.component) if serviceStatus is None: logger.warn("There is no service to pid mapping for " + self.component) status = self.LIVE_STATUS if serviceStatus else self.DEAD_STATUS livestatus = { "componentName": self.component, "msg": "", "status": status, "clusterName": self.cluster, "serviceName": self.service, "stackVersion": self.versionsHandler.read_stack_version(self.component) } active_config = self.actualConfigHandler.read_actual_component( self.component) if not active_config is None: livestatus['configurationTags'] = active_config logger.debug("The live status for component " + str(self.component) +\ " of service " + str(self.service) + " is " + str(livestatus)) return livestatus
class UpgradeExecutor: """ Class that performs the StackVersion stack upgrade""" SCRIPT_DIRS = ['pre-upgrade.d', 'upgrade.d', 'post-upgrade.d'] NAME_PARSING_FAILED_CODE = 999 def __init__(self, pythonExecutor, puppetExecutor, config): self.pythonExecutor = pythonExecutor self.puppetExecutor = puppetExecutor self.stacksDir = config.get('stack', 'upgradeScriptsDir') self.config = config versionsFileDir = config.get('agent', 'prefix') self.versionsHandler = StackVersionsFileHandler(versionsFileDir) def perform_stack_upgrade(self, command, tmpout, tmperr): logger.info("Performing stack upgrade") params = command['commandParams'] srcStack = params['source_stack_version'] tgtStack = params['target_stack_version'] component = command['role'] srcStackTuple = self.split_stack_version(srcStack) tgtStackTuple = self.split_stack_version(tgtStack) if srcStackTuple is None or tgtStackTuple is None: errorstr = "Source (%s) or target (%s) version does not match pattern \ <Name>-<Version>" % (srcStack, tgtStack) logger.info(errorstr) result = {'exitcode': 1, 'stdout': 'None', 'stderr': errorstr} elif srcStack != tgtStack: paramTuple = sum((srcStackTuple, tgtStackTuple), ()) upgradeId = "%s-%s.%s_%s-%s.%s" % paramTuple # Check stack version (do we need upgrade?) basedir = os.path.join(self.stacksDir, upgradeId, component) if not os.path.isdir(basedir): errorstr = "Upgrade %s is not supported (dir %s does not exist)" \ % (upgradeId, basedir) logger.error(errorstr) result = { 'exitcode': 1, 'stdout': errorstr, 'stderr': errorstr } else: result = {'exitcode': 0, 'stdout': '', 'stderr': ''} # Request repos update (will be executed once before running any pp file) self.puppetExecutor.discardInstalledRepos() for dir in self.SCRIPT_DIRS: if result['exitcode'] != 0: break tmpRes = self.execute_dir(command, basedir, dir, tmpout, tmperr) result = { 'exitcode': result['exitcode'] or tmpRes['exitcode'], 'stdout': "%s\n%s" % (result['stdout'], tmpRes['stdout']), 'stderr': "%s\n%s" % (result['stderr'], tmpRes['stderr']), } if result['exitcode'] == 0: logger.info("Upgrade %s successfully finished" % upgradeId) self.versionsHandler.write_stack_version( component, tgtStack) else: infostr = "target_stack_version (%s) matches current stack version" \ " for component %s, nothing to do" % (tgtStack, component) logger.info(infostr) result = {'exitcode': 0, 'stdout': infostr, 'stderr': 'None'} result = { 'exitcode': result['exitcode'], 'stdout': grep.tail(result['stdout'], grep.OUTPUT_LAST_LINES), 'stderr': grep.tail(result['stderr'], grep.OUTPUT_LAST_LINES) } return result def get_key_func(self, name): """ Returns a number from filenames like 70-foobar.* or 999 for not matching filenames """ parts = name.split('-', 1) if not parts or not parts[0].isdigit(): logger.warn("Can't parse script filename number %s" % name) return self.NAME_PARSING_FAILED_CODE # unknown element will be placed to the end of list return int(parts[0]) def split_stack_version(self, verstr): verdict = json.loads(verstr) stack_name = verdict["stackName"].strip() matchObj = re.match(r'(\d+).(\d+)', verdict["stackVersion"].strip(), re.M | re.I) if matchObj: stack_major_ver = matchObj.group(1) stack_minor_ver = matchObj.group(2) return stack_name, stack_major_ver, stack_minor_ver else: return None def execute_dir(self, command, basedir, dir, tmpout, tmperr): """ Executes *.py and *.pp files located in a given directory. Files a executed in a numeric sorting order. """ dirpath = os.path.join(basedir, dir) logger.info("Executing %s" % dirpath) if not os.path.isdir(dirpath): warnstr = "Script directory %s does not exist, skipping" % dirpath logger.warn(warnstr) result = {'exitcode': 0, 'stdout': warnstr, 'stderr': 'None'} return result fileList = os.listdir(dirpath) fileList.sort(key=self.get_key_func) formattedResult = {'exitcode': 0, 'stdout': '', 'stderr': ''} for filename in fileList: prevcode = formattedResult['exitcode'] if prevcode != 0 or self.get_key_func( filename) == self.NAME_PARSING_FAILED_CODE: break filepath = os.path.join(dirpath, filename) if filename.endswith(".pp"): logger.info("Running puppet file %s" % filepath) result = self.puppetExecutor.run_manifest( command, filepath, tmpout, tmperr) elif filename.endswith(".py"): logger.info("Running python file %s" % filepath) result = self.pythonExecutor.run_file(command, filepath, tmpout, tmperr) elif filename.endswith(".pyc"): pass # skipping compiled files else: warnstr = "Unrecognized file type, skipping: %s" % filepath logger.warn(warnstr) result = {'exitcode': 0, 'stdout': warnstr, 'stderr': 'None'} formattedResult = { 'exitcode': prevcode or result['exitcode'], 'stdout': "%s\n%s" % (formattedResult['stdout'], result['stdout']), 'stderr': "%s\n%s" % (formattedResult['stderr'], result['stderr']), } logger.debug("Result of %s: \n %s" % (dirpath, pprint.pformat(formattedResult))) return formattedResult