示例#1
0
 def __init__(self, cluster, service, component, globalConfig, config):
     self.cluster = cluster
     self.service = service
     self.component = component
     self.globalConfig = globalConfig
     versionsFileDir = config.getResolvedPath(AgentConfig.APP_TASK_DIR)
     self.versionsHandler = StackVersionsFileHandler(versionsFileDir)
示例#2
0
 def __init__(self, pythonExecutor, puppetExecutor, config):
     self.pythonExecutor = pythonExecutor
     self.puppetExecutor = puppetExecutor
     self.stacksDir = config.get('stack', 'upgradeScriptsDir')
     self.config = config
     versionsFileDir = config.get('agent', 'prefix')
     self.versionsHandler = StackVersionsFileHandler(versionsFileDir)
示例#3
0
 def __init__(self, cluster, service, component, globalConfig, config):
     self.cluster = cluster
     self.service = service
     self.component = component
     self.globalConfig = globalConfig
     versionsFileDir = config.get('agent', 'prefix')
     self.versionsHandler = StackVersionsFileHandler(versionsFileDir)
     self.actualConfigHandler = ActualConfigHandler(config)
 def __init__(self, pythonExecutor, puppetExecutor, config):
   self.pythonExecutor = pythonExecutor
   self.puppetExecutor = puppetExecutor
   self.stacksDir = config.get('stack', 'upgradeScriptsDir')
   self.config = config
   versionsFileDir = config.get('agent', 'prefix')
   self.versionsHandler = StackVersionsFileHandler(versionsFileDir)
示例#5
0
class LiveStatus:

    SERVICES = [
        "HDFS", "MAPREDUCE", "GANGLIA", "HBASE", "NAGIOS", "ZOOKEEPER",
        "OOZIE", "HCATALOG", "KERBEROS", "TEMPLETON", "HIVE", "WEBHCAT",
        "YARN", "MAPREDUCE2", "FLUME"
    ]

    CLIENT_COMPONENTS = [{
        "serviceName": "HBASE",
        "componentName": "HBASE_CLIENT"
    }, {
        "serviceName": "HDFS",
        "componentName": "HDFS_CLIENT"
    }, {
        "serviceName": "MAPREDUCE",
        "componentName": "MAPREDUCE_CLIENT"
    }, {
        "serviceName": "ZOOKEEPER",
        "componentName": "ZOOKEEPER_CLIENT"
    }, {
        "serviceName": "OOZIE",
        "componentName": "OOZIE_CLIENT"
    }, {
        "serviceName": "HCATALOG",
        "componentName": "HCAT"
    }, {
        "serviceName": "HIVE",
        "componentName": "HIVE_CLIENT"
    }, {
        "serviceName": "YARN",
        "componentName": "YARN_CLIENT"
    }, {
        "serviceName": "MAPREDUCE2",
        "componentName": "MAPREDUCE2_CLIENT"
    }, {
        "serviceName": "PIG",
        "componentName": "PIG"
    }, {
        "serviceName": "SQOOP",
        "componentName": "SQOOP"
    }]

    COMPONENTS = [
        {
            "serviceName": "HDFS",
            "componentName": "DATANODE"
        },
        {
            "serviceName": "HDFS",
            "componentName": "NAMENODE"
        },
        {
            "serviceName": "HDFS",
            "componentName": "SECONDARY_NAMENODE"
        },
        {
            "serviceName": "HDFS",
            "componentName": "JOURNALNODE"
        },
        {
            "serviceName": "HDFS",
            "componentName": "ZKFC"
        },
        {
            "serviceName": "MAPREDUCE",
            "componentName": "JOBTRACKER"
        },
        {
            "serviceName": "MAPREDUCE",
            "componentName": "TASKTRACKER"
        },
        {
            "serviceName": "GANGLIA",
            "componentName": "GANGLIA_SERVER"
        },
        {
            "serviceName": "GANGLIA",
            "componentName": "GANGLIA_MONITOR"
        },
        {
            "serviceName": "HBASE",
            "componentName": "HBASE_MASTER"
        },
        {
            "serviceName": "HBASE",
            "componentName": "HBASE_REGIONSERVER"
        },
        {
            "serviceName": "NAGIOS",
            "componentName": "NAGIOS_SERVER"
        },
        {
            "serviceName": "FLUME",
            "componentName": "FLUME_SERVER"
        },
        {
            "serviceName": "ZOOKEEPER",
            "componentName": "ZOOKEEPER_SERVER"
        },
        {
            "serviceName": "OOZIE",
            "componentName": "OOZIE_SERVER"
        },
        {
            "serviceName": "HCATALOG",
            "componentName": "HCATALOG_SERVER"
        },
        {
            "serviceName": "KERBEROS",
            "componentName": "KERBEROS_SERVER"
        },
        {
            "serviceName": "HIVE",
            "componentName": "HIVE_SERVER"
        },
        {
            "serviceName": "HIVE",
            "componentName": "HIVE_METASTORE"
        },
        {
            "serviceName": "HIVE",
            "componentName": "MYSQL_SERVER"
        },
        {
            "serviceName": "WEBHCAT",
            "componentName": "WEBHCAT_SERVER"
        },
        {
            "serviceName": "YARN",
            "componentName": "RESOURCEMANAGER"
        },
        {
            "serviceName": "YARN",
            "componentName": "NODEMANAGER"
        },
        {
            "serviceName": "MAPREDUCE2",
            "componentName": "HISTORYSERVER"
        },
    ]

    LIVE_STATUS = "STARTED"
    DEAD_STATUS = "INSTALLED"

    def __init__(self, cluster, service, component, globalConfig, config):
        self.cluster = cluster
        self.service = service
        self.component = component
        self.globalConfig = globalConfig
        versionsFileDir = config.get('agent', 'prefix')
        self.versionsHandler = StackVersionsFileHandler(versionsFileDir)
        self.actualConfigHandler = ActualConfigHandler(config)

    def belongsToService(self, component):
        #TODO: Should also check belonging of server to cluster
        return component['serviceName'] == self.service

    # Live status was stripped from heartbeat after revision e1718dd
    def build(self):
        global SERVICES, CLIENT_COMPONENTS, COMPONENTS, LIVE_STATUS, DEAD_STATUS
        statusCheck = StatusCheck(AmbariConfig.servicesToPidNames,
                                  AmbariConfig.pidPathesVars,
                                  self.globalConfig,
                                  AmbariConfig.servicesToLinuxUser)
        livestatus = None
        component = {
            "serviceName": self.service,
            "componentName": self.component
        }
        if component in self.COMPONENTS + self.CLIENT_COMPONENTS:
            # CLIENT components can't have status STARTED
            if component in self.CLIENT_COMPONENTS:
                status = self.DEAD_STATUS
            else:
                serviceStatus = statusCheck.getStatus(self.component)

                if serviceStatus is None:
                    logger.warn("There is no service to pid mapping for " +
                                self.component)
                status = self.LIVE_STATUS if serviceStatus else self.DEAD_STATUS

            livestatus = {
                "componentName":
                self.component,
                "msg":
                "",
                "status":
                status,
                "clusterName":
                self.cluster,
                "serviceName":
                self.service,
                "stackVersion":
                self.versionsHandler.read_stack_version(self.component)
            }
            active_config = self.actualConfigHandler.read_actual_component(
                self.component)
            if not active_config is None:
                livestatus['configurationTags'] = active_config

        logger.debug("The live status for component " + str(self.component) +\
                    " of service " + str(self.service) + " is " + str(livestatus))
        return livestatus
class UpgradeExecutor:

  """ Class that performs the StackVersion stack upgrade"""

  SCRIPT_DIRS = [
    'pre-upgrade.d',
    'upgrade.d',
    'post-upgrade.d'
  ]

  NAME_PARSING_FAILED_CODE = 999

  def __init__(self, pythonExecutor, puppetExecutor, config):
    self.pythonExecutor = pythonExecutor
    self.puppetExecutor = puppetExecutor
    self.stacksDir = config.get('stack', 'upgradeScriptsDir')
    self.config = config
    versionsFileDir = config.get('agent', 'prefix')
    self.versionsHandler = StackVersionsFileHandler(versionsFileDir)


  def perform_stack_upgrade(self, command, tmpout, tmperr):
    logger.info("Performing stack upgrade")
    params = command['commandParams']
    srcStack = params['source_stack_version']
    tgtStack = params['target_stack_version']
    component = command['role']

    srcStackTuple = self.split_stack_version(srcStack)
    tgtStackTuple = self.split_stack_version(tgtStack)

    if srcStackTuple is None or tgtStackTuple is None:
      errorstr = "Source (%s) or target (%s) version does not match pattern \
      <Name>-<Version>" % (srcStack, tgtStack)
      logger.info(errorstr)
      result = {
        'exitcode' : 1,
        'stdout'   : 'None',
        'stderr'   : errorstr
      }
    elif srcStack != tgtStack:
      paramTuple = sum((srcStackTuple, tgtStackTuple), ())
      upgradeId = "%s-%s.%s_%s-%s.%s" % paramTuple
      # Check stack version (do we need upgrade?)
      basedir = os.path.join(self.stacksDir, upgradeId, component)
      if not os.path.isdir(basedir):
        errorstr = "Upgrade %s is not supported (dir %s does not exist)" \
                   % (upgradeId, basedir)
        logger.error(errorstr)
        result = {
          'exitcode' : 1,
          'stdout'   : errorstr,
          'stderr'   : errorstr
        }
      else:
        result = {
          'exitcode' : 0,
          'stdout'   : '',
          'stderr'   : ''
        }
        # Request repos update (will be executed once before running any pp file)
        self.puppetExecutor.discardInstalledRepos()
        for dir in self.SCRIPT_DIRS:
          if result['exitcode'] != 0:
            break
          tmpRes = self.execute_dir(command, basedir, dir, tmpout, tmperr)

          result = {
            'exitcode' : result['exitcode'] or tmpRes['exitcode'],
            'stdout'   : "%s\n%s" % (result['stdout'], tmpRes['stdout']),
            'stderr'   : "%s\n%s" % (result['stderr'], tmpRes['stderr']),
          }

        if result['exitcode'] == 0:
          logger.info("Upgrade %s successfully finished" % upgradeId)
          self.versionsHandler.write_stack_version(component, tgtStack)
    else:
      infostr = "target_stack_version (%s) matches current stack version" \
          " for component %s, nothing to do" % (tgtStack, component)
      logger.info(infostr)
      result = {
        'exitcode' : 0,
        'stdout'   : infostr,
        'stderr'   : 'None'
      }
    result = {
      'exitcode' : result['exitcode'],
      'stdout'   : grep.tail(result['stdout'], grep.OUTPUT_LAST_LINES),
      'stderr'   : grep.tail(result['stderr'], grep.OUTPUT_LAST_LINES)
    }
    return result


  def get_key_func(self, name):
    """
    Returns a number from filenames like 70-foobar.* or 999 for not matching
    filenames
    """
    parts = name.split('-', 1)
    if not parts or not parts[0].isdigit():
      logger.warn("Can't parse script filename number %s" % name)
      return self.NAME_PARSING_FAILED_CODE # unknown element will be placed to the end of list
    return int(parts[0])


  def split_stack_version(self, verstr):
    verdict = json.loads(verstr)
    stack_name = verdict["stackName"].strip()

    matchObj = re.match( r'(\d+).(\d+)', verdict["stackVersion"].strip(), re.M|re.I)
    if matchObj:
      stack_major_ver = matchObj.group(1)
      stack_minor_ver = matchObj.group(2)
      return stack_name, stack_major_ver, stack_minor_ver
    else:
      return None


  def execute_dir(self, command, basedir, dir, tmpout, tmperr):
    """
    Executes *.py and *.pp files located in a given directory.
    Files a executed in a numeric sorting order.
    """
    dirpath = os.path.join(basedir, dir)
    logger.info("Executing %s" % dirpath)
    if not os.path.isdir(dirpath):
      warnstr = "Script directory %s does not exist, skipping" % dirpath
      logger.warn(warnstr)
      result = {
        'exitcode' : 0,
        'stdout'   : warnstr,
        'stderr'   : 'None'
      }
      return result
    fileList=os.listdir(dirpath)
    fileList.sort(key = self.get_key_func)
    formattedResult = {
      'exitcode' : 0,
      'stdout'   : '',
      'stderr'   : ''
    }
    for filename in fileList:
      prevcode = formattedResult['exitcode']
      if prevcode != 0 or self.get_key_func(filename) == self.NAME_PARSING_FAILED_CODE:
        break
      filepath = os.path.join(dirpath, filename)
      if filename.endswith(".pp"):
        logger.info("Running puppet file %s" % filepath)
        result = self.puppetExecutor.run_manifest(command, filepath,
                                                                tmpout, tmperr)
      elif filename.endswith(".py"):
        logger.info("Running python file %s" % filepath)
        result = self.pythonExecutor.run_file(command, filepath, tmpout, tmperr)
      elif filename.endswith(".pyc"):
        pass # skipping compiled files
      else:
        warnstr = "Unrecognized file type, skipping: %s" % filepath
        logger.warn(warnstr)
        result = {
          'exitcode' : 0,
          'stdout'   : warnstr,
          'stderr'   : 'None'
        }
      formattedResult = {
        'exitcode' : prevcode or result['exitcode'],
        'stdout'   : "%s\n%s" % (formattedResult['stdout'], result['stdout']),
        'stderr'   : "%s\n%s" % (formattedResult['stderr'], result['stderr']),
      }
    logger.debug("Result of %s: \n %s" % (dirpath, pprint.pformat(formattedResult)))
    return formattedResult
示例#7
0
class LiveStatus:

    SERVICES = [
        "HDFS", "MAPREDUCE", "GANGLIA", "HBASE", "NAGIOS", "ZOOKEEPER",
        "OOZIE", "HCATALOG", "KERBEROS", "TEMPLETON", "HIVE", "WEBHCAT",
        "YARN", "MAPREDUCE2", "FLUME", "TEZ", "FALCON", "STORM"
    ]

    CLIENT_COMPONENTS = [{
        "serviceName": "HBASE",
        "componentName": "HBASE_CLIENT"
    }, {
        "serviceName": "HDFS",
        "componentName": "HDFS_CLIENT"
    }, {
        "serviceName": "MAPREDUCE",
        "componentName": "MAPREDUCE_CLIENT"
    }, {
        "serviceName": "ZOOKEEPER",
        "componentName": "ZOOKEEPER_CLIENT"
    }, {
        "serviceName": "OOZIE",
        "componentName": "OOZIE_CLIENT"
    }, {
        "serviceName": "HCATALOG",
        "componentName": "HCAT"
    }, {
        "serviceName": "HIVE",
        "componentName": "HIVE_CLIENT"
    }, {
        "serviceName": "YARN",
        "componentName": "YARN_CLIENT"
    }, {
        "serviceName": "MAPREDUCE2",
        "componentName": "MAPREDUCE2_CLIENT"
    }, {
        "serviceName": "PIG",
        "componentName": "PIG"
    }, {
        "serviceName": "SQOOP",
        "componentName": "SQOOP"
    }, {
        "serviceName": "TEZ",
        "componentName": "TEZ_CLIENT"
    }, {
        "serviceName": "FALCON",
        "componentName": "FALCON_CLIENT"
    }]

    COMPONENTS = [{
        "serviceName": "HDFS",
        "componentName": "DATANODE"
    }, {
        "serviceName": "HDFS",
        "componentName": "NAMENODE"
    }, {
        "serviceName": "HDFS",
        "componentName": "SECONDARY_NAMENODE"
    }, {
        "serviceName": "HDFS",
        "componentName": "JOURNALNODE"
    }, {
        "serviceName": "HDFS",
        "componentName": "ZKFC"
    }, {
        "serviceName": "MAPREDUCE",
        "componentName": "JOBTRACKER"
    }, {
        "serviceName": "MAPREDUCE",
        "componentName": "TASKTRACKER"
    }, {
        "serviceName": "GANGLIA",
        "componentName": "GANGLIA_SERVER"
    }, {
        "serviceName": "GANGLIA",
        "componentName": "GANGLIA_MONITOR"
    }, {
        "serviceName": "HBASE",
        "componentName": "HBASE_MASTER"
    }, {
        "serviceName": "HBASE",
        "componentName": "HBASE_REGIONSERVER"
    }, {
        "serviceName": "NAGIOS",
        "componentName": "NAGIOS_SERVER"
    }, {
        "serviceName": "FLUME",
        "componentName": "FLUME_SERVER"
    }, {
        "serviceName": "ZOOKEEPER",
        "componentName": "ZOOKEEPER_SERVER"
    }, {
        "serviceName": "OOZIE",
        "componentName": "OOZIE_SERVER"
    }, {
        "serviceName": "HCATALOG",
        "componentName": "HCATALOG_SERVER"
    }, {
        "serviceName": "KERBEROS",
        "componentName": "KERBEROS_SERVER"
    }, {
        "serviceName": "HIVE",
        "componentName": "HIVE_SERVER"
    }, {
        "serviceName": "HIVE",
        "componentName": "HIVE_METASTORE"
    }, {
        "serviceName": "HIVE",
        "componentName": "MYSQL_SERVER"
    }, {
        "serviceName": "WEBHCAT",
        "componentName": "WEBHCAT_SERVER"
    }, {
        "serviceName": "YARN",
        "componentName": "RESOURCEMANAGER"
    }, {
        "serviceName": "YARN",
        "componentName": "NODEMANAGER"
    }, {
        "serviceName": "YARN",
        "componentName": "APP_TIMELINE_SERVER"
    }, {
        "serviceName": "MAPREDUCE2",
        "componentName": "HISTORYSERVER"
    }, {
        "serviceName": "FALCON",
        "componentName": "FALCON_SERVER"
    }, {
        "serviceName": "STORM",
        "componentName": "NIMBUS"
    }, {
        "serviceName": "STORM",
        "componentName": "STORM_REST_API"
    }, {
        "serviceName": "STORM",
        "componentName": "SUPERVISOR"
    }, {
        "serviceName": "STORM",
        "componentName": "STORM_UI_SERVER"
    }, {
        "serviceName": "STORM",
        "componentName": "DRPC_SERVER"
    }, {
        "serviceName": "STORM",
        "componentName": "LOGVIEWER_SERVER"
    }]

    LIVE_STATUS = "STARTED"
    DEAD_STATUS = "INSTALLED"

    def __init__(self, cluster, service, component, globalConfig, config):
        self.cluster = cluster
        self.service = service
        self.component = component
        self.globalConfig = globalConfig
        versionsFileDir = config.getResolvedPath(AgentConfig.APP_TASK_DIR)
        self.versionsHandler = StackVersionsFileHandler(versionsFileDir)

    def belongsToService(self, component):
        #TODO: Should also check belonging of server to cluster
        return component['serviceName'] == self.service

    def build(self, forsed_component_status=None):
        """
    If forsed_component_status is explicitly defined, than StatusCheck methods are
    not used. This feature has been added to support custom (ver 2.0) services.
    """
        global SERVICES, CLIENT_COMPONENTS, COMPONENTS, LIVE_STATUS, DEAD_STATUS

        livestatus = None
        component = {
            "serviceName": self.service,
            "componentName": self.component
        }
        if forsed_component_status:  # If already determined
            status = forsed_component_status  # Nothing to do
        elif component in self.CLIENT_COMPONENTS:
            status = self.DEAD_STATUS  # CLIENT components can't have status STARTED
        elif component in self.COMPONENTS:
            statusCheck = StatusCheck(AgentConfig.servicesToPidNames,
                                      AgentConfig.pidPathesVars,
                                      self.globalConfig,
                                      AgentConfig.servicesToLinuxUser)
            serviceStatus = statusCheck.getStatus(self.component)
            if serviceStatus is None:
                logger.warn("There is no service to pid mapping for " +
                            self.component)
            status = self.LIVE_STATUS if serviceStatus else self.DEAD_STATUS

        livestatus = {
            "componentName": self.component,
            "msg": "",
            "status": status,
            "clusterName": self.cluster,
            "serviceName": self.service,
            "stackVersion":
            self.versionsHandler.read_stack_version(self.component)
        }

        logger.debug("The live status for component " + str(self.component) +\
                    " of service " + str(self.service) + " is " + str(livestatus))
        return livestatus
示例#8
0
class LiveStatus:

    SERVICES = []
    CLIENT_COMPONENTS = []
    COMPONENTS = []

    LIVE_STATUS = "STARTED"
    DEAD_STATUS = "INSTALLED"

    def __init__(self, cluster, service, component, globalConfig, config,
                 configTags):
        self.cluster = cluster
        self.service = service
        self.component = component
        self.globalConfig = globalConfig
        versionsFileDir = config.get('agent', 'prefix')
        self.versionsHandler = StackVersionsFileHandler(versionsFileDir)
        self.configTags = configTags
        self.actualConfigHandler = ActualConfigHandler(config, configTags)

    def belongsToService(self, component):
        #TODO: Should also check belonging of server to cluster
        return component['serviceName'] == self.service

    def build(self, forsed_component_status=None):
        """
    If forsed_component_status is explicitly defined, than StatusCheck methods are
    not used. This feature has been added to support custom (ver 2.0) services.
    """
        global SERVICES, CLIENT_COMPONENTS, COMPONENTS, LIVE_STATUS, DEAD_STATUS

        component = {
            "serviceName": self.service,
            "componentName": self.component
        }
        if forsed_component_status:  # If already determined
            status = forsed_component_status  # Nothing to do
        elif component in self.CLIENT_COMPONENTS:
            status = self.DEAD_STATUS  # CLIENT components can't have status STARTED
        elif component in self.COMPONENTS:
            statusCheck = StatusCheck(AmbariConfig.servicesToPidNames,
                                      AmbariConfig.pidPathesVars,
                                      self.globalConfig,
                                      AmbariConfig.servicesToLinuxUser)
            serviceStatus = statusCheck.getStatus(self.component)
            if serviceStatus is None:
                logger.warn("There is no service to pid mapping for " +
                            self.component)
            status = self.LIVE_STATUS if serviceStatus else self.DEAD_STATUS

        livestatus = {
            "componentName": self.component,
            "msg": "",
            "status": status,
            "clusterName": self.cluster,
            "serviceName": self.service,
            "stackVersion":
            self.versionsHandler.read_stack_version(self.component)
        }

        active_config = self.actualConfigHandler.read_actual_component(
            self.component)
        if not active_config is None:
            livestatus['configurationTags'] = active_config

        logger.debug("The live status for component " + str(self.component) +\
                    " of service " + str(self.service) + " is " + str(livestatus))
        return livestatus
示例#9
0
class UpgradeExecutor:
    """ Class that performs the StackVersion stack upgrade"""

    SCRIPT_DIRS = ['pre-upgrade.d', 'upgrade.d', 'post-upgrade.d']

    NAME_PARSING_FAILED_CODE = 999

    def __init__(self, pythonExecutor, puppetExecutor, config):
        self.pythonExecutor = pythonExecutor
        self.puppetExecutor = puppetExecutor
        self.stacksDir = config.get('stack', 'upgradeScriptsDir')
        self.config = config
        versionsFileDir = config.get('agent', 'prefix')
        self.versionsHandler = StackVersionsFileHandler(versionsFileDir)

    def perform_stack_upgrade(self, command, tmpout, tmperr):
        logger.info("Performing stack upgrade")
        params = command['commandParams']
        srcStack = params['source_stack_version']
        tgtStack = params['target_stack_version']
        component = command['role']

        srcStackTuple = self.split_stack_version(srcStack)
        tgtStackTuple = self.split_stack_version(tgtStack)

        if srcStackTuple is None or tgtStackTuple is None:
            errorstr = "Source (%s) or target (%s) version does not match pattern \
      <Name>-<Version>" % (srcStack, tgtStack)
            logger.info(errorstr)
            result = {'exitcode': 1, 'stdout': 'None', 'stderr': errorstr}
        elif srcStack != tgtStack:
            paramTuple = sum((srcStackTuple, tgtStackTuple), ())
            upgradeId = "%s-%s.%s_%s-%s.%s" % paramTuple
            # Check stack version (do we need upgrade?)
            basedir = os.path.join(self.stacksDir, upgradeId, component)
            if not os.path.isdir(basedir):
                errorstr = "Upgrade %s is not supported (dir %s does not exist)" \
                           % (upgradeId, basedir)
                logger.error(errorstr)
                result = {
                    'exitcode': 1,
                    'stdout': errorstr,
                    'stderr': errorstr
                }
            else:
                result = {'exitcode': 0, 'stdout': '', 'stderr': ''}
                # Request repos update (will be executed once before running any pp file)
                self.puppetExecutor.discardInstalledRepos()
                for dir in self.SCRIPT_DIRS:
                    if result['exitcode'] != 0:
                        break
                    tmpRes = self.execute_dir(command, basedir, dir, tmpout,
                                              tmperr)

                    result = {
                        'exitcode': result['exitcode'] or tmpRes['exitcode'],
                        'stdout':
                        "%s\n%s" % (result['stdout'], tmpRes['stdout']),
                        'stderr':
                        "%s\n%s" % (result['stderr'], tmpRes['stderr']),
                    }

                if result['exitcode'] == 0:
                    logger.info("Upgrade %s successfully finished" % upgradeId)
                    self.versionsHandler.write_stack_version(
                        component, tgtStack)
        else:
            infostr = "target_stack_version (%s) matches current stack version" \
                " for component %s, nothing to do" % (tgtStack, component)
            logger.info(infostr)
            result = {'exitcode': 0, 'stdout': infostr, 'stderr': 'None'}
        result = {
            'exitcode': result['exitcode'],
            'stdout': grep.tail(result['stdout'], grep.OUTPUT_LAST_LINES),
            'stderr': grep.tail(result['stderr'], grep.OUTPUT_LAST_LINES)
        }
        return result

    def get_key_func(self, name):
        """
    Returns a number from filenames like 70-foobar.* or 999 for not matching
    filenames
    """
        parts = name.split('-', 1)
        if not parts or not parts[0].isdigit():
            logger.warn("Can't parse script filename number %s" % name)
            return self.NAME_PARSING_FAILED_CODE  # unknown element will be placed to the end of list
        return int(parts[0])

    def split_stack_version(self, verstr):
        verdict = json.loads(verstr)
        stack_name = verdict["stackName"].strip()

        matchObj = re.match(r'(\d+).(\d+)', verdict["stackVersion"].strip(),
                            re.M | re.I)
        if matchObj:
            stack_major_ver = matchObj.group(1)
            stack_minor_ver = matchObj.group(2)
            return stack_name, stack_major_ver, stack_minor_ver
        else:
            return None

    def execute_dir(self, command, basedir, dir, tmpout, tmperr):
        """
    Executes *.py and *.pp files located in a given directory.
    Files a executed in a numeric sorting order.
    """
        dirpath = os.path.join(basedir, dir)
        logger.info("Executing %s" % dirpath)
        if not os.path.isdir(dirpath):
            warnstr = "Script directory %s does not exist, skipping" % dirpath
            logger.warn(warnstr)
            result = {'exitcode': 0, 'stdout': warnstr, 'stderr': 'None'}
            return result
        fileList = os.listdir(dirpath)
        fileList.sort(key=self.get_key_func)
        formattedResult = {'exitcode': 0, 'stdout': '', 'stderr': ''}
        for filename in fileList:
            prevcode = formattedResult['exitcode']
            if prevcode != 0 or self.get_key_func(
                    filename) == self.NAME_PARSING_FAILED_CODE:
                break
            filepath = os.path.join(dirpath, filename)
            if filename.endswith(".pp"):
                logger.info("Running puppet file %s" % filepath)
                result = self.puppetExecutor.run_manifest(
                    command, filepath, tmpout, tmperr)
            elif filename.endswith(".py"):
                logger.info("Running python file %s" % filepath)
                result = self.pythonExecutor.run_file(command, filepath,
                                                      tmpout, tmperr)
            elif filename.endswith(".pyc"):
                pass  # skipping compiled files
            else:
                warnstr = "Unrecognized file type, skipping: %s" % filepath
                logger.warn(warnstr)
                result = {'exitcode': 0, 'stdout': warnstr, 'stderr': 'None'}
            formattedResult = {
                'exitcode': prevcode or result['exitcode'],
                'stdout':
                "%s\n%s" % (formattedResult['stdout'], result['stdout']),
                'stderr':
                "%s\n%s" % (formattedResult['stderr'], result['stderr']),
            }
        logger.debug("Result of %s: \n %s" %
                     (dirpath, pprint.pformat(formattedResult)))
        return formattedResult