Esempio n. 1
0
class PTLTestRunner(Plugin):
    """
    PTL Test Runner Plugin
    """
    name = 'PTLTestRunner'
    score = sys.maxint - 4
    logger = logging.getLogger(__name__)

    def __init__(self):
        Plugin.__init__(self)
        self.param = None
        self.lcov_bin = None
        self.lcov_data = None
        self.lcov_out = None
        self.lcov_utils = None
        self.lcov_nosrc = None
        self.lcov_baseurl = None
        self.genhtml_bin = None
        self.config = None
        self.result = None
        self.tc_failure_threshold = None
        self.cumulative_tc_failure_threshold = None
        self.__failed_tc_count = 0
        self.__tf_count = 0
        self.__failed_tc_count_msg = False

    def options(self, parser, env):
        """
        Register command line options
        """
        pass

    def set_data(self, paramfile, testparam, lcov_bin, lcov_data, lcov_out,
                 genhtml_bin, lcov_nosrc, lcov_baseurl, tc_failure_threshold,
                 cumulative_tc_failure_threshold):
        if paramfile is not None:
            _pf = open(paramfile, 'r')
            _params_from_file = _pf.readlines()
            _pf.close()
            _nparams = []
            for l in range(len(_params_from_file)):
                if _params_from_file[l].startswith('#'):
                    continue
                else:
                    _nparams.append(_params_from_file[l])
            _f = ','.join(map(lambda l: l.strip('\r\n'), _nparams))
            if testparam is not None:
                testparam += ',' + _f
            else:
                testparam = _f
        self.param = testparam
        self.lcov_bin = lcov_bin
        self.lcov_data = lcov_data
        self.lcov_out = lcov_out
        self.genhtml_bin = genhtml_bin
        self.lcov_nosrc = lcov_nosrc
        self.lcov_baseurl = lcov_baseurl
        self.tc_failure_threshold = tc_failure_threshold
        self.cumulative_tc_failure_threshold = cumulative_tc_failure_threshold

    def configure(self, options, config):
        """
        Configure the plugin and system, based on selected options
        """
        self.config = config
        self.enabled = True

    def prepareTestRunner(self, runner):
        """
        Prepare test runner
        """
        return PtlTestRunner(verbosity=3, config=self.config)

    def prepareTestResult(self, result):
        """
        Prepare test result
        """
        self.result = result

    def startContext(self, context):
        context.param = self.param
        context.start_time = datetime.datetime.now()
        if isclass(context) and issubclass(context, PBSTestSuite):
            self.result.logger.info(self.result.separator1)
            self.result.logger.info('suite name: ' + context.__name__)
            doc = context.__doc__
            if doc is not None:
                self.result.logger.info('suite docstring: \n' + doc + '\n')
            self.result.logger.info(self.result.separator1)
            self.__failed_tc_count = 0
            self.__failed_tc_count_msg = False

    def __get_timeout(self, test):
        try:
            method = getattr(test.test, getattr(test.test, '_testMethodName'))
            return getattr(method, TIMEOUT_KEY)
        except:
            if hasattr(test, 'test'):
                __conf = getattr(test.test, 'conf')
            else:
                __conf = getattr(test.context, 'conf')
            return __conf['default_testcase_timeout']

    def __set_test_end_data(self, test, err=None):
        if not hasattr(test, 'start_time'):
            test = test.context
        if err is not None:
            is_skip = issubclass(err[0], SkipTest)
            is_tctr = issubclass(err[0], TCThresholdReached)
            if not (is_skip or is_tctr):
                self.__failed_tc_count += 1
                self.__tf_count += 1
            try:
                test.err_in_string = self.result._exc_info_to_string(err, test)
            except:
                etype, value, tb = err
                test.err_in_string = ''.join(format_exception(
                    etype, value, tb))
        else:
            test.err_in_string = 'None'
        test.end_time = datetime.datetime.now()
        test.duration = test.end_time - test.start_time
        test.captured_logs = self.result.handler.get_logs()

    def startTest(self, test):
        """
        Start the test
        """
        if ((self.cumulative_tc_failure_threshold != 0)
                and (self.__tf_count >= self.cumulative_tc_failure_threshold)):
            _msg = 'Total testcases failure count exceeded cumulative'
            _msg += ' testcase failure threshold '
            _msg += '(%d)' % self.cumulative_tc_failure_threshold
            self.logger.error(_msg)
            raise KeyboardInterrupt
        if ((self.tc_failure_threshold != 0)
                and (self.__failed_tc_count >= self.tc_failure_threshold)):
            if self.__failed_tc_count_msg:
                raise TCThresholdReached
            _msg = 'Testcases failure for this testsuite count exceeded'
            _msg += ' testcase failure threshold '
            _msg += '(%d)' % self.tc_failure_threshold
            self.logger.error(_msg)
            self.__failed_tc_count_msg = True
            raise TCThresholdReached
        timeout = self.__get_timeout(test)

        def timeout_handler(signum, frame):
            raise TimeOut('Timed out after %s second' % timeout)

        old_handler = signal.signal(signal.SIGALRM, timeout_handler)
        setattr(test, 'old_sigalrm_handler', old_handler)
        signal.alarm(timeout)

    def stopTest(self, test):
        """
        Stop the test
        """
        old_sigalrm_handler = getattr(test, 'old_sigalrm_handler', None)
        if old_sigalrm_handler is not None:
            signal.signal(signal.SIGALRM, old_sigalrm_handler)
            signal.alarm(0)

    def addError(self, test, err):
        """
        Add error
        """
        if isclass(err[0]) and issubclass(err[0], TCThresholdReached):
            return True
        self.__set_test_end_data(test, err)

    def addFailure(self, test, err):
        """
        Add failure
        """
        self.__set_test_end_data(test, err)

    def addSuccess(self, test):
        """
        Add success
        """
        self.__set_test_end_data(test)

    def _cleanup(self):
        self.logger.info('Cleaning up temporary files')
        du = DshUtils()
        tmpdir = tempfile.gettempdir()
        ftd = []
        if tmpdir:
            files = du.listdir(path=tmpdir)
            bn = os.path.basename
            ftd.extend([f for f in files if bn(f).startswith('PtlPbs')])
            ftd.extend([f for f in files if bn(f).startswith('STDIN')])
            for f in ftd:
                du.rm(path=f,
                      sudo=True,
                      recursive=True,
                      force=True,
                      level=logging.DEBUG)
        os.chdir(tmpdir)
        tmppath = os.path.join(tmpdir, 'dejagnutemp%s' % os.getpid())
        if du.isdir(path=tmppath):
            du.rm(path=tmppath,
                  recursive=True,
                  sudo=True,
                  force=True,
                  level=logging.DEBUG)

    def begin(self):
        command = sys.argv
        command[0] = os.path.basename(command[0])
        self.logger.info('input command: ' + ' '.join(command))
        self.logger.info('param: ' + str(self.param))
        self.logger.info('ptl version: ' + str(ptl.__version__))
        _m = 'platform: ' + ' '.join(platform.uname()).strip()
        self.logger.info(_m)
        self.logger.info('python version: ' + str(platform.python_version()))
        self.logger.info('user: '******'-' * 80)

        if self.lcov_data is not None:
            self.lcov_utils = LcovUtils(cov_bin=self.lcov_bin,
                                        html_bin=self.genhtml_bin,
                                        cov_out=self.lcov_out,
                                        data_dir=self.lcov_data,
                                        html_nosrc=self.lcov_nosrc,
                                        html_baseurl=self.lcov_baseurl)
            # Initialize coverage analysis
            self.lcov_utils.zero_coverage()
            # The following 'dance' is done due to some oddities on lcov's
            # part, according to this the lcov readme file at
            # http://ltp.sourceforge.net/coverage/lcov/readme.php that reads:
            #
            # Note that this step only works after the application has
            # been started and stopped at least once. Otherwise lcov will
            # abort with an error mentioning that there are no data/.gcda
            # files.
            self.lcov_utils.initialize_coverage(name='PTLTestCov')
            PBSInitServices().restart()
        self._cleanup()

    def finalize(self, result):
        if self.lcov_data is not None:
            # See note above that briefly explains the 'dance' needed to get
            # reliable coverage data
            PBSInitServices().restart()
            self.lcov_utils.capture_coverage(name='PTLTestCov')
            exclude = [
                '"*work/gSOAP/*"', '"*/pbs/doc/*"', 'lex.yy.c',
                'pbs_ifl_wrap.c', 'usr/include/*', 'unsupported/*'
            ]
            self.lcov_utils.merge_coverage_traces(name='PTLTestCov',
                                                  exclude=exclude)
            self.lcov_utils.generate_html()
            self.lcov_utils.change_baseurl()
            self.logger.info('\n'.join(self.lcov_utils.summarize_coverage()))
        self._cleanup()
Esempio n. 2
0
class PTLTestRunner(Plugin):

    """
    PTL Test Runner Plugin
    """
    name = 'PTLTestRunner'
    score = sys.maxint - 4
    logger = logging.getLogger(__name__)

    def __init__(self):
        Plugin.__init__(self)
        self.param = None
        self.lcov_bin = None
        self.lcov_data = None
        self.lcov_out = None
        self.lcov_utils = None
        self.lcov_nosrc = None
        self.lcov_baseurl = None
        self.genhtml_bin = None
        self.config = None
        self.result = None
        self.tc_failure_threshold = None
        self.cumulative_tc_failure_threshold = None
        self.__failed_tc_count = 0
        self.__tf_count = 0
        self.__failed_tc_count_msg = False
        self._test_marker = 'test_'

    def options(self, parser, env):
        """
        Register command line options
        """
        pass

    def set_data(self, paramfile, testparam,
                 lcov_bin, lcov_data, lcov_out, genhtml_bin, lcov_nosrc,
                 lcov_baseurl, tc_failure_threshold,
                 cumulative_tc_failure_threshold):
        if paramfile is not None:
            _pf = open(paramfile, 'r')
            _params_from_file = _pf.readlines()
            _pf.close()
            _nparams = []
            for l in range(len(_params_from_file)):
                if _params_from_file[l].startswith('#'):
                    continue
                else:
                    _nparams.append(_params_from_file[l])
            _f = ','.join(map(lambda l: l.strip('\r\n'), _nparams))
            if testparam is not None:
                testparam += ',' + _f
            else:
                testparam = _f
        self.param = testparam
        self.lcov_bin = lcov_bin
        self.lcov_data = lcov_data
        self.lcov_out = lcov_out
        self.genhtml_bin = genhtml_bin
        self.lcov_nosrc = lcov_nosrc
        self.lcov_baseurl = lcov_baseurl
        self.tc_failure_threshold = tc_failure_threshold
        self.cumulative_tc_failure_threshold = cumulative_tc_failure_threshold

    def configure(self, options, config):
        """
        Configure the plugin and system, based on selected options
        """
        self.config = config
        self.enabled = True
        self.param_dict = self.__get_param_dictionary()

    def prepareTestRunner(self, runner):
        """
        Prepare test runner
        """
        return PtlTestRunner(verbosity=3, config=self.config)

    def prepareTestResult(self, result):
        """
        Prepare test result
        """
        self.result = result

    def startContext(self, context):
        context.param = self.param
        context.start_time = datetime.datetime.now()
        if isclass(context) and issubclass(context, PBSTestSuite):
            self.result.logger.info(self.result.separator1)
            self.result.logger.info('suite name: ' + context.__name__)
            doc = context.__doc__
            if doc is not None:
                self.result.logger.info('suite docstring: \n' + doc + '\n')
            self.result.logger.info(self.result.separator1)
            self.__failed_tc_count = 0
            self.__failed_tc_count_msg = False

    def __get_timeout(self, test):
        try:
            method = getattr(test.test, getattr(test.test, '_testMethodName'))
            return getattr(method, TIMEOUT_KEY)
        except AttributeError:
            testcase_timeout = MINIMUM_TESTCASE_TIMEOUT
            if hasattr(test, 'test'):
                if hasattr(test.test, 'conf'):
                    __conf = getattr(test.test, 'conf')
                    testcase_timeout = __conf['default_testcase_timeout']
            elif hasattr(test, 'context'):
                if hasattr(test.context, 'conf'):
                    __conf = getattr(test.context, 'conf')
                    testcase_timeout = __conf['default_testcase_timeout']
            return testcase_timeout

    def __set_test_end_data(self, test, err=None):
        if not hasattr(test, 'start_time'):
            test = test.context
        if err is not None:
            is_skip = issubclass(err[0], SkipTest)
            is_tctr = issubclass(err[0], TCThresholdReached)
            if not (is_skip or is_tctr):
                self.__failed_tc_count += 1
                self.__tf_count += 1
            try:
                test.err_in_string = self.result._exc_info_to_string(err,
                                                                     test)
            except:
                etype, value, tb = err
                test.err_in_string = ''.join(format_exception(etype, value,
                                                              tb))
        else:
            test.err_in_string = 'None'
        test.end_time = datetime.datetime.now()
        test.duration = test.end_time - test.start_time
        test.captured_logs = self.result.handler.get_logs()

    def __get_param_dictionary(self):
        """
        Method to convert data in param into dictionary of cluster
        information
        """
        tparam_contents = {}
        nomomlist = []
        shortname = (socket.gethostname()).split('.', 1)[0]
        for key in ['servers', 'moms', 'comms', 'clients']:
            tparam_contents[key] = []
        if self.param is not None:
            for h in self.param.split(','):
                if '=' in h:
                    k, v = h.split('=', 1)
                    if (k == 'server' or k == 'servers'):
                        tparam_contents['servers'].extend(v.split(':'))
                    elif (k == 'mom' or k == 'moms'):
                        tparam_contents['moms'].extend(v.split(':'))
                    elif k == 'comms':
                        tparam_contents['comms'] = v.split(':')
                    elif k == 'client':
                        tparam_contents['clients'] = v.split(':')
                    elif k == 'nomom':
                        nomomlist = v.split(':')
        for pkey in ['servers', 'moms', 'comms', 'clients']:
            if not tparam_contents[pkey]:
                tparam_contents[pkey] = set([shortname])
            else:
                tparam_contents[pkey] = set(tparam_contents[pkey])
        if nomomlist:
            tparam_contents['moms'] -= set(nomomlist)
        return tparam_contents

    @staticmethod
    def __are_requirements_matching(param_dic=None, test=None):
        """
        Validates test requirements against test cluster information
        returns True on match or False otherwise None

        :param param_dic: dictionary of cluster information from data passed
                          to param list
        :param_dic type: dic
        :param test: test object
        :test type: object

        :returns True or False or None
        """
        ts_requirements = {}
        tc_requirements = {}
        param_count = {}
        shortname = (socket.gethostname()).split('.', 1)[0]
        if test is None:
            return None
        test_name = getattr(test.test, '_testMethodName', None)
        if test_name is not None:
            method = getattr(test.test, test_name, None)
        if method is not None:
            tc_requirements = getattr(method, REQUIREMENTS_KEY, {})
            cls = method.im_class
            ts_requirements = getattr(cls, REQUIREMENTS_KEY, {})
        if not tc_requirements:
            if not ts_requirements:
                return None
        eff_tc_req = get_effective_reqs(ts_requirements, tc_requirements)
        setattr(test.test, 'requirements', eff_tc_req)
        for key in ['servers', 'moms', 'comms', 'clients']:
            param_count['num_' + key] = len(param_dic[key])
        for pk in param_count:
            if param_count[pk] < eff_tc_req[pk]:
                return False
        if set(param_dic['moms']) & set(param_dic['servers']):
            if eff_tc_req['no_mom_on_server']:
                return False
        else:
            if not eff_tc_req['no_mom_on_server']:
                return False
        if set(param_dic['comms']) & set(param_dic['servers']):
            if eff_tc_req['no_comm_on_server']:
                return False
        else:
            if not eff_tc_req['no_comm_on_server']:
                return False
        comm_mom_list = set(param_dic['moms']) & set(param_dic['comms'])
        if comm_mom_list and shortname in comm_mom_list:
            # Excluding the server hostname for flag 'no_comm_on_mom'
            comm_mom_list.remove(shortname)
        if comm_mom_list:
            if eff_tc_req['no_comm_on_mom']:
                return False
        else:
            if not eff_tc_req['no_comm_on_mom']:
                return False

    def startTest(self, test):
        """
        Start the test
        """
        if ((self.cumulative_tc_failure_threshold != 0) and
                (self.__tf_count >= self.cumulative_tc_failure_threshold)):
            _msg = 'Total testcases failure count exceeded cumulative'
            _msg += ' testcase failure threshold '
            _msg += '(%d)' % self.cumulative_tc_failure_threshold
            self.logger.error(_msg)
            raise KeyboardInterrupt
        if ((self.tc_failure_threshold != 0) and
                (self.__failed_tc_count >= self.tc_failure_threshold)):
            if self.__failed_tc_count_msg:
                raise TCThresholdReached
            _msg = 'Testcases failure for this testsuite count exceeded'
            _msg += ' testcase failure threshold '
            _msg += '(%d)' % self.tc_failure_threshold
            self.logger.error(_msg)
            self.__failed_tc_count_msg = True
            raise TCThresholdReached
        timeout = self.__get_timeout(test)
        rv = self.__are_requirements_matching(self.param_dict, test)
        if rv is False:
            # Below method call is needed in order to get the test case
            # details in the output and to have the skipped test count
            # included in total run count of the test run
            self.result.startTest(test)
            raise SkipTest('Test requirements are not matching')

        def timeout_handler(signum, frame):
            raise TimeOut('Timed out after %s second' % timeout)
        old_handler = signal.signal(signal.SIGALRM, timeout_handler)
        setattr(test, 'old_sigalrm_handler', old_handler)
        signal.alarm(timeout)

    def stopTest(self, test):
        """
        Stop the test
        """
        old_sigalrm_handler = getattr(test, 'old_sigalrm_handler', None)
        if old_sigalrm_handler is not None:
            signal.signal(signal.SIGALRM, old_sigalrm_handler)
            signal.alarm(0)

    def addError(self, test, err):
        """
        Add error
        """
        if isclass(err[0]) and issubclass(err[0], TCThresholdReached):
            return True
        self.__set_test_end_data(test, err)

    def addFailure(self, test, err):
        """
        Add failure
        """
        self.__set_test_end_data(test, err)

    def addSuccess(self, test):
        """
        Add success
        """
        self.__set_test_end_data(test)

    def _cleanup(self):
        self.logger.info('Cleaning up temporary files')
        du = DshUtils()
        root_dir = os.sep
        dirlist = set([os.path.join(root_dir, 'tmp'),
                       os.path.join(root_dir, 'var', 'tmp')])
        # get tmp dir from the environment
        for envname in 'TMPDIR', 'TEMP', 'TMP':
            dirname = os.getenv(envname)
            if dirname:
                dirlist.add(dirname)

        p = re.compile(r'^pbs\.\d+')
        for tmpdir in dirlist:
            # list the contents of each tmp dir and
            # get the file list to be deleted
            self.logger.info('Cleaning up ' + tmpdir + ' dir')
            ftd = []
            files = du.listdir(path=tmpdir)
            bn = os.path.basename
            ftd.extend([f for f in files if bn(f).startswith('PtlPbs')])
            ftd.extend([f for f in files if bn(f).startswith('STDIN')])
            ftd.extend([f for f in files if bn(f).startswith('pbsscrpt')])
            ftd.extend([f for f in files if bn(f).startswith('pbs.conf.')])
            ftd.extend([f for f in files if p.match(bn(f))])
            for f in ftd:
                du.rm(path=f, sudo=True, recursive=True, force=True,
                      level=logging.DEBUG)
        for f in du.tmpfilelist:
            du.rm(path=f, sudo=True, force=True, level=logging.DEBUG)
        del du.tmpfilelist[:]
        tmpdir = tempfile.gettempdir()
        os.chdir(tmpdir)
        tmppath = os.path.join(tmpdir, 'dejagnutemp%s' % os.getpid())
        if du.isdir(path=tmppath):
            du.rm(path=tmppath, recursive=True, sudo=True, force=True,
                  level=logging.DEBUG)

    def begin(self):
        command = sys.argv
        command[0] = os.path.basename(command[0])
        self.logger.info('input command: ' + ' '.join(command))
        self.logger.info('param: ' + str(self.param))
        self.logger.info('ptl version: ' + str(ptl.__version__))
        _m = 'platform: ' + ' '.join(platform.uname()).strip()
        self.logger.info(_m)
        self.logger.info('python version: ' + str(platform.python_version()))
        self.logger.info('user: '******'-' * 80)

        if self.lcov_data is not None:
            self.lcov_utils = LcovUtils(cov_bin=self.lcov_bin,
                                        html_bin=self.genhtml_bin,
                                        cov_out=self.lcov_out,
                                        data_dir=self.lcov_data,
                                        html_nosrc=self.lcov_nosrc,
                                        html_baseurl=self.lcov_baseurl)
            # Initialize coverage analysis
            self.lcov_utils.zero_coverage()
            # The following 'dance' is done due to some oddities on lcov's
            # part, according to this the lcov readme file at
            # http://ltp.sourceforge.net/coverage/lcov/readme.php that reads:
            #
            # Note that this step only works after the application has
            # been started and stopped at least once. Otherwise lcov will
            # abort with an error mentioning that there are no data/.gcda
            # files.
            self.lcov_utils.initialize_coverage(name='PTLTestCov')
            PBSInitServices().restart()
        self._cleanup()

    def finalize(self, result):
        if self.lcov_data is not None:
            # See note above that briefly explains the 'dance' needed to get
            # reliable coverage data
            PBSInitServices().restart()
            self.lcov_utils.capture_coverage(name='PTLTestCov')
            exclude = ['"*work/gSOAP/*"', '"*/pbs/doc/*"', 'lex.yy.c',
                       'pbs_ifl_wrap.c', 'usr/include/*', 'unsupported/*']
            self.lcov_utils.merge_coverage_traces(name='PTLTestCov',
                                                  exclude=exclude)
            self.lcov_utils.generate_html()
            self.lcov_utils.change_baseurl()
            self.logger.info('\n'.join(self.lcov_utils.summarize_coverage()))
        self._cleanup()
Esempio n. 3
0
class PTLTestRunner(Plugin):

    """
    PTL Test Runner Plugin
    """
    name = 'PTLTestRunner'
    score = sys.maxint - 4
    logger = logging.getLogger(__name__)

    def __init__(self):
        Plugin.__init__(self)
        self.param = None
        self.lcov_bin = None
        self.lcov_data = None
        self.lcov_out = None
        self.lcov_utils = None
        self.lcov_nosrc = None
        self.lcov_baseurl = None
        self.genhtml_bin = None
        self.config = None
        self.result = None
        self.tc_failure_threshold = None
        self.cumulative_tc_failure_threshold = None
        self.__failed_tc_count = 0
        self.__tf_count = 0
        self.__failed_tc_count_msg = False

    def options(self, parser, env):
        """
        Register command line options
        """
        pass

    def set_data(self, paramfile, testparam,
                 lcov_bin, lcov_data, lcov_out, genhtml_bin, lcov_nosrc,
                 lcov_baseurl, tc_failure_threshold,
                 cumulative_tc_failure_threshold):
        if paramfile is not None:
            _pf = open(paramfile, 'r')
            _params_from_file = _pf.readlines()
            _pf.close()
            _nparams = []
            for l in range(len(_params_from_file)):
                if _params_from_file[l].startswith('#'):
                    continue
                else:
                    _nparams.append(_params_from_file[l])
            _f = ','.join(map(lambda l: l.strip('\r\n'), _nparams))
            if testparam is not None:
                testparam += ',' + _f
            else:
                testparam = _f
        self.param = testparam
        self.lcov_bin = lcov_bin
        self.lcov_data = lcov_data
        self.lcov_out = lcov_out
        self.genhtml_bin = genhtml_bin
        self.lcov_nosrc = lcov_nosrc
        self.lcov_baseurl = lcov_baseurl
        self.tc_failure_threshold = tc_failure_threshold
        self.cumulative_tc_failure_threshold = cumulative_tc_failure_threshold

    def configure(self, options, config):
        """
        Configure the plugin and system, based on selected options
        """
        self.config = config
        self.enabled = True

    def prepareTestRunner(self, runner):
        """
        Prepare test runner
        """
        return PtlTestRunner(verbosity=3, config=self.config)

    def prepareTestResult(self, result):
        """
        Prepare test result
        """
        self.result = result

    def startContext(self, context):
        context.param = self.param
        context.start_time = datetime.datetime.now()
        if isclass(context) and issubclass(context, PBSTestSuite):
            self.result.logger.info(self.result.separator1)
            self.result.logger.info('suite name: ' + context.__name__)
            doc = context.__doc__
            if doc is not None:
                self.result.logger.info('suite docstring: \n' + doc + '\n')
            self.result.logger.info(self.result.separator1)
            self.__failed_tc_count = 0
            self.__failed_tc_count_msg = False

    def __get_timeout(self, test):
        try:
            method = getattr(test.test, getattr(test.test, '_testMethodName'))
            return getattr(method, TIMEOUT_KEY)
        except:
            if hasattr(test, 'test'):
                __conf = getattr(test.test, 'conf')
            else:
                __conf = getattr(test.context, 'conf')
            return __conf['default_testcase_timeout']

    def __set_test_end_data(self, test, err=None):
        if not hasattr(test, 'start_time'):
            test = test.context
        if err is not None:
            is_skip = issubclass(err[0], SkipTest)
            is_tctr = issubclass(err[0], TCThresholdReached)
            if not (is_skip or is_tctr):
                self.__failed_tc_count += 1
                self.__tf_count += 1
            try:
                test.err_in_string = self.result._exc_info_to_string(err,
                                                                     test)
            except:
                etype, value, tb = err
                test.err_in_string = ''.join(format_exception(etype, value,
                                                              tb))
        else:
            test.err_in_string = 'None'
        test.end_time = datetime.datetime.now()
        test.duration = test.end_time - test.start_time
        test.captured_logs = self.result.handler.get_logs()

    def startTest(self, test):
        """
        Start the test
        """
        if ((self.cumulative_tc_failure_threshold != 0) and
                (self.__tf_count >= self.cumulative_tc_failure_threshold)):
            _msg = 'Total testcases failure count exceeded cumulative'
            _msg += ' testcase failure threshold '
            _msg += '(%d)' % self.cumulative_tc_failure_threshold
            self.logger.error(_msg)
            raise KeyboardInterrupt
        if ((self.tc_failure_threshold != 0) and
                (self.__failed_tc_count >= self.tc_failure_threshold)):
            if self.__failed_tc_count_msg:
                raise TCThresholdReached
            _msg = 'Testcases failure for this testsuite count exceeded'
            _msg += ' testcase failure threshold '
            _msg += '(%d)' % self.tc_failure_threshold
            self.logger.error(_msg)
            self.__failed_tc_count_msg = True
            raise TCThresholdReached
        timeout = self.__get_timeout(test)

        def timeout_handler(signum, frame):
            raise TimeOut('Timed out after %s second' % timeout)
        old_handler = signal.signal(signal.SIGALRM, timeout_handler)
        setattr(test, 'old_sigalrm_handler', old_handler)
        signal.alarm(timeout)

    def stopTest(self, test):
        """
        Stop the test
        """
        old_sigalrm_handler = getattr(test, 'old_sigalrm_handler', None)
        if old_sigalrm_handler is not None:
            signal.signal(signal.SIGALRM, old_sigalrm_handler)
            signal.alarm(0)

    def addError(self, test, err):
        """
        Add error
        """
        if isclass(err[0]) and issubclass(err[0], TCThresholdReached):
            return True
        self.__set_test_end_data(test, err)

    def addFailure(self, test, err):
        """
        Add failure
        """
        self.__set_test_end_data(test, err)

    def addSuccess(self, test):
        """
        Add success
        """
        self.__set_test_end_data(test)

    def _cleanup(self):
        self.logger.info('Cleaning up temporary files')
        du = DshUtils()
        tmpdir = tempfile.gettempdir()
        ftd = []
        if tmpdir:
            files = du.listdir(path=tmpdir)
            bn = os.path.basename
            ftd.extend([f for f in files if bn(f).startswith('PtlPbs')])
            ftd.extend([f for f in files if bn(f).startswith('STDIN')])
            for f in ftd:
                du.rm(path=f, sudo=True, recursive=True, force=True,
                      level=logging.DEBUG)
        os.chdir(tmpdir)
        tmppath = os.path.join(tmpdir, 'dejagnutemp%s' % os.getpid())
        if du.isdir(path=tmppath):
            du.rm(path=tmppath, recursive=True, sudo=True, force=True,
                  level=logging.DEBUG)

    def begin(self):
        command = sys.argv
        command[0] = os.path.basename(command[0])
        self.logger.info('input command: ' + ' '.join(command))
        self.logger.info('param: ' + str(self.param))
        self.logger.info('ptl version: ' + str(ptl.__version__))
        _m = 'platform: ' + ' '.join(platform.uname()).strip()
        self.logger.info(_m)
        self.logger.info('python version: ' + str(platform.python_version()))
        self.logger.info('user: '******'-' * 80)

        if self.lcov_data is not None:
            self.lcov_utils = LcovUtils(cov_bin=self.lcov_bin,
                                        html_bin=self.genhtml_bin,
                                        cov_out=self.lcov_out,
                                        data_dir=self.lcov_data,
                                        html_nosrc=self.lcov_nosrc,
                                        html_baseurl=self.lcov_baseurl)
            # Initialize coverage analysis
            self.lcov_utils.zero_coverage()
            # The following 'dance' is done due to some oddities on lcov's
            # part, according to this the lcov readme file at
            # http://ltp.sourceforge.net/coverage/lcov/readme.php that reads:
            #
            # Note that this step only works after the application has
            # been started and stopped at least once. Otherwise lcov will
            # abort with an error mentioning that there are no data/.gcda
            # files.
            self.lcov_utils.initialize_coverage(name='PTLTestCov')
            PBSInitServices().restart()
        self._cleanup()

    def finalize(self, result):
        if self.lcov_data is not None:
            # See note above that briefly explains the 'dance' needed to get
            # reliable coverage data
            PBSInitServices().restart()
            self.lcov_utils.capture_coverage(name='PTLTestCov')
            exclude = ['"*work/gSOAP/*"', '"*/pbs/doc/*"', 'lex.yy.c',
                       'pbs_ifl_wrap.c', 'usr/include/*', 'unsupported/*']
            self.lcov_utils.merge_coverage_traces(name='PTLTestCov',
                                                  exclude=exclude)
            self.lcov_utils.generate_html()
            self.lcov_utils.change_baseurl()
            self.logger.info('\n'.join(self.lcov_utils.summarize_coverage()))
        self._cleanup()
Esempio n. 4
0
class PTLTestRunner(Plugin):
    """
    PTL Test Runner Plugin
    """
    name = 'PTLTestRunner'
    score = sys.maxsize - 4
    logger = logging.getLogger(__name__)
    timeout = None

    def __init__(self):
        Plugin.__init__(self)
        self.param = None
        self.repeat_count = 1
        self.repeat_delay = 0
        self.use_cur_setup = False
        self.lcov_bin = None
        self.lcov_data = None
        self.lcov_out = None
        self.lcov_utils = None
        self.lcov_nosrc = None
        self.lcov_baseurl = None
        self.genhtml_bin = None
        self.config = None
        self.result = None
        self.tc_failure_threshold = None
        self.cumulative_tc_failure_threshold = None
        self.__failed_tc_count = 0
        self.__tf_count = 0
        self.__failed_tc_count_msg = False
        self._test_marker = 'test_'
        self.hardware_report_timer = None

    def options(self, parser, env):
        """
        Register command line options
        """
        pass

    def set_data(self, paramfile, testparam, repeat_count, repeat_delay,
                 lcov_bin, lcov_data, lcov_out, genhtml_bin, lcov_nosrc,
                 lcov_baseurl, tc_failure_threshold,
                 cumulative_tc_failure_threshold, use_cur_setup):
        if paramfile is not None:
            _pf = open(paramfile, 'r')
            _params_from_file = _pf.readlines()
            _pf.close()
            _nparams = []
            for l in range(len(_params_from_file)):
                if _params_from_file[l].startswith('#'):
                    continue
                else:
                    _nparams.append(_params_from_file[l])
            _f = ','.join([l.strip('\r\n') for l in _nparams])
            if testparam is not None:
                testparam += ',' + _f
            else:
                testparam = _f
        self.param = testparam
        self.repeat_count = repeat_count
        self.repeat_delay = repeat_delay
        self.use_cur_setup = use_cur_setup
        self.lcov_bin = lcov_bin
        self.lcov_data = lcov_data
        self.lcov_out = lcov_out
        self.genhtml_bin = genhtml_bin
        self.lcov_nosrc = lcov_nosrc
        self.lcov_baseurl = lcov_baseurl
        self.tc_failure_threshold = tc_failure_threshold
        self.cumulative_tc_failure_threshold = cumulative_tc_failure_threshold

    def configure(self, options, config):
        """
        Configure the plugin and system, based on selected options
        """
        self.config = config
        self.enabled = True
        self.param_dict = self.__get_param_dictionary()

    def prepareTestRunner(self, runner):
        """
        Prepare test runner
        """
        return PtlTextTestRunner(verbosity=3,
                                 config=self.config,
                                 repeat_count=self.repeat_count,
                                 repeat_delay=self.repeat_delay)

    def prepareTestResult(self, result):
        """
        Prepare test result
        """
        self.result = result

    def startContext(self, context):
        context.param = self.param
        context.use_cur_setup = self.use_cur_setup
        context.start_time = datetime.datetime.now()
        if isclass(context) and issubclass(context, unittest.TestCase):
            self.result.logger.info(self.result.separator1)
            self.result.logger.info('suite name: ' + context.__name__)
            doc = context.__doc__
            if doc is not None:
                self.result.logger.info('suite docstring: \n' + doc + '\n')
            self.result.logger.info(self.result.separator1)
            self.__failed_tc_count = 0
            self.__failed_tc_count_msg = False

    def __get_timeout(self, test):
        _test = None
        if hasattr(test, 'test'):
            _test = test.test
        elif hasattr(test, 'context'):
            _test = test.context
        if _test is None:
            return MINIMUM_TESTCASE_TIMEOUT
        dflt_timeout = int(
            getattr(_test, 'conf', {}).get('default-testcase-timeout',
                                           MINIMUM_TESTCASE_TIMEOUT))
        tc_timeout = int(
            getattr(
                getattr(_test, getattr(_test, '_testMethodName', ''), None),
                TIMEOUT_KEY, 0))
        return max([dflt_timeout, tc_timeout])

    def __set_test_end_data(self, test, err=None):
        if self.hardware_report_timer is not None:
            self.hardware_report_timer.cancel()
        if not hasattr(test, 'start_time'):
            test = test.context
        if err is not None:
            is_skip = issubclass(err[0], SkipTest)
            is_tctr = issubclass(err[0], TCThresholdReached)
            if not (is_skip or is_tctr):
                self.__failed_tc_count += 1
                self.__tf_count += 1
            try:
                test.err_in_string = self.result._exc_info_to_string(err, test)
            except BaseException:
                etype, value, tb = err
                test.err_in_string = ''.join(format_exception(
                    etype, value, tb))
        else:
            test.err_in_string = 'None'
        test.end_time = datetime.datetime.now()
        test.duration = test.end_time - test.start_time
        test.captured_logs = self.result.handler.get_logs()

    def __get_param_dictionary(self):
        """
        Method to convert data in param into dictionary of cluster
        information
        """
        tparam_contents = {}
        nomomlist = []
        shortname = (socket.gethostname()).split('.', 1)[0]
        for key in ['servers', 'moms', 'comms', 'clients', 'nomom']:
            tparam_contents[key] = []
        tparam_contents['no_mom_on_server'] = False
        tparam_contents['no_comm_on_server'] = False
        tparam_contents['no_comm_on_mom'] = False
        if self.param is not None:
            for h in self.param.split(','):
                if '=' in h:
                    k, v = h.split('=', 1)
                    hosts = [x.split('@')[0] for x in v.split(':')]
                    if (k == 'server' or k == 'servers'):
                        tparam_contents['servers'].extend(hosts)
                    elif (k == 'mom' or k == 'moms'):
                        tparam_contents['moms'].extend(hosts)
                    elif k == 'comms':
                        tparam_contents['comms'] = hosts
                    elif k == 'client':
                        tparam_contents['clients'] = hosts
                    elif k == 'nomom':
                        nomomlist = hosts
                    elif k == 'no_mom_on_server':
                        tparam_contents['no_mom_on_server'] = v
                    elif k == 'no_comm_on_mom':
                        tparam_contents['no_comm_on_mom'] = v
        for pkey in ['servers', 'moms', 'comms', 'clients']:
            if not tparam_contents[pkey]:
                tparam_contents[pkey] = set([shortname])
            else:
                tparam_contents[pkey] = set(tparam_contents[pkey])
        if nomomlist:
            tparam_contents['nomom'] = set(nomomlist)
        return tparam_contents

    @staticmethod
    def __are_requirements_matching(param_dic=None, test=None):
        """
        Validates test requirements against test cluster information
        returns True on match or error message otherwise None

        :param param_dic: dictionary of cluster information from data passed
                          to param list
        :param_dic type: dic
        :param test: test object
        :test type: object

        :returns True or error message or None
        """
        logger = logging.getLogger(__name__)
        ts_requirements = {}
        tc_requirements = {}
        param_count = {}
        _servers = set(param_dic['servers'])
        _moms = set(param_dic['moms'])
        _comms = set(param_dic['comms'])
        _nomom = set(param_dic['nomom'])
        _no_mom_on_server = param_dic['no_mom_on_server']
        _no_comm_on_mom = param_dic['no_comm_on_mom']
        _no_comm_on_server = param_dic['no_comm_on_server']
        shortname = (socket.gethostname()).split('.', 1)[0]
        if test is None:
            return None
        test_name = getattr(test.test, '_testMethodName', None)
        if test_name is not None:
            method = getattr(test.test, test_name, None)
        if method is not None:
            tc_requirements = getattr(method, REQUIREMENTS_KEY, {})
            cls = method.__self__.__class__
            ts_requirements = getattr(cls, REQUIREMENTS_KEY, {})
        if not tc_requirements:
            if not ts_requirements:
                return None
        eff_tc_req = get_effective_reqs(ts_requirements, tc_requirements)
        setattr(test.test, 'requirements', eff_tc_req)
        for key in ['servers', 'moms', 'comms', 'clients']:
            param_count['num_' + key] = len(param_dic[key])
        for pk in param_count:
            if param_count[pk] < eff_tc_req[pk]:
                _msg = 'available ' + pk + " ("
                _msg += str(param_count[pk]) + ") is less than required " + pk
                _msg += " (" + str(eff_tc_req[pk]) + ")"
                logger.error(_msg)
                return _msg

        if hasattr(test, 'test'):
            _test = test.test
        elif hasattr(test, 'context'):
            _test = test.context
        else:
            return None

        name = 'moms'
        if (hasattr(_test, name) and (getattr(_test, name, None) is not None)):
            for mc in getattr(_test, name).values():
                platform = mc.platform
                if platform not in ['linux', 'shasta', 'cray'
                                    ] and mc.hostname in _moms:
                    _moms.remove(mc.hostname)
        for hostname in _moms:
            si = SystemInfo()
            si.get_system_info(hostname)
            available_sys_ram = getattr(si, 'system_ram', None)
            if available_sys_ram is None:
                _msg = 'failed to get ram info on host: ' + hostname
                logger.error(_msg)
                return _msg
            elif eff_tc_req['min_mom_ram'] >= available_sys_ram:
                _msg = hostname + ': available ram (' + str(available_sys_ram)
                _msg += ') is less than the minimum required ram ('
                _msg += str(eff_tc_req['min_mom_ram'])
                _msg += ') for test execution'
                logger.error(_msg)
                return _msg
            available_sys_disk = getattr(si, 'system_disk', None)
            if available_sys_disk is None:
                _msg = 'failed to get disk info on host: ' + hostname
                logger.error(_msg)
                return _msg
            elif eff_tc_req['min_mom_disk'] >= available_sys_disk:
                _msg = hostname + ': available disk space ('
                _msg += str(available_sys_disk)
                _msg += ') is less than the minimum required disk space ('
                _msg += str(eff_tc_req['min_mom_disk'])
                _msg += ') for test execution'
                logger.error(_msg)
                return _msg
        for hostname in param_dic['servers']:
            si = SystemInfo()
            si.get_system_info(hostname)
            available_sys_ram = getattr(si, 'system_ram', None)
            if available_sys_ram is None:
                _msg = 'failed to get ram info on host: ' + hostname
                logger.error(_msg)
                return _msg
            elif eff_tc_req['min_server_ram'] >= available_sys_ram:
                _msg = hostname + ': available ram (' + str(available_sys_ram)
                _msg += ') is less than the minimum required ram ('
                _msg += str(eff_tc_req['min_server_ram'])
                _msg += ') for test execution'
                logger.error(_msg)
                return _msg
            available_sys_disk = getattr(si, 'system_disk', None)
            if available_sys_disk is None:
                _msg = 'failed to get disk info on host: ' + hostname
                logger.error(_msg)
                return _msg
            elif eff_tc_req['min_server_disk'] >= available_sys_disk:
                _msg = hostname + ': available disk space ('
                _msg += str(available_sys_disk)
                _msg += ') is less than the minimum required disk space ('
                _msg += str(eff_tc_req['min_server_disk'])
                _msg += ') for test execution'
                logger.error(_msg)
                return _msg
        if _moms & _servers:
            if eff_tc_req['no_mom_on_server'] or \
               (_nomom - _servers) or \
               _no_mom_on_server:
                _msg = 'no mom on server'
                logger.error(_msg)
                return _msg
        if _comms & _servers:
            if eff_tc_req['no_comm_on_server'] or _no_comm_on_server:
                return False
                _msg = 'no comm on server'
                logger.error(_msg)
                return _msg
        comm_mom_list = _moms & _comms
        if comm_mom_list and shortname in comm_mom_list:
            # Excluding the server hostname for flag 'no_comm_on_mom'
            comm_mom_list.remove(shortname)
        if comm_mom_list:
            if eff_tc_req['no_comm_on_mom']:
                _msg = 'no comm on mom'
                logger.error(_msg)
                return _msg
        else:
            if not eff_tc_req['no_comm_on_mom']:
                _msg = 'no comm on server'
                logger.error(_msg)
                return _msg

    def check_hardware_status_and_core_files(self, test):
        """
        function checks hardware status and core files
        every 5 minutes
        """
        du = DshUtils()
        systems = list(self.param_dict['servers'])
        systems.extend(self.param_dict['moms'])
        systems.extend(self.param_dict['comms'])
        systems = list(set(systems))

        if hasattr(test, 'test'):
            _test = test.test
        elif hasattr(test, 'context'):
            _test = test.context
        else:
            return None

        for name in ['servers', 'moms', 'comms', 'clients']:
            mlist = None
            if (hasattr(_test, name)
                    and (getattr(_test, name, None) is not None)):
                mlist = getattr(_test, name).values()
            if mlist:
                for mc in mlist:
                    platform = mc.platform
                    if ((platform not in ['linux', 'shasta', 'cray'])
                            and (mc.hostname in systems)):
                        systems.remove(mc.hostname)

        self.hardware_report_timer = Timer(
            300, self.check_hardware_status_and_core_files, args=(test, ))
        self.hardware_report_timer.start()

        for hostname in systems:
            hr = SystemInfo()
            hr.get_system_info(hostname)
            # monitors disk
            used_disk_percent = getattr(hr, 'system_disk_used_percent', None)
            if used_disk_percent is None:
                _msg = hostname
                _msg += ": unable to get disk info"
                self.hardware_report_timer.cancel()
                raise SkipTest(_msg)
            elif 70 <= used_disk_percent < 95:
                _msg = hostname + ": disk usage is at "
                _msg += str(used_disk_percent) + "%"
                _msg += ", disk cleanup is recommended."
                self.logger.warning(_msg)
            elif used_disk_percent >= 95:
                _msg = hostname + ":disk usage > 95%, skipping the test(s)"
                self.hardware_report_timer.cancel()
                raise SkipTest(_msg)
            # checks for core files
            pbs_conf = du.parse_pbs_config(hostname)
            mom_priv_path = os.path.join(pbs_conf["PBS_HOME"], "mom_priv")
            if du.isdir(hostname=hostname, path=mom_priv_path):
                mom_priv_files = du.listdir(hostname=hostname,
                                            path=mom_priv_path,
                                            sudo=True,
                                            fullpath=False)
                if fnmatch.filter(mom_priv_files, "core*"):
                    _msg = hostname + ": core files found in "
                    _msg += mom_priv_path
                    self.logger.warning(_msg)
            server_priv_path = os.path.join(pbs_conf["PBS_HOME"],
                                            "server_priv")
            if du.isdir(hostname=hostname, path=server_priv_path):
                server_priv_files = du.listdir(hostname=hostname,
                                               path=server_priv_path,
                                               sudo=True,
                                               fullpath=False)
                if fnmatch.filter(server_priv_files, "core*"):
                    _msg = hostname + ": core files found in "
                    _msg += server_priv_path
                    self.logger.warning(_msg)
            sched_priv_path = os.path.join(pbs_conf["PBS_HOME"], "sched_priv")
            if du.isdir(hostname=hostname, path=sched_priv_path):
                sched_priv_files = du.listdir(hostname=hostname,
                                              path=sched_priv_path,
                                              sudo=True,
                                              fullpath=False)
                if fnmatch.filter(sched_priv_files, "core*"):
                    _msg = hostname + ": core files found in "
                    _msg += sched_priv_path
                    self.logger.warning(_msg)
            for u in PBS_ALL_USERS:
                user_home_files = du.listdir(hostname=hostname,
                                             path=u.home,
                                             sudo=True,
                                             fullpath=False,
                                             runas=u.name)
                if user_home_files and fnmatch.filter(user_home_files,
                                                      "core*"):
                    _msg = hostname + ": user-" + str(u)
                    _msg += ": core files found in "
                    self.logger.warning(_msg + u.home)

    def startTest(self, test):
        """
        Start the test
        """
        if ((self.cumulative_tc_failure_threshold != 0)
                and (self.__tf_count >= self.cumulative_tc_failure_threshold)):
            _msg = 'Total testcases failure count exceeded cumulative'
            _msg += ' testcase failure threshold '
            _msg += '(%d)' % self.cumulative_tc_failure_threshold
            self.logger.error(_msg)
            raise KeyboardInterrupt
        if ((self.tc_failure_threshold != 0)
                and (self.__failed_tc_count >= self.tc_failure_threshold)):
            if self.__failed_tc_count_msg:
                raise TCThresholdReached
            _msg = 'Testcases failure for this testsuite count exceeded'
            _msg += ' testcase failure threshold '
            _msg += '(%d)' % self.tc_failure_threshold
            self.logger.error(_msg)
            self.__failed_tc_count_msg = True
            raise TCThresholdReached
        rv = None
        rv = self.__are_requirements_matching(self.param_dict, test)
        if rv is not None:
            # Below method call is needed in order to get the test case
            # details in the output and to have the skipped test count
            # included in total run count of the test run
            self.result.startTest(test)
            raise SkipTest(rv)
        # function report hardware status and core files
        self.check_hardware_status_and_core_files(test)

        def timeout_handler(signum, frame):
            raise TimeOut('Timed out after %s second' % timeout)

        if PTLTestRunner.timeout is None:
            timeout = self.__get_timeout(test)
            old_handler = signal.signal(signal.SIGALRM, timeout_handler)
            setattr(test, 'old_sigalrm_handler', old_handler)
            signal.alarm(timeout)

    def stopTest(self, test):
        """
        Stop the test
        """
        old_sigalrm_handler = getattr(test, 'old_sigalrm_handler', None)
        if old_sigalrm_handler is not None:
            signal.signal(signal.SIGALRM, old_sigalrm_handler)
            signal.alarm(0)

    def addError(self, test, err):
        """
        Add error
        """
        if isclass(err[0]) and issubclass(err[0], TCThresholdReached):
            return True
        self.__set_test_end_data(test, err)

    def addFailure(self, test, err):
        """
        Add failure
        """
        self.__set_test_end_data(test, err)

    def addSuccess(self, test):
        """
        Add success
        """
        self.__set_test_end_data(test)

    def _cleanup(self):
        self.logger.info('Cleaning up temporary files')
        du = DshUtils()
        root_dir = os.sep
        dirlist = set([
            os.path.join(root_dir, 'tmp'),
            os.path.join(root_dir, 'var', 'tmp')
        ])
        # get tmp dir from the environment
        for envname in 'TMPDIR', 'TEMP', 'TMP':
            dirname = os.getenv(envname)
            if dirname:
                dirlist.add(dirname)

        p = re.compile(r'^pbs\.\d+')
        for tmpdir in dirlist:
            # list the contents of each tmp dir and
            # get the file list to be deleted
            self.logger.info('Cleaning up ' + tmpdir + ' dir')
            ftd = []
            files = du.listdir(path=tmpdir)
            bn = os.path.basename
            ftd.extend([f for f in files if bn(f).startswith('PtlPbs')])
            ftd.extend([f for f in files if bn(f).startswith('STDIN')])
            ftd.extend([f for f in files if bn(f).startswith('pbsscrpt')])
            ftd.extend([f for f in files if bn(f).startswith('pbs.conf.')])
            ftd.extend([f for f in files if p.match(bn(f))])
            for f in ftd:
                du.rm(path=f,
                      sudo=True,
                      recursive=True,
                      force=True,
                      level=logging.DEBUG)
        for f in du.tmpfilelist:
            du.rm(path=f, sudo=True, force=True, level=logging.DEBUG)
        del du.tmpfilelist[:]
        tmpdir = tempfile.gettempdir()
        os.chdir(tmpdir)
        tmppath = os.path.join(tmpdir, 'dejagnutemp%s' % os.getpid())
        if du.isdir(path=tmppath):
            du.rm(path=tmppath,
                  recursive=True,
                  sudo=True,
                  force=True,
                  level=logging.DEBUG)

    def begin(self):
        command = sys.argv
        command[0] = os.path.basename(command[0])
        self.logger.info('input command: ' + ' '.join(command))
        self.logger.info('param: ' + str(self.param))
        self.logger.info('ptl version: ' + str(ptl.__version__))
        _m = 'platform: ' + ' '.join(platform.uname()).strip()
        self.logger.info(_m)
        self.logger.info('python version: ' + str(platform.python_version()))
        self.logger.info('user: '******'-' * 80)

        if self.lcov_data is not None:
            self.lcov_utils = LcovUtils(cov_bin=self.lcov_bin,
                                        html_bin=self.genhtml_bin,
                                        cov_out=self.lcov_out,
                                        data_dir=self.lcov_data,
                                        html_nosrc=self.lcov_nosrc,
                                        html_baseurl=self.lcov_baseurl)
            # Initialize coverage analysis
            self.lcov_utils.zero_coverage()
            # The following 'dance' is done due to some oddities on lcov's
            # part, according to this the lcov readme file at
            # http://ltp.sourceforge.net/coverage/lcov/readme.php that reads:
            #
            # Note that this step only works after the application has
            # been started and stopped at least once. Otherwise lcov will
            # abort with an error mentioning that there are no data/.gcda
            # files.
            self.lcov_utils.initialize_coverage(name='PTLTestCov')
            PBSInitServices().restart()
        self._cleanup()

    def finalize(self, result):
        if self.lcov_data is not None:
            # See note above that briefly explains the 'dance' needed to get
            # reliable coverage data
            PBSInitServices().restart()
            self.lcov_utils.capture_coverage(name='PTLTestCov')
            exclude = [
                '"*work/gSOAP/*"', '"*/pbs/doc/*"', 'lex.yy.c',
                'pbs_ifl_wrap.c', 'usr/include/*', 'unsupported/*'
            ]
            self.lcov_utils.merge_coverage_traces(name='PTLTestCov',
                                                  exclude=exclude)
            self.lcov_utils.generate_html()
            self.lcov_utils.change_baseurl()
            self.logger.info('\n'.join(self.lcov_utils.summarize_coverage()))
        self._cleanup()
Esempio n. 5
0
class PTLTestRunner(Plugin):
    """
    PTL Test Runner Plugin
    """
    name = 'PTLTestRunner'
    score = sys.maxint - 4
    logger = logging.getLogger(__name__)

    def __init__(self):
        Plugin.__init__(self)
        self.param = None
        self.lcov_bin = None
        self.lcov_data = None
        self.lcov_out = None
        self.lcov_utils = None
        self.lcov_nosrc = None
        self.lcov_baseurl = None
        self.genhtml_bin = None
        self.config = None
        self.result = None
        self.tc_failure_threshold = None
        self.cumulative_tc_failure_threshold = None
        self.__failed_tc_count = 0
        self.__tf_count = 0
        self.__failed_tc_count_msg = False
        self._test_marker = 'test_'

    def options(self, parser, env):
        """
        Register command line options
        """
        pass

    def set_data(self, paramfile, testparam, lcov_bin, lcov_data, lcov_out,
                 genhtml_bin, lcov_nosrc, lcov_baseurl, tc_failure_threshold,
                 cumulative_tc_failure_threshold):
        if paramfile is not None:
            _pf = open(paramfile, 'r')
            _params_from_file = _pf.readlines()
            _pf.close()
            _nparams = []
            for l in range(len(_params_from_file)):
                if _params_from_file[l].startswith('#'):
                    continue
                else:
                    _nparams.append(_params_from_file[l])
            _f = ','.join(map(lambda l: l.strip('\r\n'), _nparams))
            if testparam is not None:
                testparam += ',' + _f
            else:
                testparam = _f
        self.param = testparam
        self.lcov_bin = lcov_bin
        self.lcov_data = lcov_data
        self.lcov_out = lcov_out
        self.genhtml_bin = genhtml_bin
        self.lcov_nosrc = lcov_nosrc
        self.lcov_baseurl = lcov_baseurl
        self.tc_failure_threshold = tc_failure_threshold
        self.cumulative_tc_failure_threshold = cumulative_tc_failure_threshold

    def configure(self, options, config):
        """
        Configure the plugin and system, based on selected options
        """
        self.config = config
        self.enabled = True
        self.param_dict = self.__get_param_dictionary()

    def prepareTestRunner(self, runner):
        """
        Prepare test runner
        """
        return PtlTestRunner(verbosity=3, config=self.config)

    def prepareTestResult(self, result):
        """
        Prepare test result
        """
        self.result = result

    def startContext(self, context):
        context.param = self.param
        context.start_time = datetime.datetime.now()
        if isclass(context) and issubclass(context, PBSTestSuite):
            self.result.logger.info(self.result.separator1)
            self.result.logger.info('suite name: ' + context.__name__)
            doc = context.__doc__
            if doc is not None:
                self.result.logger.info('suite docstring: \n' + doc + '\n')
            self.result.logger.info(self.result.separator1)
            self.__failed_tc_count = 0
            self.__failed_tc_count_msg = False

    def __get_timeout(self, test):
        try:
            method = getattr(test.test, getattr(test.test, '_testMethodName'))
            return getattr(method, TIMEOUT_KEY)
        except AttributeError:
            testcase_timeout = MINIMUM_TESTCASE_TIMEOUT
            if hasattr(test, 'test'):
                if hasattr(test.test, 'conf'):
                    __conf = getattr(test.test, 'conf')
                    testcase_timeout = __conf['default_testcase_timeout']
            elif hasattr(test, 'context'):
                if hasattr(test.context, 'conf'):
                    __conf = getattr(test.context, 'conf')
                    testcase_timeout = __conf['default_testcase_timeout']
            return testcase_timeout

    def __set_test_end_data(self, test, err=None):
        if not hasattr(test, 'start_time'):
            test = test.context
        if err is not None:
            is_skip = issubclass(err[0], SkipTest)
            is_tctr = issubclass(err[0], TCThresholdReached)
            if not (is_skip or is_tctr):
                self.__failed_tc_count += 1
                self.__tf_count += 1
            try:
                test.err_in_string = self.result._exc_info_to_string(err, test)
            except:
                etype, value, tb = err
                test.err_in_string = ''.join(format_exception(
                    etype, value, tb))
        else:
            test.err_in_string = 'None'
        test.end_time = datetime.datetime.now()
        test.duration = test.end_time - test.start_time
        test.captured_logs = self.result.handler.get_logs()

    def __get_param_dictionary(self):
        """
        Method to convert data in param into dictionary of cluster
        information
        """
        tparam_contents = {}
        nomomlist = []
        shortname = (socket.gethostname()).split('.', 1)[0]
        for key in ['servers', 'moms', 'comms', 'clients']:
            tparam_contents[key] = []
        if self.param is not None:
            for h in self.param.split(','):
                if '=' in h:
                    k, v = h.split('=', 1)
                    if (k == 'server' or k == 'servers'):
                        tparam_contents['servers'].extend(v.split(':'))
                    elif (k == 'mom' or k == 'moms'):
                        tparam_contents['moms'].extend(v.split(':'))
                    elif k == 'comms':
                        tparam_contents['comms'] = v.split(':')
                    elif k == 'client':
                        tparam_contents['clients'] = v.split(':')
                    elif k == 'nomom':
                        nomomlist = v.split(':')
        for pkey in ['servers', 'moms', 'comms', 'clients']:
            if not tparam_contents[pkey]:
                tparam_contents[pkey] = set([shortname])
            else:
                tparam_contents[pkey] = set(tparam_contents[pkey])
        if nomomlist:
            tparam_contents['moms'] -= set(nomomlist)
        return tparam_contents

    @staticmethod
    def __are_requirements_matching(param_dic=None, test=None):
        """
        Validates test requirements against test cluster information
        returns True on match or False otherwise None

        :param param_dic: dictionary of cluster information from data passed
                          to param list
        :param_dic type: dic
        :param test: test object
        :test type: object

        :returns True or False or None
        """
        ts_requirements = {}
        tc_requirements = {}
        param_count = {}
        shortname = (socket.gethostname()).split('.', 1)[0]
        if test is None:
            return None
        test_name = getattr(test.test, '_testMethodName', None)
        if test_name is not None:
            method = getattr(test.test, test_name, None)
        if method is not None:
            tc_requirements = getattr(method, REQUIREMENTS_KEY, {})
            cls = method.im_class
            ts_requirements = getattr(cls, REQUIREMENTS_KEY, {})
        if not tc_requirements:
            if not ts_requirements:
                return None
        eff_tc_req = get_effective_reqs(ts_requirements, tc_requirements)
        setattr(test.test, 'requirements', eff_tc_req)
        for key in ['servers', 'moms', 'comms', 'clients']:
            param_count['num_' + key] = len(param_dic[key])
        for pk in param_count:
            if param_count[pk] < eff_tc_req[pk]:
                return False
        if set(param_dic['moms']) & set(param_dic['servers']):
            if eff_tc_req['no_mom_on_server']:
                return False
        else:
            if not eff_tc_req['no_mom_on_server']:
                return False
        if set(param_dic['comms']) & set(param_dic['servers']):
            if eff_tc_req['no_comm_on_server']:
                return False
        else:
            if not eff_tc_req['no_comm_on_server']:
                return False
        comm_mom_list = set(param_dic['moms']) & set(param_dic['comms'])
        if comm_mom_list and shortname in comm_mom_list:
            # Excluding the server hostname for flag 'no_comm_on_mom'
            comm_mom_list.remove(shortname)
        if comm_mom_list:
            if eff_tc_req['no_comm_on_mom']:
                return False
        else:
            if not eff_tc_req['no_comm_on_mom']:
                return False

    def startTest(self, test):
        """
        Start the test
        """
        if ((self.cumulative_tc_failure_threshold != 0)
                and (self.__tf_count >= self.cumulative_tc_failure_threshold)):
            _msg = 'Total testcases failure count exceeded cumulative'
            _msg += ' testcase failure threshold '
            _msg += '(%d)' % self.cumulative_tc_failure_threshold
            self.logger.error(_msg)
            raise KeyboardInterrupt
        if ((self.tc_failure_threshold != 0)
                and (self.__failed_tc_count >= self.tc_failure_threshold)):
            if self.__failed_tc_count_msg:
                raise TCThresholdReached
            _msg = 'Testcases failure for this testsuite count exceeded'
            _msg += ' testcase failure threshold '
            _msg += '(%d)' % self.tc_failure_threshold
            self.logger.error(_msg)
            self.__failed_tc_count_msg = True
            raise TCThresholdReached
        timeout = self.__get_timeout(test)
        rv = self.__are_requirements_matching(self.param_dict, test)
        if rv is False:
            # Below method call is needed in order to get the test case
            # details in the output and to have the skipped test count
            # included in total run count of the test run
            self.result.startTest(test)
            raise SkipTest('Test requirements are not matching')

        def timeout_handler(signum, frame):
            raise TimeOut('Timed out after %s second' % timeout)

        old_handler = signal.signal(signal.SIGALRM, timeout_handler)
        setattr(test, 'old_sigalrm_handler', old_handler)
        signal.alarm(timeout)

    def stopTest(self, test):
        """
        Stop the test
        """
        old_sigalrm_handler = getattr(test, 'old_sigalrm_handler', None)
        if old_sigalrm_handler is not None:
            signal.signal(signal.SIGALRM, old_sigalrm_handler)
            signal.alarm(0)

    def addError(self, test, err):
        """
        Add error
        """
        if isclass(err[0]) and issubclass(err[0], TCThresholdReached):
            return True
        self.__set_test_end_data(test, err)

    def addFailure(self, test, err):
        """
        Add failure
        """
        self.__set_test_end_data(test, err)

    def addSuccess(self, test):
        """
        Add success
        """
        self.__set_test_end_data(test)

    def _cleanup(self):
        self.logger.info('Cleaning up temporary files')
        du = DshUtils()
        root_dir = os.sep
        dirlist = set([
            os.path.join(root_dir, 'tmp'),
            os.path.join(root_dir, 'var', 'tmp')
        ])
        # get tmp dir from the environment
        for envname in 'TMPDIR', 'TEMP', 'TMP':
            dirname = os.getenv(envname)
            if dirname:
                dirlist.add(dirname)

        p = re.compile(r'^pbs\.\d+')
        for tmpdir in dirlist:
            # list the contents of each tmp dir and
            # get the file list to be deleted
            self.logger.info('Cleaning up ' + tmpdir + ' dir')
            ftd = []
            files = du.listdir(path=tmpdir)
            bn = os.path.basename
            ftd.extend([f for f in files if bn(f).startswith('PtlPbs')])
            ftd.extend([f for f in files if bn(f).startswith('STDIN')])
            ftd.extend([f for f in files if bn(f).startswith('pbsscrpt')])
            ftd.extend([f for f in files if bn(f).startswith('pbs.conf.')])
            ftd.extend([f for f in files if p.match(bn(f))])
            for f in ftd:
                du.rm(path=f,
                      sudo=True,
                      recursive=True,
                      force=True,
                      level=logging.DEBUG)
        for f in du.tmpfilelist:
            du.rm(path=f, sudo=True, force=True, level=logging.DEBUG)
        del du.tmpfilelist[:]
        tmpdir = tempfile.gettempdir()
        os.chdir(tmpdir)
        tmppath = os.path.join(tmpdir, 'dejagnutemp%s' % os.getpid())
        if du.isdir(path=tmppath):
            du.rm(path=tmppath,
                  recursive=True,
                  sudo=True,
                  force=True,
                  level=logging.DEBUG)

    def begin(self):
        command = sys.argv
        command[0] = os.path.basename(command[0])
        self.logger.info('input command: ' + ' '.join(command))
        self.logger.info('param: ' + str(self.param))
        self.logger.info('ptl version: ' + str(ptl.__version__))
        _m = 'platform: ' + ' '.join(platform.uname()).strip()
        self.logger.info(_m)
        self.logger.info('python version: ' + str(platform.python_version()))
        self.logger.info('user: '******'-' * 80)

        if self.lcov_data is not None:
            self.lcov_utils = LcovUtils(cov_bin=self.lcov_bin,
                                        html_bin=self.genhtml_bin,
                                        cov_out=self.lcov_out,
                                        data_dir=self.lcov_data,
                                        html_nosrc=self.lcov_nosrc,
                                        html_baseurl=self.lcov_baseurl)
            # Initialize coverage analysis
            self.lcov_utils.zero_coverage()
            # The following 'dance' is done due to some oddities on lcov's
            # part, according to this the lcov readme file at
            # http://ltp.sourceforge.net/coverage/lcov/readme.php that reads:
            #
            # Note that this step only works after the application has
            # been started and stopped at least once. Otherwise lcov will
            # abort with an error mentioning that there are no data/.gcda
            # files.
            self.lcov_utils.initialize_coverage(name='PTLTestCov')
            PBSInitServices().restart()
        self._cleanup()

    def finalize(self, result):
        if self.lcov_data is not None:
            # See note above that briefly explains the 'dance' needed to get
            # reliable coverage data
            PBSInitServices().restart()
            self.lcov_utils.capture_coverage(name='PTLTestCov')
            exclude = [
                '"*work/gSOAP/*"', '"*/pbs/doc/*"', 'lex.yy.c',
                'pbs_ifl_wrap.c', 'usr/include/*', 'unsupported/*'
            ]
            self.lcov_utils.merge_coverage_traces(name='PTLTestCov',
                                                  exclude=exclude)
            self.lcov_utils.generate_html()
            self.lcov_utils.change_baseurl()
            self.logger.info('\n'.join(self.lcov_utils.summarize_coverage()))
        self._cleanup()