class PTLTestData(Plugin): """ Save post analysis data on test cases failure or error """ name = 'PTLTestData' score = sys.maxsize - 6 logger = logging.getLogger(__name__) def __init__(self): Plugin.__init__(self) self.post_data_dir = None self.max_postdata_threshold = None self.__save_data_count = 0 self.__priv_sn = '' self.du = DshUtils() def options(self, parser, env): """ Register command line options """ pass def set_data(self, post_data_dir, max_postdata_threshold): self.post_data_dir = post_data_dir self.max_postdata_threshold = max_postdata_threshold def configure(self, options, config): """ Configure the plugin and system, based on selected options """ self.config = config if self.post_data_dir is not None: self.enabled = True else: self.enabled = False def __save_home(self, test, status, err=None): if hasattr(test, 'test'): _test = test.test sn = _test.__class__.__name__ elif hasattr(test, 'context'): _test = test.context sn = _test.__name__ else: # test does not have any PBS Objects, so just return return if self.__priv_sn != sn: self.__save_data_count = 0 self.__priv_sn = sn # Saving home might take time so disable timeout # handler set by runner tn = getattr(_test, '_testMethodName', 'unknown') testlogs = getattr(test, 'captured_logs', '') datadir = os.path.join(self.post_data_dir, sn, tn) if os.path.exists(datadir): _msg = 'Old post analysis data exists at %s' % datadir _msg += ', skipping saving data for this test case' self.logger.warn(_msg) _msg = 'Please remove old directory or' _msg += ' provide different directory' self.logger.warn(_msg) return if getattr(test, 'old_sigalrm_handler', None) is not None: _h = getattr(test, 'old_sigalrm_handler') signal.signal(signal.SIGALRM, _h) signal.alarm(0) self.logger.log(logging.DEBUG2, 'Saving post analysis data...') current_host = socket.gethostname().split('.')[0] self.du.mkdir(current_host, path=datadir, mode=0o755, parents=True, logerr=False, level=logging.DEBUG2) if err is not None: if isclass(err[0]) and issubclass(err[0], SkipTest): status = 'SKIP' status_data = 'Reason = %s' % (err[1]) else: if isclass(err[0]) and issubclass(err[0], TimeOut): status = 'TIMEDOUT' status_data = getattr(test, 'err_in_string', '') else: status_data = '' logfile = os.path.join(datadir, 'logfile_' + status) f = open(logfile, 'w+') f.write(testlogs + '\n') f.write(status_data + '\n') f.write('test duration: %s\n' % str(getattr(test, 'duration', '0'))) if status in ('PASS', 'SKIP'): # Test case passed or skipped, no need to save post analysis data f.close() return if ((self.max_postdata_threshold != 0) and (self.__save_data_count >= self.max_postdata_threshold)): _msg = 'Total number of saved post analysis data for this' _msg += ' testsuite is exceeded max postdata threshold' _msg += ' (%d)' % self.max_postdata_threshold f.write(_msg + '\n') self.logger.error(_msg) f.close() return servers = getattr(_test, 'servers', None) if servers is not None: server_host = servers.values()[0].shortname else: _msg = 'Could not find Server Object in given test object' _msg += ', skipping saving post analysis data' f.write(_msg + '\n') self.logger.warning(_msg) f.close() return moms = getattr(_test, 'moms', None) comms = getattr(_test, 'comms', None) client = getattr(_test.servers.values()[0], 'client', None) server = servers.values()[0] add_hosts = [] if len(servers) > 1: for param in servers.values()[1:]: add_hosts.append(param.shortname) if moms is not None: for param in moms.values(): add_hosts.append(param.shortname) if comms is not None: for param in comms.values(): add_hosts.append(param.shortname) if client is not None: add_hosts.append(client.split('.')[0]) add_hosts = list(set(add_hosts) - set([server_host])) pbs_snapshot_path = os.path.join(server.pbs_conf["PBS_EXEC"], "sbin", "pbs_snapshot") cur_user = self.du.get_current_user() cur_user_dir = pwd.getpwnam(cur_user).pw_dir cmd = [ pbs_snapshot_path, '-H', server_host, '--daemon-logs', '2', '--accounting-logs', '2', '--with-sudo' ] if len(add_hosts) > 0: cmd += ['--additional-hosts=' + ','.join(add_hosts)] cmd += ['-o', cur_user_dir] ret = self.du.run_cmd(current_host, cmd, level=logging.DEBUG2, logerr=False) if ret['rc'] != 0: _msg = 'Failed to get analysis information ' _msg += 'on %s:' % server_host _msg += '\n\n' + '\n'.join(ret['err']) + '\n\n' f.write(_msg + '\n') self.logger.error(_msg) f.close() return else: if len(ret['out']) == 0: self.logger.error('Snapshot command failed') f.close() return snap_out = ret['out'][0] snap_out_dest = (snap_out.split(":")[1]).strip() dest = os.path.join(datadir, 'PBS_' + server_host + '.tar.gz') ret = self.du.run_copy(current_host, snap_out_dest, dest, sudo=True, level=logging.DEBUG2) self.du.rm(current_host, path=snap_out_dest, recursive=True, force=True, level=logging.DEBUG2) f.close() self.__save_data_count += 1 _msg = 'Saved post analysis data' self.logger.info(_msg) def addError(self, test, err): self.__save_home(test, 'ERROR', err) def addFailure(self, test, err): self.__save_home(test, 'FAIL', err) def addSuccess(self, test): self.__save_home(test, 'PASS')
class PTLJsonData(object): """ The intent of the class is to generate json format of PTL test data """ cur_repeat_count = 1 def __init__(self, command): self.__du = DshUtils() self.__cmd = command def get_json(self, data, prev_data=None): """ Method to generate test data in accordance to json schema :param data: dictionary of a test case details :type data: dict :param prev_data: dictionary of test run details that ran before the current test :type prev_data: dict :returns a formatted dictionary of the data """ FMT = '%H:%M:%S.%f' run_count = str(PtlTextTestRunner.cur_repeat_count) data_json = None if not prev_data: PTLJsonData.cur_repeat_count = 1 tests_start = str(data['start_time']).split()[1] data_json = { 'command': self.__cmd, 'user': self.__du.get_current_user(), 'product_version': data['pbs_version'], 'run_id': data['start_time'].strftime('%s'), 'test_conf': {}, 'machine_info': data['machinfo'], 'testsuites': {}, 'additional_data': {}, 'test_summary': {}, 'avg_measurements': {}, 'result': { 'tests_with_failures': [], 'test_suites_with_failures': [], 'start': str(data['start_time']) } } test_summary = { 'result_summary': { 'run': 0, 'succeeded': 0, 'failed': 0, 'errors': 0, 'skipped': 0, 'timedout': 0 }, 'test_start_time': str(data['start_time']), 'tests_with_failures': [], 'test_suites_with_failures': [] } data_json['test_summary'][run_count] = test_summary if data['testparam']: for param in data['testparam'].split(','): if '=' in param: par = param.split('=', 1) data_json['test_conf'][par[0]] = par[1] else: data_json['test_conf'][param] = True else: data_json = prev_data if PTLJsonData.cur_repeat_count != PtlTextTestRunner.cur_repeat_count: test_summary = { 'result_summary': { 'run': 0, 'succeeded': 0, 'failed': 0, 'errors': 0, 'skipped': 0, 'timedout': 0 }, 'test_start_time': str(data['start_time']), 'tests_with_failures': [], 'test_suites_with_failures': [] } data_json['test_summary'][run_count] = test_summary PTLJsonData.cur_repeat_count = PtlTextTestRunner.cur_repeat_count tsname = data['suite'] tcname = data['testcase'] jdata = { 'status': data['status'], 'status_data': str(data['status_data']), 'duration': str(data['duration']), 'start_time': str(data['start_time']), 'end_time': str(data['end_time']), 'measurements': [] } if 'measurements' in data: jdata['measurements'] = data['measurements'] if PtlTextTestRunner.cur_repeat_count == 1: if tsname not in data_json['testsuites']: data_json['testsuites'][tsname] = { 'module': data['module'], 'file': data['file'], 'testcases': {} } tsdoc = [] if data['suitedoc']: tsdoc = (re.sub(r"[\t\n ]+", " ", data['suitedoc'])).strip() data_json['testsuites'][tsname]['docstring'] = tsdoc tcdoc = [] if data['testdoc']: tcdoc = (re.sub(r"[\t\n ]+", " ", data['testdoc'])).strip() data_json['testsuites'][tsname]['testcases'][tcname] = { 'docstring': tcdoc, 'requirements': data['requirements'], 'results': {} } if data['testdoc']: jdata_tests = data_json['testsuites'][tsname]['testcases'] jdata_tests[tcname]['tags'] = data['tags'] jdata_tests = data_json['testsuites'][tsname]['testcases'] jdata_tests[tcname]['results'][run_count] = jdata if 'additional_data' in data: data_json['additional_data'] = data['additional_data'] data_json['test_summary'][run_count]['test_end_time'] = str( data['end_time']) run_summary = data_json['test_summary'][run_count] start = run_summary['test_start_time'].split()[1] end = str(data['end_time']).split()[1] dur = str(datetime.datetime.strptime(end, FMT) - datetime.datetime.strptime(start, FMT)) data_json['test_summary'][run_count]['tests_duration'] = dur data_json['test_summary'][run_count]['result_summary']['run'] += 1 d_ts = data_json['test_summary'][run_count] if data['status'] == 'PASS': d_ts['result_summary']['succeeded'] += 1 elif data['status'] == 'SKIP': d_ts['result_summary']['skipped'] += 1 elif data['status'] == 'TIMEDOUT': d_ts['result_summary']['timedout'] += 1 d_ts['tests_with_failures'].append(data['testcase']) if data['suite'] not in d_ts['test_suites_with_failures']: d_ts['test_suites_with_failures'].append(data['suite']) elif data['status'] == 'ERROR': d_ts['result_summary']['errors'] += 1 d_ts['tests_with_failures'].append(data['testcase']) if data['suite'] not in d_ts['test_suites_with_failures']: d_ts['test_suites_with_failures'].append(data['suite']) elif data['status'] == 'FAIL': d_ts['result_summary']['failed'] += 1 d_ts['tests_with_failures'].append(data['testcase']) if data['suite'] not in d_ts['test_suites_with_failures']: d_ts['test_suites_with_failures'].append(data['suite']) m_avg = { 'testsuites': {} } for tsname in data_json['testsuites']: m_avg['testsuites'][tsname] = { 'testcases': {} } for tcname in data_json['testsuites'][tsname]['testcases']: test_status = "PASS" m_avg['testsuites'][tsname]['testcases'][tcname] = [] t_sum = [] count = 0 j_data = data_json['testsuites'][tsname]['testcases'][tcname] measurements_data = [] for key in j_data['results'].keys(): count += 1 r_count = str(count) m_case = data_json['testsuites'][tsname]['testcases'] m = m_case[tcname]['results'][r_count]['measurements'] if j_data['results'][r_count]['status'] is not "PASS": test_status = "FAIL" m_sum = [] for i in range(len(m)): sum_mean = 0 sum_std = 0 sum_min = 0 sum_max = 0 record = [] if "test_measure" in m[i].keys(): if len(t_sum) > i: sum_mean = m[i]["test_data"]['mean'] + \ t_sum[i][0] sum_std = m[i]["test_data"]['std_dev'] + \ t_sum[i][1] sum_min = m[i]["test_data"]['minimum'] + \ t_sum[i][2] sum_max = m[i]["test_data"]['maximum'] + \ t_sum[i][3] else: measurements_data.append(m[i]) sum_mean = m[i]["test_data"]['mean'] sum_std = m[i]["test_data"]['std_dev'] sum_min = m[i]["test_data"]['minimum'] sum_max = m[i]["test_data"]['maximum'] record = [sum_mean, sum_std, sum_min, sum_max] else: if len(measurements_data) <= i: measurements_data.append(m[i]) record = [sum_mean, sum_std, sum_min, sum_max] m_sum.append(record) if len(t_sum) > len(m_sum): for v in range(len(m_sum)): t_sum[v] = m_sum[v] else: t_sum = m_sum m_list = [] if test_status == "PASS": for i in range(len(measurements_data)): m_data = {} if "test_measure" in measurements_data[i].keys(): measure = measurements_data[i]['test_measure'] m_data['test_measure'] = measure m_data['unit'] = measurements_data[i]['unit'] m_data['test_data'] = {} div = count m_data['test_data']['mean'] = t_sum[i][0] / div m_data['test_data']['std_dev'] = t_sum[i][1] / div m_data['test_data']['minimum'] = t_sum[i][2] / div m_data['test_data']['maximum'] = t_sum[i][3] / div m_list.append(m_data) m_avg['testsuites'][tsname]['testcases'][tcname] = m_list data_json["avg_measurements"] = m_avg data_json['result']['end'] = str(data['end_time']) start = data_json['result']['start'].split()[1] end = data_json['result']['end'].split()[1] dur = str(datetime.datetime.strptime(end, FMT) - datetime.datetime.strptime(start, FMT)) fail_tests = [] fail_ts = [] for count in range(PtlTextTestRunner.cur_repeat_count): r_count = str(count + 1) fail_tests.extend( data_json['test_summary'][r_count]['tests_with_failures']) fail_ts.extend(data_json['test_summary'] [r_count]['test_suites_with_failures']) data_json['result']['duration'] = dur data_json['result']['tests_with_failures'] = list(set(fail_tests)) data_json['result']['test_suites_with_failures'] = list(set(fail_ts)) return data_json
class PTLTestData(Plugin): """ Save post analysis data on test cases failure or error """ name = 'PTLTestData' score = sys.maxint - 6 logger = logging.getLogger(__name__) def __init__(self): Plugin.__init__(self) self.post_data_dir = None self.max_postdata_threshold = None self.__save_data_count = 0 self.__priv_sn = '' self.du = DshUtils() def options(self, parser, env): """ Register command line options """ pass def set_data(self, post_data_dir, max_postdata_threshold): self.post_data_dir = post_data_dir self.max_postdata_threshold = max_postdata_threshold def configure(self, options, config): """ Configure the plugin and system, based on selected options """ self.config = config if self.post_data_dir is not None: self.enabled = True else: self.enabled = False def __save_home(self, test, status, err=None): if hasattr(test, 'test'): _test = test.test sn = _test.__class__.__name__ elif hasattr(test, 'context'): _test = test.context sn = _test.__name__ else: # test does not have any PBS Objects, so just return return if self.__priv_sn != sn: self.__save_data_count = 0 self.__priv_sn = sn # Saving home might take time so disable timeout # handler set by runner tn = getattr(_test, '_testMethodName', 'unknown') testlogs = getattr(test, 'captured_logs', '') datadir = os.path.join(self.post_data_dir, sn, tn) if os.path.exists(datadir): _msg = 'Old post analysis data exists at %s' % datadir _msg += ', skipping saving data for this test case' self.logger.warn(_msg) _msg = 'Please remove old directory or' _msg += ' provide different directory' self.logger.warn(_msg) return if getattr(test, 'old_sigalrm_handler', None) is not None: _h = getattr(test, 'old_sigalrm_handler') signal.signal(signal.SIGALRM, _h) signal.alarm(0) self.logger.log(logging.DEBUG2, 'Saving post analysis data...') current_host = socket.gethostname().split('.')[0] self.du.mkdir(current_host, path=datadir, mode=0755, parents=True, logerr=False, level=logging.DEBUG2) if err is not None: if isclass(err[0]) and issubclass(err[0], SkipTest): status = 'SKIP' status_data = 'Reason = %s' % (err[1]) else: if isclass(err[0]) and issubclass(err[0], TimeOut): status = 'TIMEDOUT' status_data = getattr(test, 'err_in_string', '') else: status_data = '' logfile = os.path.join(datadir, 'logfile_' + status) f = open(logfile, 'w+') f.write(testlogs + '\n') f.write(status_data + '\n') f.write('test duration: %s\n' % str(getattr(test, 'duration', '0'))) if status in ('PASS', 'SKIP'): # Test case passed or skipped, no need to save post analysis data f.close() return if ((self.max_postdata_threshold != 0) and (self.__save_data_count >= self.max_postdata_threshold)): _msg = 'Total number of saved post analysis data for this' _msg += ' testsuite is exceeded max postdata threshold' _msg += ' (%d)' % self.max_postdata_threshold f.write(_msg + '\n') self.logger.error(_msg) f.close() return svr = getattr(_test, 'server', None) if svr is not None: svr_host = svr.hostname else: _msg = 'Could not find Server Object in given test object' _msg += ', skipping saving post analysis data' f.write(_msg + '\n') self.logger.warning(_msg) f.close() return pbs_diag = os.path.join(svr.pbs_conf['PBS_EXEC'], 'unsupported', 'pbs_diag') cur_user = self.du.get_current_user() cmd = [pbs_diag, '-f', '-d', '2'] cmd += ['-u', cur_user] cmd += ['-o', pwd.getpwnam(cur_user).pw_dir] if len(svr.jobs) > 0: cmd += ['-j', ','.join(svr.jobs.keys())] ret = self.du.run_cmd(svr_host, cmd, sudo=True, level=logging.DEBUG2) if ret['rc'] != 0: _msg = 'Failed to get diag information for ' _msg += 'on %s:' % svr_host _msg += '\n\n' + '\n'.join(ret['err']) + '\n\n' f.write(_msg + '\n') self.logger.error(_msg) f.close() return else: diag_re = r"(?P<path>\/.*\/pbs_diag_[\d]+_[\d]+\.tar\.gz).*" m = re.search(diag_re, '\n'.join(ret['out'])) if m is not None: diag_out = m.group('path') else: _msg = 'Failed to find generated diag path in below output:' _msg += '\n\n' + '-' * 80 + '\n' _msg += '\n'.join(ret['out']) + '\n' _msg += '-' * 80 + '\n\n' f.write(_msg) self.logger.error(_msg) f.close() return diag_out_dest = os.path.join(datadir, os.path.basename(diag_out)) if not self.du.is_localhost(svr_host): diag_out_r = svr_host + ':' + diag_out else: diag_out_r = diag_out ret = self.du.run_copy(current_host, diag_out_r, diag_out_dest, sudo=True, level=logging.DEBUG2) if ret['rc'] != 0: _msg = 'Failed to copy generated diag from' _msg += ' %s to %s' % (diag_out_r, diag_out_dest) f.write(_msg + '\n') self.logger.error(_msg) f.close() return else: self.du.rm(svr_host, path=diag_out, sudo=True, force=True, level=logging.DEBUG2) cores = [] dir_list = ['server_priv', 'sched_priv', 'mom_priv'] for d in dir_list: path = os.path.join(svr.pbs_conf['PBS_HOME'], d) files = self.du.listdir(hostname=svr_host, path=path, sudo=True, level=logging.DEBUG2) for _f in files: if os.path.basename(_f).startswith('core'): cores.append(_f) cores = list(set(cores)) if len(cores) > 0: cmd = ['gunzip', diag_out_dest] ret = self.du.run_cmd(current_host, cmd, sudo=True, level=logging.DEBUG2) if ret['rc'] != 0: _msg = 'Failed unzip generated diag at %s:' % diag_out_dest _msg += '\n\n' + '\n'.join(ret['err']) + '\n\n' f.write(_msg + '\n') self.logger.error(_msg) f.close() return diag_out_dest = diag_out_dest.rstrip('.gz') cmd = ['tar', '-xf', diag_out_dest, '-C', datadir] ret = self.du.run_cmd(current_host, cmd, sudo=True, level=logging.DEBUG2) if ret['rc'] != 0: _msg = 'Failed extract generated diag %s' % diag_out_dest _msg += ' to %s:' % datadir _msg += '\n\n' + '\n'.join(ret['err']) + '\n\n' f.write(_msg + '\n') self.logger.error(_msg) f.close() return self.du.rm(hostname=current_host, path=diag_out_dest, force=True, sudo=True, level=logging.DEBUG2) diag_out_dest = diag_out_dest.rstrip('.tar') for c in cores: cmd = [pbs_diag, '-g', c] ret = self.du.run_cmd(svr_host, cmd, sudo=True, level=logging.DEBUG2) if ret['rc'] != 0: _msg = 'Failed to get core file information for ' _msg += '%s on %s:' % (c, svr_host) _msg += '\n\n' + '\n'.join(ret['err']) + '\n\n' f.write(_msg + '\n') self.logger.error(_msg) else: of = os.path.join(diag_out_dest, os.path.basename(c) + '.out') _f = open(of, 'w+') _f.write('\n'.join(ret['out']) + '\n') _f.close() self.du.rm(hostname=svr_host, path=c, force=True, sudo=True, level=logging.DEBUG2) cmd = ['tar', '-cf', diag_out_dest + '.tar'] cmd += [os.path.basename(diag_out_dest)] ret = self.du.run_cmd(current_host, cmd, sudo=True, cwd=datadir, level=logging.DEBUG2) if ret['rc'] != 0: _msg = 'Failed generate tarball of diag directory' _msg += ' %s' % diag_out_dest _msg += ' after adding core(s) information in it:' _msg += '\n\n' + '\n'.join(ret['err']) + '\n\n' f.write(_msg + '\n') self.logger.error(_msg) f.close() return cmd = ['gzip', diag_out_dest + '.tar'] ret = self.du.run_cmd(current_host, cmd, sudo=True, level=logging.DEBUG2) if ret['rc'] != 0: _msg = 'Failed compress tarball of diag %s' % diag_out_dest _msg += '.tar after adding core(s) information in it:' _msg += '\n\n' + '\n'.join(ret['err']) + '\n\n' f.write(_msg + '\n') self.logger.error(_msg) f.close() return self.du.rm(current_host, diag_out_dest, sudo=True, recursive=True, force=True, level=logging.DEBUG2) else: diag_out_dest = diag_out_dest.rstrip('.tar.gz') dest = os.path.join(datadir, 'PBS_' + current_host.split('.')[0] + '.tar.gz') ret = self.du.run_copy(current_host, diag_out_dest + '.tar.gz', dest, sudo=True, level=logging.DEBUG2) if ret['rc'] != 0: _msg = 'Failed rename tarball of diag from %s' % diag_out_dest _msg += '.tar.gz to %s:' % dest _msg += '\n\n' + '\n'.join(ret['err']) + '\n\n' f.write(_msg + '\n') self.logger.error(_msg) f.close() return self.du.rm(current_host, path=diag_out_dest + '.tar.gz', force=True, sudo=True, level=logging.DEBUG2) f.close() self.__save_data_count += 1 _msg = 'Successfully saved post analysis data' self.logger.log(logging.DEBUG2, _msg) def addError(self, test, err): self.__save_home(test, 'ERROR', err) def addFailure(self, test, err): self.__save_home(test, 'FAIL', err) def addSuccess(self, test): self.__save_home(test, 'PASS')
class PTLTestData(Plugin): """ Save post analysis data on test cases failure or error """ name = 'PTLTestData' score = sys.maxint - 6 logger = logging.getLogger(__name__) def __init__(self): Plugin.__init__(self) self.post_data_dir = None self.max_postdata_threshold = None self.__save_data_count = 0 self.__priv_sn = '' self.du = DshUtils() def options(self, parser, env): """ Register command line options """ pass def set_data(self, post_data_dir, max_postdata_threshold): self.post_data_dir = post_data_dir self.max_postdata_threshold = max_postdata_threshold def configure(self, options, config): """ Configure the plugin and system, based on selected options """ self.config = config if self.post_data_dir is not None: self.enabled = True else: self.enabled = False def __save_home(self, test, status, err=None): if hasattr(test, 'test'): _test = test.test sn = _test.__class__.__name__ elif hasattr(test, 'context'): _test = test.context sn = _test.__name__ else: # test does not have any PBS Objects, so just return return if self.__priv_sn != sn: self.__save_data_count = 0 self.__priv_sn = sn # Saving home might take time so disable timeout # handler set by runner tn = getattr(_test, '_testMethodName', 'unknown') testlogs = getattr(test, 'captured_logs', '') datadir = os.path.join(self.post_data_dir, sn, tn) if os.path.exists(datadir): _msg = 'Old post analysis data exists at %s' % datadir _msg += ', skipping saving data for this test case' self.logger.warn(_msg) _msg = 'Please remove old directory or' _msg += ' provide different directory' self.logger.warn(_msg) return if getattr(test, 'old_sigalrm_handler', None) is not None: _h = getattr(test, 'old_sigalrm_handler') signal.signal(signal.SIGALRM, _h) signal.alarm(0) self.logger.log(logging.DEBUG2, 'Saving post analysis data...') current_host = socket.gethostname().split('.')[0] self.du.mkdir(current_host, path=datadir, mode=0755, parents=True, logerr=False, level=logging.DEBUG2) if err is not None: if isclass(err[0]) and issubclass(err[0], SkipTest): status = 'SKIP' status_data = 'Reason = %s' % (err[1]) else: if isclass(err[0]) and issubclass(err[0], TimeOut): status = 'TIMEDOUT' status_data = getattr(test, 'err_in_string', '') else: status_data = '' logfile = os.path.join(datadir, 'logfile_' + status) f = open(logfile, 'w+') f.write(testlogs + '\n') f.write(status_data + '\n') f.write('test duration: %s\n' % str(getattr(test, 'duration', '0'))) if status in ('PASS', 'SKIP'): # Test case passed or skipped, no need to save post analysis data f.close() return if ((self.max_postdata_threshold != 0) and (self.__save_data_count >= self.max_postdata_threshold)): _msg = 'Total number of saved post analysis data for this' _msg += ' testsuite is exceeded max postdata threshold' _msg += ' (%d)' % self.max_postdata_threshold f.write(_msg + '\n') self.logger.error(_msg) f.close() return svr = getattr(_test, 'server', None) if svr is not None: svr_host = svr.hostname else: _msg = 'Could not find Server Object in given test object' _msg += ', skipping saving post analysis data' f.write(_msg + '\n') self.logger.warning(_msg) f.close() return pbs_diag = os.path.join(svr.pbs_conf['PBS_EXEC'], 'unsupported', 'pbs_diag') cmd = [pbs_diag, '-f', '-d', '2'] cmd += ['-u', self.du.get_current_user()] if len(svr.jobs) > 0: cmd += ['-j', ','.join(svr.jobs.keys())] ret = self.du.run_cmd(svr_host, cmd, sudo=True, level=logging.DEBUG2) if ret['rc'] != 0: _msg = 'Failed to get diag information for ' _msg += 'on %s:' % svr_host _msg += '\n\n' + '\n'.join(ret['err']) + '\n\n' f.write(_msg + '\n') self.logger.error(_msg) f.close() return else: diag_re = r"(?P<path>\/.*\/pbs_diag_[\d]+_[\d]+\.tar\.gz).*" m = re.search(diag_re, '\n'.join(ret['out'])) if m is not None: diag_out = m.group('path') else: _msg = 'Failed to find generated diag path in below output:' _msg += '\n\n' + '-' * 80 + '\n' _msg += '\n'.join(ret['out']) + '\n' _msg += '-' * 80 + '\n\n' f.write(_msg) self.logger.error(_msg) f.close() return diag_out_dest = os.path.join(datadir, os.path.basename(diag_out)) if not self.du.is_localhost(svr_host): diag_out_r = svr_host + ':' + diag_out else: diag_out_r = diag_out ret = self.du.run_copy(current_host, diag_out_r, diag_out_dest, sudo=True, level=logging.DEBUG2) if ret['rc'] != 0: _msg = 'Failed to copy generated diag from' _msg += ' %s to %s' % (diag_out_r, diag_out_dest) f.write(_msg + '\n') self.logger.error(_msg) f.close() return else: self.du.rm(svr_host, path=diag_out, sudo=True, force=True, level=logging.DEBUG2) cores = [] dir_list = ['server_priv', 'sched_priv', 'mom_priv'] for d in dir_list: path = os.path.join(svr.pbs_conf['PBS_HOME'], d) files = self.du.listdir(hostname=svr_host, path=path, sudo=True, level=logging.DEBUG2) for _f in files: if os.path.basename(_f).startswith('core'): cores.append(_f) cores = list(set(cores)) if len(cores) > 0: cmd = ['gunzip', diag_out_dest] ret = self.du.run_cmd(current_host, cmd, sudo=True, level=logging.DEBUG2) if ret['rc'] != 0: _msg = 'Failed unzip generated diag at %s:' % diag_out_dest _msg += '\n\n' + '\n'.join(ret['err']) + '\n\n' f.write(_msg + '\n') self.logger.error(_msg) f.close() return diag_out_dest = diag_out_dest.rstrip('.gz') cmd = ['tar', '-xf', diag_out_dest, '-C', datadir] ret = self.du.run_cmd(current_host, cmd, sudo=True, level=logging.DEBUG2) if ret['rc'] != 0: _msg = 'Failed extract generated diag %s' % diag_out_dest _msg += ' to %s:' % datadir _msg += '\n\n' + '\n'.join(ret['err']) + '\n\n' f.write(_msg + '\n') self.logger.error(_msg) f.close() return self.du.rm(hostname=current_host, path=diag_out_dest, force=True, sudo=True, level=logging.DEBUG2) diag_out_dest = diag_out_dest.rstrip('.tar') for c in cores: cmd = [pbs_diag, '-g', c] ret = self.du.run_cmd(svr_host, cmd, sudo=True, level=logging.DEBUG2) if ret['rc'] != 0: _msg = 'Failed to get core file information for ' _msg += '%s on %s:' % (c, svr_host) _msg += '\n\n' + '\n'.join(ret['err']) + '\n\n' f.write(_msg + '\n') self.logger.error(_msg) else: of = os.path.join(diag_out_dest, os.path.basename(c) + '.out') _f = open(of, 'w+') _f.write('\n'.join(ret['out']) + '\n') _f.close() self.du.rm(hostname=svr_host, path=c, force=True, sudo=True, level=logging.DEBUG2) cmd = ['tar', '-cf', diag_out_dest + '.tar'] cmd += [os.path.basename(diag_out_dest)] ret = self.du.run_cmd(current_host, cmd, sudo=True, cwd=datadir, level=logging.DEBUG2) if ret['rc'] != 0: _msg = 'Failed generate tarball of diag directory' _msg += ' %s' % diag_out_dest _msg += ' after adding core(s) information in it:' _msg += '\n\n' + '\n'.join(ret['err']) + '\n\n' f.write(_msg + '\n') self.logger.error(_msg) f.close() return cmd = ['gzip', diag_out_dest + '.tar'] ret = self.du.run_cmd(current_host, cmd, sudo=True, level=logging.DEBUG2) if ret['rc'] != 0: _msg = 'Failed compress tarball of diag %s' % diag_out_dest _msg += '.tar after adding core(s) information in it:' _msg += '\n\n' + '\n'.join(ret['err']) + '\n\n' f.write(_msg + '\n') self.logger.error(_msg) f.close() return self.du.rm(current_host, diag_out_dest, sudo=True, recursive=True, force=True, level=logging.DEBUG2) else: diag_out_dest = diag_out_dest.rstrip('.tar.gz') dest = os.path.join(datadir, 'PBS_' + current_host.split('.')[0] + '.tar.gz') ret = self.du.run_copy(current_host, diag_out_dest + '.tar.gz', dest, sudo=True, level=logging.DEBUG2) if ret['rc'] != 0: _msg = 'Failed rename tarball of diag from %s' % diag_out_dest _msg += '.tar.gz to %s:' % dest _msg += '\n\n' + '\n'.join(ret['err']) + '\n\n' f.write(_msg + '\n') self.logger.error(_msg) f.close() return self.du.rm(current_host, path=diag_out_dest + '.tar.gz', force=True, sudo=True, level=logging.DEBUG2) f.close() self.__save_data_count += 1 _msg = 'Successfully saved post analysis data' self.logger.log(logging.DEBUG2, _msg) def addError(self, test, err): self.__save_home(test, 'ERROR', err) def addFailure(self, test, err): self.__save_home(test, 'FAIL', err) def addSuccess(self, test): self.__save_home(test, 'PASS')
class PTLTestData(Plugin): """ Save post analysis data on test cases failure or error """ name = 'PTLTestData' score = sys.maxsize - 6 logger = logging.getLogger(__name__) def __init__(self): Plugin.__init__(self) self.post_data_dir = None self.max_postdata_threshold = None self.__save_data_count = 0 self.__priv_sn = '' self.du = DshUtils() def options(self, parser, env): """ Register command line options """ pass def set_data(self, post_data_dir, max_postdata_threshold): self.post_data_dir = post_data_dir self.max_postdata_threshold = max_postdata_threshold def configure(self, options, config): """ Configure the plugin and system, based on selected options """ self.config = config if self.post_data_dir is not None: self.enabled = True else: self.enabled = False def __save_home(self, test, status, err=None): if hasattr(test, 'test'): _test = test.test sn = _test.__class__.__name__ elif hasattr(test, 'context'): _test = test.context sn = _test.__name__ else: # test does not have any PBS Objects, so just return return if self.__priv_sn != sn: self.__save_data_count = 0 self.__priv_sn = sn # Saving home might take time so disable timeout # handler set by runner tn = getattr(_test, '_testMethodName', 'unknown') testlogs = getattr(test, 'captured_logs', '') datadir = os.path.join(self.post_data_dir, sn, tn) if os.path.exists(datadir): _msg = 'Old post analysis data exists at %s' % datadir _msg += ', skipping saving data for this test case' self.logger.warn(_msg) _msg = 'Please remove old directory or' _msg += ' provide different directory' self.logger.warn(_msg) return if getattr(test, 'old_sigalrm_handler', None) is not None: _h = getattr(test, 'old_sigalrm_handler') signal.signal(signal.SIGALRM, _h) signal.alarm(0) self.logger.log(logging.DEBUG2, 'Saving post analysis data...') current_host = socket.gethostname().split('.')[0] self.du.mkdir(current_host, path=datadir, mode=0o755, parents=True, logerr=False, level=logging.DEBUG2) if err is not None: if isclass(err[0]) and issubclass(err[0], SkipTest): status = 'SKIP' status_data = 'Reason = %s' % (err[1]) else: if isclass(err[0]) and issubclass(err[0], TimeOut): status = 'TIMEDOUT' status_data = getattr(test, 'err_in_string', '') else: status_data = '' logfile = os.path.join(datadir, 'logfile_' + status) f = open(logfile, 'w+') f.write(testlogs + '\n') f.write(status_data + '\n') f.write('test duration: %s\n' % str(getattr(test, 'duration', '0'))) if status in ('PASS', 'SKIP'): # Test case passed or skipped, no need to save post analysis data f.close() return if ((self.max_postdata_threshold != 0) and (self.__save_data_count >= self.max_postdata_threshold)): _msg = 'Total number of saved post analysis data for this' _msg += ' testsuite is exceeded max postdata threshold' _msg += ' (%d)' % self.max_postdata_threshold f.write(_msg + '\n') self.logger.error(_msg) f.close() return servers = getattr(_test, 'servers', None) if servers is not None: server_host = servers.values()[0].shortname else: _msg = 'Could not find Server Object in given test object' _msg += ', skipping saving post analysis data' f.write(_msg + '\n') self.logger.warning(_msg) f.close() return moms = getattr(_test, 'moms', None) comms = getattr(_test, 'comms', None) client = getattr(_test.servers.values()[0], 'client', None) server = servers.values()[0] add_hosts = [] if len(servers) > 1: for param in servers.values()[1:]: add_hosts.append(param.shortname) if moms is not None: for param in moms.values(): add_hosts.append(param.shortname) if comms is not None: for param in comms.values(): add_hosts.append(param.shortname) if client is not None: add_hosts.append(client.split('.')[0]) add_hosts = list(set(add_hosts) - set([server_host])) pbs_snapshot_path = os.path.join( server.pbs_conf["PBS_EXEC"], "sbin", "pbs_snapshot") cur_user = self.du.get_current_user() cur_user_dir = pwd.getpwnam(cur_user).pw_dir cmd = [ pbs_snapshot_path, '-H', server_host, '--daemon-logs', '2', '--accounting-logs', '2', '--with-sudo' ] if len(add_hosts) > 0: cmd += ['--additional-hosts=' + ','.join(add_hosts)] cmd += ['-o', cur_user_dir] ret = self.du.run_cmd(current_host, cmd, level=logging.DEBUG2, logerr=False) if ret['rc'] != 0: _msg = 'Failed to get analysis information for ' _msg += 'on %s:' % servers_host _msg += '\n\n' + '\n'.join(ret['err']) + '\n\n' f.write(_msg + '\n') self.logger.error(_msg) f.close() return else: if len(ret['out']) == 0: self.logger.error('Snapshot command failed') f.close() return snap_out = ret['out'][0] snap_out_dest = (snap_out.split(":")[1]).strip() dest = os.path.join(datadir, 'PBS_' + server_host + '.tar.gz') ret = self.du.run_copy(current_host, snap_out_dest, dest, sudo=True, level=logging.DEBUG2) self.du.rm(current_host, path=snap_out_dest, recursive=True, force=True, level=logging.DEBUG2) f.close() self.__save_data_count += 1 _msg = 'Saved post analysis data' self.logger.info(_msg) def addError(self, test, err): self.__save_home(test, 'ERROR', err) def addFailure(self, test, err): self.__save_home(test, 'FAIL', err) def addSuccess(self, test): self.__save_home(test, 'PASS')
class PTLJsonData(object): """ The intent of the class is to generate json format of PTL test data """ def __init__(self, command): self.__du = DshUtils() self.__cmd = command def get_json(self, data, prev_data=None): """ Method to generate test data in accordance to json schema :param data: dictionary of a test case details :type data: dict :param prev_data: dictionary of test run details that ran before the current test :type prev_data: dict :returns a formatted dictionary of the data """ data_json = None if not prev_data: data_json = { 'command': self.__cmd, 'user': self.__du.get_current_user(), 'product_version': data['pbs_version'], 'run_id': data['start_time'].strftime('%s'), 'test_conf': {}, 'machine_info': data['machinfo'], 'testsuites': {}, 'additional_data': {}, 'test_summary': { 'result_summary': { 'run': 0, 'succeeded': 0, 'failed': 0, 'errors': 0, 'skipped': 0, 'timedout': 0 }, 'test_start_time': str(data['start_time']), 'tests_with_failures': [], 'test_suites_with_failures': [] } } if data['testparam']: for param in data['testparam'].split(','): par = param.split('=', 1) data_json['test_conf'][par[0]] = par[1] else: data_json = prev_data tsname = data['suite'] tcname = data['testcase'] if tsname not in data_json['testsuites']: data_json['testsuites'][tsname] = { 'module': data['module'], 'file': data['file'], 'testcases': {} } tsdoc = [] if data['suitedoc']: tsdoc = (re.sub(r"[\t\n ]+", " ", data['suitedoc'])).strip() data_json['testsuites'][tsname]['docstring'] = tsdoc tcshort = {} tcdoc = [] if data['testdoc']: tcdoc = (re.sub(r"[\t\n ]+", " ", data['testdoc'])).strip() tcshort['docstring'] = tcdoc if data['tags']: tcshort['tags'] = data['tags'] tcshort['results'] = { 'status': data['status'], 'status_data': str(data['status_data']), 'duration': str(data['duration']), 'start_time': str(data['start_time']), 'end_time': str(data['end_time']), 'measurements': [] } tcshort['requirements'] = {} if 'measurements' in data: tcshort['results']['measurements'] = data['measurements'] data_json['testsuites'][tsname]['testcases'][tcname] = tcshort if 'additional_data' in data: data_json['additional_data'] = data['additional_data'] data_json['test_summary']['test_end_time'] = str(data['end_time']) data_json['test_summary']['result_summary']['run'] += 1 d_ts = data_json['test_summary'] if data['status'] == 'PASS': d_ts['result_summary']['succeeded'] += 1 elif data['status'] == 'SKIP': d_ts['result_summary']['skipped'] += 1 elif data['status'] == 'TIMEDOUT': d_ts['result_summary']['timedout'] += 1 d_ts['tests_with_failures'].append(data['testcase']) if data['suite'] not in d_ts['test_suites_with_failures']: d_ts['test_suites_with_failures'].append(data['suite']) elif data['status'] == 'ERROR': d_ts['result_summary']['errors'] += 1 d_ts['tests_with_failures'].append(data['testcase']) if data['suite'] not in d_ts['test_suites_with_failures']: d_ts['test_suites_with_failures'].append(data['suite']) elif data['status'] == 'FAIL': d_ts['result_summary']['failed'] += 1 d_ts['tests_with_failures'].append(data['testcase']) if data['suite'] not in d_ts['test_suites_with_failures']: d_ts['test_suites_with_failures'].append(data['suite']) return data_json