def _fill_sample(self, report, call, item, status):
filename, lineno, _ = report.location
if self._sample is None:
self._sample = Sample()
self._sample.test_case = item.name
self._sample.test_suite = item.module.__name__
self._sample.start_time = call.start
self._sample.extras = {"filename": filename, "lineno": lineno}
self._sample.status = status
self._sample.duration = time.time() - self._sample.start_time
if call.excinfo is not None:
self._sample.error_msg = call.excinfo.exconly().strip()
self._sample.error_trace = "\n".join(
traceback.format_tb(call.excinfo.tb)).strip()
if call.excinfo.errisinstance(AssertionError):
self._sample.add_assertion(call.excinfo.exconly(), {
'args': '',
'kwargs': ''
})
self._sample.set_assertion_failed(
call.excinfo.exconly(), str(call.excinfo.value),
str(call.excinfo.getrepr(style="native")))
def beforeTest(self):
self.current_sample = Sample(
test_case=self.test_info["test_case"],
test_suite=self.test_info["suite_name"],
start_time=time.time(),
status="SKIPPED")
self.current_sample.extras.update({
"file": self.test_info["test_file"],
"full_name": self.test_info["test_fqn"],
"description": self.test_info["description"]
})
module_fqn_parts = self.test_info["module_fqn"].split('.')
for item in module_fqn_parts[:-1]:
self.current_sample.path.append(PathComponent("package", item))
self.current_sample.path.append(PathComponent("module", module_fqn_parts[-1]))
if "." in self.test_info["class_method"]: # TestClass.test_method
class_name, method_name = self.test_info["class_method"].split('.')[:2]
self.current_sample.path.extend([
PathComponent("class", class_name),
PathComponent("method", method_name)])
else: # test_func
self.current_sample.path.append(PathComponent("func", self.test_info["class_method"]))
self.log.debug("Test method path: %r", self.current_sample.path)
self.test_count += 1
def _get_subsamples(self):
recording = self._pop_events()
sample = Sample()
extr = ApiritifSampleExtractor()
trace = extr.parse_recording(recording, sample)
toplevel_sample = trace[0].to_dict()
self._filter([toplevel_sample])
return toplevel_sample
def __init__(self, writer, index, outfile_name):
super(SampleGenerator, self).__init__(target=self._write_sample)
self.writer = writer
self.index = index
self.outfile_name = outfile_name
self.sample = Sample(start_time=index,
duration=index,
test_case="Generator %s" % index)
def _get_subsamples(self):
recording = self._pop_events()
sample = Sample()
extr = ApiritifSampleExtractor()
trace = extr.parse_recording(recording, sample)
subsamples = trace[0].to_dict()['subsamples']
self._filter(subsamples)
return subsamples
def beforeTest(self, test):
"""
before test run
"""
addr = test.address(
) # file path, package.subpackage.module, class.method
test_file, module_fqn, class_method = addr
test_fqn = test.id() # [package].module.class.method
suite_name, case_name = test_fqn.split('.')[-2:]
log.debug("Addr: %r", addr)
log.debug("id: %r", test_fqn)
if class_method is None:
class_method = case_name
self.current_sample = Sample(test_case=case_name,
test_suite=suite_name,
start_time=time.time(),
status="SKIPPED")
self.current_sample.extras.update({
"file":
test_file,
"full_name":
test_fqn,
"description":
test.shortDescription()
})
module_fqn_parts = module_fqn.split('.')
for item in module_fqn_parts[:-1]:
self.current_sample.path.append(PathComponent("package", item))
self.current_sample.path.append(
PathComponent("module", module_fqn_parts[-1]))
if "." in class_method: # TestClass.test_method
class_name, method_name = class_method.split('.')[:2]
self.current_sample.path.extend([
PathComponent("class", class_name),
PathComponent("method", method_name)
])
else: # test_func
self.current_sample.path.append(PathComponent(
"func", class_method))
log.debug("Test method path: %r", self.current_sample.path)
self.test_count += 1
def end_test(self, name, attrs):
# TODO: include keywords as subsamples
sample = Sample()
sample.test_case = name
sample.test_suite = self._current_suite
sample.start_time = time.mktime(datetime.strptime(attrs['starttime'], '%Y%m%d %H:%M:%S.%f').timetuple())
sample.duration = float(attrs['elapsedtime']) / 1000
if attrs['status'] == 'PASS':
sample.status = 'PASSED'
else:
sample.status = 'FAILED'
message = attrs['message']
if '\n' in message:
lines = message.split('\n')
sample.error_msg = lines[0]
sample.error_trace = message
else:
sample.error_msg = message
self._report_sample(sample)
def beforeTest(self, test):
"""
before test run
"""
test_file, _, _ = test.address(
) # file path, module name, class.method
test_fqn = test.id() # [package].module.class.method
class_name, method_name = test_fqn.split('.')[-2:]
self.current_sample = Sample(test_case=method_name,
test_suite=class_name,
start_time=time.time(),
status="SKIPPED")
self.current_sample.extras.update({
"file":
test_file,
"full_name":
test_fqn,
"description":
test.shortDescription()
})
self.test_count += 1
def _get_subsamples(self, call, report, item):
recording = self._pop_events()
sample = Sample()
sample.test_case = item.name
if item.parent:
sample.test_suite = item.parent.name
sample.duration = report.duration
sample.status = 'PASSED' if report.passed else 'FAILED'
if call.excinfo:
sample.error_msg = str(call.excinfo.value)
sample.error_trace = traceback.format_tb(call.excinfo.tb)
extr = ApiritifSampleExtractor()
trace = extr.parse_recording(recording, sample)
toplevel_sample = trace[0].to_dict()
self._filter([toplevel_sample])
return toplevel_sample
def _fill_sample(self, report, call, item, status):
filename, lineno, _ = report.location
if self._sample is None:
self._sample = Sample()
self._sample.test_case = item.name
self._sample.test_suite = item.module.__name__
self._sample.start_time = call.start
self._sample.extras = {"filename": filename, "lineno": lineno}
self._sample.status = status
self._sample.duration = time.time() - self._sample.start_time
if call.excinfo is not None:
self._sample.error_msg = call.excinfo.exconly().strip()
self._sample.error_trace = "\n".join(
traceback.format_tb(call.excinfo.tb)).strip()
def _fill_sample(self, report, call, item, status):
filename, lineno, _ = report.location
if self._sample is None:
self._sample = Sample()
self._sample.test_case = item.name
self._sample.test_suite = item.module.__name__
self._sample.start_time = call.start
self._sample.extras = {"filename": filename, "lineno": lineno}
self._sample.status = status
self._sample.duration = time.time() - self._sample.start_time
if call.excinfo is not None:
self._sample.error_msg = call.excinfo.exconly().strip()
self._sample.error_trace = "\n".join(traceback.format_tb(call.excinfo.tb)).strip()
if call.excinfo.errisinstance(AssertionError):
self._sample.add_assertion(call.excinfo.exconly())
self._sample.set_assertion_failed(call.excinfo.exconly(), str(call.excinfo.value),
str(call.excinfo.getrepr(style="native")))
def end_test(self, name, attrs):
# TODO: include keywords as subsamples
sample = Sample()
sample.test_case = name
sample.test_suite = self._current_suite
sample.start_time = time.mktime(
datetime.strptime(attrs['starttime'],
'%Y%m%d %H:%M:%S.%f').timetuple())
sample.duration = float(attrs['elapsedtime']) / 1000.0
if attrs['status'] == 'PASS':
sample.status = 'PASSED'
else:
sample.status = 'FAILED'
message = attrs['message']
if '\n' in message:
lines = message.split('\n')
sample.error_msg = lines[0]
sample.error_trace = message
else:
sample.error_msg = message
self._report_sample(sample)
class SampleController(object):
def __init__(self, log):
self.current_sample = None # todo: recreate it from plugin's template every transaction
self.success_count = None
self.log = log
self.test_count = 0
self.success_count = 0
self.apiritif_extractor = ApiritifSampleExtractor()
self.tran_mode = False # it's regular test (without smart transaction) by default
self.start_time = None
self.end_time = None
self.test_info = {}
def beforeTest(self):
self.current_sample = Sample(
test_case=self.test_info["test_case"],
test_suite=self.test_info["suite_name"],
start_time=time.time(),
status="SKIPPED")
self.current_sample.extras.update({
"file": self.test_info["test_file"],
"full_name": self.test_info["test_fqn"],
"description": self.test_info["description"]
})
module_fqn_parts = self.test_info["module_fqn"].split('.')
for item in module_fqn_parts[:-1]:
self.current_sample.path.append(PathComponent("package", item))
self.current_sample.path.append(PathComponent("module", module_fqn_parts[-1]))
if "." in self.test_info["class_method"]: # TestClass.test_method
class_name, method_name = self.test_info["class_method"].split('.')[:2]
self.current_sample.path.extend([
PathComponent("class", class_name),
PathComponent("method", method_name)])
else: # test_func
self.current_sample.path.append(PathComponent("func", self.test_info["class_method"]))
self.log.debug("Test method path: %r", self.current_sample.path)
self.test_count += 1
def startTest(self):
self.start_time = time.time()
def stopTest(self, is_transaction=False):
if self.tran_mode == is_transaction:
self.end_time = time.time()
def addError(self, assertion_name, error_msg, error_trace, is_transaction=False):
if self.tran_mode == is_transaction:
self.current_sample.add_assertion(assertion_name, {"args": [], "kwargs": {}})
self.current_sample.set_assertion_failed(assertion_name, error_msg, error_trace)
def addFailure(self, error, is_transaction=False):
if self.tran_mode == is_transaction:
assertion_name = error[0].__name__
error_msg = str(error[1]).split('\n')[0]
error_trace = get_trace(error)
self.current_sample.add_assertion(assertion_name, {"args": [], "kwargs": {}})
self.current_sample.set_assertion_failed(assertion_name, error_msg, error_trace)
def addSuccess(self, is_transaction=False):
if self.tran_mode == is_transaction:
self.current_sample.status = "PASSED"
self.success_count += 1
def afterTest(self, is_transaction=False):
if self.tran_mode == is_transaction:
if self.end_time is None:
self.end_time = time.time()
self.current_sample.duration = self.end_time - self.current_sample.start_time
samples_processed = self._process_apiritif_samples(self.current_sample)
if not samples_processed:
self._process_sample(self.current_sample)
self.current_sample = None
def _process_apiritif_samples(self, sample):
samples = []
# get list of events
recording = apiritif.recorder.pop_events(from_ts=self.start_time, to_ts=self.end_time)
try:
if recording:
# convert requests (events) to samples
samples = self.apiritif_extractor.parse_recording(recording, sample)
except BaseException as exc:
self.log.debug("Couldn't parse recording: %s", traceback.format_exc())
self.log.warning("Couldn't parse recording: %s", exc)
for sample in samples:
self._process_sample(sample) # just write to disk
return len(samples)
def _process_sample(self, sample):
writer.add(sample, self.test_count, self.success_count)
class ApiritifPlugin(Plugin):
"""
Saves test results in a format suitable for Taurus.
:type sample_writer: LDJSONSampleWriter
"""
name = 'apiritif'
enabled = False
def __init__(self, sample_writer):
super(ApiritifPlugin, self).__init__()
self.sample_writer = sample_writer
self.test_count = 0
self.success_count = 0
self.current_sample = None
self.apiritif_extractor = ApiritifSampleExtractor()
self.start_time = None
self.end_time = None
def finalize(self, result):
"""
After all tests
"""
if not self.test_count:
raise RuntimeError("Nothing to test.")
def beforeTest(self, test):
"""
before test run
"""
addr = test.address(
) # file path, package.subpackage.module, class.method
test_file, module_fqn, class_method = addr
test_fqn = test.id() # [package].module.class.method
suite_name, case_name = test_fqn.split('.')[-2:]
log.debug("Addr: %r", addr)
log.debug("id: %r", test_fqn)
if class_method is None:
class_method = case_name
self.current_sample = Sample(test_case=case_name,
test_suite=suite_name,
start_time=time.time(),
status="SKIPPED")
self.current_sample.extras.update({
"file":
test_file,
"full_name":
test_fqn,
"description":
test.shortDescription()
})
module_fqn_parts = module_fqn.split('.')
for item in module_fqn_parts[:-1]:
self.current_sample.path.append(PathComponent("package", item))
self.current_sample.path.append(
PathComponent("module", module_fqn_parts[-1]))
if "." in class_method: # TestClass.test_method
class_name, method_name = class_method.split('.')[:2]
self.current_sample.path.extend([
PathComponent("class", class_name),
PathComponent("method", method_name)
])
else: # test_func
self.current_sample.path.append(PathComponent(
"func", class_method))
log.debug("Test method path: %r", self.current_sample.path)
self.test_count += 1
def startTest(self, test):
self.start_time = time.time()
def stopTest(self, test):
self.end_time = time.time()
def afterTest(self, test):
"""
after the test has been run
:param test:
:return:
"""
if self.end_time is None:
self.end_time = time.time()
self.current_sample.duration = self.end_time - self.current_sample.start_time
samples_processed = self._process_apiritif_samples(self.current_sample)
if not samples_processed:
self._process_sample(self.current_sample)
self.current_sample = None
def _process_apiritif_samples(self, sample):
samples_processed = 0
recording = apiritif.recorder.pop_events(from_ts=self.start_time,
to_ts=self.end_time)
if not recording:
return samples_processed
try:
samples = self.apiritif_extractor.parse_recording(
recording, sample)
except BaseException as exc:
log.debug("Couldn't parse recording: %s", traceback.format_exc())
log.warning("Couldn't parse recording: %s", exc)
samples = []
for sample in samples:
samples_processed += 1
self._process_sample(sample)
return samples_processed
def _process_sample(self, sample):
self.sample_writer.add(sample, self.test_count, self.success_count)
def addError(self, test, error):
"""
when a test raises an uncaught exception
:param test:
:param error:
:return:
"""
# test_dict will be None if startTest wasn't called (i.e. exception in setUp/setUpClass)
# status=BROKEN
if self.current_sample is not None:
assertion_name = error[0].__name__
error_msg = str(error[1]).split('\n')[0]
error_trace = self._get_trace(error)
self.current_sample.add_assertion(assertion_name)
self.current_sample.set_assertion_failed(assertion_name, error_msg,
error_trace)
@staticmethod
def _get_trace(error):
if sys.version > '3':
# noinspection PyArgumentList
exct, excv, trace = error
if isinstance(excv, str):
excv = exct(excv)
lines = traceback.format_exception(exct, excv, trace, chain=True)
else:
lines = traceback.format_exception(*error)
return ''.join(lines).rstrip()
def addFailure(self, test, error):
"""
when a test fails
:param test:
:param error:
:return:
"""
# status=FAILED
assertion_name = error[0].__name__
error_msg = str(error[1]).split('\n')[0]
error_trace = self._get_trace(error)
self.current_sample.add_assertion(assertion_name)
self.current_sample.set_assertion_failed(assertion_name, error_msg,
error_trace)
def addSkip(self, test):
"""
when a test is skipped
:param test:
:return:
"""
self.current_sample.status = "SKIPPED"
def addSuccess(self, test):
"""
when a test passes
:param test:
:return:
"""
self.current_sample.status = "PASSED"
self.success_count += 1
class RecordingPlugin(object):
def __init__(self, report_path):
self._report_path = report_path
self._report_fds = None
self.test_count = 0
self.failed_tests = 0
self.passed_tests = 0
self._sample = None
self.start_time = None
self.end_time = None
self.apiritif_extractor = ApiritifSampleExtractor()
def prepare(self):
if self._report_fds is None:
self._report_fds = open(self._report_path, 'w')
def post_process(self):
if self._report_fds is not None:
self._report_fds.close()
self._report_fds = None
def _write_sample(self, sample):
if self._report_fds is None:
raise ValueError("Plugin wasn't prepared")
self._report_fds.write("%s\n" % json.dumps(sample.to_dict()))
self._report_fds.flush()
def _write_stdout_report(self, label):
report_pattern = "%s,Total:%d Passed:%d Failed:%d\n"
sys.stdout.write(report_pattern % (label, self.test_count, self.passed_tests, self.failed_tests))
sys.stdout.flush()
def _fill_sample(self, report, call, item, status):
filename, lineno, _ = report.location
if self._sample is None:
self._sample = Sample()
self._sample.test_case = item.name
self._sample.test_suite = item.module.__name__
self._sample.start_time = call.start
self._sample.extras = {"filename": filename, "lineno": lineno}
self._sample.status = status
self._sample.duration = time.time() - self._sample.start_time
if call.excinfo is not None:
self._sample.error_msg = call.excinfo.exconly().strip()
self._sample.error_trace = "\n".join(traceback.format_tb(call.excinfo.tb)).strip()
if call.excinfo.errisinstance(AssertionError):
self._sample.add_assertion(call.excinfo.exconly())
self._sample.set_assertion_failed(call.excinfo.exconly(), str(call.excinfo.value),
str(call.excinfo.getrepr(style="native")))
def _report_sample(self, label):
if self._sample.status == "PASSED":
self.passed_tests += 1
elif self._sample.status in ["BROKEN", "FAILED"]:
self.failed_tests += 1
samples_processed = self._process_apiritif_samples(self._sample, label)
if not samples_processed:
self._process_sample(self._sample, label)
self._sample = None
def _process_sample(self, sample, label):
self._write_sample(sample)
self._write_stdout_report(label)
def _process_apiritif_samples(self, sample, label):
samples_processed = 0
recording = apiritif.recorder.pop_events(from_ts=self.start_time, to_ts=self.end_time)
if not recording:
return samples_processed
try:
samples = self.apiritif_extractor.parse_recording(recording, sample)
except BaseException as exc:
print("Warning: Couldn't parse recording: %s" % exc)
samples = []
for sample in samples:
samples_processed += 1
self._process_sample(sample, label)
return samples_processed
@pytest.mark.hookwrapper
def pytest_runtest_makereport(self, item, call):
outcome = (yield)
report = outcome.get_result()
filename, lineno, test_name = report.location
if report.when == 'call':
if report.passed:
self._fill_sample(report, call, item, "PASSED")
elif report.failed:
self._fill_sample(report, call, item, "FAILED")
elif report.skipped:
self._fill_sample(report, call, item, "SKIPPED")
elif report.when == 'setup':
self.test_count += 1
self.start_time = time.time()
if report.failed:
self._fill_sample(report, call, item, "BROKEN")
elif report.skipped:
self._fill_sample(report, call, item, "SKIPPED")
elif report.when == 'teardown':
if not report.passed:
if self._sample.status not in ["FAILED", "BROKEN"]:
self._fill_sample(report, call, item, "BROKEN")
else:
self._sample.status = "BROKEN"
self.end_time = time.time()
self._report_sample(test_name)
class RecordingPlugin(object):
def __init__(self, report_path):
self._report_path = report_path
self._report_fds = None
self.test_count = 0
self.failed_tests = 0
self.passed_tests = 0
self._sample = None
self.start_time = None
self.end_time = None
self.apiritif_extractor = ApiritifSampleExtractor()
def prepare(self):
if self._report_fds is None:
self._report_fds = open(self._report_path, 'w')
def post_process(self):
if self._report_fds is not None:
self._report_fds.close()
self._report_fds = None
def _write_sample(self, sample):
if self._report_fds is None:
raise ValueError("Plugin wasn't prepared")
self._report_fds.write("%s\n" % json.dumps(sample.to_dict()))
self._report_fds.flush()
def _write_stdout_report(self, label):
report_pattern = "%s,Total:%d Passed:%d Failed:%d\n"
sys.stdout.write(
report_pattern %
(label, self.test_count, self.passed_tests, self.failed_tests))
sys.stdout.flush()
def _fill_sample(self, report, call, item, status):
filename, lineno, _ = report.location
if self._sample is None:
self._sample = Sample()
self._sample.test_case = item.name
self._sample.test_suite = item.module.__name__
self._sample.start_time = call.start
self._sample.extras = {"filename": filename, "lineno": lineno}
self._sample.status = status
self._sample.duration = time.time() - self._sample.start_time
if call.excinfo is not None:
self._sample.error_msg = call.excinfo.exconly().strip()
self._sample.error_trace = "\n".join(
traceback.format_tb(call.excinfo.tb)).strip()
if call.excinfo.errisinstance(AssertionError):
self._sample.add_assertion(call.excinfo.exconly())
self._sample.set_assertion_failed(
call.excinfo.exconly(), str(call.excinfo.value),
str(call.excinfo.getrepr(style="native")))
def _report_sample(self, label):
if self._sample.status == "PASSED":
self.passed_tests += 1
elif self._sample.status in ["BROKEN", "FAILED"]:
self.failed_tests += 1
samples_processed = self._process_apiritif_samples(self._sample, label)
if not samples_processed:
self._process_sample(self._sample, label)
self._sample = None
def _process_sample(self, sample, label):
self._write_sample(sample)
self._write_stdout_report(label)
def _process_apiritif_samples(self, sample, label):
samples_processed = 0
recording = apiritif.recorder.pop_events(from_ts=self.start_time,
to_ts=self.end_time)
if not recording:
return samples_processed
try:
samples = self.apiritif_extractor.parse_recording(
recording, sample)
except BaseException as exc:
print("Warning: Couldn't parse recording: %s" % exc)
samples = []
for sample in samples:
samples_processed += 1
self._process_sample(sample, label)
return samples_processed
@pytest.mark.hookwrapper
def pytest_runtest_makereport(self, item, call):
outcome = (yield)
report = outcome.get_result()
filename, lineno, test_name = report.location
if report.when == 'call':
if report.passed:
self._fill_sample(report, call, item, "PASSED")
elif report.failed:
self._fill_sample(report, call, item, "FAILED")
elif report.skipped:
self._fill_sample(report, call, item, "SKIPPED")
elif report.when == 'setup':
self.test_count += 1
self.start_time = time.time()
if report.failed:
self._fill_sample(report, call, item, "BROKEN")
elif report.skipped:
self._fill_sample(report, call, item, "SKIPPED")
elif report.when == 'teardown':
if not report.passed:
if self._sample.status not in ["FAILED", "BROKEN"]:
self._fill_sample(report, call, item, "BROKEN")
else:
self._sample.status = "BROKEN"
self.end_time = time.time()
self._report_sample(test_name)
