def test_test_suite_started():
stream = StreamStub()
messages = TeamcityServiceMessages(output=stream, now=lambda: fixed_date)
messages.testSuiteStarted('suite emotion')
assert stream.observed_output.strip() == textwrap.dedent("""\
##teamcity[testSuiteStarted timestamp='2000-11-02T10:23:01.556' name='suite emotion']
""").strip().encode('utf-8')
def test_test_suite_started():
stream = StreamStub()
messages = TeamcityServiceMessages(output=stream, now=lambda: fixed_date)
messages.testSuiteStarted('suite emotion')
assert stream.observed_output.strip() == textwrap.dedent("""\
##teamcity[testSuiteStarted timestamp='2000-11-02T10:23:01.556' name='suite emotion']
""").strip().encode('utf-8')
class EchoTeamCityMessages(object):
def __init__(self, ):
self.tw = py.io.TerminalWriter(py.std.sys.stdout)
self.teamcity = TeamcityServiceMessages(self.tw)
self.currentSuite = None
def format_names(self, name):
split = '.py'
file, testname = name.split(split, 1)
if not testname:
testname = file
file = 'NO_TEST_FILE_FOUND'
testname = testname.replace("::()::", ".")
testname = testname.replace("::", ".")
testname = testname.strip(".")
return "".join([file, split]), testname
def pytest_runtest_logstart(self, nodeid, location):
file, testname = self.format_names(nodeid)
if not file == self.currentSuite:
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
self.currentSuite = file
self.teamcity.testSuiteStarted(self.currentSuite)
self.teamcity.testStarted(testname)
def pytest_runtest_logreport(self, report):
file, testname = self.format_names(report.nodeid)
if report.when == "call":
for (secname, data) in report.sections:
if 'stdout' in secname:
self.teamcity.testStdOut(testname, out=data)
elif 'stderr' in secname:
self.teamcity.testStdErr(testname, out=data)
if report.passed:
if report.when == "call": # ignore setup/teardown
duration = timedelta(seconds=report.duration)
self.teamcity.testFinished(testname, testDuration=duration)
elif report.failed:
if report.when == "call":
self.teamcity.testFailed(testname, str(report.location), str(report.longrepr))
duration = timedelta(seconds=report.duration)
self.teamcity.testFinished(testname, testDuration=duration) # report finished after the failure
elif report.skipped:
self.teamcity.testIgnored(testname, str(report.longrepr))
self.teamcity.testFinished(testname) # report finished after the skip
def pytest_sessionfinish(self, session, exitstatus, __multicall__):
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
class EchoTeamCityMessages(object):
def __init__(self, ):
self.tw = py.io.TerminalWriter(py.std.sys.stdout)
self.teamcity = TeamcityServiceMessages(self.tw)
self.currentSuite = None
def format_names(self, name):
split = '.py'
file, testname = name.split(split, 1)
if not testname:
testname = file
file = 'NO_TEST_FILE_FOUND'
testname = testname.replace("::()::", ".")
testname = testname.replace("::", ".")
testname = testname.strip(".")
return ("".join([file, split]), testname)
def pytest_runtest_logstart(self, nodeid, location):
file, testname = self.format_names(nodeid)
if not file == self.currentSuite:
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
self.currentSuite = file
self.teamcity.testSuiteStarted(self.currentSuite)
self.teamcity.testStarted(testname)
def pytest_runtest_logreport(self, report):
file, testname = self.format_names(report.nodeid)
if report.passed:
if report.when == "call": # ignore setup/teardown
self.teamcity.testFinished(testname)
elif report.failed:
if report.when == "call":
self.teamcity.testFailed(testname, str(report.location),
str(report.longrepr))
self.teamcity.testFinished(
testname) # report finished after the failure
elif report.skipped:
self.teamcity.testIgnored(testname, str(report.longrepr))
self.teamcity.testFinished(
testname) # report finished after the skip
def pytest_sessionfinish(self, session, exitstatus, __multicall__):
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
class TeamCityReport(Report):
'''TeamCity streaming reporter for a test suite'''
def __init__(self, file, info, sync=True, **kwargs):
self.messages = TeamcityServiceMessages(file)
self.messages.message('compile-info',
name=info[0],
date=info[1],
time=info[2])
self.sync = sync
def __call__(self, index, args, info):
cls = TeamCityLazyReport if self.sync else TeamCityTestReport
return cls(self.messages, args, info[0])
def __enter__(self):
self.messages.testSuiteStarted('default-suite')
return self
def __exit__(self, value, cls, traceback):
self.messages.testSuiteFinished('default-suite')
def get_file_results(self):
self._deferred_print.sort()
messages = TeamcityServiceMessages()
normalized_filename = self.filename.replace("\\", "/")
suite_name = 'pep8: %s' % normalized_filename
messages.testSuiteStarted(suite_name)
for line_number, offset, code, text, doc in self._deferred_print:
position = '%(path)s:%(row)d:%(col)d' % {
'path': normalized_filename,
'row': self.line_offset + line_number,
'col': offset + 1,
}
error_message = '%s: %s' % (code, text)
test_name = '%s: %s' % (code, position)
messages.testStarted(test_name)
if line_number > len(self.lines):
line = ''
else:
line = self.lines[line_number - 1]
details = [
line.rstrip(),
re.sub(r'\S', ' ', line[:offset]) + '^',
]
if doc:
details.append(doc.strip())
details = '\n'.join(details)
messages.testFailed(test_name, error_message, details)
messages.testFinished(test_name)
messages.testSuiteFinished(suite_name)
return self.file_errors
def get_file_results(self):
self._deferred_print.sort()
messages = TeamcityServiceMessages()
normalized_filename = self.filename.replace("\\", "/")
suite_name = 'pep8: %s' % normalized_filename
messages.testSuiteStarted(suite_name)
for line_number, offset, code, text, doc in self._deferred_print:
position = '%(path)s:%(row)d:%(col)d' % {
'path': normalized_filename,
'row': self.line_offset + line_number,
'col': offset + 1,
}
error_message = '%s: %s' % (code, text)
test_name = '%s: %s' % (code, position)
messages.testStarted(test_name)
if line_number > len(self.lines):
line = ''
else:
line = self.lines[line_number - 1]
details = [
line.rstrip(),
re.sub(r'\S', ' ', line[:offset]) + '^',
]
if doc:
details.append(doc.strip())
details = '\n'.join(details)
messages.testFailed(test_name, error_message, details)
messages.testFinished(test_name)
messages.testSuiteFinished(suite_name)
return self.file_errors
class EchoTeamCityMessages(object):
def __init__(self, ):
self.tw = py.io.TerminalWriter(py.std.sys.stdout)
self.teamcity = TeamcityServiceMessages(self.tw)
self.currentSuite = None
def format_names(self, name):
if name.find("::") > 0:
file, testname = name.split("::", 1)
else:
file, testname = name, "top_level"
testname = testname.replace("::()::", ".")
testname = testname.replace("::", ".")
testname = testname.strip(".")
file = file.replace(".", "_").replace(os.sep, ".").replace("/", ".")
return file, testname
def pytest_runtest_logstart(self, nodeid, location):
file, testname = self.format_names(nodeid)
if not file == self.currentSuite:
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
self.currentSuite = file
self.teamcity.testSuiteStarted(self.currentSuite)
self.teamcity.testStarted(testname)
def pytest_runtest_logreport(self, report):
file, testname = self.format_names(report.nodeid)
if report.passed:
if report.when == "call": # ignore setup/teardown
duration = timedelta(seconds=report.duration)
self.teamcity.testFinished(testname, testDuration=duration)
elif report.failed:
if report.when in ("call", "setup"):
self.teamcity.testFailed(testname, str(report.location), str(report.longrepr))
duration = timedelta(seconds=report.duration)
self.teamcity.testFinished(testname, testDuration=duration) # report finished after the failure
elif report.when == "teardown":
name = testname + "_teardown"
self.teamcity.testStarted(name)
self.teamcity.testFailed(name, str(report.location), str(report.longrepr))
self.teamcity.testFinished(name)
elif report.skipped:
self.teamcity.testIgnored(testname, str(report.longrepr))
self.teamcity.testFinished(testname) # report finished after the skip
def pytest_collectreport(self, report):
if report.failed:
file, testname = self.format_names(report.nodeid)
name = file + "_collect"
self.teamcity.testStarted(name)
self.teamcity.testFailed(name, str(report.location), str(report.longrepr))
self.teamcity.testFinished(name)
def pytest_sessionfinish(self, session, exitstatus, __multicall__):
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
class TeamcityFormatter(Formatter):
"""
Stateful TC reporter.
Since we can't fetch all steps from the very beginning (even skipped tests are reported)
we store tests and features on each call.
To hook into test reporting override _report_suite_started and/or _report_test_started
"""
def __init__(self, stream_opener, config):
super(TeamcityFormatter, self).__init__(stream_opener, config)
assert version.LooseVersion(behave_version) >= version.LooseVersion(
"1.2.6"), "Only 1.2.6+ is supported"
self._messages = TeamcityServiceMessages()
self.__feature = None
self.__scenario = None
self.__steps = deque()
self.__scenario_opened = False
self.__feature_opened = False
self.__test_start_time = None
def feature(self, feature):
assert isinstance(feature, Feature)
assert not self.__feature, "Prev. feature not closed"
self.__feature = feature
def scenario(self, scenario):
assert isinstance(scenario, Scenario)
self.__scenario = scenario
self.__scenario_opened = False
def step(self, step):
assert isinstance(step, Step)
self.__steps.append(step)
def match(self, match):
if not self.__feature_opened:
self._report_suite_started(self.__feature,
_suite_name(self.__feature))
self.__feature_opened = True
if not self.__scenario_opened:
self._report_suite_started(self.__scenario,
_suite_name(self.__scenario))
self.__scenario_opened = True
assert self.__steps, "No steps left"
step = self.__steps.popleft()
self._report_test_started(step, _step_name(step))
self.__test_start_time = datetime.datetime.now()
def _report_suite_started(self, suite, suite_name):
"""
:param suite: behave suite
:param suite_name: suite name that must be reported, be sure to use it instead of suite.name
"""
self._messages.testSuiteStarted(suite_name)
def _report_test_started(self, test, test_name):
"""
Suite name is always stripped, be sure to strip() it too
:param test: behave test
:param test_name: test name that must be reported, be sure to use it instead of test.name
"""
self._messages.testStarted(test_name)
def result(self, step):
assert isinstance(step, Step)
step_name = _step_name(step)
if step.status == Status.failed:
try:
error = traceback.format_exc(step.exc_traceback)
if error != step.error_message:
self._messages.testStdErr(step_name, error)
except Exception:
pass # exception shall not prevent error message
self._messages.testFailed(step_name, message=step.error_message)
if step.status == Status.undefined:
self._messages.testFailed(step_name, message="Undefined")
if step.status == Status.skipped:
self._messages.testIgnored(step_name)
self._messages.testFinished(step_name,
testDuration=datetime.datetime.now() -
self.__test_start_time)
if not self.__steps:
self.__close_scenario()
elif step.status in [Status.failed, Status.undefined]:
# Broken background/undefined step stops whole scenario
reason = "Undefined step" if step.status == Status.undefined else "Prev. step failed"
self.__skip_rest_of_scenario(reason)
def __skip_rest_of_scenario(self, reason):
while self.__steps:
step = self.__steps.popleft()
self._report_test_started(step, _step_name(step))
self._messages.testIgnored(
_step_name(step),
message="{0}. Rest part of scenario is skipped".format(reason))
self._messages.testFinished(_step_name(step))
self.__close_scenario()
def __close_scenario(self):
if self.__scenario:
self._messages.testSuiteFinished(_suite_name(self.__scenario))
self.__scenario = None
def eof(self):
self.__skip_rest_of_scenario("")
self._messages.testSuiteFinished(_suite_name(self.__feature))
self.__feature = None
self.__feature_opened = False
class TeamcityFormatter(Formatter):
"""
Stateful TC reporter.
Since we can't fetch all steps from the very beginning (even skipped tests are reported)
we store tests and features on each call.
To hook into test reporting override _report_suite_started and/or _report_test_started
"""
def __init__(self, stream_opener, config):
super(TeamcityFormatter, self).__init__(stream_opener, config)
assert version.LooseVersion(behave_version) >= version.LooseVersion("1.2.6"), "Only 1.2.6+ is supported"
self._messages = TeamcityServiceMessages()
self.__feature = None
self.__scenario = None
self.__steps = deque()
self.__scenario_opened = False
self.__feature_opened = False
self.__test_start_time = None
def feature(self, feature):
assert isinstance(feature, Feature)
assert not self.__feature, "Prev. feature not closed"
self.__feature = feature
def scenario(self, scenario):
assert isinstance(scenario, Scenario)
self.__scenario = scenario
self.__scenario_opened = False
self.__steps.clear()
def step(self, step):
assert isinstance(step, Step)
self.__steps.append(step)
def match(self, match):
if not self.__feature_opened:
self._report_suite_started(self.__feature, _suite_name(self.__feature))
self.__feature_opened = True
if not self.__scenario_opened:
self._report_suite_started(self.__scenario, _suite_name(self.__scenario))
self.__scenario_opened = True
assert self.__steps, "No steps left"
step = self.__steps.popleft()
self._report_test_started(step, _step_name(step))
self.__test_start_time = datetime.datetime.now()
def _report_suite_started(self, suite, suite_name):
"""
:param suite: behave suite
:param suite_name: suite name that must be reported, be sure to use it instead of suite.name
"""
self._messages.testSuiteStarted(suite_name)
def _report_test_started(self, test, test_name):
"""
Suite name is always stripped, be sure to strip() it too
:param test: behave test
:param test_name: test name that must be reported, be sure to use it instead of test.name
"""
self._messages.testStarted(test_name)
def result(self, step):
assert isinstance(step, Step)
step_name = _step_name(step)
if step.status == Status.failed:
try:
error = traceback.format_exc(step.exc_traceback)
if error != step.error_message:
self._messages.testStdErr(step_name, error)
except Exception:
pass # exception shall not prevent error message
self._messages.testFailed(step_name, message=step.error_message)
if step.status == Status.undefined:
self._messages.testFailed(step_name, message="Undefined")
if step.status == Status.skipped:
self._messages.testIgnored(step_name)
self._messages.testFinished(step_name, testDuration=datetime.datetime.now() - self.__test_start_time)
if not self.__steps:
self.__close_scenario()
elif step.status in [Status.failed, Status.undefined]:
# Broken background/undefined step stops whole scenario
reason = "Undefined step" if step.status == Status.undefined else "Prev. step failed"
self.__skip_rest_of_scenario(reason)
def __skip_rest_of_scenario(self, reason):
while self.__steps:
step = self.__steps.popleft()
self._report_test_started(step, _step_name(step))
self._messages.testIgnored(_step_name(step),
message="{0}. Rest part of scenario is skipped".format(reason))
self._messages.testFinished(_step_name(step))
self.__close_scenario()
def __close_scenario(self):
if self.__scenario:
self._messages.testSuiteFinished(_suite_name(self.__scenario))
self.__scenario = None
def eof(self):
self.__skip_rest_of_scenario("")
self._messages.testSuiteFinished(_suite_name(self.__feature))
self.__feature = None
self.__feature_opened = False
class EchoTeamCityMessages(object):
def __init__(self, ):
self.tw = py.io.TerminalWriter(py.std.sys.stdout)
self.teamcity = TeamcityServiceMessages(self.tw)
self.currentSuite = None
def format_names(self, name):
if name.find("::") > 0:
file, testname = name.split("::", 1)
else:
file, testname = name, "top_level"
testname = testname.replace("::()::", ".")
testname = testname.replace("::", ".")
testname = testname.strip(".")
file = file.replace(".", "_").replace(os.sep, ".").replace("/", ".")
return file, testname
def pytest_runtest_logstart(self, nodeid, location):
file, testname = self.format_names(nodeid)
if not file == self.currentSuite:
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
self.currentSuite = file
self.teamcity.testSuiteStarted(self.currentSuite)
self.teamcity.testStarted(testname)
def pytest_runtest_logreport(self, report):
file, testname = self.format_names(report.nodeid)
if report.passed:
if report.when == "call": # ignore setup/teardown
duration = timedelta(seconds=report.duration)
self.teamcity.testFinished(testname, testDuration=duration)
elif report.failed:
if report.when in ("call", "setup"):
self.teamcity.testFailed(testname, str(report.location),
str(report.longrepr))
duration = timedelta(seconds=report.duration)
self.teamcity.testFinished(
testname,
testDuration=duration) # report finished after the failure
elif report.when == "teardown":
name = testname + "_teardown"
self.teamcity.testStarted(name)
self.teamcity.testFailed(name, str(report.location),
str(report.longrepr))
self.teamcity.testFinished(name)
elif report.skipped:
self.teamcity.testIgnored(testname, str(report.longrepr))
self.teamcity.testFinished(
testname) # report finished after the skip
def pytest_collectreport(self, report):
if report.failed:
file, testname = self.format_names(report.nodeid)
name = file + "_collect"
self.teamcity.testStarted(name)
self.teamcity.testFailed(name, str(report.location),
str(report.longrepr))
self.teamcity.testFinished(name)
def pytest_sessionfinish(self, session, exitstatus, __multicall__):
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
Print the teamcity service messages for associating op/s benchmark result
to the tests.
Requires the teamcity-messages python library:
pip3 install teamcity-messages
""".format(sys.argv[0]))
sys.exit(1)
suite_name = sys.argv[1]
with open(sys.argv[2], encoding='utf-8') as f:
json_results = json.load(f)
teamcity_messages = TeamcityServiceMessages()
teamcity_messages.testSuiteStarted(suite_name)
def testMetadata_number_message(test_name, param_name, param_value):
teamcity_messages.message(
'testMetadata',
type='number',
testName=test_name,
name=param_name,
value='{:.2f}'.format(param_value),
)
for result in json_results.get('results', []):
test_name = result['name']
