def test_test_stderr():
stream = StreamStub()
messages = TeamcityServiceMessages(output=stream, now=lambda: fixed_date)
messages.testStdErr(testName='only a test', out='out')
assert stream.observed_output.strip() == textwrap.dedent("""\
##teamcity[testStdErr timestamp='2000-11-02T10:23:01.556' name='only a test' out='out']
""").strip().encode('utf-8')
def test_test_stderr():
stream = StreamStub()
messages = TeamcityServiceMessages(output=stream, now=lambda: fixed_date)
messages.testStdErr(testName='only a test', out='out')
assert stream.observed_output.strip() == textwrap.dedent("""\
##teamcity[testStdErr timestamp='2000-11-02T10:23:01.556' name='only a test' out='out']
""").strip().encode('utf-8')
class EchoTeamCityMessages(object):
def __init__(self, ):
self.tw = py.io.TerminalWriter(py.std.sys.stdout)
self.teamcity = TeamcityServiceMessages(self.tw)
self.currentSuite = None
def format_names(self, name):
split = '.py'
file, testname = name.split(split, 1)
if not testname:
testname = file
file = 'NO_TEST_FILE_FOUND'
testname = testname.replace("::()::", ".")
testname = testname.replace("::", ".")
testname = testname.strip(".")
return "".join([file, split]), testname
def pytest_runtest_logstart(self, nodeid, location):
file, testname = self.format_names(nodeid)
if not file == self.currentSuite:
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
self.currentSuite = file
self.teamcity.testSuiteStarted(self.currentSuite)
self.teamcity.testStarted(testname)
def pytest_runtest_logreport(self, report):
file, testname = self.format_names(report.nodeid)
if report.when == "call":
for (secname, data) in report.sections:
if 'stdout' in secname:
self.teamcity.testStdOut(testname, out=data)
elif 'stderr' in secname:
self.teamcity.testStdErr(testname, out=data)
if report.passed:
if report.when == "call": # ignore setup/teardown
duration = timedelta(seconds=report.duration)
self.teamcity.testFinished(testname, testDuration=duration)
elif report.failed:
if report.when == "call":
self.teamcity.testFailed(testname, str(report.location), str(report.longrepr))
duration = timedelta(seconds=report.duration)
self.teamcity.testFinished(testname, testDuration=duration) # report finished after the failure
elif report.skipped:
self.teamcity.testIgnored(testname, str(report.longrepr))
self.teamcity.testFinished(testname) # report finished after the skip
def pytest_sessionfinish(self, session, exitstatus, __multicall__):
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
class TeamcityTestResult(TestResult):
separator2 = "\n"
# noinspection PyUnusedLocal
def __init__(self, stream=_real_stdout, descriptions=None, verbosity=None):
super(TeamcityTestResult, self).__init__()
# Some code may ask for self.failfast, see unittest2.case.TestCase.subTest
self.failfast = getattr(self, "failfast", False)
self.test_started_datetime_map = {}
self.failed_tests = set()
self.subtest_failures = {}
self.messages = TeamcityServiceMessages(_real_stdout)
@staticmethod
def get_test_id(test):
if is_string(test):
return test
# Force test_id for doctests
if get_class_fullname(test) != "doctest.DocTestCase":
desc = test.shortDescription()
test_method_name = getattr(test, "_testMethodName", "")
if desc and desc != test.id() and desc != test_method_name:
return "%s (%s)" % (test.id(), desc.replace('.', '_'))
return test.id()
def addSuccess(self, test):
super(TeamcityTestResult, self).addSuccess(test)
def addExpectedFailure(self, test, err):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addExpectedFailure"):
_super.addExpectedFailure(test, err)
err = convert_error_to_string(err)
test_id = self.get_test_id(test)
self.messages.testIgnored(test_id,
message="Expected failure: " + err,
flowId=test_id)
def get_subtest_block_id(self, test, subtest):
test_id = self.get_test_id(test)
subtest_id = self.get_test_id(subtest)
if subtest_id.startswith(test_id):
block_id = subtest_id[len(test_id):].strip()
else:
block_id = subtest_id
if len(block_id) == 0:
block_id = test_id
return block_id
def addSkip(self, test, reason=""):
if sys.version_info >= (2, 7):
super(TeamcityTestResult, self).addSkip(test, reason)
if reason:
reason_str = ": " + str(reason)
else:
reason_str = ""
test_class_name = get_class_fullname(test)
if test_class_name == "unittest.case._SubTest" or test_class_name == "unittest2.case._SubTest":
parent_test = test.test_case
parent_test_id = self.get_test_id(parent_test)
subtest = test
block_id = self.get_subtest_block_id(parent_test, subtest)
self.messages.subTestBlockOpened(block_id,
subTestResult="Skip",
flowId=parent_test_id)
self.messages.testStdOut(parent_test_id,
out="SubTest skipped" + reason_str + "\n",
flowId=parent_test_id)
self.messages.blockClosed(block_id, flowId=parent_test_id)
else:
test_id = self.get_test_id(test)
self.messages.testIgnored(test_id,
message="Skipped" + reason_str,
flowId=test_id)
def addUnexpectedSuccess(self, test):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addUnexpectedSuccess"):
_super.addUnexpectedSuccess(test)
test_id = self.get_test_id(test)
self.messages.testFailed(
test_id,
message='Failure',
details=
"Test should not succeed since it's marked with @unittest.expectedFailure",
flowId=test_id)
def addError(self, test, err, *k):
super(TeamcityTestResult, self).addError(test, err)
test_class = get_class_fullname(test)
if test_class == "unittest.suite._ErrorHolder" or test_class == "unittest2.suite._ErrorHolder":
# This is a standalone error
test_name = test.id()
# patch setUpModule (__main__) -> __main__.setUpModule
test_name = re.sub(r'^(.*) \((.*)\)$', r'\2.\1', test_name)
self.messages.testStarted(test_name, flowId=test_name)
# noinspection PyTypeChecker
self.report_fail(test_name, 'Failure', err)
self.messages.testFinished(test_name, flowId=test_name)
elif get_class_fullname(err[0]) == "unittest2.case.SkipTest":
message = ""
if hasattr(err[1], "message"):
message = getattr(err[1], "message", "")
elif hasattr(err[1], "args"):
message = getattr(err[1], "args", [""])[0]
self.addSkip(test, message)
else:
self.report_fail(test, 'Error', err)
def addFailure(self, test, err, *k):
super(TeamcityTestResult, self).addFailure(test, err)
self.report_fail(test, 'Failure', err)
def addSubTest(self, test, subtest, err):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addSubTest"):
_super.addSubTest(test, subtest, err)
test_id = self.get_test_id(test)
subtest_id = self.get_test_id(subtest)
if subtest_id.startswith(test_id):
# Replace "." -> "_" since '.' is a test hierarchy separator
# See i.e. https://github.com/JetBrains/teamcity-messages/issues/134 (https://youtrack.jetbrains.com/issue/PY-23846)
block_id = subtest_id[len(test_id):].strip().replace(".", "_")
else:
block_id = subtest_id
if len(block_id) == 0:
block_id = subtest_id
if err is not None:
self.add_subtest_failure(test_id, block_id)
if issubclass(err[0], test.failureException):
self.messages.subTestBlockOpened(block_id,
subTestResult="Failure",
flowId=test_id)
self.messages.testStdErr(test_id,
out="SubTest failure: %s\n" %
convert_error_to_string(err),
flowId=test_id)
self.messages.blockClosed(block_id, flowId=test_id)
else:
self.messages.subTestBlockOpened(block_id,
subTestResult="Error",
flowId=test_id)
self.messages.testStdErr(test_id,
out="SubTest error: %s\n" %
convert_error_to_string(err),
flowId=test_id)
self.messages.blockClosed(block_id, flowId=test_id)
else:
self.messages.subTestBlockOpened(block_id,
subTestResult="Success",
flowId=test_id)
self.messages.blockClosed(block_id, flowId=test_id)
def add_subtest_failure(self, test_id, subtest_block_id):
fail_array = self.subtest_failures.get(test_id, [])
fail_array.append(subtest_block_id)
self.subtest_failures[test_id] = fail_array
def get_subtest_failure(self, test_id):
fail_array = self.subtest_failures.get(test_id, [])
return ", ".join(fail_array)
def report_fail(self, test, fail_type, err):
test_id = self.get_test_id(test)
if is_string(err):
details = err
elif get_class_fullname(err) == "twisted.python.failure.Failure":
details = err.getTraceback()
else:
details = convert_error_to_string(err)
subtest_failures = self.get_subtest_failure(test_id)
if subtest_failures:
details = "Failed subtests list: " + subtest_failures + "\n\n" + details.strip(
)
details = details.strip()
self.messages.testFailed(test_id,
message=fail_type,
details=details,
flowId=test_id)
self.failed_tests.add(test_id)
def startTest(self, test):
super(TeamcityTestResult, self).startTest(test)
test_id = self.get_test_id(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id,
captureStandardOutput='true',
flowId=test_id)
def stopTest(self, test):
test_id = self.get_test_id(test)
if getattr(self, 'buffer', None):
# Do not allow super() method to print output by itself
self._mirrorOutput = False
output = sys.stdout.getvalue()
if output:
for chunk in split_output(limit_output(output)):
self.messages.testStdOut(test_id, chunk, flowId=test_id)
error = sys.stderr.getvalue()
if error:
for chunk in split_output(limit_output(error)):
self.messages.testStdErr(test_id, chunk, flowId=test_id)
super(TeamcityTestResult, self).stopTest(test)
if test_id not in self.failed_tests:
subtest_failures = self.get_subtest_failure(test_id)
if subtest_failures:
self.report_fail(test, "One or more subtests failed", "")
time_diff = datetime.datetime.now(
) - self.test_started_datetime_map[test_id]
self.messages.testFinished(test_id,
testDuration=time_diff,
flowId=test_id)
def printErrors(self):
pass
class TeamcityFormatter(Formatter):
"""
Stateful TC reporter.
Since we can't fetch all steps from the very beginning (even skipped tests are reported)
we store tests and features on each call.
To hook into test reporting override _report_suite_started and/or _report_test_started
"""
def __init__(self, stream_opener, config):
super(TeamcityFormatter, self).__init__(stream_opener, config)
assert version.LooseVersion(behave_version) >= version.LooseVersion(
"1.2.6"), "Only 1.2.6+ is supported"
self._messages = TeamcityServiceMessages()
self.__feature = None
self.__scenario = None
self.__steps = deque()
self.__scenario_opened = False
self.__feature_opened = False
self.__test_start_time = None
def feature(self, feature):
assert isinstance(feature, Feature)
assert not self.__feature, "Prev. feature not closed"
self.__feature = feature
def scenario(self, scenario):
assert isinstance(scenario, Scenario)
self.__scenario = scenario
self.__scenario_opened = False
def step(self, step):
assert isinstance(step, Step)
self.__steps.append(step)
def match(self, match):
if not self.__feature_opened:
self._report_suite_started(self.__feature,
_suite_name(self.__feature))
self.__feature_opened = True
if not self.__scenario_opened:
self._report_suite_started(self.__scenario,
_suite_name(self.__scenario))
self.__scenario_opened = True
assert self.__steps, "No steps left"
step = self.__steps.popleft()
self._report_test_started(step, _step_name(step))
self.__test_start_time = datetime.datetime.now()
def _report_suite_started(self, suite, suite_name):
"""
:param suite: behave suite
:param suite_name: suite name that must be reported, be sure to use it instead of suite.name
"""
self._messages.testSuiteStarted(suite_name)
def _report_test_started(self, test, test_name):
"""
Suite name is always stripped, be sure to strip() it too
:param test: behave test
:param test_name: test name that must be reported, be sure to use it instead of test.name
"""
self._messages.testStarted(test_name)
def result(self, step):
assert isinstance(step, Step)
step_name = _step_name(step)
if step.status == Status.failed:
try:
error = traceback.format_exc(step.exc_traceback)
if error != step.error_message:
self._messages.testStdErr(step_name, error)
except Exception:
pass # exception shall not prevent error message
self._messages.testFailed(step_name, message=step.error_message)
if step.status == Status.undefined:
self._messages.testFailed(step_name, message="Undefined")
if step.status == Status.skipped:
self._messages.testIgnored(step_name)
self._messages.testFinished(step_name,
testDuration=datetime.datetime.now() -
self.__test_start_time)
if not self.__steps:
self.__close_scenario()
elif step.status in [Status.failed, Status.undefined]:
# Broken background/undefined step stops whole scenario
reason = "Undefined step" if step.status == Status.undefined else "Prev. step failed"
self.__skip_rest_of_scenario(reason)
def __skip_rest_of_scenario(self, reason):
while self.__steps:
step = self.__steps.popleft()
self._report_test_started(step, _step_name(step))
self._messages.testIgnored(
_step_name(step),
message="{0}. Rest part of scenario is skipped".format(reason))
self._messages.testFinished(_step_name(step))
self.__close_scenario()
def __close_scenario(self):
if self.__scenario:
self._messages.testSuiteFinished(_suite_name(self.__scenario))
self.__scenario = None
def eof(self):
self.__skip_rest_of_scenario("")
self._messages.testSuiteFinished(_suite_name(self.__feature))
self.__feature = None
self.__feature_opened = False
class TeamcityTestResult(TestResult):
separator2 = "\n"
def __init__(self, stream=_real_stdout, descriptions=None, verbosity=None):
super(TeamcityTestResult, self).__init__()
self.test_started_datetime_map = {}
self.failed_tests = set()
self.subtest_failures = {}
self.messages = TeamcityServiceMessages(_real_stdout)
def get_test_id(self, test):
if is_string(test):
return test
# Force test_id for doctests
if get_class_fullname(test) != "doctest.DocTestCase":
desc = test.shortDescription()
test_method_name = getattr(test, "_testMethodName", "")
if desc and desc != test.id() and desc != test_method_name:
return "%s (%s)" % (test.id(), desc.replace('.', '_'))
return test.id()
def addSuccess(self, test):
super(TeamcityTestResult, self).addSuccess(test)
def addExpectedFailure(self, test, err):
super(TeamcityTestResult, self).addExpectedFailure(test, err)
err = convert_error_to_string(err)
test_id = self.get_test_id(test)
self.messages.testIgnored(test_id,
message="Expected failure: " + err,
flowId=test_id)
def addSkip(self, test, reason=""):
if sys.version_info >= (2, 7):
super(TeamcityTestResult, self).addSkip(test, reason)
test_id = self.get_test_id(test)
if reason:
reason_str = ": " + str(reason)
else:
reason_str = ""
self.messages.testIgnored(test_id,
message="Skipped" + reason_str,
flowId=test_id)
def addUnexpectedSuccess(self, test):
super(TeamcityTestResult, self).addUnexpectedSuccess(test)
test_id = self.get_test_id(test)
self.messages.testFailed(
test_id,
message='Failure',
details=
"Test should not succeed since it's marked with @unittest.expectedFailure",
flowId=test_id)
def addError(self, test, err, *k):
super(TeamcityTestResult, self).addError(test, err)
if get_class_fullname(test) == "unittest.suite._ErrorHolder":
# This is a standalone error
test_name = test.id()
# patch setUpModule (__main__) -> __main__.setUpModule
test_name = re.sub(r'^(.*) \((.*)\)$', r'\2.\1', test_name)
self.messages.testStarted(test_name, flowId=test_name)
self.report_fail(test_name, 'Failure', err)
self.messages.testFinished(test_name, flowId=test_name)
elif get_class_fullname(err[0]) == "unittest2.case.SkipTest":
message = ""
if hasattr(err[1], "message"):
message = getattr(err[1], "message", "")
elif hasattr(err[1], "args"):
message = getattr(err[1], "args", [""])[0]
self.addSkip(test, message)
else:
self.report_fail(test, 'Error', err)
def addFailure(self, test, err, *k):
super(TeamcityTestResult, self).addFailure(test, err)
self.report_fail(test, 'Failure', err)
def addSubTest(self, test, subtest, err):
super(TeamcityTestResult, self).addSubTest(test, subtest, err)
test_id = self.get_test_id(test)
if err is not None:
if issubclass(err[0], test.failureException):
self.add_subtest_failure(test_id, self.get_test_id(subtest),
err)
self.messages.testStdErr(test_id,
out="%s: failure\n" %
self.get_test_id(subtest),
flowId=test_id)
else:
self.add_subtest_failure(test_id, self.get_test_id(subtest),
err)
self.messages.testStdErr(test_id,
out="%s: error\n" %
self.get_test_id(subtest),
flowId=test_id)
else:
self.messages.testStdOut(test_id,
out="%s: ok\n" %
self.get_test_id(subtest),
flowId=test_id)
def add_subtest_failure(self, test_id, subtest_id, err):
fail_array = self.subtest_failures.get(test_id, [])
fail_array.append("%s:\n%s" %
(subtest_id, convert_error_to_string(err)))
self.subtest_failures[test_id] = fail_array
def get_subtest_failure(self, test_id):
fail_array = self.subtest_failures.get(test_id, [])
return "\n".join(fail_array)
def report_fail(self, test, fail_type, err):
test_id = self.get_test_id(test)
if is_string(err):
details = err
elif get_class_fullname(err) == "twisted.python.failure.Failure":
details = err.getTraceback()
else:
details = convert_error_to_string(err)
subtest_failure = self.get_subtest_failure(test_id)
if subtest_failure:
details = subtest_failure + "\n" + details
self.messages.testFailed(test_id,
message=fail_type,
details=details,
flowId=test_id)
self.failed_tests.add(test_id)
def startTest(self, test):
super(TeamcityTestResult, self).startTest(test)
test_id = self.get_test_id(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id,
captureStandardOutput='true',
flowId=test_id)
def stopTest(self, test):
test_id = self.get_test_id(test)
if self.buffer:
# Do not allow super() method to print output by itself
self._mirrorOutput = False
output = sys.stdout.getvalue()
if output:
for chunk in split_output(limit_output(output)):
self.messages.testStdOut(test_id, chunk, flowId=test_id)
error = sys.stderr.getvalue()
if error:
for chunk in split_output(limit_output(error)):
self.messages.testStdErr(test_id, chunk, flowId=test_id)
super(TeamcityTestResult, self).stopTest(test)
if test_id not in self.failed_tests and self.subtest_failures.get(
test_id, []):
self.report_fail(test, "Subtest failed", "")
time_diff = datetime.datetime.now(
) - self.test_started_datetime_map[test_id]
self.messages.testFinished(test_id,
testDuration=time_diff,
flowId=test_id)
def printErrors(self):
pass
class TeamcityTestResult(TestResult):
separator2 = "\n"
# noinspection PyUnusedLocal
def __init__(self, stream=_real_stdout, descriptions=None, verbosity=None):
super(TeamcityTestResult, self).__init__()
# Some code may ask for self.failfast, see unittest2.case.TestCase.subTest
self.failfast = getattr(self, "failfast", False)
self.test_started_datetime_map = {}
self.failed_tests = set()
self.subtest_failures = {}
self.messages = TeamcityServiceMessages(_real_stdout)
self.current_test_id = None
@staticmethod
def get_test_id_with_description(test):
if is_string(test):
return test
test_class_fullname = get_class_fullname(test)
test_id = test.id()
if test_class_fullname in _ERROR_HOLDERS_FQN:
# patch setUpModule (__main__) -> __main__.setUpModule
return re.sub(r'^(.*) \((.*)\)$', r'\2.\1', test_id)
# Force test_id for doctests
if test_class_fullname != "doctest.DocTestCase":
desc = test.shortDescription()
test_method_name = getattr(test, "_testMethodName", "")
if desc and desc != test_id and desc != test_method_name:
return "%s (%s)" % (test_id, desc.replace('.', '_'))
return test_id
def addSuccess(self, test):
super(TeamcityTestResult, self).addSuccess(test)
def addExpectedFailure(self, test, err):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addExpectedFailure"):
_super.addExpectedFailure(test, err)
err = convert_error_to_string(err)
test_id = self.get_test_id_with_description(test)
self.messages.testIgnored(test_id,
message="Expected failure: " + err,
flowId=test_id)
def get_subtest_block_id(self, test, subtest):
test_id = self.get_test_id_with_description(test)
subtest_id = self.get_test_id_with_description(subtest)
if subtest_id.startswith(test_id):
block_id = subtest_id[len(test_id):].strip()
else:
block_id = subtest_id
if len(block_id) == 0:
block_id = test_id
return block_id
def addSkip(self, test, reason=""):
if sys.version_info >= (2, 7):
super(TeamcityTestResult, self).addSkip(test, reason)
if reason:
if isinstance(reason, Exception):
reason_str = ": " + get_exception_message(reason)
else:
reason_str = ": " + to_unicode(reason)
else:
reason_str = ""
test_class_name = get_class_fullname(test)
if test_class_name == "unittest.case._SubTest" or test_class_name == "unittest2.case._SubTest":
parent_test = test.test_case
parent_test_id = self.get_test_id_with_description(parent_test)
subtest = test
block_id = self.get_subtest_block_id(parent_test, subtest)
self.messages.subTestBlockOpened(block_id,
subTestResult="Skip",
flowId=parent_test_id)
self.messages.testStdOut(parent_test_id,
out="SubTest skipped" + reason_str + "\n",
flowId=parent_test_id)
self.messages.blockClosed(block_id, flowId=parent_test_id)
else:
test_id = self.get_test_id_with_description(test)
if test_id not in self.test_started_datetime_map:
# Test ignored without startTest. Handle start and finish events ourselves
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testIgnored(test_id,
message="Skipped" + reason_str,
flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
else:
self.messages.testIgnored(test_id,
message="Skipped" + reason_str,
flowId=test_id)
def addUnexpectedSuccess(self, test):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addUnexpectedSuccess"):
_super.addUnexpectedSuccess(test)
test_id = self.get_test_id_with_description(test)
self.messages.testFailed(
test_id,
message='Failure',
details=
"Test should not succeed since it's marked with @unittest.expectedFailure",
flowId=test_id)
def addError(self, test, err, *k):
super(TeamcityTestResult, self).addError(test, err)
test_class = get_class_fullname(test)
if test_class in _ERROR_HOLDERS_FQN:
# This is a standalone error
test_id = self.get_test_id_with_description(test)
self.messages.testStarted(test_id, flowId=test_id)
self.report_fail(test, 'Failure', err)
self.messages.testFinished(test_id, flowId=test_id)
elif get_class_fullname(err[0]) == "unittest2.case.SkipTest":
message = ""
if hasattr(err[1], "message"):
message = getattr(err[1], "message", "")
elif hasattr(err[1], "args"):
message = getattr(err[1], "args", [""])[0]
self.addSkip(test, message)
else:
self.report_fail(test, 'Error', err)
def addFailure(self, test, err, *k):
super(TeamcityTestResult, self).addFailure(test, err)
self.report_fail(test, 'Failure', err)
def addSubTest(self, test, subtest, err):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addSubTest"):
_super.addSubTest(test, subtest, err)
# test_id_with_desc may contain description which breaks process of fetching id from subtest
test_id_with_desc = self.get_test_id_with_description(test)
test_id = test.id()
subtest_id = subtest.id()
if subtest_id.startswith(test_id):
# Replace "." -> "_" since '.' is a test hierarchy separator
# See i.e. https://github.com/JetBrains/teamcity-messages/issues/134 (https://youtrack.jetbrains.com/issue/PY-23846)
block_id = subtest_id[len(test_id):].strip().replace(".", "_")
else:
block_id = subtest_id
if len(block_id) == 0:
block_id = subtest_id
if err is not None:
self.add_subtest_failure(test_id_with_desc, block_id)
if issubclass(err[0], test.failureException):
self.messages.subTestBlockOpened(block_id,
subTestResult="Failure",
flowId=test_id_with_desc)
self.messages.testStdErr(test_id_with_desc,
out="SubTest failure: %s\n" %
convert_error_to_string(err),
flowId=test_id_with_desc)
self.messages.blockClosed(block_id, flowId=test_id_with_desc)
else:
self.messages.subTestBlockOpened(block_id,
subTestResult="Error",
flowId=test_id_with_desc)
self.messages.testStdErr(test_id_with_desc,
out="SubTest error: %s\n" %
convert_error_to_string(err),
flowId=test_id_with_desc)
self.messages.blockClosed(block_id, flowId=test_id_with_desc)
else:
self.messages.subTestBlockOpened(block_id,
subTestResult="Success",
flowId=test_id_with_desc)
self.messages.blockClosed(block_id, flowId=test_id_with_desc)
def add_subtest_failure(self, test_id, subtest_block_id):
fail_array = self.subtest_failures.get(test_id, [])
fail_array.append(subtest_block_id)
self.subtest_failures[test_id] = fail_array
def get_subtest_failure(self, test_id):
fail_array = self.subtest_failures.get(test_id, [])
return ", ".join(fail_array)
def report_fail(self, test, fail_type, err):
test_id = self.get_test_id_with_description(test)
diff_failed = None
try:
from .jb_local_exc_store import get_exception
error = get_exception()
if isinstance(error, EqualsAssertionError):
diff_failed = error
except Exception:
pass
if is_string(err):
details = err
else:
try:
details = err.getTraceback()
except AttributeError:
frames_to_skip_from_tail = 2 if diff_failed else 0
details = convert_error_to_string(err,
frames_to_skip_from_tail)
subtest_failures = self.get_subtest_failure(test_id)
if subtest_failures:
details = "Failed subtests list: " + subtest_failures + "\n\n" + details.strip(
)
details = details.strip()
if diff_failed:
self.messages.testFailed(test_id,
message=diff_failed.msg,
details=details,
flowId=test_id,
comparison_failure=diff_failed)
else:
self.messages.testFailed(test_id,
message=fail_type,
details=details,
flowId=test_id)
self.failed_tests.add(test_id)
def startTest(self, test):
test_id = self.get_test_id_with_description(test)
self.current_test_id = test_id
super(TeamcityTestResult, self).startTest(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id,
captureStandardOutput='true',
flowId=test_id)
def _dump_test_stderr(self, data):
if self.current_test_id is not None:
dump_test_stderr(self.messages, self.current_test_id,
self.current_test_id, data)
else:
_real_stderr.write(data)
def _dump_test_stdout(self, data):
if self.current_test_id is not None:
dump_test_stdout(self.messages, self.current_test_id,
self.current_test_id, data)
else:
_real_stdout.write(data)
def _setupStdout(self):
if getattr(self, 'buffer', None):
self._stderr_buffer = FlushingStringIO(self._dump_test_stderr)
self._stdout_buffer = FlushingStringIO(self._dump_test_stdout)
sys.stdout = self._stdout_buffer
sys.stderr = self._stderr_buffer
def stopTest(self, test):
test_id = self.get_test_id_with_description(test)
if getattr(self, 'buffer', None):
# Do not allow super() method to print output by itself
self._mirrorOutput = False
output = sys.stdout.getvalue()
if output:
dump_test_stdout(self.messages, test_id, test_id, output)
error = sys.stderr.getvalue()
if error:
dump_test_stderr(self.messages, test_id, test_id, error)
super(TeamcityTestResult, self).stopTest(test)
self.current_test_id = None
if test_id not in self.failed_tests:
subtest_failures = self.get_subtest_failure(test_id)
if subtest_failures:
self.report_fail(test, "One or more subtests failed", "")
try:
time_diff = datetime.datetime.now(
) - self.test_started_datetime_map[test_id]
except KeyError:
time_diff = None
self.messages.testFinished(test_id,
testDuration=time_diff,
flowId=test_id)
def printErrors(self):
pass
class TeamcityTestResult(TestResult):
separator2 = "\n"
def __init__(self, stream=_real_stdout, descriptions=None, verbosity=None):
super(TeamcityTestResult, self).__init__()
self.test_started_datetime_map = {}
self.failed_tests = set()
self.subtest_failures = {}
self.messages = TeamcityServiceMessages(stream)
def get_test_id(self, test):
if is_string(test):
return test
# Force test_id for doctests
if get_class_fullname(test) != "doctest.DocTestCase":
desc = test.shortDescription()
if desc and desc != test.id():
return "%s (%s)" % (test.id(), desc.replace('.', '_'))
return test.id()
def addSuccess(self, test):
super(TeamcityTestResult, self).addSuccess(test)
def addExpectedFailure(self, test, err):
super(TeamcityTestResult, self).addExpectedFailure(test, err)
err = convert_error_to_string(err)
test_id = self.get_test_id(test)
self.messages.testIgnored(test_id, message="Expected failure: " + err, flowId=test_id)
def addSkip(self, test, reason=""):
if sys.version_info >= (2, 7):
super(TeamcityTestResult, self).addSkip(test, reason)
test_id = self.get_test_id(test)
if reason:
reason_str = ": " + reason
else:
reason_str = ""
self.messages.testIgnored(test_id, message="Skipped" + reason_str, flowId=test_id)
def addUnexpectedSuccess(self, test):
super(TeamcityTestResult, self).addUnexpectedSuccess(test)
test_id = self.get_test_id(test)
self.messages.testFailed(test_id, message='Failure',
details="Test should not succeed since it's marked with @unittest.expectedFailure",
flowId=test_id)
def addError(self, test, err, *k):
super(TeamcityTestResult, self).addError(test, err)
if get_class_fullname(test) == "unittest.suite._ErrorHolder":
# This is a standalone error
test_name = test.id()
# patch setUpModule (__main__) -> __main__.setUpModule
test_name = re.sub(r'^(.*) \((.*)\)$', r'\2.\1', test_name)
self.messages.testStarted(test_name, flowId=test_name)
self.report_fail(test_name, 'Failure', err)
self.messages.testFinished(test_name, flowId=test_name)
elif get_class_fullname(err[0]) == "unittest2.case.SkipTest":
message = ""
if hasattr(err[1], "message"):
message = getattr(err[1], "message", "")
elif hasattr(err[1], "args"):
message = getattr(err[1], "args", [""])[0]
self.addSkip(test, message)
else:
self.report_fail(test, 'Error', err)
def addFailure(self, test, err, *k):
super(TeamcityTestResult, self).addFailure(test, err)
self.report_fail(test, 'Failure', err)
def addSubTest(self, test, subtest, err):
super(TeamcityTestResult, self).addSubTest(test, subtest, err)
test_id = self.get_test_id(test)
if err is not None:
if issubclass(err[0], test.failureException):
self.add_subtest_failure(test_id, self.get_test_id(subtest), err)
self.messages.testStdErr(test_id, out="%s: failure\n" % self.get_test_id(subtest), flowId=test_id)
else:
self.add_subtest_failure(test_id, self.get_test_id(subtest), err)
self.messages.testStdErr(test_id, out="%s: error\n" % self.get_test_id(subtest), flowId=test_id)
else:
self.messages.testStdOut(test_id, out="%s: ok\n" % self.get_test_id(subtest), flowId=test_id)
def add_subtest_failure(self, test_id, subtest_id, err):
fail_array = self.subtest_failures.get(test_id, [])
fail_array.append("%s:\n%s" % (subtest_id, convert_error_to_string(err)))
self.subtest_failures[test_id] = fail_array
def get_subtest_failure(self, test_id):
fail_array = self.subtest_failures.get(test_id, [])
return "\n".join(fail_array)
def report_fail(self, test, fail_type, err):
test_id = self.get_test_id(test)
if is_string(err):
details = err
else:
details = convert_error_to_string(err)
subtest_failure = self.get_subtest_failure(test_id)
if subtest_failure:
details = subtest_failure + "\n" + details
self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id)
self.failed_tests.add(test_id)
def startTest(self, test):
super(TeamcityTestResult, self).startTest(test)
test_id = self.get_test_id(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id, captureStandardOutput='true', flowId=test_id)
def stopTest(self, test):
super(TeamcityTestResult, self).stopTest(test)
test_id = self.get_test_id(test)
if test_id not in self.failed_tests and self.subtest_failures.get(test_id, []):
self.report_fail(test, "Subtest failed", "")
time_diff = datetime.datetime.now() - self.test_started_datetime_map[test_id]
self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id)
def printErrors(self):
pass
class TeamcityTestResult(TestResult):
separator2 = "\n"
# noinspection PyUnusedLocal
def __init__(self, stream=_real_stdout, descriptions=None, verbosity=None):
super(TeamcityTestResult, self).__init__()
# Some code may ask for self.failfast, see unittest2.case.TestCase.subTest
self.failfast = getattr(self, "failfast", False)
self.test_started_datetime_map = {}
self.failed_tests = set()
self.subtest_failures = {}
self.messages = TeamcityServiceMessages(_real_stdout)
self.current_test_id = None
@staticmethod
def get_test_id(test):
if is_string(test):
return test
test_class_fullname = get_class_fullname(test)
test_id = test.id()
if test_class_fullname in _ERROR_HOLDERS_FQN:
# patch setUpModule (__main__) -> __main__.setUpModule
return re.sub(r'^(.*) \((.*)\)$', r'\2.\1', test_id)
# Force test_id for doctests
if test_class_fullname != "doctest.DocTestCase":
desc = test.shortDescription()
test_method_name = getattr(test, "_testMethodName", "")
if desc and desc != test_id and desc != test_method_name:
return "%s (%s)" % (test_id, desc.replace('.', '_'))
return test_id
def addSuccess(self, test):
super(TeamcityTestResult, self).addSuccess(test)
def addExpectedFailure(self, test, err):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addExpectedFailure"):
_super.addExpectedFailure(test, err)
err = convert_error_to_string(err)
test_id = self.get_test_id(test)
self.messages.testIgnored(test_id, message="Expected failure: " + err, flowId=test_id)
def get_subtest_block_id(self, test, subtest):
test_id = self.get_test_id(test)
subtest_id = self.get_test_id(subtest)
if subtest_id.startswith(test_id):
block_id = subtest_id[len(test_id):].strip()
else:
block_id = subtest_id
if len(block_id) == 0:
block_id = test_id
return block_id
def addSkip(self, test, reason=""):
if sys.version_info >= (2, 7):
super(TeamcityTestResult, self).addSkip(test, reason)
if reason:
if isinstance(reason, Exception):
reason_str = ": " + get_exception_message(reason)
else:
reason_str = ": " + to_unicode(reason)
else:
reason_str = ""
test_class_name = get_class_fullname(test)
if test_class_name == "unittest.case._SubTest" or test_class_name == "unittest2.case._SubTest":
parent_test = test.test_case
parent_test_id = self.get_test_id(parent_test)
subtest = test
block_id = self.get_subtest_block_id(parent_test, subtest)
self.messages.subTestBlockOpened(block_id, subTestResult="Skip", flowId=parent_test_id)
self.messages.testStdOut(parent_test_id, out="SubTest skipped" + reason_str + "\n", flowId=parent_test_id)
self.messages.blockClosed(block_id, flowId=parent_test_id)
else:
test_id = self.get_test_id(test)
if test_id not in self.test_started_datetime_map:
# Test ignored without startTest. Handle start and finish events ourselves
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testIgnored(test_id, message="Skipped" + reason_str, flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
else:
self.messages.testIgnored(test_id, message="Skipped" + reason_str, flowId=test_id)
def addUnexpectedSuccess(self, test):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addUnexpectedSuccess"):
_super.addUnexpectedSuccess(test)
test_id = self.get_test_id(test)
self.messages.testFailed(test_id, message='Failure',
details="Test should not succeed since it's marked with @unittest.expectedFailure",
flowId=test_id)
def addError(self, test, err, *k):
super(TeamcityTestResult, self).addError(test, err)
test_class = get_class_fullname(test)
if test_class in _ERROR_HOLDERS_FQN:
# This is a standalone error
test_id = self.get_test_id(test)
self.messages.testStarted(test_id, flowId=test_id)
self.report_fail(test, 'Failure', err)
self.messages.testFinished(test_id, flowId=test_id)
elif get_class_fullname(err[0]) == "unittest2.case.SkipTest":
message = ""
if hasattr(err[1], "message"):
message = getattr(err[1], "message", "")
elif hasattr(err[1], "args"):
message = getattr(err[1], "args", [""])[0]
self.addSkip(test, message)
else:
self.report_fail(test, 'Error', err)
def addFailure(self, test, err, *k):
super(TeamcityTestResult, self).addFailure(test, err)
self.report_fail(test, 'Failure', err)
def addSubTest(self, test, subtest, err):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addSubTest"):
_super.addSubTest(test, subtest, err)
test_id = self.get_test_id(test)
subtest_id = self.get_test_id(subtest)
if subtest_id.startswith(test_id):
# Replace "." -> "_" since '.' is a test hierarchy separator
# See i.e. https://github.com/JetBrains/teamcity-messages/issues/134 (https://youtrack.jetbrains.com/issue/PY-23846)
block_id = subtest_id[len(test_id):].strip().replace(".", "_")
else:
block_id = subtest_id
if len(block_id) == 0:
block_id = subtest_id
if err is not None:
self.add_subtest_failure(test_id, block_id)
if issubclass(err[0], test.failureException):
self.messages.subTestBlockOpened(block_id, subTestResult="Failure", flowId=test_id)
self.messages.testStdErr(test_id, out="SubTest failure: %s\n" % convert_error_to_string(err), flowId=test_id)
self.messages.blockClosed(block_id, flowId=test_id)
else:
self.messages.subTestBlockOpened(block_id, subTestResult="Error", flowId=test_id)
self.messages.testStdErr(test_id, out="SubTest error: %s\n" % convert_error_to_string(err), flowId=test_id)
self.messages.blockClosed(block_id, flowId=test_id)
else:
self.messages.subTestBlockOpened(block_id, subTestResult="Success", flowId=test_id)
self.messages.blockClosed(block_id, flowId=test_id)
def add_subtest_failure(self, test_id, subtest_block_id):
fail_array = self.subtest_failures.get(test_id, [])
fail_array.append(subtest_block_id)
self.subtest_failures[test_id] = fail_array
def get_subtest_failure(self, test_id):
fail_array = self.subtest_failures.get(test_id, [])
return ", ".join(fail_array)
def report_fail(self, test, fail_type, err):
test_id = self.get_test_id(test)
diff_failed = None
try:
error = err[1]
if isinstance(error, EqualsAssertionError):
diff_failed = error
except Exception:
pass
if is_string(err):
details = err
else:
try:
details = err.getTraceback()
except AttributeError:
frames_to_skip_from_tail = 2 if diff_failed else 0
details = convert_error_to_string(err, frames_to_skip_from_tail)
subtest_failures = self.get_subtest_failure(test_id)
if subtest_failures:
details = "Failed subtests list: " + subtest_failures + "\n\n" + details.strip()
details = details.strip()
if diff_failed:
self.messages.testFailed(test_id,
message=diff_failed.msg,
details=details,
flowId=test_id,
comparison_failure=diff_failed)
else:
self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id)
self.failed_tests.add(test_id)
def startTest(self, test):
test_id = self.get_test_id(test)
self.current_test_id = test_id
super(TeamcityTestResult, self).startTest(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id, captureStandardOutput='true', flowId=test_id)
def _dump_test_stderr(self, data):
if self.current_test_id is not None:
dump_test_stderr(self.messages, self.current_test_id, self.current_test_id, data)
else:
_real_stderr.write(data)
def _dump_test_stdout(self, data):
if self.current_test_id is not None:
dump_test_stdout(self.messages, self.current_test_id, self.current_test_id, data)
else:
_real_stdout.write(data)
def _setupStdout(self):
if getattr(self, 'buffer', None):
self._stderr_buffer = FlushingStringIO(self._dump_test_stderr)
self._stdout_buffer = FlushingStringIO(self._dump_test_stdout)
sys.stdout = self._stdout_buffer
sys.stderr = self._stderr_buffer
def stopTest(self, test):
test_id = self.get_test_id(test)
if getattr(self, 'buffer', None):
# Do not allow super() method to print output by itself
self._mirrorOutput = False
output = sys.stdout.getvalue()
if output:
dump_test_stdout(self.messages, test_id, test_id, output)
error = sys.stderr.getvalue()
if error:
dump_test_stderr(self.messages, test_id, test_id, error)
super(TeamcityTestResult, self).stopTest(test)
self.current_test_id = None
if test_id not in self.failed_tests:
subtest_failures = self.get_subtest_failure(test_id)
if subtest_failures:
self.report_fail(test, "One or more subtests failed", "")
try:
time_diff = datetime.datetime.now() - self.test_started_datetime_map[test_id]
except KeyError:
time_diff = None
self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id)
def printErrors(self):
pass
class EchoTeamCityMessages(object):
def __init__(self, output_capture_enabled, coverage_controller):
self.coverage_controller = coverage_controller
self.output_capture_enabled = output_capture_enabled
self.teamcity = TeamcityServiceMessages()
self.test_start_reported_mark = set()
self.max_reported_output_size = 1 * 1024 * 1024
self.reported_output_chunk_size = 50000
def get_id_from_location(self, location):
if type(location) is not tuple or len(location) != 3 or not hasattr(
location[2], "startswith"):
return None
def convert_file_to_id(filename):
filename = re.sub(r"\.pyc?$", "", filename)
return filename.replace(os.sep, ".").replace("/", ".")
def add_prefix_to_filename_id(filename_id, prefix):
dot_location = filename_id.rfind('.')
if dot_location <= 0 or dot_location >= len(filename_id) - 1:
return None
return filename_id[:dot_location +
1] + prefix + filename_id[dot_location + 1:]
pylint_prefix = '[pylint] '
if location[2].startswith(pylint_prefix):
id_from_file = convert_file_to_id(location[2][len(pylint_prefix):])
return id_from_file + ".Pylint"
if location[2] == "PEP8-check":
id_from_file = convert_file_to_id(location[0])
return id_from_file + ".PEP8"
return None
def format_test_id(self, nodeid, location):
id_from_location = self.get_id_from_location(location)
if id_from_location is not None:
return id_from_location
test_id = nodeid
if test_id:
if test_id.find("::") < 0:
test_id += "::top_level"
else:
test_id = "top_level"
first_bracket = test_id.find("[")
if first_bracket > 0:
# [] -> (), make it look like nose parameterized tests
params = "(" + test_id[first_bracket + 1:]
if params.endswith("]"):
params = params[:-1] + ")"
test_id = test_id[:first_bracket]
if test_id.endswith("::"):
test_id = test_id[:-2]
else:
params = ""
test_id = test_id.replace("::()::", "::")
test_id = re.sub(r"\.pyc?::", r"::", test_id)
test_id = test_id.replace(".", "_").replace(os.sep, ".").replace(
"/", ".").replace('::', '.')
if params:
params = params.replace(".", "_")
test_id += params
return test_id
def format_location(self, location):
if type(location) is tuple and len(location) == 3:
return "%s:%s (%s)" % (str(location[0]), str(
location[1]), str(location[2]))
return str(location)
def pytest_collection_modifyitems(self, session, config, items):
self.teamcity.testCount(len(items))
def pytest_runtest_logstart(self, nodeid, location):
self.ensure_test_start_reported(self.format_test_id(nodeid, location))
def ensure_test_start_reported(self, test_id):
if test_id not in self.test_start_reported_mark:
if self.output_capture_enabled:
capture_standard_output = "false"
else:
capture_standard_output = "true"
self.teamcity.testStarted(
test_id,
flowId=test_id,
captureStandardOutput=capture_standard_output)
self.test_start_reported_mark.add(test_id)
def report_has_output(self, report):
for (secname, data) in report.sections:
if report.when in secname and ('stdout' in secname
or 'stderr' in secname):
return True
return False
def report_test_output(self, report, test_id):
for (secname, data) in report.sections:
# https://github.com/JetBrains/teamcity-messages/issues/112
# CollectReport doesn't have 'when' property
if hasattr(report, "when") and report.when not in secname:
continue
if not data:
continue
if 'stdout' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdOut(test_id,
out=chunk,
flowId=test_id)
elif 'stderr' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdErr(test_id,
out=chunk,
flowId=test_id)
def report_test_finished(self, test_id, duration=None):
self.teamcity.testFinished(test_id,
testDuration=duration,
flowId=test_id)
self.test_start_reported_mark.remove(test_id)
def report_test_failure(self,
test_id,
report,
message=None,
report_output=True):
if hasattr(report, 'duration'):
duration = timedelta(seconds=report.duration)
else:
duration = None
if message is None:
message = self.format_location(report.location)
self.ensure_test_start_reported(test_id)
if report_output:
self.report_test_output(report, test_id)
self.teamcity.testFailed(test_id,
message,
str(report.longrepr),
flowId=test_id)
self.report_test_finished(test_id, duration)
def report_test_skip(self, test_id, report):
if type(report.longrepr) is tuple and len(report.longrepr) == 3:
reason = report.longrepr[2]
else:
reason = str(report.longrepr)
if hasattr(report, 'duration'):
duration = timedelta(seconds=report.duration)
else:
duration = None
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id)
self.teamcity.testIgnored(test_id, reason, flowId=test_id)
self.report_test_finished(test_id, duration)
def pytest_runtest_logreport(self, report):
"""
:type report: _pytest.runner.TestReport
"""
test_id = self.format_test_id(report.nodeid, report.location)
duration = timedelta(seconds=report.duration)
if report.passed:
# Do not report passed setup/teardown if no output
if report.when == 'call':
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id)
self.report_test_finished(test_id, duration)
else:
if self.report_has_output(report):
block_name = "test " + report.when
self.teamcity.blockOpened(block_name, flowId=test_id)
self.report_test_output(report, test_id)
self.teamcity.blockClosed(block_name, flowId=test_id)
elif report.failed:
if report.when == 'call':
self.report_test_failure(test_id, report)
elif report.when == 'setup':
if self.report_has_output(report):
self.teamcity.blockOpened("test setup", flowId=test_id)
self.report_test_output(report, test_id)
self.teamcity.blockClosed("test setup", flowId=test_id)
self.report_test_failure(test_id,
report,
message="test setup failed",
report_output=False)
elif report.when == 'teardown':
# Report failed teardown as a separate test as original test is already finished
self.report_test_failure(test_id + "_teardown", report)
elif report.skipped:
self.report_test_skip(test_id, report)
def pytest_collectreport(self, report):
test_id = self.format_test_id(report.nodeid,
report.location) + "_collect"
if report.failed:
self.report_test_failure(test_id, report)
elif report.skipped:
self.report_test_skip(test_id, report)
def pytest_terminal_summary(self):
if self.coverage_controller is not None:
try:
self._report_coverage()
except:
tb = traceback.format_exc()
self.teamcity.customMessage(
"Coverage statistics reporting failed",
"ERROR",
errorDetails=tb)
def _report_coverage(self):
from coverage.misc import NotPython
from coverage.report import Reporter
from coverage.results import Numbers
class _CoverageReporter(Reporter):
def __init__(self, coverage, config, messages):
super(_CoverageReporter, self).__init__(coverage, config)
self.branches = coverage.data.has_arcs()
self.messages = messages
def report(self, morfs, outfile=None):
if hasattr(self, 'find_code_units'):
self.find_code_units(morfs)
else:
self.find_file_reporters(morfs)
total = Numbers()
if hasattr(self, 'code_units'):
units = self.code_units
else:
units = self.file_reporters
for cu in units:
try:
analysis = self.coverage._analyze(cu)
nums = analysis.numbers
total += nums
except KeyboardInterrupt:
raise
except:
if self.config.ignore_errors:
continue
err = sys.exc_info()
typ, msg = err[:2]
if typ is NotPython and not cu.should_be_python():
continue
test_id = cu.name
details = convert_error_to_string(err)
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testFailed(
test_id,
message="Coverage analysis failed",
details=details,
flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
if total.n_files > 0:
covered = total.n_executed
total_statements = total.n_statements
if self.branches:
covered += total.n_executed_branches
total_statements += total.n_branches
self.messages.buildStatisticLinesCovered(covered)
self.messages.buildStatisticTotalLines(total_statements)
self.messages.buildStatisticLinesUncovered(
total_statements - covered)
reporter = _CoverageReporter(
self.coverage_controller.cov,
self.coverage_controller.cov.config,
self.teamcity,
)
reporter.report(None)
class EchoTeamCityMessages(object):
def __init__(self, output_capture_enabled, coverage_controller):
self.coverage_controller = coverage_controller
self.output_capture_enabled = output_capture_enabled
self.teamcity = TeamcityServiceMessages()
self.test_start_reported_mark = set()
self.max_reported_output_size = 1 * 1024 * 1024
self.reported_output_chunk_size = 50000
def format_test_id(self, nodeid):
test_id = nodeid
if test_id.find("::") < 0:
test_id += "::top_level"
test_id = test_id.replace("::()::", "::")
test_id = re.sub(r"\.pyc?::", r"::", test_id)
test_id = test_id.replace(".", "_").replace(os.sep, ".").replace(
"/", ".").replace('::', '.')
return test_id
def pytest_runtest_logstart(self, nodeid, location):
self.ensure_test_start_reported(self.format_test_id(nodeid))
def ensure_test_start_reported(self, test_id):
if test_id not in self.test_start_reported_mark:
self.teamcity.testStarted(test_id,
flowId=test_id,
captureStandardOutput='false' if
self.output_capture_enabled else 'true')
self.test_start_reported_mark.add(test_id)
def report_has_output(self, report):
for (secname, data) in report.sections:
if report.when in secname and ('stdout' in secname
or 'stderr' in secname):
return True
return False
def report_test_output(self, report, test_id, when):
for (secname, data) in report.sections:
if when not in secname:
continue
if not data:
continue
if 'stdout' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdOut(test_id,
out=chunk,
flowId=test_id)
elif 'stderr' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdErr(test_id,
out=chunk,
flowId=test_id)
def pytest_runtest_logreport(self, report):
"""
:type report: _pytest.runner.TestReport
"""
orig_test_id = self.format_test_id(report.nodeid)
suffix = '' if report.when == 'call' else ('_' + report.when)
test_id = orig_test_id + suffix
duration = timedelta(seconds=report.duration)
if report.passed:
# Do not report passed setup/teardown if no output
if report.when == 'call' or self.report_has_output(report):
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id, report.when)
self.teamcity.testFinished(test_id,
testDuration=duration,
flowId=test_id)
elif report.failed:
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id, report.when)
self.teamcity.testFailed(test_id,
str(report.location),
str(report.longrepr),
flowId=test_id)
self.teamcity.testFinished(test_id,
testDuration=duration,
flowId=test_id)
# If test setup failed, report test failure as well
if report.when == 'setup':
self.ensure_test_start_reported(orig_test_id)
self.teamcity.testFailed(
orig_test_id,
"test setup failed, see %s test failure" % test_id,
flowId=orig_test_id)
self.teamcity.testFinished(orig_test_id, flowId=orig_test_id)
elif report.skipped:
if type(report.longrepr) is tuple and len(report.longrepr) == 3:
reason = report.longrepr[2]
else:
reason = str(report.longrepr)
self.ensure_test_start_reported(orig_test_id)
self.report_test_output(report, orig_test_id, report.when)
self.teamcity.testIgnored(orig_test_id,
reason,
flowId=orig_test_id)
self.teamcity.testFinished(orig_test_id, flowId=orig_test_id)
def pytest_collectreport(self, report):
if report.failed:
test_id = self.format_test_id(report.nodeid) + "_collect"
self.teamcity.testStarted(test_id, flowId=test_id)
self.teamcity.testFailed(test_id,
str(report.location),
str(report.longrepr),
flowId=test_id)
self.teamcity.testFinished(test_id, flowId=test_id)
def pytest_terminal_summary(self):
if self.coverage_controller is not None:
try:
self._report_coverage()
except:
tb = traceback.format_exc()
self.teamcity.customMessage(
"Coverage statistics reporting failed",
"ERROR",
errorDetails=tb)
def _report_coverage(self):
from coverage.misc import NotPython
from coverage.report import Reporter
from coverage.results import Numbers
class _CoverageReporter(Reporter):
def __init__(self, coverage, config, messages):
super(_CoverageReporter, self).__init__(coverage, config)
self.branches = coverage.data.has_arcs()
self.messages = messages
def report(self, morfs, outfile=None):
self.find_code_units(morfs)
total = Numbers()
for cu in self.code_units:
try:
analysis = self.coverage._analyze(cu)
nums = analysis.numbers
total += nums
except KeyboardInterrupt:
raise
except:
if self.config.ignore_errors:
continue
err = sys.exc_info()
typ, msg = err[:2]
if typ is NotPython and not cu.should_be_python():
continue
test_id = cu.name
details = convert_error_to_string(err)
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testFailed(
test_id,
message="Coverage analysis failed",
details=details,
flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
if total.n_files > 0:
covered = total.n_executed + (total.n_executed_branches
if self.branches else 0)
total_statements = total.n_statements + (
total.n_branches if self.branches else 0)
self.messages.buildStatisticLinesCovered(covered)
self.messages.buildStatisticTotalLines(total_statements)
self.messages.buildStatisticLinesUncovered(
total_statements - covered)
reporter = _CoverageReporter(
self.coverage_controller.cov,
self.coverage_controller.cov.config,
self.teamcity,
)
reporter.report(None)
class TeamcityFormatter(Formatter):
"""
Stateful TC reporter.
Since we can't fetch all steps from the very beginning (even skipped tests are reported)
we store tests and features on each call.
To hook into test reporting override _report_suite_started and/or _report_test_started
"""
def __init__(self, stream_opener, config):
super(TeamcityFormatter, self).__init__(stream_opener, config)
assert version.LooseVersion(behave_version) >= version.LooseVersion("1.2.6"), "Only 1.2.6+ is supported"
self._messages = TeamcityServiceMessages()
self.__feature = None
self.__scenario = None
self.__steps = deque()
self.__scenario_opened = False
self.__feature_opened = False
self.__test_start_time = None
def feature(self, feature):
assert isinstance(feature, Feature)
assert not self.__feature, "Prev. feature not closed"
self.__feature = feature
def scenario(self, scenario):
assert isinstance(scenario, Scenario)
self.__scenario = scenario
self.__scenario_opened = False
self.__steps.clear()
def step(self, step):
assert isinstance(step, Step)
self.__steps.append(step)
def match(self, match):
if not self.__feature_opened:
self._report_suite_started(self.__feature, _suite_name(self.__feature))
self.__feature_opened = True
if not self.__scenario_opened:
self._report_suite_started(self.__scenario, _suite_name(self.__scenario))
self.__scenario_opened = True
assert self.__steps, "No steps left"
step = self.__steps.popleft()
self._report_test_started(step, _step_name(step))
self.__test_start_time = datetime.datetime.now()
def _report_suite_started(self, suite, suite_name):
"""
:param suite: behave suite
:param suite_name: suite name that must be reported, be sure to use it instead of suite.name
"""
self._messages.testSuiteStarted(suite_name)
def _report_test_started(self, test, test_name):
"""
Suite name is always stripped, be sure to strip() it too
:param test: behave test
:param test_name: test name that must be reported, be sure to use it instead of test.name
"""
self._messages.testStarted(test_name)
def result(self, step):
assert isinstance(step, Step)
step_name = _step_name(step)
if step.status == Status.failed:
try:
error = traceback.format_exc(step.exc_traceback)
if error != step.error_message:
self._messages.testStdErr(step_name, error)
except Exception:
pass # exception shall not prevent error message
self._messages.testFailed(step_name, message=step.error_message)
if step.status == Status.undefined:
self._messages.testFailed(step_name, message="Undefined")
if step.status == Status.skipped:
self._messages.testIgnored(step_name)
self._messages.testFinished(step_name, testDuration=datetime.datetime.now() - self.__test_start_time)
if not self.__steps:
self.__close_scenario()
elif step.status in [Status.failed, Status.undefined]:
# Broken background/undefined step stops whole scenario
reason = "Undefined step" if step.status == Status.undefined else "Prev. step failed"
self.__skip_rest_of_scenario(reason)
def __skip_rest_of_scenario(self, reason):
while self.__steps:
step = self.__steps.popleft()
self._report_test_started(step, _step_name(step))
self._messages.testIgnored(_step_name(step),
message="{0}. Rest part of scenario is skipped".format(reason))
self._messages.testFinished(_step_name(step))
self.__close_scenario()
def __close_scenario(self):
if self.__scenario:
self._messages.testSuiteFinished(_suite_name(self.__scenario))
self.__scenario = None
def eof(self):
self.__skip_rest_of_scenario("")
self._messages.testSuiteFinished(_suite_name(self.__feature))
self.__feature = None
self.__feature_opened = False
class EchoTeamCityMessages(object):
def __init__(self, output_capture_enabled, coverage_controller):
self.coverage_controller = coverage_controller
self.output_capture_enabled = output_capture_enabled
self.teamcity = TeamcityServiceMessages()
self.test_start_reported_mark = set()
self.max_reported_output_size = 1 * 1024 * 1024
self.reported_output_chunk_size = 50000
def get_id_from_location(self, location):
if type(location) is not tuple or len(location) != 3 or not hasattr(location[2], "startswith"):
return None
def convert_file_to_id(filename):
filename = re.sub(r"\.pyc?$", "", filename)
return filename.replace(os.sep, ".").replace("/", ".")
def add_prefix_to_filename_id(filename_id, prefix):
dot_location = filename_id.rfind('.')
if dot_location <= 0 or dot_location >= len(filename_id) - 1:
return None
return filename_id[:dot_location + 1] + prefix + filename_id[dot_location + 1:]
pylint_prefix = '[pylint] '
if location[2].startswith(pylint_prefix):
id_from_file = convert_file_to_id(location[2][len(pylint_prefix):])
return id_from_file + ".Pylint"
if location[2] == "PEP8-check":
id_from_file = convert_file_to_id(location[0])
return id_from_file + ".PEP8"
return None
def format_test_id(self, nodeid, location):
id_from_location = self.get_id_from_location(location)
if id_from_location is not None:
return id_from_location
test_id = nodeid
if test_id.find("::") < 0:
test_id += "::top_level"
test_id = test_id.replace("::()::", "::")
test_id = re.sub(r"\.pyc?::", r"::", test_id)
test_id = test_id.replace(".", "_").replace(os.sep, ".").replace("/", ".").replace('::', '.')
return test_id
def format_location(self, location):
if type(location) is tuple and len(location) == 3:
return "%s:%s (%s)" % (str(location[0]), str(location[1]), str(location[2]))
return str(location)
def pytest_runtest_logstart(self, nodeid, location):
self.ensure_test_start_reported(self.format_test_id(nodeid, location))
def ensure_test_start_reported(self, test_id):
if test_id not in self.test_start_reported_mark:
if self.output_capture_enabled:
capture_standard_output = "false"
else:
capture_standard_output = "true"
self.teamcity.testStarted(test_id, flowId=test_id, captureStandardOutput=capture_standard_output)
self.test_start_reported_mark.add(test_id)
def report_has_output(self, report):
for (secname, data) in report.sections:
if report.when in secname and ('stdout' in secname or 'stderr' in secname):
return True
return False
def report_test_output(self, report, test_id):
for (secname, data) in report.sections:
if report.when not in secname:
continue
if not data:
continue
if 'stdout' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdOut(test_id, out=chunk, flowId=test_id)
elif 'stderr' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdErr(test_id, out=chunk, flowId=test_id)
def report_test_finished(self, test_id, duration=None):
self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id)
self.test_start_reported_mark.remove(test_id)
def report_test_failure(self, test_id, report, message=None, report_output=True):
if hasattr(report, 'duration'):
duration = timedelta(seconds=report.duration)
else:
duration = None
if message is None:
message = self.format_location(report.location)
self.ensure_test_start_reported(test_id)
if report_output:
self.report_test_output(report, test_id)
self.teamcity.testFailed(test_id, message, str(report.longrepr), flowId=test_id)
self.report_test_finished(test_id, duration)
def pytest_runtest_logreport(self, report):
"""
:type report: _pytest.runner.TestReport
"""
test_id = self.format_test_id(report.nodeid, report.location)
duration = timedelta(seconds=report.duration)
if report.passed:
# Do not report passed setup/teardown if no output
if report.when == 'call':
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id)
self.report_test_finished(test_id, duration)
else:
if self.report_has_output(report):
block_name = "test " + report.when
self.teamcity.blockOpened(block_name, flowId=test_id)
self.report_test_output(report, test_id)
self.teamcity.blockClosed(block_name, flowId=test_id)
elif report.failed:
if report.when == 'call':
self.report_test_failure(test_id, report)
elif report.when == 'setup':
if self.report_has_output(report):
self.teamcity.blockOpened("test setup", flowId=test_id)
self.report_test_output(report, test_id)
self.teamcity.blockClosed("test setup", flowId=test_id)
self.report_test_failure(test_id, report, message="test setup failed", report_output=False)
elif report.when == 'teardown':
# Report failed teardown as a separate test as original test is already finished
self.report_test_failure(test_id + "_teardown", report)
elif report.skipped:
if type(report.longrepr) is tuple and len(report.longrepr) == 3:
reason = report.longrepr[2]
else:
reason = str(report.longrepr)
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id)
self.teamcity.testIgnored(test_id, reason, flowId=test_id)
self.report_test_finished(test_id, duration)
def pytest_collectreport(self, report):
if report.failed:
test_id = self.format_test_id(report.nodeid, report.location) + "_collect"
self.report_test_failure(test_id, report)
def pytest_terminal_summary(self):
if self.coverage_controller is not None:
try:
self._report_coverage()
except:
tb = traceback.format_exc()
self.teamcity.customMessage("Coverage statistics reporting failed", "ERROR", errorDetails=tb)
def _report_coverage(self):
from coverage.misc import NotPython
from coverage.report import Reporter
from coverage.results import Numbers
class _CoverageReporter(Reporter):
def __init__(self, coverage, config, messages):
super(_CoverageReporter, self).__init__(coverage, config)
self.branches = coverage.data.has_arcs()
self.messages = messages
def report(self, morfs, outfile=None):
if hasattr(self, 'find_code_units'):
self.find_code_units(morfs)
else:
self.find_file_reporters(morfs)
total = Numbers()
if hasattr(self, 'code_units'):
units = self.code_units
else:
units = self.file_reporters
for cu in units:
try:
analysis = self.coverage._analyze(cu)
nums = analysis.numbers
total += nums
except KeyboardInterrupt:
raise
except:
if self.config.ignore_errors:
continue
err = sys.exc_info()
typ, msg = err[:2]
if typ is NotPython and not cu.should_be_python():
continue
test_id = cu.name
details = convert_error_to_string(err)
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testFailed(test_id, message="Coverage analysis failed", details=details, flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
if total.n_files > 0:
covered = total.n_executed
total_statements = total.n_statements
if self.branches:
covered += total.n_executed_branches
total_statements += total.n_branches
self.messages.buildStatisticLinesCovered(covered)
self.messages.buildStatisticTotalLines(total_statements)
self.messages.buildStatisticLinesUncovered(total_statements - covered)
reporter = _CoverageReporter(
self.coverage_controller.cov,
self.coverage_controller.cov.config,
self.teamcity,
)
reporter.report(None)
class EchoTeamCityMessages(object):
def __init__(self, output_capture_enabled, coverage_controller):
self.coverage_controller = coverage_controller
self.output_capture_enabled = output_capture_enabled
self.teamcity = TeamcityServiceMessages()
self.test_start_reported_mark = set()
self.max_reported_output_size = 1 * 1024 * 1024
self.reported_output_chunk_size = 50000
def format_test_id(self, nodeid):
test_id = nodeid
if test_id.find("::") < 0:
test_id += "::top_level"
test_id = test_id.replace("::()::", "::")
test_id = re.sub(r"\.pyc?::", r"::", test_id)
test_id = test_id.replace(".", "_").replace(os.sep, ".").replace("/", ".").replace('::', '.')
return test_id
def pytest_runtest_logstart(self, nodeid, location):
self.ensure_test_start_reported(self.format_test_id(nodeid))
def ensure_test_start_reported(self, test_id):
if test_id not in self.test_start_reported_mark:
self.teamcity.testStarted(test_id, flowId=test_id, captureStandardOutput='false' if self.output_capture_enabled else 'true')
self.test_start_reported_mark.add(test_id)
def report_has_output(self, report):
for (secname, data) in report.sections:
if report.when in secname and ('stdout' in secname or 'stderr' in secname):
return True
return False
def report_test_output(self, report, test_id, when):
for (secname, data) in report.sections:
if when not in secname:
continue
if not data:
continue
if 'stdout' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdOut(test_id, out=chunk, flowId=test_id)
elif 'stderr' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdErr(test_id, out=chunk, flowId=test_id)
def pytest_runtest_logreport(self, report):
"""
:type report: _pytest.runner.TestReport
"""
orig_test_id = self.format_test_id(report.nodeid)
suffix = '' if report.when == 'call' else ('_' + report.when)
test_id = orig_test_id + suffix
duration = timedelta(seconds=report.duration)
if report.passed:
# Do not report passed setup/teardown if no output
if report.when == 'call' or self.report_has_output(report):
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id, report.when)
self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id)
elif report.failed:
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id, report.when)
self.teamcity.testFailed(test_id, str(report.location), str(report.longrepr), flowId=test_id)
self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id)
# If test setup failed, report test failure as well
if report.when == 'setup':
self.ensure_test_start_reported(orig_test_id)
self.teamcity.testFailed(orig_test_id, "test setup failed, see %s test failure" % test_id, flowId=orig_test_id)
self.teamcity.testFinished(orig_test_id, flowId=orig_test_id)
elif report.skipped:
if type(report.longrepr) is tuple and len(report.longrepr) == 3:
reason = report.longrepr[2]
else:
reason = str(report.longrepr)
self.ensure_test_start_reported(orig_test_id)
self.report_test_output(report, orig_test_id, report.when)
self.teamcity.testIgnored(orig_test_id, reason, flowId=orig_test_id)
self.teamcity.testFinished(orig_test_id, flowId=orig_test_id)
def pytest_collectreport(self, report):
if report.failed:
test_id = self.format_test_id(report.nodeid) + "_collect"
self.teamcity.testStarted(test_id, flowId=test_id)
self.teamcity.testFailed(test_id, str(report.location), str(report.longrepr), flowId=test_id)
self.teamcity.testFinished(test_id, flowId=test_id)
def pytest_terminal_summary(self):
if self.coverage_controller is not None:
try:
self._report_coverage()
except:
tb = traceback.format_exc()
self.teamcity.customMessage("Coverage statistics reporting failed", "ERROR", errorDetails=tb)
def _report_coverage(self):
from coverage.misc import NotPython
from coverage.report import Reporter
from coverage.results import Numbers
class _CoverageReporter(Reporter):
def __init__(self, coverage, config, messages):
super(_CoverageReporter, self).__init__(coverage, config)
self.branches = coverage.data.has_arcs()
self.messages = messages
def report(self, morfs, outfile=None):
self.find_code_units(morfs)
total = Numbers()
for cu in self.code_units:
try:
analysis = self.coverage._analyze(cu)
nums = analysis.numbers
total += nums
except KeyboardInterrupt:
raise
except:
if self.config.ignore_errors:
continue
err = sys.exc_info()
typ, msg = err[:2]
if typ is NotPython and not cu.should_be_python():
continue
test_id = cu.name
details = convert_error_to_string(err)
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testFailed(test_id, message="Coverage analysis failed", details=details, flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
if total.n_files > 0:
covered = total.n_executed + (total.n_executed_branches if self.branches else 0)
total_statements = total.n_statements + (total.n_branches if self.branches else 0)
self.messages.buildStatisticLinesCovered(covered)
self.messages.buildStatisticTotalLines(total_statements)
self.messages.buildStatisticLinesUncovered(total_statements - covered)
reporter = _CoverageReporter(
self.coverage_controller.cov,
self.coverage_controller.cov.config,
self.teamcity,
)
reporter.report(None)
