def test_test_ignored():
stream = StreamStub()
messages = TeamcityServiceMessages(output=stream, now=lambda: fixed_date)
messages.testIgnored(testName='only a test', message='some message')
assert stream.observed_output.strip() == textwrap.dedent("""\
##teamcity[testIgnored timestamp='2000-11-02T10:23:01.556' message='some message' name='only a test']
""").strip().encode('utf-8')
def _run_tests_for_shell(
tc: TeamCity,
tc_msg: TeamcityServiceMessages,
shell_name: str,
tests_info: AutoTestsInfo,
) -> Optional[int]:
build_id = None
if is_shell_uses_package(shell_name, tests_info):
if tests_info.re_run_builds:
success, build_url = is_last_build_successful(
tc, shell_name, tests_info)
if success:
tc_msg.testIgnored(
shell_name,
f"{shell_name} last auto tests for this package and commit "
f"id was successful, skip it. {build_url}",
)
else:
build_id, build_url = trigger_auto_tests_build(
tc, shell_name, tests_info)
click.echo(
f"{shell_name} Re run automation tests. {build_url}")
else:
build_id, build_url = trigger_auto_tests_build(
tc, shell_name, tests_info)
click.echo(
f"{shell_name} Automation tests build triggered. {build_url}")
else:
tc_msg.testIgnored(
shell_name,
f"{shell_name} is not uses package with this version, skipped tests",
)
return build_id
def test_test_ignored():
stream = StreamStub()
messages = TeamcityServiceMessages(output=stream, now=lambda: fixed_date)
messages.testIgnored(testName='only a test', message='some message')
assert stream.observed_output.strip() == textwrap.dedent("""\
##teamcity[testIgnored timestamp='2000-11-02T10:23:01.556' message='some message' name='only a test']
""").strip().encode('utf-8')
class EchoTeamCityMessages(object):
def __init__(self, ):
self.tw = py.io.TerminalWriter(py.std.sys.stdout)
self.teamcity = TeamcityServiceMessages(self.tw)
self.currentSuite = None
def format_names(self, name):
split = '.py'
file, testname = name.split(split, 1)
if not testname:
testname = file
file = 'NO_TEST_FILE_FOUND'
testname = testname.replace("::()::", ".")
testname = testname.replace("::", ".")
testname = testname.strip(".")
return "".join([file, split]), testname
def pytest_runtest_logstart(self, nodeid, location):
file, testname = self.format_names(nodeid)
if not file == self.currentSuite:
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
self.currentSuite = file
self.teamcity.testSuiteStarted(self.currentSuite)
self.teamcity.testStarted(testname)
def pytest_runtest_logreport(self, report):
file, testname = self.format_names(report.nodeid)
if report.when == "call":
for (secname, data) in report.sections:
if 'stdout' in secname:
self.teamcity.testStdOut(testname, out=data)
elif 'stderr' in secname:
self.teamcity.testStdErr(testname, out=data)
if report.passed:
if report.when == "call": # ignore setup/teardown
duration = timedelta(seconds=report.duration)
self.teamcity.testFinished(testname, testDuration=duration)
elif report.failed:
if report.when == "call":
self.teamcity.testFailed(testname, str(report.location), str(report.longrepr))
duration = timedelta(seconds=report.duration)
self.teamcity.testFinished(testname, testDuration=duration) # report finished after the failure
elif report.skipped:
self.teamcity.testIgnored(testname, str(report.longrepr))
self.teamcity.testFinished(testname) # report finished after the skip
def pytest_sessionfinish(self, session, exitstatus, __multicall__):
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
class EchoTeamCityMessages(object):
def __init__(self, ):
self.tw = py.io.TerminalWriter(py.std.sys.stdout)
self.teamcity = TeamcityServiceMessages(self.tw)
self.currentSuite = None
def format_names(self, name):
split = '.py'
file, testname = name.split(split, 1)
if not testname:
testname = file
file = 'NO_TEST_FILE_FOUND'
testname = testname.replace("::()::", ".")
testname = testname.replace("::", ".")
testname = testname.strip(".")
return ("".join([file, split]), testname)
def pytest_runtest_logstart(self, nodeid, location):
file, testname = self.format_names(nodeid)
if not file == self.currentSuite:
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
self.currentSuite = file
self.teamcity.testSuiteStarted(self.currentSuite)
self.teamcity.testStarted(testname)
def pytest_runtest_logreport(self, report):
file, testname = self.format_names(report.nodeid)
if report.passed:
if report.when == "call": # ignore setup/teardown
self.teamcity.testFinished(testname)
elif report.failed:
if report.when == "call":
self.teamcity.testFailed(testname, str(report.location),
str(report.longrepr))
self.teamcity.testFinished(
testname) # report finished after the failure
elif report.skipped:
self.teamcity.testIgnored(testname, str(report.longrepr))
self.teamcity.testFinished(
testname) # report finished after the skip
def pytest_sessionfinish(self, session, exitstatus, __multicall__):
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
class TeamcityReport(Plugin):
name = 'teamcity-report'
score = 10000
def __init__(self):
super(TeamcityReport, self).__init__()
self.messages = TeamcityServiceMessages(_real_stdout)
self.test_started_datetime_map = {}
self.config = None
self.total_tests = 0
self.enabled = False
def get_test_id(self, test):
if is_string(test):
return test
# Handle special "tests"
test_class_name = get_class_fullname(test)
if test_class_name == CONTEXT_SUITE_FQN:
if inspect.ismodule(test.context):
module_name = test.context.__name__
return module_name + "." + test.error_context
elif inspect.isclass(test.context):
class_name = get_class_fullname(test.context)
return class_name + "." + test.error_context
test_id = test.id()
real_test = getattr(test, "test", test)
real_test_class_name = get_class_fullname(real_test)
test_arg = getattr(real_test, "arg", tuple())
if (type(test_arg) is tuple or type(test_arg) is list) and len(test_arg) > 0:
# As written in nose.case.FunctionTestCase#__str__ or nose.case.MethodTestCase#__str__
test_arg_str = "%s" % (test_arg,)
if test_id.endswith(test_arg_str):
# Replace '.' in test args with '_' to preserve test hierarchy on TeamCity
test_id = test_id[:len(test_id) - len(test_arg_str)] + test_arg_str.replace('.', '_')
# Force test_id for doctests
if real_test_class_name != "doctest.DocTestCase" and real_test_class_name != "nose.plugins.doctests.DocTestCase":
desc = test.shortDescription()
if desc and desc != test.id():
return "%s (%s)" % (test_id, desc.replace('.', '_'))
return test_id
def configure(self, options, conf):
self.enabled = is_running_under_teamcity()
self.config = conf
if self._capture_plugin_enabled():
capture_plugin = self._get_capture_plugin()
old_before_test = capture_plugin.beforeTest
old_after_test = capture_plugin.afterTest
old_format_error = capture_plugin.formatError
def newCaptureBeforeTest(test):
old_before_test(test)
test_id = self.get_test_id(test)
capture_plugin._buf = FlushingStringIO(lambda data: dump_test_stdout(self.messages, test_id, test_id, data))
sys.stdout = capture_plugin._buf
def newCaptureAfterTest(test):
if isinstance(capture_plugin._buf, FlushingStringIO):
capture_plugin._buf.flush()
old_after_test(test)
def newCaptureFormatError(test, err):
if isinstance(capture_plugin._buf, FlushingStringIO):
capture_plugin._buf.flush()
old_format_error(test, err)
capture_plugin.beforeTest = newCaptureBeforeTest
capture_plugin.afterTest = newCaptureAfterTest
capture_plugin.formatError = newCaptureFormatError
def options(self, parser, env=os.environ):
pass
def _get_capture_plugin(self):
"""
:rtype: nose.plugins.capture.Capture
"""
for plugin in self.config.plugins.plugins:
if plugin.name == "capture":
return plugin
return None
def _capture_plugin_enabled(self):
plugin = self._get_capture_plugin()
return plugin is not None and plugin.enabled
def _capture_plugin_buffer(self):
plugin = self._get_capture_plugin()
if plugin is None:
return None
return getattr(plugin, "buffer", None)
def _captureStandardOutput_value(self):
if self._capture_plugin_enabled():
return 'false'
else:
return 'true'
def report_fail(self, test, fail_type, err):
# workaround nose bug on python 3
if is_string(err[1]):
err = (err[0], Exception(err[1]), err[2])
test_id = self.get_test_id(test)
details = convert_error_to_string(err)
start_index = details.find(_captured_output_start_marker)
end_index = details.find(_captured_output_end_marker)
if 0 <= start_index < end_index:
# do not log test output twice, see report_finish for actual output handling
details = details[:start_index] + details[end_index + len(_captured_output_end_marker):]
try:
error = err[1]
if isinstance(error, EqualsAssertionError):
details = convert_error_to_string(err, 2)
self.messages.testFailed(test_id, message=error.msg, details=details, flowId=test_id, comparison_failure=error)
return
except Exception:
pass
self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id)
def report_finish(self, test):
test_id = self.get_test_id(test)
if test_id in self.test_started_datetime_map:
time_diff = datetime.datetime.now() - self.test_started_datetime_map[test_id]
self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id)
else:
self.messages.testFinished(test_id, flowId=test_id)
def prepareTestLoader(self, loader):
"""Insert ourselves into loader calls to count tests.
The top-level loader call often returns lazy results, like a LazySuite.
This is a problem, as we would destroy the suite by iterating over it
to count the tests. Consequently, we monkey-patch the top-level loader
call to do the load twice: once for the actual test running and again
to yield something we can iterate over to do the count.
from https://github.com/erikrose/nose-progressive/
:type loader: nose.loader.TestLoader
"""
# TODO: If there's ever a practical need, also patch loader.suiteClass
# or even TestProgram.createTests. createTests seems to be main top-
# level caller of loader methods, and nose.core.collector() (which
# isn't even called in nose) is an alternate one.
#
# nose 1.3.4 contains required fix:
# Another fix for Python 3.4: Call super in LazySuite to access _removed_tests variable
if hasattr(loader, 'loadTestsFromNames') and nose.__versioninfo__ >= (1, 3, 4):
old_loadTestsFromNames = loader.loadTestsFromNames
def _loadTestsFromNames(*args, **kwargs):
suite = old_loadTestsFromNames(*args, **kwargs)
self.total_tests += suite.countTestCases()
# Clear out the loader's cache. Otherwise, it never finds any tests
# for the actual test run:
loader._visitedPaths = set()
return old_loadTestsFromNames(*args, **kwargs)
loader.loadTestsFromNames = _loadTestsFromNames
# noinspection PyUnusedLocal
def prepareTestRunner(self, runner):
if self.total_tests:
self.messages.testCount(self.total_tests)
def addError(self, test, err):
test_class_name = get_class_fullname(test)
test_id = self.get_test_id(test)
if issubclass(err[0], SkipTest):
self.messages.testIgnored(test_id, message=("SKIPPED: %s" % str(err[1])), flowId=test_id)
self.report_finish(test)
elif issubclass(err[0], DeprecatedTest):
self.messages.testIgnored(test_id, message="Deprecated", flowId=test_id)
self.report_finish(test)
elif test_class_name == CONTEXT_SUITE_FQN:
self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(), flowId=test_id)
self.report_fail(test, 'error in ' + test.error_context + ' context', err)
self.messages.testFinished(test_id, flowId=test_id)
else:
self.report_fail(test, 'Error', err)
self.report_finish(test)
def addFailure(self, test, err):
self.report_fail(test, 'Failure', err)
self.report_finish(test)
def startTest(self, test):
test_id = self.get_test_id(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(), flowId=test_id)
def addSuccess(self, test):
self.report_finish(test)
class TeamcityTestResult(TestResult):
separator2 = "\n"
def __init__(self, stream=_real_stdout, descriptions=None, verbosity=None):
super(TeamcityTestResult, self).__init__()
self.test_started_datetime_map = {}
self.failed_tests = set()
self.subtest_failures = {}
self.messages = TeamcityServiceMessages(stream)
def get_test_id(self, test):
if is_string(test):
return test
# Force test_id for doctests
if get_class_fullname(test) != "doctest.DocTestCase":
desc = test.shortDescription()
if desc and desc != test.id():
return "%s (%s)" % (test.id(), desc.replace('.', '_'))
return test.id()
def addSuccess(self, test):
super(TeamcityTestResult, self).addSuccess(test)
def addExpectedFailure(self, test, err):
super(TeamcityTestResult, self).addExpectedFailure(test, err)
err = convert_error_to_string(err)
test_id = self.get_test_id(test)
self.messages.testIgnored(test_id, message="Expected failure: " + err, flowId=test_id)
def addSkip(self, test, reason=""):
if sys.version_info >= (2, 7):
super(TeamcityTestResult, self).addSkip(test, reason)
test_id = self.get_test_id(test)
if reason:
reason_str = ": " + reason
else:
reason_str = ""
self.messages.testIgnored(test_id, message="Skipped" + reason_str, flowId=test_id)
def addUnexpectedSuccess(self, test):
super(TeamcityTestResult, self).addUnexpectedSuccess(test)
test_id = self.get_test_id(test)
self.messages.testFailed(test_id, message='Failure',
details="Test should not succeed since it's marked with @unittest.expectedFailure",
flowId=test_id)
def addError(self, test, err, *k):
super(TeamcityTestResult, self).addError(test, err)
if get_class_fullname(test) == "unittest.suite._ErrorHolder":
# This is a standalone error
test_name = test.id()
# patch setUpModule (__main__) -> __main__.setUpModule
test_name = re.sub(r'^(.*) \((.*)\)$', r'\2.\1', test_name)
self.messages.testStarted(test_name, flowId=test_name)
self.report_fail(test_name, 'Failure', err)
self.messages.testFinished(test_name, flowId=test_name)
elif get_class_fullname(err[0]) == "unittest2.case.SkipTest":
message = ""
if hasattr(err[1], "message"):
message = getattr(err[1], "message", "")
elif hasattr(err[1], "args"):
message = getattr(err[1], "args", [""])[0]
self.addSkip(test, message)
else:
self.report_fail(test, 'Error', err)
def addFailure(self, test, err, *k):
super(TeamcityTestResult, self).addFailure(test, err)
self.report_fail(test, 'Failure', err)
def addSubTest(self, test, subtest, err):
super(TeamcityTestResult, self).addSubTest(test, subtest, err)
test_id = self.get_test_id(test)
if err is not None:
if issubclass(err[0], test.failureException):
self.add_subtest_failure(test_id, self.get_test_id(subtest), err)
self.messages.testStdErr(test_id, out="%s: failure\n" % self.get_test_id(subtest), flowId=test_id)
else:
self.add_subtest_failure(test_id, self.get_test_id(subtest), err)
self.messages.testStdErr(test_id, out="%s: error\n" % self.get_test_id(subtest), flowId=test_id)
else:
self.messages.testStdOut(test_id, out="%s: ok\n" % self.get_test_id(subtest), flowId=test_id)
def add_subtest_failure(self, test_id, subtest_id, err):
fail_array = self.subtest_failures.get(test_id, [])
fail_array.append("%s:\n%s" % (subtest_id, convert_error_to_string(err)))
self.subtest_failures[test_id] = fail_array
def get_subtest_failure(self, test_id):
fail_array = self.subtest_failures.get(test_id, [])
return "\n".join(fail_array)
def report_fail(self, test, fail_type, err):
test_id = self.get_test_id(test)
if is_string(err):
details = err
else:
details = convert_error_to_string(err)
subtest_failure = self.get_subtest_failure(test_id)
if subtest_failure:
details = subtest_failure + "\n" + details
self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id)
self.failed_tests.add(test_id)
def startTest(self, test):
super(TeamcityTestResult, self).startTest(test)
test_id = self.get_test_id(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id, captureStandardOutput='true', flowId=test_id)
def stopTest(self, test):
super(TeamcityTestResult, self).stopTest(test)
test_id = self.get_test_id(test)
if test_id not in self.failed_tests and self.subtest_failures.get(test_id, []):
self.report_fail(test, "Subtest failed", "")
time_diff = datetime.datetime.now() - self.test_started_datetime_map[test_id]
self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id)
def printErrors(self):
pass
class EchoTeamCityMessages(object):
def __init__(self, output_capture_enabled, coverage_controller,
skip_passed_output, swap_diff):
self.coverage_controller = coverage_controller
self.output_capture_enabled = output_capture_enabled
self.skip_passed_output = skip_passed_output
self.teamcity = TeamcityServiceMessages()
self.test_start_reported_mark = set()
self.max_reported_output_size = 1 * 1024 * 1024
self.reported_output_chunk_size = 50000
self.swap_diff = swap_diff
def get_id_from_location(self, location):
if type(location) is not tuple or len(location) != 3 or not hasattr(
location[2], "startswith"):
return None
def convert_file_to_id(filename):
filename = re.sub(r"\.pyc?$", "", filename)
return filename.replace(os.sep, ".").replace("/", ".")
def add_prefix_to_filename_id(filename_id, prefix):
dot_location = filename_id.rfind('.')
if dot_location <= 0 or dot_location >= len(filename_id) - 1:
return None
return filename_id[:dot_location +
1] + prefix + filename_id[dot_location + 1:]
pylint_prefix = '[pylint] '
if location[2].startswith(pylint_prefix):
id_from_file = convert_file_to_id(location[2][len(pylint_prefix):])
return id_from_file + ".Pylint"
if location[2] == "PEP8-check":
id_from_file = convert_file_to_id(location[0])
return id_from_file + ".PEP8"
return None
def format_test_id(self, nodeid, location):
id_from_location = self.get_id_from_location(location)
if id_from_location is not None:
return id_from_location
test_id = nodeid
if test_id:
if test_id.find("::") < 0:
test_id += "::top_level"
else:
test_id = "top_level"
first_bracket = test_id.find("[")
if first_bracket > 0:
# [] -> (), make it look like nose parameterized tests
params = "(" + test_id[first_bracket + 1:]
if params.endswith("]"):
params = params[:-1] + ")"
test_id = test_id[:first_bracket]
if test_id.endswith("::"):
test_id = test_id[:-2]
else:
params = ""
test_id = test_id.replace("::()::", "::")
test_id = re.sub(r"\.pyc?::", r"::", test_id)
test_id = test_id.replace(".", "_").replace(os.sep, ".").replace(
"/", ".").replace('::', '.')
if params:
params = params.replace(".", "_")
test_id += params
return test_id
def format_location(self, location):
if type(location) is tuple and len(location) == 3:
return "%s:%s (%s)" % (str(location[0]), str(
location[1]), str(location[2]))
return str(location)
def pytest_collection_finish(self, session):
self.teamcity.testCount(len(session.items))
def pytest_runtest_logstart(self, nodeid, location):
# test name fetched from location passed as metainfo to PyCharm
# it will be used to run specific test
# See IDEA-176950, PY-31836
test_name = location[2]
if test_name:
test_name = str(test_name).split(".")[-1]
self.ensure_test_start_reported(self.format_test_id(nodeid, location),
test_name)
def ensure_test_start_reported(self, test_id, metainfo=None):
if test_id not in self.test_start_reported_mark:
if self.output_capture_enabled:
capture_standard_output = "false"
else:
capture_standard_output = "true"
self.teamcity.testStarted(
test_id,
flowId=test_id,
captureStandardOutput=capture_standard_output,
metainfo=metainfo)
self.test_start_reported_mark.add(test_id)
def report_has_output(self, report):
for (secname, data) in report.sections:
if report.when in secname and ('stdout' in secname
or 'stderr' in secname):
return True
return False
def report_test_output(self, report, test_id):
for (secname, data) in report.sections:
# https://github.com/JetBrains/teamcity-messages/issues/112
# CollectReport didn't have 'when' property, but now it has.
# But we still need output on 'collect' state
if hasattr(
report, "when"
) and report.when not in secname and report.when != 'collect':
continue
if not data:
continue
if 'stdout' in secname:
dump_test_stdout(self.teamcity, test_id, test_id, data)
elif 'stderr' in secname:
dump_test_stderr(self.teamcity, test_id, test_id, data)
def report_test_finished(self, test_id, duration=None):
self.teamcity.testFinished(test_id,
testDuration=duration,
flowId=test_id)
self.test_start_reported_mark.remove(test_id)
def report_test_failure(self,
test_id,
report,
message=None,
report_output=True):
if hasattr(report, 'duration'):
duration = timedelta(seconds=report.duration)
else:
duration = None
if message is None:
message = self.format_location(report.location)
self.ensure_test_start_reported(test_id)
if report_output:
self.report_test_output(report, test_id)
diff_error = None
try:
err_message = str(report.longrepr.reprcrash.message)
diff_name = diff_tools.EqualsAssertionError.__name__
# There is a string like "foo.bar.DiffError: [serialized_data]"
if diff_name in err_message:
serialized_data = err_message[err_message.index(diff_name) +
len(diff_name) + 1:]
diff_error = diff_tools.deserialize_error(serialized_data)
# AssertionError is patched in py.test, we can try to fetch diff from it
# In general case message starts with "AssertionError: ", but can also starts with "assert" for top-level
# function. To support both cases we unify them
if err_message.startswith("assert"):
err_message = "AssertionError: " + err_message
if err_message.startswith("AssertionError:"):
diff_error = fetch_diff_error_from_message(
err_message, self.swap_diff)
except Exception:
pass
if not diff_error:
from .jb_local_exc_store import get_exception
diff_error = get_exception()
if diff_error:
# Cut everything after postfix: it is internal view of DiffError
strace = str(report.longrepr)
data_postfix = "_ _ _ _ _"
# Error message in pytest must be in "file.py:22 AssertionError" format
# This message goes to strace
# With custom error we must add real exception class explicitly
if data_postfix in strace:
strace = strace[0:strace.index(data_postfix)].strip()
if strace.endswith(":") and diff_error.real_exception:
strace += " " + type(diff_error.real_exception).__name__
self.teamcity.testFailed(
test_id,
diff_error.msg if diff_error.msg else message,
strace,
flowId=test_id,
comparison_failure=diff_error)
else:
self.teamcity.testFailed(test_id,
message,
str(report.longrepr),
flowId=test_id)
self.report_test_finished(test_id, duration)
def report_test_skip(self, test_id, report):
if type(report.longrepr) is tuple and len(report.longrepr) == 3:
reason = report.longrepr[2]
else:
reason = str(report.longrepr)
if hasattr(report, 'duration'):
duration = timedelta(seconds=report.duration)
else:
duration = None
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id)
self.teamcity.testIgnored(test_id, reason, flowId=test_id)
self.report_test_finished(test_id, duration)
def pytest_assertrepr_compare(self, config, op, left, right):
if op in ('==', '!='):
return [
'{0} {1} {2}'.format(pprint.pformat(left), op,
pprint.pformat(right))
]
def pytest_runtest_logreport(self, report):
"""
:type report: _pytest.runner.TestReport
"""
test_id = self.format_test_id(report.nodeid, report.location)
duration = timedelta(seconds=report.duration)
if report.passed:
# Do not report passed setup/teardown if no output
if report.when == 'call':
self.ensure_test_start_reported(test_id)
if not self.skip_passed_output:
self.report_test_output(report, test_id)
self.report_test_finished(test_id, duration)
else:
if self.report_has_output(
report) and not self.skip_passed_output:
block_name = "test " + report.when
self.teamcity.blockOpened(block_name, flowId=test_id)
self.report_test_output(report, test_id)
self.teamcity.blockClosed(block_name, flowId=test_id)
elif report.failed:
if report.when == 'call':
self.report_test_failure(test_id, report)
elif report.when == 'setup':
if self.report_has_output(report):
self.teamcity.blockOpened("test setup", flowId=test_id)
self.report_test_output(report, test_id)
self.teamcity.blockClosed("test setup", flowId=test_id)
self.report_test_failure(test_id,
report,
message="test setup failed",
report_output=False)
elif report.when == 'teardown':
# Report failed teardown as a separate test as original test is already finished
self.report_test_failure(test_id + "_teardown", report)
elif report.skipped:
self.report_test_skip(test_id, report)
def pytest_collectreport(self, report):
test_id = self.format_test_id(report.nodeid,
report.location) + "_collect"
if report.failed:
self.report_test_failure(test_id, report)
elif report.skipped:
self.report_test_skip(test_id, report)
def pytest_terminal_summary(self):
if self.coverage_controller is not None:
try:
self._report_coverage()
except Exception:
tb = traceback.format_exc()
self.teamcity.customMessage(
"Coverage statistics reporting failed",
"ERROR",
errorDetails=tb)
def _report_coverage(self):
from coverage.misc import NotPython
from coverage.results import Numbers
class _Reporter(object):
def __init__(self, coverage, config):
try:
from coverage.report import Reporter
except ImportError:
# Support for coverage >= 5.0.1.
from coverage.report import get_analysis_to_report
class Reporter(object):
def __init__(self, coverage, config):
self.coverage = coverage
self.config = config
self._file_reporters = []
def find_file_reporters(self, morfs):
return [
fr for fr, _ in get_analysis_to_report(
self.coverage, morfs)
]
self._reporter = Reporter(coverage, config)
def find_file_reporters(self, morfs):
self.file_reporters = self._reporter.find_file_reporters(morfs)
def __getattr__(self, name):
return getattr(self._reporter, name)
class _CoverageReporter(_Reporter):
def __init__(self, coverage, config, messages):
super(_CoverageReporter, self).__init__(coverage, config)
if hasattr(coverage, 'data'):
self.branches = coverage.data.has_arcs()
else:
self.branches = coverage.get_data().has_arcs()
self.messages = messages
def report(self, morfs, outfile=None):
if hasattr(self, 'find_code_units'):
self.find_code_units(morfs)
else:
self.find_file_reporters(morfs)
total = Numbers()
if hasattr(self, 'code_units'):
units = self.code_units
else:
units = self.file_reporters
for cu in units:
try:
analysis = self.coverage._analyze(cu)
nums = analysis.numbers
total += nums
except KeyboardInterrupt:
raise
except Exception:
if self.config.ignore_errors:
continue
err = sys.exc_info()
typ, msg = err[:2]
if typ is NotPython and not cu.should_be_python():
continue
test_id = cu.name
details = convert_error_to_string(err)
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testFailed(
test_id,
message="Coverage analysis failed",
details=details,
flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
if total.n_files > 0:
covered = total.n_executed
total_statements = total.n_statements
if self.branches:
covered += total.n_executed_branches
total_statements += total.n_branches
self.messages.buildStatisticLinesCovered(covered)
self.messages.buildStatisticTotalLines(total_statements)
self.messages.buildStatisticLinesUncovered(
total_statements - covered)
reporter = _CoverageReporter(
self.coverage_controller.cov,
self.coverage_controller.cov.config,
self.teamcity,
)
reporter.report(None)
class TeamcityTestResult(TestResult):
separator2 = "\n"
# noinspection PyUnusedLocal
def __init__(self, stream=_real_stdout, descriptions=None, verbosity=None):
super(TeamcityTestResult, self).__init__()
# Some code may ask for self.failfast, see unittest2.case.TestCase.subTest
self.failfast = getattr(self, "failfast", False)
self.test_started_datetime_map = {}
self.failed_tests = set()
self.subtest_failures = {}
self.messages = TeamcityServiceMessages(_real_stdout)
self.current_test_id = None
@staticmethod
def get_test_id(test):
if is_string(test):
return test
test_class_fullname = get_class_fullname(test)
test_id = test.id()
if test_class_fullname in _ERROR_HOLDERS_FQN:
# patch setUpModule (__main__) -> __main__.setUpModule
return re.sub(r'^(.*) \((.*)\)$', r'\2.\1', test_id)
# Force test_id for doctests
if test_class_fullname != "doctest.DocTestCase":
desc = test.shortDescription()
test_method_name = getattr(test, "_testMethodName", "")
if desc and desc != test_id and desc != test_method_name:
return "%s (%s)" % (test_id, desc.replace('.', '_'))
return test_id
def addSuccess(self, test):
super(TeamcityTestResult, self).addSuccess(test)
def addExpectedFailure(self, test, err):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addExpectedFailure"):
_super.addExpectedFailure(test, err)
err = convert_error_to_string(err)
test_id = self.get_test_id(test)
self.messages.testIgnored(test_id, message="Expected failure: " + err, flowId=test_id)
def get_subtest_block_id(self, test, subtest):
test_id = self.get_test_id(test)
subtest_id = self.get_test_id(subtest)
if subtest_id.startswith(test_id):
block_id = subtest_id[len(test_id):].strip()
else:
block_id = subtest_id
if len(block_id) == 0:
block_id = test_id
return block_id
def addSkip(self, test, reason=""):
if sys.version_info >= (2, 7):
super(TeamcityTestResult, self).addSkip(test, reason)
if reason:
if isinstance(reason, Exception):
reason_str = ": " + get_exception_message(reason)
else:
reason_str = ": " + to_unicode(reason)
else:
reason_str = ""
test_class_name = get_class_fullname(test)
if test_class_name == "unittest.case._SubTest" or test_class_name == "unittest2.case._SubTest":
parent_test = test.test_case
parent_test_id = self.get_test_id(parent_test)
subtest = test
block_id = self.get_subtest_block_id(parent_test, subtest)
self.messages.subTestBlockOpened(block_id, subTestResult="Skip", flowId=parent_test_id)
self.messages.testStdOut(parent_test_id, out="SubTest skipped" + reason_str + "\n", flowId=parent_test_id)
self.messages.blockClosed(block_id, flowId=parent_test_id)
else:
test_id = self.get_test_id(test)
if test_id not in self.test_started_datetime_map:
# Test ignored without startTest. Handle start and finish events ourselves
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testIgnored(test_id, message="Skipped" + reason_str, flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
else:
self.messages.testIgnored(test_id, message="Skipped" + reason_str, flowId=test_id)
def addUnexpectedSuccess(self, test):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addUnexpectedSuccess"):
_super.addUnexpectedSuccess(test)
test_id = self.get_test_id(test)
self.messages.testFailed(test_id, message='Failure',
details="Test should not succeed since it's marked with @unittest.expectedFailure",
flowId=test_id)
def addError(self, test, err, *k):
super(TeamcityTestResult, self).addError(test, err)
test_class = get_class_fullname(test)
if test_class in _ERROR_HOLDERS_FQN:
# This is a standalone error
test_id = self.get_test_id(test)
self.messages.testStarted(test_id, flowId=test_id)
self.report_fail(test, 'Failure', err)
self.messages.testFinished(test_id, flowId=test_id)
elif get_class_fullname(err[0]) == "unittest2.case.SkipTest":
message = ""
if hasattr(err[1], "message"):
message = getattr(err[1], "message", "")
elif hasattr(err[1], "args"):
message = getattr(err[1], "args", [""])[0]
self.addSkip(test, message)
else:
self.report_fail(test, 'Error', err)
def addFailure(self, test, err, *k):
super(TeamcityTestResult, self).addFailure(test, err)
self.report_fail(test, 'Failure', err)
def addSubTest(self, test, subtest, err):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addSubTest"):
_super.addSubTest(test, subtest, err)
test_id = self.get_test_id(test)
subtest_id = self.get_test_id(subtest)
if subtest_id.startswith(test_id):
# Replace "." -> "_" since '.' is a test hierarchy separator
# See i.e. https://github.com/JetBrains/teamcity-messages/issues/134 (https://youtrack.jetbrains.com/issue/PY-23846)
block_id = subtest_id[len(test_id):].strip().replace(".", "_")
else:
block_id = subtest_id
if len(block_id) == 0:
block_id = subtest_id
if err is not None:
self.add_subtest_failure(test_id, block_id)
if issubclass(err[0], test.failureException):
self.messages.subTestBlockOpened(block_id, subTestResult="Failure", flowId=test_id)
self.messages.testStdErr(test_id, out="SubTest failure: %s\n" % convert_error_to_string(err), flowId=test_id)
self.messages.blockClosed(block_id, flowId=test_id)
else:
self.messages.subTestBlockOpened(block_id, subTestResult="Error", flowId=test_id)
self.messages.testStdErr(test_id, out="SubTest error: %s\n" % convert_error_to_string(err), flowId=test_id)
self.messages.blockClosed(block_id, flowId=test_id)
else:
self.messages.subTestBlockOpened(block_id, subTestResult="Success", flowId=test_id)
self.messages.blockClosed(block_id, flowId=test_id)
def add_subtest_failure(self, test_id, subtest_block_id):
fail_array = self.subtest_failures.get(test_id, [])
fail_array.append(subtest_block_id)
self.subtest_failures[test_id] = fail_array
def get_subtest_failure(self, test_id):
fail_array = self.subtest_failures.get(test_id, [])
return ", ".join(fail_array)
def report_fail(self, test, fail_type, err):
test_id = self.get_test_id(test)
diff_failed = None
try:
error = err[1]
if isinstance(error, EqualsAssertionError):
diff_failed = error
except Exception:
pass
if is_string(err):
details = err
else:
try:
details = err.getTraceback()
except AttributeError:
frames_to_skip_from_tail = 2 if diff_failed else 0
details = convert_error_to_string(err, frames_to_skip_from_tail)
subtest_failures = self.get_subtest_failure(test_id)
if subtest_failures:
details = "Failed subtests list: " + subtest_failures + "\n\n" + details.strip()
details = details.strip()
if diff_failed:
self.messages.testFailed(test_id,
message=diff_failed.msg,
details=details,
flowId=test_id,
comparison_failure=diff_failed)
else:
self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id)
self.failed_tests.add(test_id)
def startTest(self, test):
test_id = self.get_test_id(test)
self.current_test_id = test_id
super(TeamcityTestResult, self).startTest(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id, captureStandardOutput='true', flowId=test_id)
def _dump_test_stderr(self, data):
if self.current_test_id is not None:
dump_test_stderr(self.messages, self.current_test_id, self.current_test_id, data)
else:
_real_stderr.write(data)
def _dump_test_stdout(self, data):
if self.current_test_id is not None:
dump_test_stdout(self.messages, self.current_test_id, self.current_test_id, data)
else:
_real_stdout.write(data)
def _setupStdout(self):
if getattr(self, 'buffer', None):
self._stderr_buffer = FlushingStringIO(self._dump_test_stderr)
self._stdout_buffer = FlushingStringIO(self._dump_test_stdout)
sys.stdout = self._stdout_buffer
sys.stderr = self._stderr_buffer
def stopTest(self, test):
test_id = self.get_test_id(test)
if getattr(self, 'buffer', None):
# Do not allow super() method to print output by itself
self._mirrorOutput = False
output = sys.stdout.getvalue()
if output:
dump_test_stdout(self.messages, test_id, test_id, output)
error = sys.stderr.getvalue()
if error:
dump_test_stderr(self.messages, test_id, test_id, error)
super(TeamcityTestResult, self).stopTest(test)
self.current_test_id = None
if test_id not in self.failed_tests:
subtest_failures = self.get_subtest_failure(test_id)
if subtest_failures:
self.report_fail(test, "One or more subtests failed", "")
try:
time_diff = datetime.datetime.now() - self.test_started_datetime_map[test_id]
except KeyError:
time_diff = None
self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id)
def printErrors(self):
pass
class TeamcityReport(object):
name = 'teamcity-report'
score = 10000
def __init__(self):
super(TeamcityReport, self).__init__()
self.messages = TeamcityServiceMessages(_real_stdout)
self.test_started_datetime_map = {}
self.enabled = False
def get_test_id(self, test):
if is_string(test):
return test
# Handle special "tests"
test_class_name = get_class_fullname(test)
if test_class_name == CONTEXT_SUITE_FQN:
if inspect.ismodule(test.context):
module_name = test.context.__name__
return module_name + "." + test.error_context
elif inspect.isclass(test.context):
class_name = get_class_fullname(test.context)
return class_name + "." + test.error_context
test_id = test.id()
real_test = getattr(test, "test", test)
real_test_class_name = get_class_fullname(real_test)
test_arg = getattr(real_test, "arg", tuple())
if (type(test_arg) is tuple or type(test_arg) is list) and len(test_arg) > 0:
# As written in nose.case.FunctionTestCase#__str__ or nose.case.MethodTestCase#__str__
test_arg_str = "%s" % (test_arg,)
if test_id.endswith(test_arg_str):
# Replace '.' in test args with '_' to preserve test hierarchy on TeamCity
test_id = test_id[:len(test_id) - len(test_arg_str)] + test_arg_str.replace('.', '_')
# Force test_id for doctests
if real_test_class_name != "doctest.DocTestCase" and real_test_class_name != "nose.plugins.doctests.DocTestCase":
desc = test.shortDescription()
if desc and desc != test.id():
return "%s (%s)" % (test_id, desc.replace('.', '_'))
return test_id
def configure(self, options, conf):
self.enabled = is_running_under_teamcity()
def options(self, parser, env=os.environ):
pass
def report_fail(self, test, fail_type, err):
# workaround nose bug on python 3
if is_string(err[1]):
err = (err[0], Exception(err[1]), err[2])
test_id = self.get_test_id(test)
details = convert_error_to_string(err)
start_index = details.find(_captured_output_start_marker)
end_index = details.find(_captured_output_end_marker)
if 0 <= start_index < end_index:
captured_output = details[start_index + len(_captured_output_start_marker):end_index]
details = details[:start_index] + details[end_index + len(_captured_output_end_marker):]
for chunk in split_output(limit_output(captured_output)):
self.messages.testStdOut(test_id, chunk, flowId=test_id)
self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id)
def addError(self, test, err):
test_class_name = get_class_fullname(test)
test_id = self.get_test_id(test)
if issubclass(err[0], SkipTest):
self.messages.testIgnored(test_id, message=("SKIPPED: %s" % str(err[1])), flowId=test_id)
elif issubclass(err[0], DeprecatedTest):
self.messages.testIgnored(test_id, message="Deprecated", flowId=test_id)
elif test_class_name == CONTEXT_SUITE_FQN:
self.messages.testStarted(test_id, captureStandardOutput='true', flowId=test_id)
self.report_fail(test, 'error in ' + test.error_context + ' context', err)
self.messages.testFinished(test_id, flowId=test_id)
else:
self.report_fail(test, 'Error', err)
def addFailure(self, test, err):
self.report_fail(test, 'Failure', err)
def startTest(self, test):
test_id = self.get_test_id(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id, captureStandardOutput='true', flowId=test_id)
def addSuccess(self, test):
test_id = self.get_test_id(test)
time_diff = datetime.datetime.now() - self.test_started_datetime_map[test_id]
self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id)
class EchoTeamCityMessages(object):
def __init__(self, output_capture_enabled, coverage_controller):
self.coverage_controller = coverage_controller
self.output_capture_enabled = output_capture_enabled
self.teamcity = TeamcityServiceMessages()
self.test_start_reported_mark = set()
self.max_reported_output_size = 1 * 1024 * 1024
self.reported_output_chunk_size = 50000
def get_id_from_location(self, location):
if type(location) is not tuple or len(location) != 3 or not hasattr(location[2], "startswith"):
return None
def convert_file_to_id(filename):
filename = re.sub(r"\.pyc?$", "", filename)
return filename.replace(os.sep, ".").replace("/", ".")
def add_prefix_to_filename_id(filename_id, prefix):
dot_location = filename_id.rfind('.')
if dot_location <= 0 or dot_location >= len(filename_id) - 1:
return None
return filename_id[:dot_location + 1] + prefix + filename_id[dot_location + 1:]
pylint_prefix = '[pylint] '
if location[2].startswith(pylint_prefix):
id_from_file = convert_file_to_id(location[2][len(pylint_prefix):])
return id_from_file + ".Pylint"
if location[2] == "PEP8-check":
id_from_file = convert_file_to_id(location[0])
return id_from_file + ".PEP8"
return None
def format_test_id(self, nodeid, location):
id_from_location = self.get_id_from_location(location)
if id_from_location is not None:
return id_from_location
test_id = nodeid
if test_id.find("::") < 0:
test_id += "::top_level"
test_id = test_id.replace("::()::", "::")
test_id = re.sub(r"\.pyc?::", r"::", test_id)
test_id = test_id.replace(".", "_").replace(os.sep, ".").replace("/", ".").replace('::', '.')
return test_id
def format_location(self, location):
if type(location) is tuple and len(location) == 3:
return "%s:%s (%s)" % (str(location[0]), str(location[1]), str(location[2]))
return str(location)
def pytest_runtest_logstart(self, nodeid, location):
self.ensure_test_start_reported(self.format_test_id(nodeid, location))
def ensure_test_start_reported(self, test_id):
if test_id not in self.test_start_reported_mark:
if self.output_capture_enabled:
capture_standard_output = "false"
else:
capture_standard_output = "true"
self.teamcity.testStarted(test_id, flowId=test_id, captureStandardOutput=capture_standard_output)
self.test_start_reported_mark.add(test_id)
def report_has_output(self, report):
for (secname, data) in report.sections:
if report.when in secname and ('stdout' in secname or 'stderr' in secname):
return True
return False
def report_test_output(self, report, test_id):
for (secname, data) in report.sections:
if report.when not in secname:
continue
if not data:
continue
if 'stdout' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdOut(test_id, out=chunk, flowId=test_id)
elif 'stderr' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdErr(test_id, out=chunk, flowId=test_id)
def report_test_finished(self, test_id, duration=None):
self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id)
self.test_start_reported_mark.remove(test_id)
def report_test_failure(self, test_id, report, message=None, report_output=True):
if hasattr(report, 'duration'):
duration = timedelta(seconds=report.duration)
else:
duration = None
if message is None:
message = self.format_location(report.location)
self.ensure_test_start_reported(test_id)
if report_output:
self.report_test_output(report, test_id)
self.teamcity.testFailed(test_id, message, str(report.longrepr), flowId=test_id)
self.report_test_finished(test_id, duration)
def pytest_runtest_logreport(self, report):
"""
:type report: _pytest.runner.TestReport
"""
test_id = self.format_test_id(report.nodeid, report.location)
duration = timedelta(seconds=report.duration)
if report.passed:
# Do not report passed setup/teardown if no output
if report.when == 'call':
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id)
self.report_test_finished(test_id, duration)
else:
if self.report_has_output(report):
block_name = "test " + report.when
self.teamcity.blockOpened(block_name, flowId=test_id)
self.report_test_output(report, test_id)
self.teamcity.blockClosed(block_name, flowId=test_id)
elif report.failed:
if report.when == 'call':
self.report_test_failure(test_id, report)
elif report.when == 'setup':
if self.report_has_output(report):
self.teamcity.blockOpened("test setup", flowId=test_id)
self.report_test_output(report, test_id)
self.teamcity.blockClosed("test setup", flowId=test_id)
self.report_test_failure(test_id, report, message="test setup failed", report_output=False)
elif report.when == 'teardown':
# Report failed teardown as a separate test as original test is already finished
self.report_test_failure(test_id + "_teardown", report)
elif report.skipped:
if type(report.longrepr) is tuple and len(report.longrepr) == 3:
reason = report.longrepr[2]
else:
reason = str(report.longrepr)
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id)
self.teamcity.testIgnored(test_id, reason, flowId=test_id)
self.report_test_finished(test_id, duration)
def pytest_collectreport(self, report):
if report.failed:
test_id = self.format_test_id(report.nodeid, report.location) + "_collect"
self.report_test_failure(test_id, report)
def pytest_terminal_summary(self):
if self.coverage_controller is not None:
try:
self._report_coverage()
except:
tb = traceback.format_exc()
self.teamcity.customMessage("Coverage statistics reporting failed", "ERROR", errorDetails=tb)
def _report_coverage(self):
from coverage.misc import NotPython
from coverage.report import Reporter
from coverage.results import Numbers
class _CoverageReporter(Reporter):
def __init__(self, coverage, config, messages):
super(_CoverageReporter, self).__init__(coverage, config)
self.branches = coverage.data.has_arcs()
self.messages = messages
def report(self, morfs, outfile=None):
if hasattr(self, 'find_code_units'):
self.find_code_units(morfs)
else:
self.find_file_reporters(morfs)
total = Numbers()
if hasattr(self, 'code_units'):
units = self.code_units
else:
units = self.file_reporters
for cu in units:
try:
analysis = self.coverage._analyze(cu)
nums = analysis.numbers
total += nums
except KeyboardInterrupt:
raise
except:
if self.config.ignore_errors:
continue
err = sys.exc_info()
typ, msg = err[:2]
if typ is NotPython and not cu.should_be_python():
continue
test_id = cu.name
details = convert_error_to_string(err)
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testFailed(test_id, message="Coverage analysis failed", details=details, flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
if total.n_files > 0:
covered = total.n_executed
total_statements = total.n_statements
if self.branches:
covered += total.n_executed_branches
total_statements += total.n_branches
self.messages.buildStatisticLinesCovered(covered)
self.messages.buildStatisticTotalLines(total_statements)
self.messages.buildStatisticLinesUncovered(total_statements - covered)
reporter = _CoverageReporter(
self.coverage_controller.cov,
self.coverage_controller.cov.config,
self.teamcity,
)
reporter.report(None)
class TeamcityFormatter(Formatter):
"""
Stateful TC reporter.
Since we can't fetch all steps from the very beginning (even skipped tests are reported)
we store tests and features on each call.
To hook into test reporting override _report_suite_started and/or _report_test_started
"""
def __init__(self, stream_opener, config):
super(TeamcityFormatter, self).__init__(stream_opener, config)
assert version.LooseVersion(behave_version) >= version.LooseVersion(
"1.2.6"), "Only 1.2.6+ is supported"
self._messages = TeamcityServiceMessages()
self.__feature = None
self.__scenario = None
self.__steps = deque()
self.__scenario_opened = False
self.__feature_opened = False
self.__test_start_time = None
def feature(self, feature):
assert isinstance(feature, Feature)
assert not self.__feature, "Prev. feature not closed"
self.__feature = feature
def scenario(self, scenario):
assert isinstance(scenario, Scenario)
self.__scenario = scenario
self.__scenario_opened = False
def step(self, step):
assert isinstance(step, Step)
self.__steps.append(step)
def match(self, match):
if not self.__feature_opened:
self._report_suite_started(self.__feature,
_suite_name(self.__feature))
self.__feature_opened = True
if not self.__scenario_opened:
self._report_suite_started(self.__scenario,
_suite_name(self.__scenario))
self.__scenario_opened = True
assert self.__steps, "No steps left"
step = self.__steps.popleft()
self._report_test_started(step, _step_name(step))
self.__test_start_time = datetime.datetime.now()
def _report_suite_started(self, suite, suite_name):
"""
:param suite: behave suite
:param suite_name: suite name that must be reported, be sure to use it instead of suite.name
"""
self._messages.testSuiteStarted(suite_name)
def _report_test_started(self, test, test_name):
"""
Suite name is always stripped, be sure to strip() it too
:param test: behave test
:param test_name: test name that must be reported, be sure to use it instead of test.name
"""
self._messages.testStarted(test_name)
def result(self, step):
assert isinstance(step, Step)
step_name = _step_name(step)
if step.status == Status.failed:
try:
error = traceback.format_exc(step.exc_traceback)
if error != step.error_message:
self._messages.testStdErr(step_name, error)
except Exception:
pass # exception shall not prevent error message
self._messages.testFailed(step_name, message=step.error_message)
if step.status == Status.undefined:
self._messages.testFailed(step_name, message="Undefined")
if step.status == Status.skipped:
self._messages.testIgnored(step_name)
self._messages.testFinished(step_name,
testDuration=datetime.datetime.now() -
self.__test_start_time)
if not self.__steps:
self.__close_scenario()
elif step.status in [Status.failed, Status.undefined]:
# Broken background/undefined step stops whole scenario
reason = "Undefined step" if step.status == Status.undefined else "Prev. step failed"
self.__skip_rest_of_scenario(reason)
def __skip_rest_of_scenario(self, reason):
while self.__steps:
step = self.__steps.popleft()
self._report_test_started(step, _step_name(step))
self._messages.testIgnored(
_step_name(step),
message="{0}. Rest part of scenario is skipped".format(reason))
self._messages.testFinished(_step_name(step))
self.__close_scenario()
def __close_scenario(self):
if self.__scenario:
self._messages.testSuiteFinished(_suite_name(self.__scenario))
self.__scenario = None
def eof(self):
self.__skip_rest_of_scenario("")
self._messages.testSuiteFinished(_suite_name(self.__feature))
self.__feature = None
self.__feature_opened = False
class EchoTeamCityMessages(object):
def __init__(self, ):
self.tw = py.io.TerminalWriter(py.std.sys.stdout)
self.teamcity = TeamcityServiceMessages(self.tw)
self.currentSuite = None
def format_names(self, name):
if name.find("::") > 0:
file, testname = name.split("::", 1)
else:
file, testname = name, "top_level"
testname = testname.replace("::()::", ".")
testname = testname.replace("::", ".")
testname = testname.strip(".")
file = file.replace(".", "_").replace(os.sep, ".").replace("/", ".")
return file, testname
def pytest_runtest_logstart(self, nodeid, location):
file, testname = self.format_names(nodeid)
if not file == self.currentSuite:
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
self.currentSuite = file
self.teamcity.testSuiteStarted(self.currentSuite)
self.teamcity.testStarted(testname)
def pytest_runtest_logreport(self, report):
file, testname = self.format_names(report.nodeid)
if report.passed:
if report.when == "call": # ignore setup/teardown
duration = timedelta(seconds=report.duration)
self.teamcity.testFinished(testname, testDuration=duration)
elif report.failed:
if report.when in ("call", "setup"):
self.teamcity.testFailed(testname, str(report.location), str(report.longrepr))
duration = timedelta(seconds=report.duration)
self.teamcity.testFinished(testname, testDuration=duration) # report finished after the failure
elif report.when == "teardown":
name = testname + "_teardown"
self.teamcity.testStarted(name)
self.teamcity.testFailed(name, str(report.location), str(report.longrepr))
self.teamcity.testFinished(name)
elif report.skipped:
self.teamcity.testIgnored(testname, str(report.longrepr))
self.teamcity.testFinished(testname) # report finished after the skip
def pytest_collectreport(self, report):
if report.failed:
file, testname = self.format_names(report.nodeid)
name = file + "_collect"
self.teamcity.testStarted(name)
self.teamcity.testFailed(name, str(report.location), str(report.longrepr))
self.teamcity.testFinished(name)
def pytest_sessionfinish(self, session, exitstatus, __multicall__):
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
class TeamcityFormatter(Formatter):
"""
Stateful TC reporter.
Since we can't fetch all steps from the very beginning (even skipped tests are reported)
we store tests and features on each call.
To hook into test reporting override _report_suite_started and/or _report_test_started
"""
def __init__(self, stream_opener, config):
super(TeamcityFormatter, self).__init__(stream_opener, config)
assert version.LooseVersion(behave_version) >= version.LooseVersion("1.2.6"), "Only 1.2.6+ is supported"
self._messages = TeamcityServiceMessages()
self.__feature = None
self.__scenario = None
self.__steps = deque()
self.__scenario_opened = False
self.__feature_opened = False
self.__test_start_time = None
def feature(self, feature):
assert isinstance(feature, Feature)
assert not self.__feature, "Prev. feature not closed"
self.__feature = feature
def scenario(self, scenario):
assert isinstance(scenario, Scenario)
self.__scenario = scenario
self.__scenario_opened = False
self.__steps.clear()
def step(self, step):
assert isinstance(step, Step)
self.__steps.append(step)
def match(self, match):
if not self.__feature_opened:
self._report_suite_started(self.__feature, _suite_name(self.__feature))
self.__feature_opened = True
if not self.__scenario_opened:
self._report_suite_started(self.__scenario, _suite_name(self.__scenario))
self.__scenario_opened = True
assert self.__steps, "No steps left"
step = self.__steps.popleft()
self._report_test_started(step, _step_name(step))
self.__test_start_time = datetime.datetime.now()
def _report_suite_started(self, suite, suite_name):
"""
:param suite: behave suite
:param suite_name: suite name that must be reported, be sure to use it instead of suite.name
"""
self._messages.testSuiteStarted(suite_name)
def _report_test_started(self, test, test_name):
"""
Suite name is always stripped, be sure to strip() it too
:param test: behave test
:param test_name: test name that must be reported, be sure to use it instead of test.name
"""
self._messages.testStarted(test_name)
def result(self, step):
assert isinstance(step, Step)
step_name = _step_name(step)
if step.status == Status.failed:
try:
error = traceback.format_exc(step.exc_traceback)
if error != step.error_message:
self._messages.testStdErr(step_name, error)
except Exception:
pass # exception shall not prevent error message
self._messages.testFailed(step_name, message=step.error_message)
if step.status == Status.undefined:
self._messages.testFailed(step_name, message="Undefined")
if step.status == Status.skipped:
self._messages.testIgnored(step_name)
self._messages.testFinished(step_name, testDuration=datetime.datetime.now() - self.__test_start_time)
if not self.__steps:
self.__close_scenario()
elif step.status in [Status.failed, Status.undefined]:
# Broken background/undefined step stops whole scenario
reason = "Undefined step" if step.status == Status.undefined else "Prev. step failed"
self.__skip_rest_of_scenario(reason)
def __skip_rest_of_scenario(self, reason):
while self.__steps:
step = self.__steps.popleft()
self._report_test_started(step, _step_name(step))
self._messages.testIgnored(_step_name(step),
message="{0}. Rest part of scenario is skipped".format(reason))
self._messages.testFinished(_step_name(step))
self.__close_scenario()
def __close_scenario(self):
if self.__scenario:
self._messages.testSuiteFinished(_suite_name(self.__scenario))
self.__scenario = None
def eof(self):
self.__skip_rest_of_scenario("")
self._messages.testSuiteFinished(_suite_name(self.__feature))
self.__feature = None
self.__feature_opened = False
class EchoTeamCityMessages(object):
def __init__(self, output_capture_enabled, coverage_controller, skip_passed_output, swap_diff):
self.coverage_controller = coverage_controller
self.output_capture_enabled = output_capture_enabled
self.skip_passed_output = skip_passed_output
self.teamcity = TeamcityServiceMessages()
self.test_start_reported_mark = set()
self.max_reported_output_size = 1 * 1024 * 1024
self.reported_output_chunk_size = 50000
self.swap_diff = swap_diff
def get_id_from_location(self, location):
if type(location) is not tuple or len(location) != 3 or not hasattr(location[2], "startswith"):
return None
def convert_file_to_id(filename):
filename = re.sub(r"\.pyc?$", "", filename)
return filename.replace(os.sep, ".").replace("/", ".")
def add_prefix_to_filename_id(filename_id, prefix):
dot_location = filename_id.rfind('.')
if dot_location <= 0 or dot_location >= len(filename_id) - 1:
return None
return filename_id[:dot_location + 1] + prefix + filename_id[dot_location + 1:]
pylint_prefix = '[pylint] '
if location[2].startswith(pylint_prefix):
id_from_file = convert_file_to_id(location[2][len(pylint_prefix):])
return id_from_file + ".Pylint"
if location[2] == "PEP8-check":
id_from_file = convert_file_to_id(location[0])
return id_from_file + ".PEP8"
return None
def format_test_id(self, nodeid, location):
id_from_location = self.get_id_from_location(location)
if id_from_location is not None:
return id_from_location
test_id = nodeid
if test_id:
if test_id.find("::") < 0:
test_id += "::top_level"
else:
test_id = "top_level"
first_bracket = test_id.find("[")
if first_bracket > 0:
# [] -> (), make it look like nose parameterized tests
params = "(" + test_id[first_bracket + 1:]
if params.endswith("]"):
params = params[:-1] + ")"
test_id = test_id[:first_bracket]
if test_id.endswith("::"):
test_id = test_id[:-2]
else:
params = ""
test_id = test_id.replace("::()::", "::")
test_id = re.sub(r"\.pyc?::", r"::", test_id)
test_id = test_id.replace(".", "_").replace(os.sep, ".").replace("/", ".").replace('::', '.')
if params:
params = params.replace(".", "_")
test_id += params
return test_id
def format_location(self, location):
if type(location) is tuple and len(location) == 3:
return "%s:%s (%s)" % (str(location[0]), str(location[1]), str(location[2]))
return str(location)
def pytest_collection_modifyitems(self, session, config, items):
self.teamcity.testCount(len(items))
def pytest_runtest_logstart(self, nodeid, location):
# test name fetched from location passed as metainfo to PyCharm
# it will be used to run specific test using "-k"
# See IDEA-176950
# We only need method/function name because only it could be used as -k
test_name = location[2]
if test_name:
test_name = str(test_name).split(".")[-1]
self.ensure_test_start_reported(self.format_test_id(nodeid, location), test_name)
def ensure_test_start_reported(self, test_id, metainfo=None):
if test_id not in self.test_start_reported_mark:
if self.output_capture_enabled:
capture_standard_output = "false"
else:
capture_standard_output = "true"
self.teamcity.testStarted(test_id, flowId=test_id, captureStandardOutput=capture_standard_output, metainfo=metainfo)
self.test_start_reported_mark.add(test_id)
def report_has_output(self, report):
for (secname, data) in report.sections:
if report.when in secname and ('stdout' in secname or 'stderr' in secname):
return True
return False
def report_test_output(self, report, test_id):
for (secname, data) in report.sections:
# https://github.com/JetBrains/teamcity-messages/issues/112
# CollectReport doesn't have 'when' property
if hasattr(report, "when") and report.when not in secname:
continue
if not data:
continue
if 'stdout' in secname:
dump_test_stdout(self.teamcity, test_id, test_id, data)
elif 'stderr' in secname:
dump_test_stderr(self.teamcity, test_id, test_id, data)
def report_test_finished(self, test_id, duration=None):
self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id)
self.test_start_reported_mark.remove(test_id)
def report_test_failure(self, test_id, report, message=None, report_output=True):
if hasattr(report, 'duration'):
duration = timedelta(seconds=report.duration)
else:
duration = None
if message is None:
message = self.format_location(report.location)
self.ensure_test_start_reported(test_id)
if report_output:
self.report_test_output(report, test_id)
diff_error = None
try:
err_message = str(report.longrepr.reprcrash.message)
diff_name = diff_tools.EqualsAssertionError.__name__
# There is a string like "foo.bar.DiffError: [serialized_data]"
if diff_name in err_message:
serialized_data = err_message[err_message.index(diff_name) + len(diff_name) + 1:]
diff_error = diff_tools.deserialize_error(serialized_data)
# AssertionError is patched in py.test, we can try to fetch diff from it
# In general case message starts with "AssertionError: ", but can also starts with "assert" for top-level
# function. To support both cases we unify them
if err_message.startswith("assert"):
err_message = "AssertionError: " + err_message
if err_message.startswith("AssertionError:"):
diff_error = fetch_diff_error_from_message(err_message, self.swap_diff)
except Exception:
pass
if diff_error:
# Cut everything after postfix: it is internal view of DiffError
strace = str(report.longrepr)
data_postfix = "_ _ _ _ _"
if data_postfix in strace:
strace = strace[0:strace.index(data_postfix)]
self.teamcity.testFailed(test_id, diff_error.msg if diff_error.msg else message, strace,
flowId=test_id,
comparison_failure=diff_error
)
else:
self.teamcity.testFailed(test_id, message, str(report.longrepr), flowId=test_id)
self.report_test_finished(test_id, duration)
def report_test_skip(self, test_id, report):
if type(report.longrepr) is tuple and len(report.longrepr) == 3:
reason = report.longrepr[2]
else:
reason = str(report.longrepr)
if hasattr(report, 'duration'):
duration = timedelta(seconds=report.duration)
else:
duration = None
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id)
self.teamcity.testIgnored(test_id, reason, flowId=test_id)
self.report_test_finished(test_id, duration)
def pytest_runtest_logreport(self, report):
"""
:type report: _pytest.runner.TestReport
"""
test_id = self.format_test_id(report.nodeid, report.location)
duration = timedelta(seconds=report.duration)
if report.passed:
# Do not report passed setup/teardown if no output
if report.when == 'call':
self.ensure_test_start_reported(test_id)
if not self.skip_passed_output:
self.report_test_output(report, test_id)
self.report_test_finished(test_id, duration)
else:
if self.report_has_output(report) and not self.skip_passed_output:
block_name = "test " + report.when
self.teamcity.blockOpened(block_name, flowId=test_id)
self.report_test_output(report, test_id)
self.teamcity.blockClosed(block_name, flowId=test_id)
elif report.failed:
if report.when == 'call':
self.report_test_failure(test_id, report)
elif report.when == 'setup':
if self.report_has_output(report):
self.teamcity.blockOpened("test setup", flowId=test_id)
self.report_test_output(report, test_id)
self.teamcity.blockClosed("test setup", flowId=test_id)
self.report_test_failure(test_id, report, message="test setup failed", report_output=False)
elif report.when == 'teardown':
# Report failed teardown as a separate test as original test is already finished
self.report_test_failure(test_id + "_teardown", report)
elif report.skipped:
self.report_test_skip(test_id, report)
def pytest_collectreport(self, report):
test_id = self.format_test_id(report.nodeid, report.location) + "_collect"
if report.failed:
self.report_test_failure(test_id, report)
elif report.skipped:
self.report_test_skip(test_id, report)
def pytest_terminal_summary(self):
if self.coverage_controller is not None:
try:
self._report_coverage()
except Exception:
tb = traceback.format_exc()
self.teamcity.customMessage("Coverage statistics reporting failed", "ERROR", errorDetails=tb)
def _report_coverage(self):
from coverage.misc import NotPython
from coverage.report import Reporter
from coverage.results import Numbers
class _CoverageReporter(Reporter):
def __init__(self, coverage, config, messages):
super(_CoverageReporter, self).__init__(coverage, config)
self.branches = coverage.data.has_arcs()
self.messages = messages
def report(self, morfs, outfile=None):
if hasattr(self, 'find_code_units'):
self.find_code_units(morfs)
else:
self.find_file_reporters(morfs)
total = Numbers()
if hasattr(self, 'code_units'):
units = self.code_units
else:
units = self.file_reporters
for cu in units:
try:
analysis = self.coverage._analyze(cu)
nums = analysis.numbers
total += nums
except KeyboardInterrupt:
raise
except Exception:
if self.config.ignore_errors:
continue
err = sys.exc_info()
typ, msg = err[:2]
if typ is NotPython and not cu.should_be_python():
continue
test_id = cu.name
details = convert_error_to_string(err)
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testFailed(test_id, message="Coverage analysis failed", details=details, flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
if total.n_files > 0:
covered = total.n_executed
total_statements = total.n_statements
if self.branches:
covered += total.n_executed_branches
total_statements += total.n_branches
self.messages.buildStatisticLinesCovered(covered)
self.messages.buildStatisticTotalLines(total_statements)
self.messages.buildStatisticLinesUncovered(total_statements - covered)
reporter = _CoverageReporter(
self.coverage_controller.cov,
self.coverage_controller.cov.config,
self.teamcity,
)
reporter.report(None)
class TeamcityTestResult(TestResult):
def __init__(self, stream=sys.stdout):
TestResult.__init__(self)
self.output = stream
self.test_started_datetime = None
self.test_name = None
self.createMessages()
def createMessages(self):
self.messages = TeamcityServiceMessages(self.output)
def formatErr(self, err):
try:
exctype, value, tb = err
return ''.join(traceback.format_exception(exctype, value, tb))
except:
tb = traceback.format_exc()
return "*FAILED TO GET TRACEBACK*: " + tb
def getTestName(self, test):
return test.shortDescription() or str(test)
def addSuccess(self, test, *k):
TestResult.addSuccess(self, test)
self.output.write("ok\n")
def addError(self, test, err, *k):
TestResult.addError(self, test, err)
err = self.formatErr(err)
if self.getTestName(test) != self.test_name:
sys.stderr.write("INTERNAL ERROR: addError(%s) outside of test\n" % self.getTestName(test))
sys.stderr.write("Error: %s\n" % err)
return
self.messages.testFailed(self.getTestName(test),
message='Error', details=err)
def addSkip(self, test, reason):
TestResult.addSkip(self, test, reason)
self.output.write("skipped %s - %s\n" % (self.getTestName(test), reason))
#TODO: "testIgnored" should be replaced by "testSkipped" when implemented
self.messages.testIgnored(self.getTestName(test), reason)
def addFailure(self, test, err, *k):
# workaround nose bug on python 3
if _is_string(err[1]):
err = (err[0], Exception(err[1]), err[2])
TestResult.addFailure(self, test, err)
err = self.formatErr(err)
if self.getTestName(test) != self.test_name:
sys.stderr.write("INTERNAL ERROR: addFailure(%s) outside of test\n" % self.getTestName(test))
sys.stderr.write("Error: %s\n" % err)
return
self.messages.testFailed(self.getTestName(test),
message='Failure', details=err)
def startTest(self, test):
self.test_started_datetime = datetime.datetime.now()
self.test_name = self.getTestName(test)
self.messages.testStarted(self.test_name)
def stopTest(self, test):
time_diff = datetime.datetime.now() - self.test_started_datetime
if self.getTestName(test) != self.test_name:
sys.stderr.write("INTERNAL ERROR: stopTest(%s) after startTest(%s)" % (self.getTestName(test), self.test_name))
self.messages.testFinished(self.test_name, time_diff)
self.test_name = None
class EchoTeamCityMessages(object):
def __init__(self, output_capture_enabled, coverage_controller):
self.coverage_controller = coverage_controller
self.output_capture_enabled = output_capture_enabled
self.teamcity = TeamcityServiceMessages()
self.test_start_reported_mark = set()
self.max_reported_output_size = 1 * 1024 * 1024
self.reported_output_chunk_size = 50000
def format_test_id(self, nodeid):
test_id = nodeid
if test_id.find("::") < 0:
test_id += "::top_level"
test_id = test_id.replace("::()::", "::")
test_id = re.sub(r"\.pyc?::", r"::", test_id)
test_id = test_id.replace(".", "_").replace(os.sep, ".").replace("/", ".").replace('::', '.')
return test_id
def pytest_runtest_logstart(self, nodeid, location):
self.ensure_test_start_reported(self.format_test_id(nodeid))
def ensure_test_start_reported(self, test_id):
if test_id not in self.test_start_reported_mark:
self.teamcity.testStarted(test_id, flowId=test_id, captureStandardOutput='false' if self.output_capture_enabled else 'true')
self.test_start_reported_mark.add(test_id)
def report_has_output(self, report):
for (secname, data) in report.sections:
if report.when in secname and ('stdout' in secname or 'stderr' in secname):
return True
return False
def report_test_output(self, report, test_id, when):
for (secname, data) in report.sections:
if when not in secname:
continue
if not data:
continue
if 'stdout' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdOut(test_id, out=chunk, flowId=test_id)
elif 'stderr' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdErr(test_id, out=chunk, flowId=test_id)
def pytest_runtest_logreport(self, report):
"""
:type report: _pytest.runner.TestReport
"""
orig_test_id = self.format_test_id(report.nodeid)
suffix = '' if report.when == 'call' else ('_' + report.when)
test_id = orig_test_id + suffix
duration = timedelta(seconds=report.duration)
if report.passed:
# Do not report passed setup/teardown if no output
if report.when == 'call' or self.report_has_output(report):
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id, report.when)
self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id)
elif report.failed:
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id, report.when)
self.teamcity.testFailed(test_id, str(report.location), str(report.longrepr), flowId=test_id)
self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id)
# If test setup failed, report test failure as well
if report.when == 'setup':
self.ensure_test_start_reported(orig_test_id)
self.teamcity.testFailed(orig_test_id, "test setup failed, see %s test failure" % test_id, flowId=orig_test_id)
self.teamcity.testFinished(orig_test_id, flowId=orig_test_id)
elif report.skipped:
if type(report.longrepr) is tuple and len(report.longrepr) == 3:
reason = report.longrepr[2]
else:
reason = str(report.longrepr)
self.ensure_test_start_reported(orig_test_id)
self.report_test_output(report, orig_test_id, report.when)
self.teamcity.testIgnored(orig_test_id, reason, flowId=orig_test_id)
self.teamcity.testFinished(orig_test_id, flowId=orig_test_id)
def pytest_collectreport(self, report):
if report.failed:
test_id = self.format_test_id(report.nodeid) + "_collect"
self.teamcity.testStarted(test_id, flowId=test_id)
self.teamcity.testFailed(test_id, str(report.location), str(report.longrepr), flowId=test_id)
self.teamcity.testFinished(test_id, flowId=test_id)
def pytest_terminal_summary(self):
if self.coverage_controller is not None:
try:
self._report_coverage()
except:
tb = traceback.format_exc()
self.teamcity.customMessage("Coverage statistics reporting failed", "ERROR", errorDetails=tb)
def _report_coverage(self):
from coverage.misc import NotPython
from coverage.report import Reporter
from coverage.results import Numbers
class _CoverageReporter(Reporter):
def __init__(self, coverage, config, messages):
super(_CoverageReporter, self).__init__(coverage, config)
self.branches = coverage.data.has_arcs()
self.messages = messages
def report(self, morfs, outfile=None):
self.find_code_units(morfs)
total = Numbers()
for cu in self.code_units:
try:
analysis = self.coverage._analyze(cu)
nums = analysis.numbers
total += nums
except KeyboardInterrupt:
raise
except:
if self.config.ignore_errors:
continue
err = sys.exc_info()
typ, msg = err[:2]
if typ is NotPython and not cu.should_be_python():
continue
test_id = cu.name
details = convert_error_to_string(err)
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testFailed(test_id, message="Coverage analysis failed", details=details, flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
if total.n_files > 0:
covered = total.n_executed + (total.n_executed_branches if self.branches else 0)
total_statements = total.n_statements + (total.n_branches if self.branches else 0)
self.messages.buildStatisticLinesCovered(covered)
self.messages.buildStatisticTotalLines(total_statements)
self.messages.buildStatisticLinesUncovered(total_statements - covered)
reporter = _CoverageReporter(
self.coverage_controller.cov,
self.coverage_controller.cov.config,
self.teamcity,
)
reporter.report(None)
class TeamcityTestResult(TestResult):
separator2 = "\n"
# noinspection PyUnusedLocal
def __init__(self, stream=_real_stdout, descriptions=None, verbosity=None):
super(TeamcityTestResult, self).__init__()
# Some code may ask for self.failfast, see unittest2.case.TestCase.subTest
self.failfast = getattr(self, "failfast", False)
self.test_started_datetime_map = {}
self.failed_tests = set()
self.subtest_failures = {}
self.messages = TeamcityServiceMessages(_real_stdout)
self.current_test_id = None
@staticmethod
def get_test_id_with_description(test):
if is_string(test):
return test
test_class_fullname = get_class_fullname(test)
test_id = test.id()
if test_class_fullname in _ERROR_HOLDERS_FQN:
# patch setUpModule (__main__) -> __main__.setUpModule
return re.sub(r'^(.*) \((.*)\)$', r'\2.\1', test_id)
# Force test_id for doctests
if test_class_fullname != "doctest.DocTestCase":
desc = test.shortDescription()
test_method_name = getattr(test, "_testMethodName", "")
if desc and desc != test_id and desc != test_method_name:
return "%s (%s)" % (test_id, desc.replace('.', '_'))
return test_id
def addSuccess(self, test):
super(TeamcityTestResult, self).addSuccess(test)
def addExpectedFailure(self, test, err):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addExpectedFailure"):
_super.addExpectedFailure(test, err)
err = convert_error_to_string(err)
test_id = self.get_test_id_with_description(test)
self.messages.testIgnored(test_id,
message="Expected failure: " + err,
flowId=test_id)
def get_subtest_block_id(self, test, subtest):
test_id = self.get_test_id_with_description(test)
subtest_id = self.get_test_id_with_description(subtest)
if subtest_id.startswith(test_id):
block_id = subtest_id[len(test_id):].strip()
else:
block_id = subtest_id
if len(block_id) == 0:
block_id = test_id
return block_id
def addSkip(self, test, reason=""):
if sys.version_info >= (2, 7):
super(TeamcityTestResult, self).addSkip(test, reason)
if reason:
if isinstance(reason, Exception):
reason_str = ": " + get_exception_message(reason)
else:
reason_str = ": " + to_unicode(reason)
else:
reason_str = ""
test_class_name = get_class_fullname(test)
if test_class_name == "unittest.case._SubTest" or test_class_name == "unittest2.case._SubTest":
parent_test = test.test_case
parent_test_id = self.get_test_id_with_description(parent_test)
subtest = test
block_id = self.get_subtest_block_id(parent_test, subtest)
self.messages.subTestBlockOpened(block_id,
subTestResult="Skip",
flowId=parent_test_id)
self.messages.testStdOut(parent_test_id,
out="SubTest skipped" + reason_str + "\n",
flowId=parent_test_id)
self.messages.blockClosed(block_id, flowId=parent_test_id)
else:
test_id = self.get_test_id_with_description(test)
if test_id not in self.test_started_datetime_map:
# Test ignored without startTest. Handle start and finish events ourselves
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testIgnored(test_id,
message="Skipped" + reason_str,
flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
else:
self.messages.testIgnored(test_id,
message="Skipped" + reason_str,
flowId=test_id)
def addUnexpectedSuccess(self, test):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addUnexpectedSuccess"):
_super.addUnexpectedSuccess(test)
test_id = self.get_test_id_with_description(test)
self.messages.testFailed(
test_id,
message='Failure',
details=
"Test should not succeed since it's marked with @unittest.expectedFailure",
flowId=test_id)
def addError(self, test, err, *k):
super(TeamcityTestResult, self).addError(test, err)
test_class = get_class_fullname(test)
if test_class in _ERROR_HOLDERS_FQN:
# This is a standalone error
test_id = self.get_test_id_with_description(test)
self.messages.testStarted(test_id, flowId=test_id)
self.report_fail(test, 'Failure', err)
self.messages.testFinished(test_id, flowId=test_id)
elif get_class_fullname(err[0]) == "unittest2.case.SkipTest":
message = ""
if hasattr(err[1], "message"):
message = getattr(err[1], "message", "")
elif hasattr(err[1], "args"):
message = getattr(err[1], "args", [""])[0]
self.addSkip(test, message)
else:
self.report_fail(test, 'Error', err)
def addFailure(self, test, err, *k):
super(TeamcityTestResult, self).addFailure(test, err)
self.report_fail(test, 'Failure', err)
def addSubTest(self, test, subtest, err):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addSubTest"):
_super.addSubTest(test, subtest, err)
# test_id_with_desc may contain description which breaks process of fetching id from subtest
test_id_with_desc = self.get_test_id_with_description(test)
test_id = test.id()
subtest_id = subtest.id()
if subtest_id.startswith(test_id):
# Replace "." -> "_" since '.' is a test hierarchy separator
# See i.e. https://github.com/JetBrains/teamcity-messages/issues/134 (https://youtrack.jetbrains.com/issue/PY-23846)
block_id = subtest_id[len(test_id):].strip().replace(".", "_")
else:
block_id = subtest_id
if len(block_id) == 0:
block_id = subtest_id
if err is not None:
self.add_subtest_failure(test_id_with_desc, block_id)
if issubclass(err[0], test.failureException):
self.messages.subTestBlockOpened(block_id,
subTestResult="Failure",
flowId=test_id_with_desc)
self.messages.testStdErr(test_id_with_desc,
out="SubTest failure: %s\n" %
convert_error_to_string(err),
flowId=test_id_with_desc)
self.messages.blockClosed(block_id, flowId=test_id_with_desc)
else:
self.messages.subTestBlockOpened(block_id,
subTestResult="Error",
flowId=test_id_with_desc)
self.messages.testStdErr(test_id_with_desc,
out="SubTest error: %s\n" %
convert_error_to_string(err),
flowId=test_id_with_desc)
self.messages.blockClosed(block_id, flowId=test_id_with_desc)
else:
self.messages.subTestBlockOpened(block_id,
subTestResult="Success",
flowId=test_id_with_desc)
self.messages.blockClosed(block_id, flowId=test_id_with_desc)
def add_subtest_failure(self, test_id, subtest_block_id):
fail_array = self.subtest_failures.get(test_id, [])
fail_array.append(subtest_block_id)
self.subtest_failures[test_id] = fail_array
def get_subtest_failure(self, test_id):
fail_array = self.subtest_failures.get(test_id, [])
return ", ".join(fail_array)
def report_fail(self, test, fail_type, err):
test_id = self.get_test_id_with_description(test)
diff_failed = None
try:
from .jb_local_exc_store import get_exception
error = get_exception()
if isinstance(error, EqualsAssertionError):
diff_failed = error
except Exception:
pass
if is_string(err):
details = err
else:
try:
details = err.getTraceback()
except AttributeError:
frames_to_skip_from_tail = 2 if diff_failed else 0
details = convert_error_to_string(err,
frames_to_skip_from_tail)
subtest_failures = self.get_subtest_failure(test_id)
if subtest_failures:
details = "Failed subtests list: " + subtest_failures + "\n\n" + details.strip(
)
details = details.strip()
if diff_failed:
self.messages.testFailed(test_id,
message=diff_failed.msg,
details=details,
flowId=test_id,
comparison_failure=diff_failed)
else:
self.messages.testFailed(test_id,
message=fail_type,
details=details,
flowId=test_id)
self.failed_tests.add(test_id)
def startTest(self, test):
test_id = self.get_test_id_with_description(test)
self.current_test_id = test_id
super(TeamcityTestResult, self).startTest(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id,
captureStandardOutput='true',
flowId=test_id)
def _dump_test_stderr(self, data):
if self.current_test_id is not None:
dump_test_stderr(self.messages, self.current_test_id,
self.current_test_id, data)
else:
_real_stderr.write(data)
def _dump_test_stdout(self, data):
if self.current_test_id is not None:
dump_test_stdout(self.messages, self.current_test_id,
self.current_test_id, data)
else:
_real_stdout.write(data)
def _setupStdout(self):
if getattr(self, 'buffer', None):
self._stderr_buffer = FlushingStringIO(self._dump_test_stderr)
self._stdout_buffer = FlushingStringIO(self._dump_test_stdout)
sys.stdout = self._stdout_buffer
sys.stderr = self._stderr_buffer
def stopTest(self, test):
test_id = self.get_test_id_with_description(test)
if getattr(self, 'buffer', None):
# Do not allow super() method to print output by itself
self._mirrorOutput = False
output = sys.stdout.getvalue()
if output:
dump_test_stdout(self.messages, test_id, test_id, output)
error = sys.stderr.getvalue()
if error:
dump_test_stderr(self.messages, test_id, test_id, error)
super(TeamcityTestResult, self).stopTest(test)
self.current_test_id = None
if test_id not in self.failed_tests:
subtest_failures = self.get_subtest_failure(test_id)
if subtest_failures:
self.report_fail(test, "One or more subtests failed", "")
try:
time_diff = datetime.datetime.now(
) - self.test_started_datetime_map[test_id]
except KeyError:
time_diff = None
self.messages.testFinished(test_id,
testDuration=time_diff,
flowId=test_id)
def printErrors(self):
pass
class TeamcityTestResult(TestResult):
separator2 = "\n"
# noinspection PyUnusedLocal
def __init__(self, stream=_real_stdout, descriptions=None, verbosity=None):
super(TeamcityTestResult, self).__init__()
# Some code may ask for self.failfast, see unittest2.case.TestCase.subTest
self.failfast = getattr(self, "failfast", False)
self.test_started_datetime_map = {}
self.failed_tests = set()
self.subtest_failures = {}
self.messages = TeamcityServiceMessages(_real_stdout)
@staticmethod
def get_test_id(test):
if is_string(test):
return test
# Force test_id for doctests
if get_class_fullname(test) != "doctest.DocTestCase":
desc = test.shortDescription()
test_method_name = getattr(test, "_testMethodName", "")
if desc and desc != test.id() and desc != test_method_name:
return "%s (%s)" % (test.id(), desc.replace('.', '_'))
return test.id()
def addSuccess(self, test):
super(TeamcityTestResult, self).addSuccess(test)
def addExpectedFailure(self, test, err):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addExpectedFailure"):
_super.addExpectedFailure(test, err)
err = convert_error_to_string(err)
test_id = self.get_test_id(test)
self.messages.testIgnored(test_id,
message="Expected failure: " + err,
flowId=test_id)
def get_subtest_block_id(self, test, subtest):
test_id = self.get_test_id(test)
subtest_id = self.get_test_id(subtest)
if subtest_id.startswith(test_id):
block_id = subtest_id[len(test_id):].strip()
else:
block_id = subtest_id
if len(block_id) == 0:
block_id = test_id
return block_id
def addSkip(self, test, reason=""):
if sys.version_info >= (2, 7):
super(TeamcityTestResult, self).addSkip(test, reason)
if reason:
reason_str = ": " + str(reason)
else:
reason_str = ""
test_class_name = get_class_fullname(test)
if test_class_name == "unittest.case._SubTest" or test_class_name == "unittest2.case._SubTest":
parent_test = test.test_case
parent_test_id = self.get_test_id(parent_test)
subtest = test
block_id = self.get_subtest_block_id(parent_test, subtest)
self.messages.subTestBlockOpened(block_id,
subTestResult="Skip",
flowId=parent_test_id)
self.messages.testStdOut(parent_test_id,
out="SubTest skipped" + reason_str + "\n",
flowId=parent_test_id)
self.messages.blockClosed(block_id, flowId=parent_test_id)
else:
test_id = self.get_test_id(test)
self.messages.testIgnored(test_id,
message="Skipped" + reason_str,
flowId=test_id)
def addUnexpectedSuccess(self, test):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addUnexpectedSuccess"):
_super.addUnexpectedSuccess(test)
test_id = self.get_test_id(test)
self.messages.testFailed(
test_id,
message='Failure',
details=
"Test should not succeed since it's marked with @unittest.expectedFailure",
flowId=test_id)
def addError(self, test, err, *k):
super(TeamcityTestResult, self).addError(test, err)
test_class = get_class_fullname(test)
if test_class == "unittest.suite._ErrorHolder" or test_class == "unittest2.suite._ErrorHolder":
# This is a standalone error
test_name = test.id()
# patch setUpModule (__main__) -> __main__.setUpModule
test_name = re.sub(r'^(.*) \((.*)\)$', r'\2.\1', test_name)
self.messages.testStarted(test_name, flowId=test_name)
# noinspection PyTypeChecker
self.report_fail(test_name, 'Failure', err)
self.messages.testFinished(test_name, flowId=test_name)
elif get_class_fullname(err[0]) == "unittest2.case.SkipTest":
message = ""
if hasattr(err[1], "message"):
message = getattr(err[1], "message", "")
elif hasattr(err[1], "args"):
message = getattr(err[1], "args", [""])[0]
self.addSkip(test, message)
else:
self.report_fail(test, 'Error', err)
def addFailure(self, test, err, *k):
super(TeamcityTestResult, self).addFailure(test, err)
self.report_fail(test, 'Failure', err)
def addSubTest(self, test, subtest, err):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addSubTest"):
_super.addSubTest(test, subtest, err)
test_id = self.get_test_id(test)
subtest_id = self.get_test_id(subtest)
if subtest_id.startswith(test_id):
# Replace "." -> "_" since '.' is a test hierarchy separator
# See i.e. https://github.com/JetBrains/teamcity-messages/issues/134 (https://youtrack.jetbrains.com/issue/PY-23846)
block_id = subtest_id[len(test_id):].strip().replace(".", "_")
else:
block_id = subtest_id
if len(block_id) == 0:
block_id = subtest_id
if err is not None:
self.add_subtest_failure(test_id, block_id)
if issubclass(err[0], test.failureException):
self.messages.subTestBlockOpened(block_id,
subTestResult="Failure",
flowId=test_id)
self.messages.testStdErr(test_id,
out="SubTest failure: %s\n" %
convert_error_to_string(err),
flowId=test_id)
self.messages.blockClosed(block_id, flowId=test_id)
else:
self.messages.subTestBlockOpened(block_id,
subTestResult="Error",
flowId=test_id)
self.messages.testStdErr(test_id,
out="SubTest error: %s\n" %
convert_error_to_string(err),
flowId=test_id)
self.messages.blockClosed(block_id, flowId=test_id)
else:
self.messages.subTestBlockOpened(block_id,
subTestResult="Success",
flowId=test_id)
self.messages.blockClosed(block_id, flowId=test_id)
def add_subtest_failure(self, test_id, subtest_block_id):
fail_array = self.subtest_failures.get(test_id, [])
fail_array.append(subtest_block_id)
self.subtest_failures[test_id] = fail_array
def get_subtest_failure(self, test_id):
fail_array = self.subtest_failures.get(test_id, [])
return ", ".join(fail_array)
def report_fail(self, test, fail_type, err):
test_id = self.get_test_id(test)
if is_string(err):
details = err
elif get_class_fullname(err) == "twisted.python.failure.Failure":
details = err.getTraceback()
else:
details = convert_error_to_string(err)
subtest_failures = self.get_subtest_failure(test_id)
if subtest_failures:
details = "Failed subtests list: " + subtest_failures + "\n\n" + details.strip(
)
details = details.strip()
self.messages.testFailed(test_id,
message=fail_type,
details=details,
flowId=test_id)
self.failed_tests.add(test_id)
def startTest(self, test):
super(TeamcityTestResult, self).startTest(test)
test_id = self.get_test_id(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id,
captureStandardOutput='true',
flowId=test_id)
def stopTest(self, test):
test_id = self.get_test_id(test)
if getattr(self, 'buffer', None):
# Do not allow super() method to print output by itself
self._mirrorOutput = False
output = sys.stdout.getvalue()
if output:
for chunk in split_output(limit_output(output)):
self.messages.testStdOut(test_id, chunk, flowId=test_id)
error = sys.stderr.getvalue()
if error:
for chunk in split_output(limit_output(error)):
self.messages.testStdErr(test_id, chunk, flowId=test_id)
super(TeamcityTestResult, self).stopTest(test)
if test_id not in self.failed_tests:
subtest_failures = self.get_subtest_failure(test_id)
if subtest_failures:
self.report_fail(test, "One or more subtests failed", "")
time_diff = datetime.datetime.now(
) - self.test_started_datetime_map[test_id]
self.messages.testFinished(test_id,
testDuration=time_diff,
flowId=test_id)
def printErrors(self):
pass
class TeamcityTestResult(TestResult):
separator2 = "\n"
def __init__(self, stream=_real_stdout, descriptions=None, verbosity=None):
super(TeamcityTestResult, self).__init__()
self.test_started_datetime_map = {}
self.failed_tests = set()
self.subtest_failures = {}
self.messages = TeamcityServiceMessages(_real_stdout)
def get_test_id(self, test):
if is_string(test):
return test
# Force test_id for doctests
if get_class_fullname(test) != "doctest.DocTestCase":
desc = test.shortDescription()
test_method_name = getattr(test, "_testMethodName", "")
if desc and desc != test.id() and desc != test_method_name:
return "%s (%s)" % (test.id(), desc.replace('.', '_'))
return test.id()
def addSuccess(self, test):
super(TeamcityTestResult, self).addSuccess(test)
def addExpectedFailure(self, test, err):
super(TeamcityTestResult, self).addExpectedFailure(test, err)
err = convert_error_to_string(err)
test_id = self.get_test_id(test)
self.messages.testIgnored(test_id,
message="Expected failure: " + err,
flowId=test_id)
def addSkip(self, test, reason=""):
if sys.version_info >= (2, 7):
super(TeamcityTestResult, self).addSkip(test, reason)
test_id = self.get_test_id(test)
if reason:
reason_str = ": " + str(reason)
else:
reason_str = ""
self.messages.testIgnored(test_id,
message="Skipped" + reason_str,
flowId=test_id)
def addUnexpectedSuccess(self, test):
super(TeamcityTestResult, self).addUnexpectedSuccess(test)
test_id = self.get_test_id(test)
self.messages.testFailed(
test_id,
message='Failure',
details=
"Test should not succeed since it's marked with @unittest.expectedFailure",
flowId=test_id)
def addError(self, test, err, *k):
super(TeamcityTestResult, self).addError(test, err)
if get_class_fullname(test) == "unittest.suite._ErrorHolder":
# This is a standalone error
test_name = test.id()
# patch setUpModule (__main__) -> __main__.setUpModule
test_name = re.sub(r'^(.*) \((.*)\)$', r'\2.\1', test_name)
self.messages.testStarted(test_name, flowId=test_name)
self.report_fail(test_name, 'Failure', err)
self.messages.testFinished(test_name, flowId=test_name)
elif get_class_fullname(err[0]) == "unittest2.case.SkipTest":
message = ""
if hasattr(err[1], "message"):
message = getattr(err[1], "message", "")
elif hasattr(err[1], "args"):
message = getattr(err[1], "args", [""])[0]
self.addSkip(test, message)
else:
self.report_fail(test, 'Error', err)
def addFailure(self, test, err, *k):
super(TeamcityTestResult, self).addFailure(test, err)
self.report_fail(test, 'Failure', err)
def addSubTest(self, test, subtest, err):
super(TeamcityTestResult, self).addSubTest(test, subtest, err)
test_id = self.get_test_id(test)
if err is not None:
if issubclass(err[0], test.failureException):
self.add_subtest_failure(test_id, self.get_test_id(subtest),
err)
self.messages.testStdErr(test_id,
out="%s: failure\n" %
self.get_test_id(subtest),
flowId=test_id)
else:
self.add_subtest_failure(test_id, self.get_test_id(subtest),
err)
self.messages.testStdErr(test_id,
out="%s: error\n" %
self.get_test_id(subtest),
flowId=test_id)
else:
self.messages.testStdOut(test_id,
out="%s: ok\n" %
self.get_test_id(subtest),
flowId=test_id)
def add_subtest_failure(self, test_id, subtest_id, err):
fail_array = self.subtest_failures.get(test_id, [])
fail_array.append("%s:\n%s" %
(subtest_id, convert_error_to_string(err)))
self.subtest_failures[test_id] = fail_array
def get_subtest_failure(self, test_id):
fail_array = self.subtest_failures.get(test_id, [])
return "\n".join(fail_array)
def report_fail(self, test, fail_type, err):
test_id = self.get_test_id(test)
if is_string(err):
details = err
elif get_class_fullname(err) == "twisted.python.failure.Failure":
details = err.getTraceback()
else:
details = convert_error_to_string(err)
subtest_failure = self.get_subtest_failure(test_id)
if subtest_failure:
details = subtest_failure + "\n" + details
self.messages.testFailed(test_id,
message=fail_type,
details=details,
flowId=test_id)
self.failed_tests.add(test_id)
def startTest(self, test):
super(TeamcityTestResult, self).startTest(test)
test_id = self.get_test_id(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id,
captureStandardOutput='true',
flowId=test_id)
def stopTest(self, test):
test_id = self.get_test_id(test)
if self.buffer:
# Do not allow super() method to print output by itself
self._mirrorOutput = False
output = sys.stdout.getvalue()
if output:
for chunk in split_output(limit_output(output)):
self.messages.testStdOut(test_id, chunk, flowId=test_id)
error = sys.stderr.getvalue()
if error:
for chunk in split_output(limit_output(error)):
self.messages.testStdErr(test_id, chunk, flowId=test_id)
super(TeamcityTestResult, self).stopTest(test)
if test_id not in self.failed_tests and self.subtest_failures.get(
test_id, []):
self.report_fail(test, "Subtest failed", "")
time_diff = datetime.datetime.now(
) - self.test_started_datetime_map[test_id]
self.messages.testFinished(test_id,
testDuration=time_diff,
flowId=test_id)
def printErrors(self):
pass
class TeamcityReport(Plugin):
name = 'teamcity-report'
score = 10000
def __init__(self):
super(TeamcityReport, self).__init__()
self.messages = TeamcityServiceMessages(_real_stdout)
self.test_started_datetime_map = {}
self.config = None
self.total_tests = 0
self.enabled = False
def get_test_id(self, test):
if is_string(test):
return test
# Handle special "tests"
test_class_name = get_class_fullname(test)
if test_class_name == CONTEXT_SUITE_FQN:
if inspect.ismodule(test.context):
module_name = test.context.__name__
return module_name + "." + test.error_context
elif inspect.isclass(test.context):
class_name = get_class_fullname(test.context)
return class_name + "." + test.error_context
test_id = test.id()
real_test = getattr(test, "test", test)
real_test_class_name = get_class_fullname(real_test)
test_arg = getattr(real_test, "arg", tuple())
if (type(test_arg) is tuple or type(test_arg) is list) and len(test_arg) > 0:
# As written in nose.case.FunctionTestCase#__str__ or nose.case.MethodTestCase#__str__
test_arg_str = "%s" % (test_arg,)
if test_id.endswith(test_arg_str):
# Replace '.' in test args with '_' to preserve test hierarchy on TeamCity
test_id = test_id[:len(test_id) - len(test_arg_str)] + test_arg_str.replace('.', '_')
# Force test_id for doctests
if real_test_class_name != "doctest.DocTestCase" and real_test_class_name != "nose.plugins.doctests.DocTestCase":
desc = test.shortDescription()
if desc and desc != test.id():
return "%s (%s)" % (test_id, desc.replace('.', '_'))
return test_id
def configure(self, options, conf):
self.enabled = is_running_under_teamcity()
self.config = conf
if self._capture_plugin_enabled():
capture_plugin = self._get_capture_plugin()
old_before_test = capture_plugin.beforeTest
old_after_test = capture_plugin.afterTest
old_format_error = capture_plugin.formatError
def newCaptureBeforeTest(test):
rv = old_before_test(test)
test_id = self.get_test_id(test)
capture_plugin._buf = FlushingStringIO(lambda data: dump_test_stdout(self.messages, test_id, test_id, data))
sys.stdout = capture_plugin._buf
return rv
def newCaptureAfterTest(test):
if isinstance(capture_plugin._buf, FlushingStringIO):
capture_plugin._buf.flush()
return old_after_test(test)
def newCaptureFormatError(test, err):
if isinstance(capture_plugin._buf, FlushingStringIO):
capture_plugin._buf.flush()
return old_format_error(test, err)
capture_plugin.beforeTest = newCaptureBeforeTest
capture_plugin.afterTest = newCaptureAfterTest
capture_plugin.formatError = newCaptureFormatError
def options(self, parser, env=os.environ):
pass
def _get_capture_plugin(self):
"""
:rtype: nose.plugins.capture.Capture
"""
for plugin in self.config.plugins.plugins:
if plugin.name == "capture":
return plugin
return None
def _capture_plugin_enabled(self):
plugin = self._get_capture_plugin()
return plugin is not None and plugin.enabled
def _capture_plugin_buffer(self):
plugin = self._get_capture_plugin()
if plugin is None:
return None
return getattr(plugin, "buffer", None)
def _captureStandardOutput_value(self):
if self._capture_plugin_enabled():
return 'false'
else:
return 'true'
def report_started(self, test):
test_id = self.get_test_id(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(), flowId=test_id)
def report_fail(self, test, fail_type, err):
# workaround nose bug on python 3
if is_string(err[1]):
err = (err[0], Exception(err[1]), err[2])
test_id = self.get_test_id(test)
details = convert_error_to_string(err)
start_index = details.find(_captured_output_start_marker)
end_index = details.find(_captured_output_end_marker)
if 0 <= start_index < end_index:
# do not log test output twice, see report_finish for actual output handling
details = details[:start_index] + details[end_index + len(_captured_output_end_marker):]
try:
error = err[1]
if isinstance(error, EqualsAssertionError):
details = convert_error_to_string(err, 2)
self.messages.testFailed(test_id, message=error.msg, details=details, flowId=test_id, comparison_failure=error)
return
except Exception:
pass
self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id)
def report_finish(self, test):
test_id = self.get_test_id(test)
if test_id in self.test_started_datetime_map:
time_diff = datetime.datetime.now() - self.test_started_datetime_map[test_id]
self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id)
else:
self.messages.testFinished(test_id, flowId=test_id)
def prepareTestLoader(self, loader):
"""Insert ourselves into loader calls to count tests.
The top-level loader call often returns lazy results, like a LazySuite.
This is a problem, as we would destroy the suite by iterating over it
to count the tests. Consequently, we monkey-patch the top-level loader
call to do the load twice: once for the actual test running and again
to yield something we can iterate over to do the count.
from https://github.com/erikrose/nose-progressive/
:type loader: nose.loader.TestLoader
"""
# TODO: If there's ever a practical need, also patch loader.suiteClass
# or even TestProgram.createTests. createTests seems to be main top-
# level caller of loader methods, and nose.core.collector() (which
# isn't even called in nose) is an alternate one.
#
# nose 1.3.4 contains required fix:
# Another fix for Python 3.4: Call super in LazySuite to access _removed_tests variable
if hasattr(loader, 'loadTestsFromNames') and nose.__versioninfo__ >= (1, 3, 4):
old_loadTestsFromNames = loader.loadTestsFromNames
def _loadTestsFromNames(*args, **kwargs):
suite = old_loadTestsFromNames(*args, **kwargs)
self.total_tests += suite.countTestCases()
# Clear out the loader's cache. Otherwise, it never finds any tests
# for the actual test run:
loader._visitedPaths = set()
return old_loadTestsFromNames(*args, **kwargs)
loader.loadTestsFromNames = _loadTestsFromNames
# noinspection PyUnusedLocal
def prepareTestRunner(self, runner):
if self.total_tests:
self.messages.testCount(self.total_tests)
def addError(self, test, err):
test_class_name = get_class_fullname(test)
test_id = self.get_test_id(test)
if issubclass(err[0], SkipTest):
self.messages.testIgnored(test_id, message=("SKIPPED: %s" % str(err[1])), flowId=test_id)
self.report_finish(test)
elif issubclass(err[0], DeprecatedTest):
self.messages.testIgnored(test_id, message="Deprecated", flowId=test_id)
self.report_finish(test)
elif test_class_name == CONTEXT_SUITE_FQN:
self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(), flowId=test_id)
self.report_fail(test, 'error in ' + test.error_context + ' context', err)
self.messages.testFinished(test_id, flowId=test_id)
else:
# some test cases may report errors in pre setup when startTest was not called yet
# example: https://github.com/JetBrains/teamcity-messages/issues/153
if test_id not in self.test_started_datetime_map:
self.report_started(test)
self.report_fail(test, 'Error', err)
self.report_finish(test)
def addFailure(self, test, err):
self.report_fail(test, 'Failure', err)
self.report_finish(test)
def startTest(self, test):
test_id = self.get_test_id(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(), flowId=test_id)
def addSuccess(self, test):
self.report_finish(test)
class TeamcityTestResult(unittest.TestResult):
def __init__(self, stream=sys.stdout):
super(TeamcityTestResult, self).__init__()
self.output = stream
self.test_started_datetime = datetime.datetime.now()
self.test_name = None
self.messages = None
self.createMessages()
self.test_succeeded = False
self.subtest_errors = []
self.subtest_failures = []
def createMessages(self):
self.messages = TeamcityServiceMessages(self.output)
def formatErr(self, err):
try:
exctype, value, tb = err
return ''.join(traceback.format_exception(exctype, value, tb))
except Exception:
tb = traceback.format_exc()
return "*FAILED TO GET TRACEBACK*: " + tb
def getTestName(self, test):
desc = test.shortDescription()
if desc:
return "%s (%s)" % (test.id(), desc)
return test.id()
def addSuccess(self, test, *args):
super(TeamcityTestResult, self).addSuccess(test)
self.output.write("ok\n")
def addError(self, test, err, *k):
super(TeamcityTestResult, self).addError(test, err)
err = self.formatErr(err)
if self.getTestName(test) != self.test_name:
sys.stderr.write("INTERNAL ERROR: addError(%s) outside of test\n" % self.getTestName(test))
sys.stderr.write("Error: %s\n" % err)
return
self.test_succeeded = False
self.messages.testFailed(self.getTestName(test), message='Error', details=err)
def addFailure(self, test, err, *k):
# workaround nose bug on python 3
if _is_string(err[1]):
err = (err[0], Exception(err[1]), err[2])
super(TeamcityTestResult, self).addFailure(test, err)
err = self.formatErr(err)
if self.getTestName(test) != self.test_name:
sys.stderr.write("INTERNAL ERROR: addFailure(%s) outside of test\n" % self.getTestName(test))
sys.stderr.write("Error: %s\n" % err)
return
self.test_succeeded = False
self.messages.testFailed(self.getTestName(test), message='Failure', details=err)
def addSkip(self, test, reason):
super(TeamcityTestResult, self).addSkip(test, reason)
if self.getTestName(test) != self.test_name:
sys.stderr.write("INTERNAL ERROR: addSkip(%s) outside of test\n" % self.getTestName(test))
sys.stderr.write("Reason: %s\n" % reason)
return
self.messages.testIgnored(self.getTestName(test), reason)
def addExpectedFailure(self, test, err):
super(TeamcityTestResult, self).addExpectedFailure(test, err)
if self.getTestName(test) != self.test_name:
err = self.formatErr(err)
sys.stderr.write("INTERNAL ERROR: addExpectedFailure(%s) outside of test\n" % self.getTestName(test))
sys.stderr.write("Error: %s\n" % err)
return
self.output.write("ok (failure expected)\n")
def addUnexpectedSuccess(self, test):
super(TeamcityTestResult, self).addUnexpectedSuccess(test)
if self.getTestName(test) != self.test_name:
sys.stderr.write("INTERNAL ERROR: addUnexpectedSuccess(%s) outside of test\n" % self.getTestName(test))
return
self.test_succeeded = False
self.messages.testFailed(self.getTestName(test), message='Unexpected success')
def addSubTest(self, test, subtest, err):
super(TeamcityTestResult, self).addSubTest(test, subtest, err)
if err is not None:
if issubclass(err[0], test.failureException):
self.subtest_failures.append("%s:\n%s" % (self.getTestName(subtest), self.formatErr(err)))
self.output.write("%s: failure\n" % self.getTestName(subtest))
else:
self.subtest_errors.append("%s:\n%s" % (self.getTestName(subtest), self.formatErr(err)))
self.output.write("%s: error\n" % self.getTestName(subtest))
else:
self.output.write("%s: ok\n" % self.getTestName(subtest))
def startTest(self, test):
self.test_started_datetime = datetime.datetime.now()
self.test_name = self.getTestName(test)
self.test_succeeded = True # we assume it succeeded, and set it to False when we send a failure message
self.subtest_errors = []
self.subtest_failures = []
self.messages.testStarted(self.test_name)
def stopTest(self, test):
# Record the test as a failure after the entire test was run and any subtest has errors
if self.test_succeeded and (self.subtest_errors or self.subtest_failures):
self.messages.testFailed(self.getTestName(test), message='Subtest error',
details="\n\n".join(self.subtest_failures) + "\n".join(self.subtest_errors))
time_diff = datetime.datetime.now() - self.test_started_datetime
if self.getTestName(test) != self.test_name:
sys.stderr.write(
"INTERNAL ERROR: stopTest(%s) after startTest(%s)" % (self.getTestName(test), self.test_name))
self.messages.testFinished(self.test_name, time_diff)
self.test_name = None
self.output.flush()
class EchoTeamCityMessages(object):
def __init__(self, ):
self.tw = py.io.TerminalWriter(py.std.sys.stdout)
self.teamcity = TeamcityServiceMessages(self.tw)
self.currentSuite = None
def format_names(self, name):
if name.find("::") > 0:
file, testname = name.split("::", 1)
else:
file, testname = name, "top_level"
testname = testname.replace("::()::", ".")
testname = testname.replace("::", ".")
testname = testname.strip(".")
file = file.replace(".", "_").replace(os.sep, ".").replace("/", ".")
return file, testname
def pytest_runtest_logstart(self, nodeid, location):
file, testname = self.format_names(nodeid)
if not file == self.currentSuite:
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
self.currentSuite = file
self.teamcity.testSuiteStarted(self.currentSuite)
self.teamcity.testStarted(testname)
def pytest_runtest_logreport(self, report):
file, testname = self.format_names(report.nodeid)
if report.passed:
if report.when == "call": # ignore setup/teardown
duration = timedelta(seconds=report.duration)
self.teamcity.testFinished(testname, testDuration=duration)
elif report.failed:
if report.when in ("call", "setup"):
self.teamcity.testFailed(testname, str(report.location),
str(report.longrepr))
duration = timedelta(seconds=report.duration)
self.teamcity.testFinished(
testname,
testDuration=duration) # report finished after the failure
elif report.when == "teardown":
name = testname + "_teardown"
self.teamcity.testStarted(name)
self.teamcity.testFailed(name, str(report.location),
str(report.longrepr))
self.teamcity.testFinished(name)
elif report.skipped:
self.teamcity.testIgnored(testname, str(report.longrepr))
self.teamcity.testFinished(
testname) # report finished after the skip
def pytest_collectreport(self, report):
if report.failed:
file, testname = self.format_names(report.nodeid)
name = file + "_collect"
self.teamcity.testStarted(name)
self.teamcity.testFailed(name, str(report.location),
str(report.longrepr))
self.teamcity.testFinished(name)
def pytest_sessionfinish(self, session, exitstatus, __multicall__):
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
class TeamcityReport(object):
name = 'teamcity-report'
score = 10000
def __init__(self):
super(TeamcityReport, self).__init__()
self.messages = TeamcityServiceMessages(_real_stdout)
self.test_started_datetime_map = {}
self.enabled = False
def get_test_id(self, test):
if is_string(test):
return test
# Handle special "tests"
test_class_name = get_class_fullname(test)
if test_class_name == CONTEXT_SUITE_FQN:
if inspect.ismodule(test.context):
module_name = test.context.__name__
return module_name + "." + test.error_context
elif inspect.isclass(test.context):
class_name = get_class_fullname(test.context)
return class_name + "." + test.error_context
test_id = test.id()
real_test = getattr(test, "test", test)
real_test_class_name = get_class_fullname(real_test)
test_arg = getattr(real_test, "arg", tuple())
if (type(test_arg) is tuple or type(test_arg) is list) and len(test_arg) > 0:
# As written in nose.case.FunctionTestCase#__str__ or nose.case.MethodTestCase#__str__
test_arg_str = "%s" % (test_arg,)
if test_id.endswith(test_arg_str):
# Replace '.' in test args with '_' to preserve test hierarchy on TeamCity
test_id = test_id[:len(test_id) - len(test_arg_str)] + test_arg_str.replace('.', '_')
# Force test_id for doctests
if real_test_class_name != "doctest.DocTestCase" and real_test_class_name != "nose.plugins.doctests.DocTestCase":
desc = test.shortDescription()
if desc and desc != test.id():
return "%s (%s)" % (test_id, desc.replace('.', '_'))
return test_id
def configure(self, options, conf):
self.enabled = is_running_under_teamcity()
def options(self, parser, env=os.environ):
pass
def _get_capture_plugin(self, test):
"""
:type test: nose.case.Test
:rtype: nose.plugins.base.Plugin
"""
for plugin in test.config.plugins.plugins:
if plugin.name == "capture":
return plugin
return None
def _capture_plugin_enabled(self, test):
"""
:type test: nose.case.Test
"""
plugin = self._get_capture_plugin(test)
return plugin is not None and plugin.enabled
def _capture_plugin_buffer(self, test):
"""
:type test: nose.case.Test
"""
plugin = self._get_capture_plugin(test)
if plugin is None:
return None
return getattr(plugin, "buffer", None)
def _captureStandardOutput_value(self, test):
"""
:type test: nose.case.Test
"""
if self._capture_plugin_enabled(test):
return 'false'
else:
return 'true'
def report_fail(self, test, fail_type, err):
# workaround nose bug on python 3
if is_string(err[1]):
err = (err[0], Exception(err[1]), err[2])
test_id = self.get_test_id(test)
details = convert_error_to_string(err)
start_index = details.find(_captured_output_start_marker)
end_index = details.find(_captured_output_end_marker)
if 0 <= start_index < end_index:
# do not log test output twice, see report_finish for actual output handling
details = details[:start_index] + details[end_index + len(_captured_output_end_marker):]
self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id)
def report_finish(self, test):
test_id = self.get_test_id(test)
captured_output = getattr(test, "capturedOutput", None)
if captured_output is None and self._capture_plugin_enabled(test):
# nose capture does not fill 'capturedOutput' property on successful tests
captured_output = self._capture_plugin_buffer(test)
if captured_output:
for chunk in split_output(limit_output(captured_output)):
self.messages.testStdOut(test_id, chunk, flowId=test_id)
if test_id in self.test_started_datetime_map:
time_diff = datetime.datetime.now() - self.test_started_datetime_map[test_id]
self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id)
else:
self.messages.testFinished(test_id, flowId=test_id)
def addError(self, test, err):
test_class_name = get_class_fullname(test)
test_id = self.get_test_id(test)
if issubclass(err[0], SkipTest):
self.messages.testIgnored(test_id, message=("SKIPPED: %s" % str(err[1])), flowId=test_id)
self.report_finish(test)
elif issubclass(err[0], DeprecatedTest):
self.messages.testIgnored(test_id, message="Deprecated", flowId=test_id)
self.report_finish(test)
elif test_class_name == CONTEXT_SUITE_FQN:
self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(test), flowId=test_id)
self.report_fail(test, 'error in ' + test.error_context + ' context', err)
self.messages.testFinished(test_id, flowId=test_id)
else:
self.report_fail(test, 'Error', err)
self.report_finish(test)
def addFailure(self, test, err):
self.report_fail(test, 'Failure', err)
self.report_finish(test)
def startTest(self, test):
test_id = self.get_test_id(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(test), flowId=test_id)
def addSuccess(self, test):
self.report_finish(test)
class EchoTeamCityMessages(object):
def __init__(self, output_capture_enabled, coverage_controller):
self.coverage_controller = coverage_controller
self.output_capture_enabled = output_capture_enabled
self.teamcity = TeamcityServiceMessages()
self.test_start_reported_mark = set()
self.max_reported_output_size = 1 * 1024 * 1024
self.reported_output_chunk_size = 50000
def format_test_id(self, nodeid):
test_id = nodeid
if test_id.find("::") < 0:
test_id += "::top_level"
test_id = test_id.replace("::()::", "::")
test_id = re.sub(r"\.pyc?::", r"::", test_id)
test_id = test_id.replace(".", "_").replace(os.sep, ".").replace(
"/", ".").replace('::', '.')
return test_id
def pytest_runtest_logstart(self, nodeid, location):
self.ensure_test_start_reported(self.format_test_id(nodeid))
def ensure_test_start_reported(self, test_id):
if test_id not in self.test_start_reported_mark:
self.teamcity.testStarted(test_id,
flowId=test_id,
captureStandardOutput='false' if
self.output_capture_enabled else 'true')
self.test_start_reported_mark.add(test_id)
def report_has_output(self, report):
for (secname, data) in report.sections:
if report.when in secname and ('stdout' in secname
or 'stderr' in secname):
return True
return False
def report_test_output(self, report, test_id, when):
for (secname, data) in report.sections:
if when not in secname:
continue
if not data:
continue
if 'stdout' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdOut(test_id,
out=chunk,
flowId=test_id)
elif 'stderr' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdErr(test_id,
out=chunk,
flowId=test_id)
def pytest_runtest_logreport(self, report):
"""
:type report: _pytest.runner.TestReport
"""
orig_test_id = self.format_test_id(report.nodeid)
suffix = '' if report.when == 'call' else ('_' + report.when)
test_id = orig_test_id + suffix
duration = timedelta(seconds=report.duration)
if report.passed:
# Do not report passed setup/teardown if no output
if report.when == 'call' or self.report_has_output(report):
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id, report.when)
self.teamcity.testFinished(test_id,
testDuration=duration,
flowId=test_id)
elif report.failed:
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id, report.when)
self.teamcity.testFailed(test_id,
str(report.location),
str(report.longrepr),
flowId=test_id)
self.teamcity.testFinished(test_id,
testDuration=duration,
flowId=test_id)
# If test setup failed, report test failure as well
if report.when == 'setup':
self.ensure_test_start_reported(orig_test_id)
self.teamcity.testFailed(
orig_test_id,
"test setup failed, see %s test failure" % test_id,
flowId=orig_test_id)
self.teamcity.testFinished(orig_test_id, flowId=orig_test_id)
elif report.skipped:
if type(report.longrepr) is tuple and len(report.longrepr) == 3:
reason = report.longrepr[2]
else:
reason = str(report.longrepr)
self.ensure_test_start_reported(orig_test_id)
self.report_test_output(report, orig_test_id, report.when)
self.teamcity.testIgnored(orig_test_id,
reason,
flowId=orig_test_id)
self.teamcity.testFinished(orig_test_id, flowId=orig_test_id)
def pytest_collectreport(self, report):
if report.failed:
test_id = self.format_test_id(report.nodeid) + "_collect"
self.teamcity.testStarted(test_id, flowId=test_id)
self.teamcity.testFailed(test_id,
str(report.location),
str(report.longrepr),
flowId=test_id)
self.teamcity.testFinished(test_id, flowId=test_id)
def pytest_terminal_summary(self):
if self.coverage_controller is not None:
try:
self._report_coverage()
except:
tb = traceback.format_exc()
self.teamcity.customMessage(
"Coverage statistics reporting failed",
"ERROR",
errorDetails=tb)
def _report_coverage(self):
from coverage.misc import NotPython
from coverage.report import Reporter
from coverage.results import Numbers
class _CoverageReporter(Reporter):
def __init__(self, coverage, config, messages):
super(_CoverageReporter, self).__init__(coverage, config)
self.branches = coverage.data.has_arcs()
self.messages = messages
def report(self, morfs, outfile=None):
self.find_code_units(morfs)
total = Numbers()
for cu in self.code_units:
try:
analysis = self.coverage._analyze(cu)
nums = analysis.numbers
total += nums
except KeyboardInterrupt:
raise
except:
if self.config.ignore_errors:
continue
err = sys.exc_info()
typ, msg = err[:2]
if typ is NotPython and not cu.should_be_python():
continue
test_id = cu.name
details = convert_error_to_string(err)
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testFailed(
test_id,
message="Coverage analysis failed",
details=details,
flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
if total.n_files > 0:
covered = total.n_executed + (total.n_executed_branches
if self.branches else 0)
total_statements = total.n_statements + (
total.n_branches if self.branches else 0)
self.messages.buildStatisticLinesCovered(covered)
self.messages.buildStatisticTotalLines(total_statements)
self.messages.buildStatisticLinesUncovered(
total_statements - covered)
reporter = _CoverageReporter(
self.coverage_controller.cov,
self.coverage_controller.cov.config,
self.teamcity,
)
reporter.report(None)
class TeamcityReport(object):
name = 'teamcity-report'
score = 10000
def __init__(self):
super(TeamcityReport, self).__init__()
self.messages = TeamcityServiceMessages(_real_stdout)
self.test_started_datetime_map = {}
self.enabled = False
def get_test_id(self, test):
if is_string(test):
return test
# Handle special "tests"
test_class_name = get_class_fullname(test)
if test_class_name == CONTEXT_SUITE_FQN:
if inspect.ismodule(test.context):
module_name = test.context.__name__
return module_name + "." + test.error_context
elif inspect.isclass(test.context):
class_name = get_class_fullname(test.context)
return class_name + "." + test.error_context
test_id = test.id()
real_test = getattr(test, "test", test)
real_test_class_name = get_class_fullname(real_test)
test_arg = getattr(real_test, "arg", tuple())
if (type(test_arg) is tuple
or type(test_arg) is list) and len(test_arg) > 0:
# As written in nose.case.FunctionTestCase#__str__ or nose.case.MethodTestCase#__str__
test_arg_str = "%s" % (test_arg, )
if test_id.endswith(test_arg_str):
# Replace '.' in test args with '_' to preserve test hierarchy on TeamCity
test_id = test_id[:len(test_id) -
len(test_arg_str)] + test_arg_str.replace(
'.', '_')
# Force test_id for doctests
if real_test_class_name != "doctest.DocTestCase" and real_test_class_name != "nose.plugins.doctests.DocTestCase":
desc = test.shortDescription()
if desc and desc != test.id():
return "%s (%s)" % (test_id, desc.replace('.', '_'))
return test_id
def configure(self, options, conf):
self.enabled = is_running_under_teamcity()
def options(self, parser, env=os.environ):
pass
def report_fail(self, test, fail_type, err):
# workaround nose bug on python 3
if is_string(err[1]):
err = (err[0], Exception(err[1]), err[2])
test_id = self.get_test_id(test)
details = convert_error_to_string(err)
start_index = details.find(_captured_output_start_marker)
end_index = details.find(_captured_output_end_marker)
if 0 <= start_index < end_index:
captured_output = details[start_index +
len(_captured_output_start_marker
):end_index]
details = details[:start_index] + details[
end_index + len(_captured_output_end_marker):]
for chunk in split_output(limit_output(captured_output)):
self.messages.testStdOut(test_id, chunk, flowId=test_id)
self.messages.testFailed(test_id,
message=fail_type,
details=details,
flowId=test_id)
def addError(self, test, err):
test_class_name = get_class_fullname(test)
test_id = self.get_test_id(test)
if issubclass(err[0], SkipTest):
self.messages.testIgnored(test_id,
message="Skipped",
flowId=test_id)
elif issubclass(err[0], DeprecatedTest):
self.messages.testIgnored(test_id,
message="Deprecated",
flowId=test_id)
elif test_class_name == CONTEXT_SUITE_FQN:
self.messages.testStarted(test_id,
captureStandardOutput='true',
flowId=test_id)
self.report_fail(test,
'error in ' + test.error_context + ' context',
err)
self.messages.testFinished(test_id, flowId=test_id)
else:
self.report_fail(test, 'Error', err)
def addFailure(self, test, err):
self.report_fail(test, 'Failure', err)
def startTest(self, test):
test_id = self.get_test_id(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id,
captureStandardOutput='true',
flowId=test_id)
def stopTest(self, test):
test_id = self.get_test_id(test)
time_diff = datetime.datetime.now(
) - self.test_started_datetime_map[test_id]
self.messages.testFinished(test_id,
testDuration=time_diff,
flowId=test_id)
class EchoTeamCityMessages(object):
def __init__(self, output_capture_enabled, coverage_controller):
self.coverage_controller = coverage_controller
self.output_capture_enabled = output_capture_enabled
self.teamcity = TeamcityServiceMessages()
self.test_start_reported_mark = set()
self.max_reported_output_size = 1 * 1024 * 1024
self.reported_output_chunk_size = 50000
def get_id_from_location(self, location):
if type(location) is not tuple or len(location) != 3 or not hasattr(
location[2], "startswith"):
return None
def convert_file_to_id(filename):
filename = re.sub(r"\.pyc?$", "", filename)
return filename.replace(os.sep, ".").replace("/", ".")
def add_prefix_to_filename_id(filename_id, prefix):
dot_location = filename_id.rfind('.')
if dot_location <= 0 or dot_location >= len(filename_id) - 1:
return None
return filename_id[:dot_location +
1] + prefix + filename_id[dot_location + 1:]
pylint_prefix = '[pylint] '
if location[2].startswith(pylint_prefix):
id_from_file = convert_file_to_id(location[2][len(pylint_prefix):])
return id_from_file + ".Pylint"
if location[2] == "PEP8-check":
id_from_file = convert_file_to_id(location[0])
return id_from_file + ".PEP8"
return None
def format_test_id(self, nodeid, location):
id_from_location = self.get_id_from_location(location)
if id_from_location is not None:
return id_from_location
test_id = nodeid
if test_id:
if test_id.find("::") < 0:
test_id += "::top_level"
else:
test_id = "top_level"
first_bracket = test_id.find("[")
if first_bracket > 0:
# [] -> (), make it look like nose parameterized tests
params = "(" + test_id[first_bracket + 1:]
if params.endswith("]"):
params = params[:-1] + ")"
test_id = test_id[:first_bracket]
if test_id.endswith("::"):
test_id = test_id[:-2]
else:
params = ""
test_id = test_id.replace("::()::", "::")
test_id = re.sub(r"\.pyc?::", r"::", test_id)
test_id = test_id.replace(".", "_").replace(os.sep, ".").replace(
"/", ".").replace('::', '.')
if params:
params = params.replace(".", "_")
test_id += params
return test_id
def format_location(self, location):
if type(location) is tuple and len(location) == 3:
return "%s:%s (%s)" % (str(location[0]), str(
location[1]), str(location[2]))
return str(location)
def pytest_collection_modifyitems(self, session, config, items):
self.teamcity.testCount(len(items))
def pytest_runtest_logstart(self, nodeid, location):
self.ensure_test_start_reported(self.format_test_id(nodeid, location))
def ensure_test_start_reported(self, test_id):
if test_id not in self.test_start_reported_mark:
if self.output_capture_enabled:
capture_standard_output = "false"
else:
capture_standard_output = "true"
self.teamcity.testStarted(
test_id,
flowId=test_id,
captureStandardOutput=capture_standard_output)
self.test_start_reported_mark.add(test_id)
def report_has_output(self, report):
for (secname, data) in report.sections:
if report.when in secname and ('stdout' in secname
or 'stderr' in secname):
return True
return False
def report_test_output(self, report, test_id):
for (secname, data) in report.sections:
# https://github.com/JetBrains/teamcity-messages/issues/112
# CollectReport doesn't have 'when' property
if hasattr(report, "when") and report.when not in secname:
continue
if not data:
continue
if 'stdout' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdOut(test_id,
out=chunk,
flowId=test_id)
elif 'stderr' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdErr(test_id,
out=chunk,
flowId=test_id)
def report_test_finished(self, test_id, duration=None):
self.teamcity.testFinished(test_id,
testDuration=duration,
flowId=test_id)
self.test_start_reported_mark.remove(test_id)
def report_test_failure(self,
test_id,
report,
message=None,
report_output=True):
if hasattr(report, 'duration'):
duration = timedelta(seconds=report.duration)
else:
duration = None
if message is None:
message = self.format_location(report.location)
self.ensure_test_start_reported(test_id)
if report_output:
self.report_test_output(report, test_id)
self.teamcity.testFailed(test_id,
message,
str(report.longrepr),
flowId=test_id)
self.report_test_finished(test_id, duration)
def report_test_skip(self, test_id, report):
if type(report.longrepr) is tuple and len(report.longrepr) == 3:
reason = report.longrepr[2]
else:
reason = str(report.longrepr)
if hasattr(report, 'duration'):
duration = timedelta(seconds=report.duration)
else:
duration = None
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id)
self.teamcity.testIgnored(test_id, reason, flowId=test_id)
self.report_test_finished(test_id, duration)
def pytest_runtest_logreport(self, report):
"""
:type report: _pytest.runner.TestReport
"""
test_id = self.format_test_id(report.nodeid, report.location)
duration = timedelta(seconds=report.duration)
if report.passed:
# Do not report passed setup/teardown if no output
if report.when == 'call':
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id)
self.report_test_finished(test_id, duration)
else:
if self.report_has_output(report):
block_name = "test " + report.when
self.teamcity.blockOpened(block_name, flowId=test_id)
self.report_test_output(report, test_id)
self.teamcity.blockClosed(block_name, flowId=test_id)
elif report.failed:
if report.when == 'call':
self.report_test_failure(test_id, report)
elif report.when == 'setup':
if self.report_has_output(report):
self.teamcity.blockOpened("test setup", flowId=test_id)
self.report_test_output(report, test_id)
self.teamcity.blockClosed("test setup", flowId=test_id)
self.report_test_failure(test_id,
report,
message="test setup failed",
report_output=False)
elif report.when == 'teardown':
# Report failed teardown as a separate test as original test is already finished
self.report_test_failure(test_id + "_teardown", report)
elif report.skipped:
self.report_test_skip(test_id, report)
def pytest_collectreport(self, report):
test_id = self.format_test_id(report.nodeid,
report.location) + "_collect"
if report.failed:
self.report_test_failure(test_id, report)
elif report.skipped:
self.report_test_skip(test_id, report)
def pytest_terminal_summary(self):
if self.coverage_controller is not None:
try:
self._report_coverage()
except:
tb = traceback.format_exc()
self.teamcity.customMessage(
"Coverage statistics reporting failed",
"ERROR",
errorDetails=tb)
def _report_coverage(self):
from coverage.misc import NotPython
from coverage.report import Reporter
from coverage.results import Numbers
class _CoverageReporter(Reporter):
def __init__(self, coverage, config, messages):
super(_CoverageReporter, self).__init__(coverage, config)
self.branches = coverage.data.has_arcs()
self.messages = messages
def report(self, morfs, outfile=None):
if hasattr(self, 'find_code_units'):
self.find_code_units(morfs)
else:
self.find_file_reporters(morfs)
total = Numbers()
if hasattr(self, 'code_units'):
units = self.code_units
else:
units = self.file_reporters
for cu in units:
try:
analysis = self.coverage._analyze(cu)
nums = analysis.numbers
total += nums
except KeyboardInterrupt:
raise
except:
if self.config.ignore_errors:
continue
err = sys.exc_info()
typ, msg = err[:2]
if typ is NotPython and not cu.should_be_python():
continue
test_id = cu.name
details = convert_error_to_string(err)
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testFailed(
test_id,
message="Coverage analysis failed",
details=details,
flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
if total.n_files > 0:
covered = total.n_executed
total_statements = total.n_statements
if self.branches:
covered += total.n_executed_branches
total_statements += total.n_branches
self.messages.buildStatisticLinesCovered(covered)
self.messages.buildStatisticTotalLines(total_statements)
self.messages.buildStatisticLinesUncovered(
total_statements - covered)
reporter = _CoverageReporter(
self.coverage_controller.cov,
self.coverage_controller.cov.config,
self.teamcity,
)
reporter.report(None)
