def test_test_suite_started(): stream = StreamStub() messages = TeamcityServiceMessages(output=stream, now=lambda: fixed_date) messages.testSuiteStarted('suite emotion') assert stream.observed_output.strip() == textwrap.dedent("""\ ##teamcity[testSuiteStarted timestamp='2000-11-02T10:23:01.556' name='suite emotion'] """).strip().encode('utf-8')
class EchoTeamCityMessages(object): def __init__(self, ): self.tw = py.io.TerminalWriter(py.std.sys.stdout) self.teamcity = TeamcityServiceMessages(self.tw) self.currentSuite = None def format_names(self, name): split = '.py' file, testname = name.split(split, 1) if not testname: testname = file file = 'NO_TEST_FILE_FOUND' testname = testname.replace("::()::", ".") testname = testname.replace("::", ".") testname = testname.strip(".") return "".join([file, split]), testname def pytest_runtest_logstart(self, nodeid, location): file, testname = self.format_names(nodeid) if not file == self.currentSuite: if self.currentSuite: self.teamcity.testSuiteFinished(self.currentSuite) self.currentSuite = file self.teamcity.testSuiteStarted(self.currentSuite) self.teamcity.testStarted(testname) def pytest_runtest_logreport(self, report): file, testname = self.format_names(report.nodeid) if report.when == "call": for (secname, data) in report.sections: if 'stdout' in secname: self.teamcity.testStdOut(testname, out=data) elif 'stderr' in secname: self.teamcity.testStdErr(testname, out=data) if report.passed: if report.when == "call": # ignore setup/teardown duration = timedelta(seconds=report.duration) self.teamcity.testFinished(testname, testDuration=duration) elif report.failed: if report.when == "call": self.teamcity.testFailed(testname, str(report.location), str(report.longrepr)) duration = timedelta(seconds=report.duration) self.teamcity.testFinished(testname, testDuration=duration) # report finished after the failure elif report.skipped: self.teamcity.testIgnored(testname, str(report.longrepr)) self.teamcity.testFinished(testname) # report finished after the skip def pytest_sessionfinish(self, session, exitstatus, __multicall__): if self.currentSuite: self.teamcity.testSuiteFinished(self.currentSuite)
class EchoTeamCityMessages(object): def __init__(self, ): self.tw = py.io.TerminalWriter(py.std.sys.stdout) self.teamcity = TeamcityServiceMessages(self.tw) self.currentSuite = None def format_names(self, name): split = '.py' file, testname = name.split(split, 1) if not testname: testname = file file = 'NO_TEST_FILE_FOUND' testname = testname.replace("::()::", ".") testname = testname.replace("::", ".") testname = testname.strip(".") return ("".join([file, split]), testname) def pytest_runtest_logstart(self, nodeid, location): file, testname = self.format_names(nodeid) if not file == self.currentSuite: if self.currentSuite: self.teamcity.testSuiteFinished(self.currentSuite) self.currentSuite = file self.teamcity.testSuiteStarted(self.currentSuite) self.teamcity.testStarted(testname) def pytest_runtest_logreport(self, report): file, testname = self.format_names(report.nodeid) if report.passed: if report.when == "call": # ignore setup/teardown self.teamcity.testFinished(testname) elif report.failed: if report.when == "call": self.teamcity.testFailed(testname, str(report.location), str(report.longrepr)) self.teamcity.testFinished( testname) # report finished after the failure elif report.skipped: self.teamcity.testIgnored(testname, str(report.longrepr)) self.teamcity.testFinished( testname) # report finished after the skip def pytest_sessionfinish(self, session, exitstatus, __multicall__): if self.currentSuite: self.teamcity.testSuiteFinished(self.currentSuite)
class TeamCityReport(Report): '''TeamCity streaming reporter for a test suite''' def __init__(self, file, info, sync=True, **kwargs): self.messages = TeamcityServiceMessages(file) self.messages.message('compile-info', name=info[0], date=info[1], time=info[2]) self.sync = sync def __call__(self, index, args, info): cls = TeamCityLazyReport if self.sync else TeamCityTestReport return cls(self.messages, args, info[0]) def __enter__(self): self.messages.testSuiteStarted('default-suite') return self def __exit__(self, value, cls, traceback): self.messages.testSuiteFinished('default-suite')
def get_file_results(self): self._deferred_print.sort() messages = TeamcityServiceMessages() normalized_filename = self.filename.replace("\\", "/") suite_name = 'pep8: %s' % normalized_filename messages.testSuiteStarted(suite_name) for line_number, offset, code, text, doc in self._deferred_print: position = '%(path)s:%(row)d:%(col)d' % { 'path': normalized_filename, 'row': self.line_offset + line_number, 'col': offset + 1, } error_message = '%s: %s' % (code, text) test_name = '%s: %s' % (code, position) messages.testStarted(test_name) if line_number > len(self.lines): line = '' else: line = self.lines[line_number - 1] details = [ line.rstrip(), re.sub(r'\S', ' ', line[:offset]) + '^', ] if doc: details.append(doc.strip()) details = '\n'.join(details) messages.testFailed(test_name, error_message, details) messages.testFinished(test_name) messages.testSuiteFinished(suite_name) return self.file_errors
class EchoTeamCityMessages(object): def __init__(self, ): self.tw = py.io.TerminalWriter(py.std.sys.stdout) self.teamcity = TeamcityServiceMessages(self.tw) self.currentSuite = None def format_names(self, name): if name.find("::") > 0: file, testname = name.split("::", 1) else: file, testname = name, "top_level" testname = testname.replace("::()::", ".") testname = testname.replace("::", ".") testname = testname.strip(".") file = file.replace(".", "_").replace(os.sep, ".").replace("/", ".") return file, testname def pytest_runtest_logstart(self, nodeid, location): file, testname = self.format_names(nodeid) if not file == self.currentSuite: if self.currentSuite: self.teamcity.testSuiteFinished(self.currentSuite) self.currentSuite = file self.teamcity.testSuiteStarted(self.currentSuite) self.teamcity.testStarted(testname) def pytest_runtest_logreport(self, report): file, testname = self.format_names(report.nodeid) if report.passed: if report.when == "call": # ignore setup/teardown duration = timedelta(seconds=report.duration) self.teamcity.testFinished(testname, testDuration=duration) elif report.failed: if report.when in ("call", "setup"): self.teamcity.testFailed(testname, str(report.location), str(report.longrepr)) duration = timedelta(seconds=report.duration) self.teamcity.testFinished(testname, testDuration=duration) # report finished after the failure elif report.when == "teardown": name = testname + "_teardown" self.teamcity.testStarted(name) self.teamcity.testFailed(name, str(report.location), str(report.longrepr)) self.teamcity.testFinished(name) elif report.skipped: self.teamcity.testIgnored(testname, str(report.longrepr)) self.teamcity.testFinished(testname) # report finished after the skip def pytest_collectreport(self, report): if report.failed: file, testname = self.format_names(report.nodeid) name = file + "_collect" self.teamcity.testStarted(name) self.teamcity.testFailed(name, str(report.location), str(report.longrepr)) self.teamcity.testFinished(name) def pytest_sessionfinish(self, session, exitstatus, __multicall__): if self.currentSuite: self.teamcity.testSuiteFinished(self.currentSuite)
class TeamcityFormatter(Formatter): """ Stateful TC reporter. Since we can't fetch all steps from the very beginning (even skipped tests are reported) we store tests and features on each call. To hook into test reporting override _report_suite_started and/or _report_test_started """ def __init__(self, stream_opener, config): super(TeamcityFormatter, self).__init__(stream_opener, config) assert version.LooseVersion(behave_version) >= version.LooseVersion( "1.2.6"), "Only 1.2.6+ is supported" self._messages = TeamcityServiceMessages() self.__feature = None self.__scenario = None self.__steps = deque() self.__scenario_opened = False self.__feature_opened = False self.__test_start_time = None def feature(self, feature): assert isinstance(feature, Feature) assert not self.__feature, "Prev. feature not closed" self.__feature = feature def scenario(self, scenario): assert isinstance(scenario, Scenario) self.__scenario = scenario self.__scenario_opened = False def step(self, step): assert isinstance(step, Step) self.__steps.append(step) def match(self, match): if not self.__feature_opened: self._report_suite_started(self.__feature, _suite_name(self.__feature)) self.__feature_opened = True if not self.__scenario_opened: self._report_suite_started(self.__scenario, _suite_name(self.__scenario)) self.__scenario_opened = True assert self.__steps, "No steps left" step = self.__steps.popleft() self._report_test_started(step, _step_name(step)) self.__test_start_time = datetime.datetime.now() def _report_suite_started(self, suite, suite_name): """ :param suite: behave suite :param suite_name: suite name that must be reported, be sure to use it instead of suite.name """ self._messages.testSuiteStarted(suite_name) def _report_test_started(self, test, test_name): """ Suite name is always stripped, be sure to strip() it too :param test: behave test :param test_name: test name that must be reported, be sure to use it instead of test.name """ self._messages.testStarted(test_name) def result(self, step): assert isinstance(step, Step) step_name = _step_name(step) if step.status == Status.failed: try: error = traceback.format_exc(step.exc_traceback) if error != step.error_message: self._messages.testStdErr(step_name, error) except Exception: pass # exception shall not prevent error message self._messages.testFailed(step_name, message=step.error_message) if step.status == Status.undefined: self._messages.testFailed(step_name, message="Undefined") if step.status == Status.skipped: self._messages.testIgnored(step_name) self._messages.testFinished(step_name, testDuration=datetime.datetime.now() - self.__test_start_time) if not self.__steps: self.__close_scenario() elif step.status in [Status.failed, Status.undefined]: # Broken background/undefined step stops whole scenario reason = "Undefined step" if step.status == Status.undefined else "Prev. step failed" self.__skip_rest_of_scenario(reason) def __skip_rest_of_scenario(self, reason): while self.__steps: step = self.__steps.popleft() self._report_test_started(step, _step_name(step)) self._messages.testIgnored( _step_name(step), message="{0}. Rest part of scenario is skipped".format(reason)) self._messages.testFinished(_step_name(step)) self.__close_scenario() def __close_scenario(self): if self.__scenario: self._messages.testSuiteFinished(_suite_name(self.__scenario)) self.__scenario = None def eof(self): self.__skip_rest_of_scenario("") self._messages.testSuiteFinished(_suite_name(self.__feature)) self.__feature = None self.__feature_opened = False
class TeamcityFormatter(Formatter): """ Stateful TC reporter. Since we can't fetch all steps from the very beginning (even skipped tests are reported) we store tests and features on each call. To hook into test reporting override _report_suite_started and/or _report_test_started """ def __init__(self, stream_opener, config): super(TeamcityFormatter, self).__init__(stream_opener, config) assert version.LooseVersion(behave_version) >= version.LooseVersion("1.2.6"), "Only 1.2.6+ is supported" self._messages = TeamcityServiceMessages() self.__feature = None self.__scenario = None self.__steps = deque() self.__scenario_opened = False self.__feature_opened = False self.__test_start_time = None def feature(self, feature): assert isinstance(feature, Feature) assert not self.__feature, "Prev. feature not closed" self.__feature = feature def scenario(self, scenario): assert isinstance(scenario, Scenario) self.__scenario = scenario self.__scenario_opened = False self.__steps.clear() def step(self, step): assert isinstance(step, Step) self.__steps.append(step) def match(self, match): if not self.__feature_opened: self._report_suite_started(self.__feature, _suite_name(self.__feature)) self.__feature_opened = True if not self.__scenario_opened: self._report_suite_started(self.__scenario, _suite_name(self.__scenario)) self.__scenario_opened = True assert self.__steps, "No steps left" step = self.__steps.popleft() self._report_test_started(step, _step_name(step)) self.__test_start_time = datetime.datetime.now() def _report_suite_started(self, suite, suite_name): """ :param suite: behave suite :param suite_name: suite name that must be reported, be sure to use it instead of suite.name """ self._messages.testSuiteStarted(suite_name) def _report_test_started(self, test, test_name): """ Suite name is always stripped, be sure to strip() it too :param test: behave test :param test_name: test name that must be reported, be sure to use it instead of test.name """ self._messages.testStarted(test_name) def result(self, step): assert isinstance(step, Step) step_name = _step_name(step) if step.status == Status.failed: try: error = traceback.format_exc(step.exc_traceback) if error != step.error_message: self._messages.testStdErr(step_name, error) except Exception: pass # exception shall not prevent error message self._messages.testFailed(step_name, message=step.error_message) if step.status == Status.undefined: self._messages.testFailed(step_name, message="Undefined") if step.status == Status.skipped: self._messages.testIgnored(step_name) self._messages.testFinished(step_name, testDuration=datetime.datetime.now() - self.__test_start_time) if not self.__steps: self.__close_scenario() elif step.status in [Status.failed, Status.undefined]: # Broken background/undefined step stops whole scenario reason = "Undefined step" if step.status == Status.undefined else "Prev. step failed" self.__skip_rest_of_scenario(reason) def __skip_rest_of_scenario(self, reason): while self.__steps: step = self.__steps.popleft() self._report_test_started(step, _step_name(step)) self._messages.testIgnored(_step_name(step), message="{0}. Rest part of scenario is skipped".format(reason)) self._messages.testFinished(_step_name(step)) self.__close_scenario() def __close_scenario(self): if self.__scenario: self._messages.testSuiteFinished(_suite_name(self.__scenario)) self.__scenario = None def eof(self): self.__skip_rest_of_scenario("") self._messages.testSuiteFinished(_suite_name(self.__feature)) self.__feature = None self.__feature_opened = False
class EchoTeamCityMessages(object): def __init__(self, ): self.tw = py.io.TerminalWriter(py.std.sys.stdout) self.teamcity = TeamcityServiceMessages(self.tw) self.currentSuite = None def format_names(self, name): if name.find("::") > 0: file, testname = name.split("::", 1) else: file, testname = name, "top_level" testname = testname.replace("::()::", ".") testname = testname.replace("::", ".") testname = testname.strip(".") file = file.replace(".", "_").replace(os.sep, ".").replace("/", ".") return file, testname def pytest_runtest_logstart(self, nodeid, location): file, testname = self.format_names(nodeid) if not file == self.currentSuite: if self.currentSuite: self.teamcity.testSuiteFinished(self.currentSuite) self.currentSuite = file self.teamcity.testSuiteStarted(self.currentSuite) self.teamcity.testStarted(testname) def pytest_runtest_logreport(self, report): file, testname = self.format_names(report.nodeid) if report.passed: if report.when == "call": # ignore setup/teardown duration = timedelta(seconds=report.duration) self.teamcity.testFinished(testname, testDuration=duration) elif report.failed: if report.when in ("call", "setup"): self.teamcity.testFailed(testname, str(report.location), str(report.longrepr)) duration = timedelta(seconds=report.duration) self.teamcity.testFinished( testname, testDuration=duration) # report finished after the failure elif report.when == "teardown": name = testname + "_teardown" self.teamcity.testStarted(name) self.teamcity.testFailed(name, str(report.location), str(report.longrepr)) self.teamcity.testFinished(name) elif report.skipped: self.teamcity.testIgnored(testname, str(report.longrepr)) self.teamcity.testFinished( testname) # report finished after the skip def pytest_collectreport(self, report): if report.failed: file, testname = self.format_names(report.nodeid) name = file + "_collect" self.teamcity.testStarted(name) self.teamcity.testFailed(name, str(report.location), str(report.longrepr)) self.teamcity.testFinished(name) def pytest_sessionfinish(self, session, exitstatus, __multicall__): if self.currentSuite: self.teamcity.testSuiteFinished(self.currentSuite)
Print the teamcity service messages for associating op/s benchmark result to the tests. Requires the teamcity-messages python library: pip3 install teamcity-messages """.format(sys.argv[0])) sys.exit(1) suite_name = sys.argv[1] with open(sys.argv[2], encoding='utf-8') as f: json_results = json.load(f) teamcity_messages = TeamcityServiceMessages() teamcity_messages.testSuiteStarted(suite_name) def testMetadata_number_message(test_name, param_name, param_value): teamcity_messages.message( 'testMetadata', type='number', testName=test_name, name=param_name, value='{:.2f}'.format(param_value), ) for result in json_results.get('results', []): test_name = result['name']