def test_test_started(): stream = StreamStub() messages = TeamcityServiceMessages(output=stream, now=lambda: fixed_date) messages.testStarted('only a test') assert stream.observed_output.strip() == textwrap.dedent("""\ ##teamcity[testStarted timestamp='2000-11-02T10:23:01.556' name='only a test'] """).strip().encode('utf-8')
def test_test_started(): stream = StreamStub() messages = TeamcityServiceMessages(output=stream, now=lambda: fixed_date) messages.testStarted('only a test') assert stream.observed_output.strip() == textwrap.dedent("""\ ##teamcity[testStarted timestamp='2000-11-02T10:23:01.556' name='only a test'] """).strip().encode('utf-8')
def _wait_build_finish( tc: TeamCity, tc_msg: TeamcityServiceMessages, triggered_builds: dict[str, int]) -> tuple[dict[str, bool], list[Exception]]: builds_statuses = {} errors = [] start_time = datetime.now() while triggered_builds: time.sleep(BUILDS_CHECK_DELAY) for shell_name, build_id in triggered_builds.copy().items(): try: build = tc.builds.get(f"id:{build_id}") if is_build_finished(build): tc_msg.testStarted(shell_name) is_success = is_build_success(build) builds_statuses[shell_name] = is_success triggered_builds.pop(shell_name) if not is_success: tc_msg.testFailed( shell_name, f"{shell_name} Automation tests is finished" f" with status {build.status}", ) tc_msg.testFinished(shell_name, testDuration=datetime.now() - start_time) except Exception as e: errors.append(e) click.echo(e, err=True) return builds_statuses, errors
def test_handling_eagain_ioerror(): stream = StreamStub(raise_ioerror=errno.EAGAIN) messages = TeamcityServiceMessages(output=stream, now=lambda: fixed_date) assert stream.raise_ioerror == errno.EAGAIN messages.testStarted('only a test') assert stream.raise_ioerror is None assert stream.observed_output == b('##teamcity[' 'testStarted' ' timestamp=\'2000-11-02T10:23:01.556\'' ' name=\'only a test\'' ']\n')
class EchoTeamCityMessages(object): def __init__(self, ): self.tw = py.io.TerminalWriter(py.std.sys.stdout) self.teamcity = TeamcityServiceMessages(self.tw) self.currentSuite = None def format_names(self, name): split = '.py' file, testname = name.split(split, 1) if not testname: testname = file file = 'NO_TEST_FILE_FOUND' testname = testname.replace("::()::", ".") testname = testname.replace("::", ".") testname = testname.strip(".") return "".join([file, split]), testname def pytest_runtest_logstart(self, nodeid, location): file, testname = self.format_names(nodeid) if not file == self.currentSuite: if self.currentSuite: self.teamcity.testSuiteFinished(self.currentSuite) self.currentSuite = file self.teamcity.testSuiteStarted(self.currentSuite) self.teamcity.testStarted(testname) def pytest_runtest_logreport(self, report): file, testname = self.format_names(report.nodeid) if report.when == "call": for (secname, data) in report.sections: if 'stdout' in secname: self.teamcity.testStdOut(testname, out=data) elif 'stderr' in secname: self.teamcity.testStdErr(testname, out=data) if report.passed: if report.when == "call": # ignore setup/teardown duration = timedelta(seconds=report.duration) self.teamcity.testFinished(testname, testDuration=duration) elif report.failed: if report.when == "call": self.teamcity.testFailed(testname, str(report.location), str(report.longrepr)) duration = timedelta(seconds=report.duration) self.teamcity.testFinished(testname, testDuration=duration) # report finished after the failure elif report.skipped: self.teamcity.testIgnored(testname, str(report.longrepr)) self.teamcity.testFinished(testname) # report finished after the skip def pytest_sessionfinish(self, session, exitstatus, __multicall__): if self.currentSuite: self.teamcity.testSuiteFinished(self.currentSuite)
def test_handling_eagain_ioerror(): stream = StreamStub(raise_ioerror=errno.EAGAIN) messages = TeamcityServiceMessages(output=stream, now=lambda: fixed_date) assert stream.raise_ioerror == errno.EAGAIN messages.testStarted('only a test') assert stream.raise_ioerror is None assert stream.observed_output == b( '##teamcity[' 'testStarted' ' timestamp=\'2000-11-02T10:23:01.556\'' ' name=\'only a test\'' ']\n' )
class TeamcityTestResult(TestResult): def __init__(self, stream=sys.stdout): TestResult.__init__(self) self.output = stream self.createMessages() def createMessages(self): self.messages = TeamcityServiceMessages(self.output) def formatErr(self, err): exctype, value, tb = err return ''.join(traceback.format_exception(exctype, value, tb)) def getTestName(self, test): return test.shortDescription() or str(test) def addSuccess(self, test, *k): TestResult.addSuccess(self, test) self.output.write("ok\n") def addError(self, test, err, *k): TestResult.addError(self, test, err) err = self.formatErr(err) self.messages.testFailed(self.getTestName(test), message='Error', details=err) def addFailure(self, test, err, *k): TestResult.addFailure(self, test, err) err = self.formatErr(err) self.messages.testFailed(self.getTestName(test), message='Failure', details=err) def startTest(self, test): self.messages.testStarted(self.getTestName(test)) def stopTest(self, test): self.messages.testFinished(self.getTestName(test))
class EchoTeamCityMessages(object): def __init__(self, ): self.tw = py.io.TerminalWriter(py.std.sys.stdout) self.teamcity = TeamcityServiceMessages(self.tw) self.currentSuite = None def format_names(self, name): split = '.py' file, testname = name.split(split, 1) if not testname: testname = file file = 'NO_TEST_FILE_FOUND' testname = testname.replace("::()::", ".") testname = testname.replace("::", ".") testname = testname.strip(".") return ("".join([file, split]), testname) def pytest_runtest_logstart(self, nodeid, location): file, testname = self.format_names(nodeid) if not file == self.currentSuite: if self.currentSuite: self.teamcity.testSuiteFinished(self.currentSuite) self.currentSuite = file self.teamcity.testSuiteStarted(self.currentSuite) self.teamcity.testStarted(testname) def pytest_runtest_logreport(self, report): file, testname = self.format_names(report.nodeid) if report.passed: if report.when == "call": # ignore setup/teardown self.teamcity.testFinished(testname) elif report.failed: if report.when == "call": self.teamcity.testFailed(testname, str(report.location), str(report.longrepr)) self.teamcity.testFinished( testname) # report finished after the failure elif report.skipped: self.teamcity.testIgnored(testname, str(report.longrepr)) self.teamcity.testFinished( testname) # report finished after the skip def pytest_sessionfinish(self, session, exitstatus, __multicall__): if self.currentSuite: self.teamcity.testSuiteFinished(self.currentSuite)
class TeamcityTestResult(TestResult): def __init__(self, stream=sys.stdout): TestResult.__init__(self) self.output = stream self.createMessages() def createMessages(self): self.messages = TeamcityServiceMessages(self.output) def formatErr(self, err): exctype, value, tb = err return ''.join(traceback.format_exception(exctype, value, tb)) def getTestName(self, test): return test.shortDescription() or str(test) def addSuccess(self, test, *k): TestResult.addSuccess(self, test) self.output.write("ok\n") def addError(self, test, err, *k): TestResult.addError(self, test, err) err = self.formatErr(err) self.messages.testFailed(self.getTestName(test), message='Error', details=err) def addFailure(self, test, err, *k): TestResult.addFailure(self, test, err) err = self.formatErr(err) self.messages.testFailed(self.getTestName(test), message='Failure', details=err) def startTest(self, test): self.messages.testStarted(self.getTestName(test)) def stopTest(self, test): self.messages.testFinished(self.getTestName(test))
def get_file_results(self): self._deferred_print.sort() messages = TeamcityServiceMessages() normalized_filename = self.filename.replace("\\", "/") suite_name = 'pep8: %s' % normalized_filename messages.testSuiteStarted(suite_name) for line_number, offset, code, text, doc in self._deferred_print: position = '%(path)s:%(row)d:%(col)d' % { 'path': normalized_filename, 'row': self.line_offset + line_number, 'col': offset + 1, } error_message = '%s: %s' % (code, text) test_name = '%s: %s' % (code, position) messages.testStarted(test_name) if line_number > len(self.lines): line = '' else: line = self.lines[line_number - 1] details = [ line.rstrip(), re.sub(r'\S', ' ', line[:offset]) + '^', ] if doc: details.append(doc.strip()) details = '\n'.join(details) messages.testFailed(test_name, error_message, details) messages.testFinished(test_name) messages.testSuiteFinished(suite_name) return self.file_errors
def get_file_results(self): self._deferred_print.sort() messages = TeamcityServiceMessages() normalized_filename = self.filename.replace("\\", "/") suite_name = 'pep8: %s' % normalized_filename messages.testSuiteStarted(suite_name) for line_number, offset, code, text, doc in self._deferred_print: position = '%(path)s:%(row)d:%(col)d' % { 'path': normalized_filename, 'row': self.line_offset + line_number, 'col': offset + 1, } error_message = '%s: %s' % (code, text) test_name = '%s: %s' % (code, position) messages.testStarted(test_name) if line_number > len(self.lines): line = '' else: line = self.lines[line_number - 1] details = [ line.rstrip(), re.sub(r'\S', ' ', line[:offset]) + '^', ] if doc: details.append(doc.strip()) details = '\n'.join(details) messages.testFailed(test_name, error_message, details) messages.testFinished(test_name) messages.testSuiteFinished(suite_name) return self.file_errors
def format(self, error): normalized_filename = error.filename.replace("\\", "/") position = '%s:%d:%d' % ( normalized_filename, error.line_number, error.column_number) error_message = '%s %s' % (error.code, error.text) test_name = 'pep8: %s: %s' % (position, error_message) line = error.physical_line offset = error.column_number details = [ line.rstrip(), re.sub(r'\S', ' ', line[:offset]) + '^', ] details = '\n'.join(details) bytesio = BytesIO() messages = TeamcityServiceMessages(output=bytesio) messages.testStarted(test_name) messages.testFailed(test_name, error_message, details) messages.testFinished(test_name) return bytesio.getvalue().decode('UTF-8')
def format(self, error): normalized_filename = error.filename.replace("\\", "/") position = '%s:%d:%d' % (normalized_filename, error.line_number, error.column_number) error_message = '%s %s' % (error.code, error.text) test_name = 'pep8: %s: %s' % (position, error_message) line = error.physical_line offset = error.column_number details = [ line.rstrip(), re.sub(r'\S', ' ', line[:offset]) + '^', ] details = '\n'.join(details) bytesio = BytesIO() messages = TeamcityServiceMessages(output=bytesio) messages.testStarted(test_name) messages.testFailed(test_name, error_message, details) messages.testFinished(test_name) return bytesio.getvalue().decode('UTF-8')
class TeamcityTestResult(TestResult): separator2 = "\n" # noinspection PyUnusedLocal def __init__(self, stream=_real_stdout, descriptions=None, verbosity=None): super(TeamcityTestResult, self).__init__() # Some code may ask for self.failfast, see unittest2.case.TestCase.subTest self.failfast = getattr(self, "failfast", False) self.test_started_datetime_map = {} self.failed_tests = set() self.subtest_failures = {} self.messages = TeamcityServiceMessages(_real_stdout) self.current_test_id = None @staticmethod def get_test_id_with_description(test): if is_string(test): return test test_class_fullname = get_class_fullname(test) test_id = test.id() if test_class_fullname in _ERROR_HOLDERS_FQN: # patch setUpModule (__main__) -> __main__.setUpModule return re.sub(r'^(.*) \((.*)\)$', r'\2.\1', test_id) # Force test_id for doctests if test_class_fullname != "doctest.DocTestCase": desc = test.shortDescription() test_method_name = getattr(test, "_testMethodName", "") if desc and desc != test_id and desc != test_method_name: return "%s (%s)" % (test_id, desc.replace('.', '_')) return test_id def addSuccess(self, test): super(TeamcityTestResult, self).addSuccess(test) def addExpectedFailure(self, test, err): _super = super(TeamcityTestResult, self) if hasattr(_super, "addExpectedFailure"): _super.addExpectedFailure(test, err) err = convert_error_to_string(err) test_id = self.get_test_id_with_description(test) self.messages.testIgnored(test_id, message="Expected failure: " + err, flowId=test_id) def get_subtest_block_id(self, test, subtest): test_id = self.get_test_id_with_description(test) subtest_id = self.get_test_id_with_description(subtest) if subtest_id.startswith(test_id): block_id = subtest_id[len(test_id):].strip() else: block_id = subtest_id if len(block_id) == 0: block_id = test_id return block_id def addSkip(self, test, reason=""): if sys.version_info >= (2, 7): super(TeamcityTestResult, self).addSkip(test, reason) if reason: if isinstance(reason, Exception): reason_str = ": " + get_exception_message(reason) else: reason_str = ": " + to_unicode(reason) else: reason_str = "" test_class_name = get_class_fullname(test) if test_class_name == "unittest.case._SubTest" or test_class_name == "unittest2.case._SubTest": parent_test = test.test_case parent_test_id = self.get_test_id_with_description(parent_test) subtest = test block_id = self.get_subtest_block_id(parent_test, subtest) self.messages.subTestBlockOpened(block_id, subTestResult="Skip", flowId=parent_test_id) self.messages.testStdOut(parent_test_id, out="SubTest skipped" + reason_str + "\n", flowId=parent_test_id) self.messages.blockClosed(block_id, flowId=parent_test_id) else: test_id = self.get_test_id_with_description(test) if test_id not in self.test_started_datetime_map: # Test ignored without startTest. Handle start and finish events ourselves self.messages.testStarted(test_id, flowId=test_id) self.messages.testIgnored(test_id, message="Skipped" + reason_str, flowId=test_id) self.messages.testFinished(test_id, flowId=test_id) else: self.messages.testIgnored(test_id, message="Skipped" + reason_str, flowId=test_id) def addUnexpectedSuccess(self, test): _super = super(TeamcityTestResult, self) if hasattr(_super, "addUnexpectedSuccess"): _super.addUnexpectedSuccess(test) test_id = self.get_test_id_with_description(test) self.messages.testFailed( test_id, message='Failure', details= "Test should not succeed since it's marked with @unittest.expectedFailure", flowId=test_id) def addError(self, test, err, *k): super(TeamcityTestResult, self).addError(test, err) test_class = get_class_fullname(test) if test_class in _ERROR_HOLDERS_FQN: # This is a standalone error test_id = self.get_test_id_with_description(test) self.messages.testStarted(test_id, flowId=test_id) self.report_fail(test, 'Failure', err) self.messages.testFinished(test_id, flowId=test_id) elif get_class_fullname(err[0]) == "unittest2.case.SkipTest": message = "" if hasattr(err[1], "message"): message = getattr(err[1], "message", "") elif hasattr(err[1], "args"): message = getattr(err[1], "args", [""])[0] self.addSkip(test, message) else: self.report_fail(test, 'Error', err) def addFailure(self, test, err, *k): super(TeamcityTestResult, self).addFailure(test, err) self.report_fail(test, 'Failure', err) def addSubTest(self, test, subtest, err): _super = super(TeamcityTestResult, self) if hasattr(_super, "addSubTest"): _super.addSubTest(test, subtest, err) # test_id_with_desc may contain description which breaks process of fetching id from subtest test_id_with_desc = self.get_test_id_with_description(test) test_id = test.id() subtest_id = subtest.id() if subtest_id.startswith(test_id): # Replace "." -> "_" since '.' is a test hierarchy separator # See i.e. https://github.com/JetBrains/teamcity-messages/issues/134 (https://youtrack.jetbrains.com/issue/PY-23846) block_id = subtest_id[len(test_id):].strip().replace(".", "_") else: block_id = subtest_id if len(block_id) == 0: block_id = subtest_id if err is not None: self.add_subtest_failure(test_id_with_desc, block_id) if issubclass(err[0], test.failureException): self.messages.subTestBlockOpened(block_id, subTestResult="Failure", flowId=test_id_with_desc) self.messages.testStdErr(test_id_with_desc, out="SubTest failure: %s\n" % convert_error_to_string(err), flowId=test_id_with_desc) self.messages.blockClosed(block_id, flowId=test_id_with_desc) else: self.messages.subTestBlockOpened(block_id, subTestResult="Error", flowId=test_id_with_desc) self.messages.testStdErr(test_id_with_desc, out="SubTest error: %s\n" % convert_error_to_string(err), flowId=test_id_with_desc) self.messages.blockClosed(block_id, flowId=test_id_with_desc) else: self.messages.subTestBlockOpened(block_id, subTestResult="Success", flowId=test_id_with_desc) self.messages.blockClosed(block_id, flowId=test_id_with_desc) def add_subtest_failure(self, test_id, subtest_block_id): fail_array = self.subtest_failures.get(test_id, []) fail_array.append(subtest_block_id) self.subtest_failures[test_id] = fail_array def get_subtest_failure(self, test_id): fail_array = self.subtest_failures.get(test_id, []) return ", ".join(fail_array) def report_fail(self, test, fail_type, err): test_id = self.get_test_id_with_description(test) diff_failed = None try: from .jb_local_exc_store import get_exception error = get_exception() if isinstance(error, EqualsAssertionError): diff_failed = error except Exception: pass if is_string(err): details = err else: try: details = err.getTraceback() except AttributeError: frames_to_skip_from_tail = 2 if diff_failed else 0 details = convert_error_to_string(err, frames_to_skip_from_tail) subtest_failures = self.get_subtest_failure(test_id) if subtest_failures: details = "Failed subtests list: " + subtest_failures + "\n\n" + details.strip( ) details = details.strip() if diff_failed: self.messages.testFailed(test_id, message=diff_failed.msg, details=details, flowId=test_id, comparison_failure=diff_failed) else: self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id) self.failed_tests.add(test_id) def startTest(self, test): test_id = self.get_test_id_with_description(test) self.current_test_id = test_id super(TeamcityTestResult, self).startTest(test) self.test_started_datetime_map[test_id] = datetime.datetime.now() self.messages.testStarted(test_id, captureStandardOutput='true', flowId=test_id) def _dump_test_stderr(self, data): if self.current_test_id is not None: dump_test_stderr(self.messages, self.current_test_id, self.current_test_id, data) else: _real_stderr.write(data) def _dump_test_stdout(self, data): if self.current_test_id is not None: dump_test_stdout(self.messages, self.current_test_id, self.current_test_id, data) else: _real_stdout.write(data) def _setupStdout(self): if getattr(self, 'buffer', None): self._stderr_buffer = FlushingStringIO(self._dump_test_stderr) self._stdout_buffer = FlushingStringIO(self._dump_test_stdout) sys.stdout = self._stdout_buffer sys.stderr = self._stderr_buffer def stopTest(self, test): test_id = self.get_test_id_with_description(test) if getattr(self, 'buffer', None): # Do not allow super() method to print output by itself self._mirrorOutput = False output = sys.stdout.getvalue() if output: dump_test_stdout(self.messages, test_id, test_id, output) error = sys.stderr.getvalue() if error: dump_test_stderr(self.messages, test_id, test_id, error) super(TeamcityTestResult, self).stopTest(test) self.current_test_id = None if test_id not in self.failed_tests: subtest_failures = self.get_subtest_failure(test_id) if subtest_failures: self.report_fail(test, "One or more subtests failed", "") try: time_diff = datetime.datetime.now( ) - self.test_started_datetime_map[test_id] except KeyError: time_diff = None self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id) def printErrors(self): pass
class TeamcityReport(Plugin): name = 'teamcity-report' score = 10000 def __init__(self): super(TeamcityReport, self).__init__() self.messages = TeamcityServiceMessages(_real_stdout) self.test_started_datetime_map = {} self.config = None self.total_tests = 0 self.enabled = False def get_test_id(self, test): if is_string(test): return test # Handle special "tests" test_class_name = get_class_fullname(test) if test_class_name == CONTEXT_SUITE_FQN: if inspect.ismodule(test.context): module_name = test.context.__name__ return module_name + "." + test.error_context elif inspect.isclass(test.context): class_name = get_class_fullname(test.context) return class_name + "." + test.error_context test_id = test.id() real_test = getattr(test, "test", test) real_test_class_name = get_class_fullname(real_test) test_arg = getattr(real_test, "arg", tuple()) if (type(test_arg) is tuple or type(test_arg) is list) and len(test_arg) > 0: # As written in nose.case.FunctionTestCase#__str__ or nose.case.MethodTestCase#__str__ test_arg_str = "%s" % (test_arg,) if test_id.endswith(test_arg_str): # Replace '.' in test args with '_' to preserve test hierarchy on TeamCity test_id = test_id[:len(test_id) - len(test_arg_str)] + test_arg_str.replace('.', '_') # Force test_id for doctests if real_test_class_name != "doctest.DocTestCase" and real_test_class_name != "nose.plugins.doctests.DocTestCase": desc = test.shortDescription() if desc and desc != test.id(): return "%s (%s)" % (test_id, desc.replace('.', '_')) return test_id def configure(self, options, conf): self.enabled = is_running_under_teamcity() self.config = conf if self._capture_plugin_enabled(): capture_plugin = self._get_capture_plugin() old_before_test = capture_plugin.beforeTest old_after_test = capture_plugin.afterTest old_format_error = capture_plugin.formatError def newCaptureBeforeTest(test): old_before_test(test) test_id = self.get_test_id(test) capture_plugin._buf = FlushingStringIO(lambda data: dump_test_stdout(self.messages, test_id, test_id, data)) sys.stdout = capture_plugin._buf def newCaptureAfterTest(test): if isinstance(capture_plugin._buf, FlushingStringIO): capture_plugin._buf.flush() old_after_test(test) def newCaptureFormatError(test, err): if isinstance(capture_plugin._buf, FlushingStringIO): capture_plugin._buf.flush() old_format_error(test, err) capture_plugin.beforeTest = newCaptureBeforeTest capture_plugin.afterTest = newCaptureAfterTest capture_plugin.formatError = newCaptureFormatError def options(self, parser, env=os.environ): pass def _get_capture_plugin(self): """ :rtype: nose.plugins.capture.Capture """ for plugin in self.config.plugins.plugins: if plugin.name == "capture": return plugin return None def _capture_plugin_enabled(self): plugin = self._get_capture_plugin() return plugin is not None and plugin.enabled def _capture_plugin_buffer(self): plugin = self._get_capture_plugin() if plugin is None: return None return getattr(plugin, "buffer", None) def _captureStandardOutput_value(self): if self._capture_plugin_enabled(): return 'false' else: return 'true' def report_fail(self, test, fail_type, err): # workaround nose bug on python 3 if is_string(err[1]): err = (err[0], Exception(err[1]), err[2]) test_id = self.get_test_id(test) details = convert_error_to_string(err) start_index = details.find(_captured_output_start_marker) end_index = details.find(_captured_output_end_marker) if 0 <= start_index < end_index: # do not log test output twice, see report_finish for actual output handling details = details[:start_index] + details[end_index + len(_captured_output_end_marker):] try: error = err[1] if isinstance(error, EqualsAssertionError): details = convert_error_to_string(err, 2) self.messages.testFailed(test_id, message=error.msg, details=details, flowId=test_id, comparison_failure=error) return except Exception: pass self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id) def report_finish(self, test): test_id = self.get_test_id(test) if test_id in self.test_started_datetime_map: time_diff = datetime.datetime.now() - self.test_started_datetime_map[test_id] self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id) else: self.messages.testFinished(test_id, flowId=test_id) def prepareTestLoader(self, loader): """Insert ourselves into loader calls to count tests. The top-level loader call often returns lazy results, like a LazySuite. This is a problem, as we would destroy the suite by iterating over it to count the tests. Consequently, we monkey-patch the top-level loader call to do the load twice: once for the actual test running and again to yield something we can iterate over to do the count. from https://github.com/erikrose/nose-progressive/ :type loader: nose.loader.TestLoader """ # TODO: If there's ever a practical need, also patch loader.suiteClass # or even TestProgram.createTests. createTests seems to be main top- # level caller of loader methods, and nose.core.collector() (which # isn't even called in nose) is an alternate one. # # nose 1.3.4 contains required fix: # Another fix for Python 3.4: Call super in LazySuite to access _removed_tests variable if hasattr(loader, 'loadTestsFromNames') and nose.__versioninfo__ >= (1, 3, 4): old_loadTestsFromNames = loader.loadTestsFromNames def _loadTestsFromNames(*args, **kwargs): suite = old_loadTestsFromNames(*args, **kwargs) self.total_tests += suite.countTestCases() # Clear out the loader's cache. Otherwise, it never finds any tests # for the actual test run: loader._visitedPaths = set() return old_loadTestsFromNames(*args, **kwargs) loader.loadTestsFromNames = _loadTestsFromNames # noinspection PyUnusedLocal def prepareTestRunner(self, runner): if self.total_tests: self.messages.testCount(self.total_tests) def addError(self, test, err): test_class_name = get_class_fullname(test) test_id = self.get_test_id(test) if issubclass(err[0], SkipTest): self.messages.testIgnored(test_id, message=("SKIPPED: %s" % str(err[1])), flowId=test_id) self.report_finish(test) elif issubclass(err[0], DeprecatedTest): self.messages.testIgnored(test_id, message="Deprecated", flowId=test_id) self.report_finish(test) elif test_class_name == CONTEXT_SUITE_FQN: self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(), flowId=test_id) self.report_fail(test, 'error in ' + test.error_context + ' context', err) self.messages.testFinished(test_id, flowId=test_id) else: self.report_fail(test, 'Error', err) self.report_finish(test) def addFailure(self, test, err): self.report_fail(test, 'Failure', err) self.report_finish(test) def startTest(self, test): test_id = self.get_test_id(test) self.test_started_datetime_map[test_id] = datetime.datetime.now() self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(), flowId=test_id) def addSuccess(self, test): self.report_finish(test)
class TeamcityTestResult(TestResult): def __init__(self, stream=sys.stdout): TestResult.__init__(self) self.output = stream self.test_started_datetime = None self.test_name = None self.createMessages() def createMessages(self): self.messages = TeamcityServiceMessages(self.output) def formatErr(self, err): try: exctype, value, tb = err return ''.join(traceback.format_exception(exctype, value, tb)) except: tb = traceback.format_exc() return "*FAILED TO GET TRACEBACK*: " + tb def getTestName(self, test): return test.shortDescription() or str(test) def addSuccess(self, test, *k): TestResult.addSuccess(self, test) self.output.write("ok\n") def addError(self, test, err, *k): TestResult.addError(self, test, err) err = self.formatErr(err) if self.getTestName(test) != self.test_name: sys.stderr.write("INTERNAL ERROR: addError(%s) outside of test\n" % self.getTestName(test)) sys.stderr.write("Error: %s\n" % err) return self.messages.testFailed(self.getTestName(test), message='Error', details=err) def addSkip(self, test, reason): TestResult.addSkip(self, test, reason) self.output.write("skipped %s - %s\n" % (self.getTestName(test), reason)) #TODO: "testIgnored" should be replaced by "testSkipped" when implemented self.messages.testIgnored(self.getTestName(test), reason) def addFailure(self, test, err, *k): # workaround nose bug on python 3 if _is_string(err[1]): err = (err[0], Exception(err[1]), err[2]) TestResult.addFailure(self, test, err) err = self.formatErr(err) if self.getTestName(test) != self.test_name: sys.stderr.write("INTERNAL ERROR: addFailure(%s) outside of test\n" % self.getTestName(test)) sys.stderr.write("Error: %s\n" % err) return self.messages.testFailed(self.getTestName(test), message='Failure', details=err) def startTest(self, test): self.test_started_datetime = datetime.datetime.now() self.test_name = self.getTestName(test) self.messages.testStarted(self.test_name) def stopTest(self, test): time_diff = datetime.datetime.now() - self.test_started_datetime if self.getTestName(test) != self.test_name: sys.stderr.write("INTERNAL ERROR: stopTest(%s) after startTest(%s)" % (self.getTestName(test), self.test_name)) self.messages.testFinished(self.test_name, time_diff) self.test_name = None
class TeamcityTestResult(TestResult): separator2 = "\n" def __init__(self, stream=_real_stdout, descriptions=None, verbosity=None): super(TeamcityTestResult, self).__init__() self.test_started_datetime_map = {} self.failed_tests = set() self.subtest_failures = {} self.messages = TeamcityServiceMessages(stream) def get_test_id(self, test): if is_string(test): return test # Force test_id for doctests if get_class_fullname(test) != "doctest.DocTestCase": desc = test.shortDescription() if desc and desc != test.id(): return "%s (%s)" % (test.id(), desc.replace('.', '_')) return test.id() def addSuccess(self, test): super(TeamcityTestResult, self).addSuccess(test) def addExpectedFailure(self, test, err): super(TeamcityTestResult, self).addExpectedFailure(test, err) err = convert_error_to_string(err) test_id = self.get_test_id(test) self.messages.testIgnored(test_id, message="Expected failure: " + err, flowId=test_id) def addSkip(self, test, reason=""): if sys.version_info >= (2, 7): super(TeamcityTestResult, self).addSkip(test, reason) test_id = self.get_test_id(test) if reason: reason_str = ": " + reason else: reason_str = "" self.messages.testIgnored(test_id, message="Skipped" + reason_str, flowId=test_id) def addUnexpectedSuccess(self, test): super(TeamcityTestResult, self).addUnexpectedSuccess(test) test_id = self.get_test_id(test) self.messages.testFailed(test_id, message='Failure', details="Test should not succeed since it's marked with @unittest.expectedFailure", flowId=test_id) def addError(self, test, err, *k): super(TeamcityTestResult, self).addError(test, err) if get_class_fullname(test) == "unittest.suite._ErrorHolder": # This is a standalone error test_name = test.id() # patch setUpModule (__main__) -> __main__.setUpModule test_name = re.sub(r'^(.*) \((.*)\)$', r'\2.\1', test_name) self.messages.testStarted(test_name, flowId=test_name) self.report_fail(test_name, 'Failure', err) self.messages.testFinished(test_name, flowId=test_name) elif get_class_fullname(err[0]) == "unittest2.case.SkipTest": message = "" if hasattr(err[1], "message"): message = getattr(err[1], "message", "") elif hasattr(err[1], "args"): message = getattr(err[1], "args", [""])[0] self.addSkip(test, message) else: self.report_fail(test, 'Error', err) def addFailure(self, test, err, *k): super(TeamcityTestResult, self).addFailure(test, err) self.report_fail(test, 'Failure', err) def addSubTest(self, test, subtest, err): super(TeamcityTestResult, self).addSubTest(test, subtest, err) test_id = self.get_test_id(test) if err is not None: if issubclass(err[0], test.failureException): self.add_subtest_failure(test_id, self.get_test_id(subtest), err) self.messages.testStdErr(test_id, out="%s: failure\n" % self.get_test_id(subtest), flowId=test_id) else: self.add_subtest_failure(test_id, self.get_test_id(subtest), err) self.messages.testStdErr(test_id, out="%s: error\n" % self.get_test_id(subtest), flowId=test_id) else: self.messages.testStdOut(test_id, out="%s: ok\n" % self.get_test_id(subtest), flowId=test_id) def add_subtest_failure(self, test_id, subtest_id, err): fail_array = self.subtest_failures.get(test_id, []) fail_array.append("%s:\n%s" % (subtest_id, convert_error_to_string(err))) self.subtest_failures[test_id] = fail_array def get_subtest_failure(self, test_id): fail_array = self.subtest_failures.get(test_id, []) return "\n".join(fail_array) def report_fail(self, test, fail_type, err): test_id = self.get_test_id(test) if is_string(err): details = err else: details = convert_error_to_string(err) subtest_failure = self.get_subtest_failure(test_id) if subtest_failure: details = subtest_failure + "\n" + details self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id) self.failed_tests.add(test_id) def startTest(self, test): super(TeamcityTestResult, self).startTest(test) test_id = self.get_test_id(test) self.test_started_datetime_map[test_id] = datetime.datetime.now() self.messages.testStarted(test_id, captureStandardOutput='true', flowId=test_id) def stopTest(self, test): super(TeamcityTestResult, self).stopTest(test) test_id = self.get_test_id(test) if test_id not in self.failed_tests and self.subtest_failures.get(test_id, []): self.report_fail(test, "Subtest failed", "") time_diff = datetime.datetime.now() - self.test_started_datetime_map[test_id] self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id) def printErrors(self): pass
class VpcClusterUpgradeTest: log = logging.getLogger(__name__) def __init__(self, num_masters: int, num_agents: int, num_public_agents: int, stable_installer_url: str, installer_url: str, aws_region: str, aws_access_key_id: str, aws_secret_access_key: str, default_os_user: str, config_yaml_override_install: str, config_yaml_override_upgrade: str, dcos_api_session_factory_install: VpcClusterUpgradeTestDcosApiSessionFactory, dcos_api_session_factory_upgrade: VpcClusterUpgradeTestDcosApiSessionFactory): self.dcos_api_session_factory_install = dcos_api_session_factory_install self.dcos_api_session_factory_upgrade = dcos_api_session_factory_upgrade self.num_masters = num_masters self.num_agents = num_agents self.num_public_agents = num_public_agents self.stable_installer_url = stable_installer_url self.installer_url = installer_url self.aws_region = aws_region self.aws_access_key_id = aws_access_key_id self.aws_secret_access_key = aws_secret_access_key self.default_os_user = default_os_user self.config_yaml_override_install = config_yaml_override_install self.config_yaml_override_upgrade = config_yaml_override_upgrade self.teamcity_msg = TeamcityServiceMessages() # the two following properties are set when running setup_cluster_workload, here we default them to empty # values. self.test_app_ids = [] self.tasks_start = [] @staticmethod def app_task_ids(dcos_api, app_id): """Return a list of Mesos task IDs for app_id's running tasks.""" assert app_id.startswith('/') response = dcos_api.marathon.get('/v2/apps' + app_id + '/tasks') response.raise_for_status() tasks = response.json()['tasks'] return [task['id'] for task in tasks] @staticmethod def get_master_task_state(dcos_api, task_id): """Returns the JSON blob associated with the task from /master/state.""" response = dcos_api.get('/mesos/master/state') response.raise_for_status() master_state = response.json() for framework in master_state['frameworks']: for task in framework['tasks']: if task_id in task['id']: return task def parse_dns_log(self, dns_log_content): """Return a list of (timestamp, status) tuples from dns_log_content.""" dns_log = [line.strip().split(' ') for line in dns_log_content.strip().split('\n')] if any(len(entry) != 2 or entry[1] not in ['SUCCESS', 'FAILURE'] for entry in dns_log): message = 'Malformed DNS log.' self.log.debug(message + ' DNS log content:\n' + dns_log_content) raise Exception(message) return dns_log def log_test(self, test_name: str, call: Callable[[], None]) -> None: try: self.teamcity_msg.testStarted(test_name) call() except Exception: # we want this except to be broad so that we can keep any Exception from taking # everything with it and not asserting the other tests self.teamcity_msg.testFailed(test_name, details=traceback.format_exc()) finally: self.teamcity_msg.testFinished(test_name) @staticmethod @retrying.retry( wait_fixed=(1 * 1000), stop_max_delay=(120 * 1000), retry_on_result=lambda x: not x) def wait_for_dns(dcos_api, hostname): """Return True if Mesos-DNS has at least one entry for hostname.""" hosts = dcos_api.get('/mesos_dns/v1/hosts/' + hostname).json() return any(h['host'] != '' and h['ip'] != '' for h in hosts) def setup_cluster_workload(self, dcos_api: DcosApiSession, healthcheck_app: dict, dns_app: dict): # Deploy test apps. # TODO(branden): We ought to be able to deploy these apps concurrently. See # https://mesosphere.atlassian.net/browse/DCOS-13360. with logger.scope("deploy apps"): dcos_api.marathon.deploy_app(healthcheck_app) dcos_api.marathon.ensure_deployments_complete() # This is a hack to make sure we don't deploy dns_app before the name it's # trying to resolve is available. self.wait_for_dns(dcos_api, dns_app['env']['RESOLVE_NAME']) dcos_api.marathon.deploy_app(dns_app, check_health=False) dcos_api.marathon.ensure_deployments_complete() test_apps = [healthcheck_app, dns_app] self.test_app_ids = [app['id'] for app in test_apps] self.tasks_start = {app_id: sorted(self.app_task_ids(dcos_api, app_id)) for app_id in self.test_app_ids} self.log.debug('Test app tasks at start:\n' + pprint.pformat(self.tasks_start)) for app in test_apps: assert app['instances'] == len(self.tasks_start[app['id']]) # Save the master's state of the task to compare with # the master's view after the upgrade. # See this issue for why we check for a difference: # https://issues.apache.org/jira/browse/MESOS-1718 self.task_state_start = self.get_master_task_state(dcos_api, self.tasks_start[self.test_app_ids[0]][0]) def verify_apps_state(self, dcos_api: DcosApiSession, dns_app: dict): with logger.scope("verify apps state"): # nested methods here so we can "close" over external state def marathon_app_tasks_survive_upgrade(): # Verify that the tasks we started are still running. tasks_end = {app_id: sorted(self.app_task_ids(dcos_api, app_id)) for app_id in self.test_app_ids} self.log.debug('Test app tasks at end:\n' + pprint.pformat(tasks_end)) if not self.tasks_start == tasks_end: self.teamcity_msg.testFailed( "test_upgrade_vpc.marathon_app_tasks_survive_upgrade", details="expected: {}\nactual: {}".format(self.tasks_start, tasks_end)) def test_mesos_task_state_remains_consistent(): # Verify that the "state" of the task does not change. task_state_end = self.get_master_task_state(dcos_api, self.tasks_start[self.test_app_ids[0]][0]) if not self.task_state_start == task_state_end: self.teamcity_msg.testFailed( "test_upgrade_vpc.test_mesos_task_state_remains_consistent", details="expected: {}\nactual: {}".format(self.task_state_start, task_state_end)) def test_app_dns_survive_upgrade(): # Verify DNS didn't fail. marathon_framework_id = dcos_api.marathon.get('/v2/info').json()['frameworkId'] dns_app_task = dcos_api.marathon.get('/v2/apps' + dns_app['id'] + '/tasks').json()['tasks'][0] dns_log = self.parse_dns_log(dcos_api.mesos_sandbox_file( dns_app_task['slaveId'], marathon_framework_id, dns_app_task['id'], dns_app['env']['DNS_LOG_FILENAME'], )) dns_failure_times = [entry[0] for entry in dns_log if entry[1] != 'SUCCESS'] if len(dns_failure_times) > 0: message = 'Failed to resolve Marathon app hostname {} at least once.'.format( dns_app['env']['RESOLVE_NAME']) err_msg = message + ' Hostname failed to resolve at these times:\n' + '\n'.join(dns_failure_times) self.log.debug(err_msg) self.teamcity_msg.testFailed("test_upgrade_vpc.test_app_dns_survive_upgrade", details=err_msg) self.log_test("test_upgrade_vpc.marathon_app_tasks_survive_upgrade", marathon_app_tasks_survive_upgrade) self.log_test( "test_upgrade_vpc.test_mesos_task_state_remains_consistent", test_mesos_task_state_remains_consistent ) self.log_test("test_upgrade_vpc.test_app_dns_survive_upgrade", test_app_dns_survive_upgrade) def run_test(self) -> int: stack_name = 'dcos-ci-test-upgrade-' + random_id(10) test_id = uuid.uuid4().hex healthcheck_app_id = TEST_APP_NAME_FMT.format('healthcheck-' + test_id) dns_app_id = TEST_APP_NAME_FMT.format('dns-' + test_id) with logger.scope("create vpc cf stack '{}'".format(stack_name)): bw = test_util.aws.BotoWrapper( region=self.aws_region, aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key) ssh_key = bw.create_key_pair(stack_name) write_string('ssh_key', ssh_key) vpc, ssh_info = test_util.aws.VpcCfStack.create( stack_name=stack_name, instance_type='m4.xlarge', instance_os='cent-os-7-dcos-prereqs', # An instance for each cluster node plus the bootstrap. instance_count=(self.num_masters + self.num_agents + self.num_public_agents + 1), admin_location='0.0.0.0/0', key_pair_name=stack_name, boto_wrapper=bw ) vpc.wait_for_complete() cluster = test_util.cluster.Cluster.from_vpc( vpc, ssh_info, ssh_key=ssh_key, num_masters=self.num_masters, num_agents=self.num_agents, num_public_agents=self.num_public_agents, ) with logger.scope("install dcos"): # Use the CLI installer to set exhibitor_storage_backend = zookeeper. test_util.cluster.install_dcos(cluster, self.stable_installer_url, api=False, add_config_path=self.config_yaml_override_install) master_list = [h.private_ip for h in cluster.masters] dcos_api_install = self.dcos_api_session_factory_install.apply( 'http://{ip}'.format(ip=cluster.masters[0].public_ip), master_list, master_list, [h.private_ip for h in cluster.agents], [h.private_ip for h in cluster.public_agents], self.default_os_user) dcos_api_install.wait_for_dcos() installed_version = dcos_api_install.get_version() healthcheck_app = create_marathon_healthcheck_app(healthcheck_app_id) dns_app = create_marathon_dns_app(dns_app_id, healthcheck_app_id) self.setup_cluster_workload(dcos_api_install, healthcheck_app, dns_app) with logger.scope("upgrade cluster"): test_util.cluster.upgrade_dcos(cluster, self.installer_url, installed_version, add_config_path=self.config_yaml_override_upgrade) with cluster.ssher.tunnel(cluster.bootstrap_host) as bootstrap_host_tunnel: bootstrap_host_tunnel.remote_cmd(['sudo', 'rm', '-rf', cluster.ssher.home_dir + '/*']) # this method invocation looks like it is the same as the one above, and that is partially correct. # the arguments to the invocation are the same, but the thing that changes is the lifecycle of the cluster # the client is being created to interact with. This client is specifically for the cluster after the # upgrade has taken place, and can account for any possible settings that may change for the client under # the hood when it probes the cluster. dcos_api_upgrade = self.dcos_api_session_factory_upgrade.apply( 'http://{ip}'.format(ip=cluster.masters[0].public_ip), master_list, master_list, [h.private_ip for h in cluster.agents], [h.private_ip for h in cluster.public_agents], self.default_os_user) dcos_api_upgrade.wait_for_dcos() # here we wait for DC/OS to be "up" so that we can auth this new client self.verify_apps_state(dcos_api_upgrade, dns_app) with logger.scope("run integration tests"): # copied from test_util/test_aws_cf.py:96 add_env = [] prefix = 'TEST_ADD_ENV_' for k, v in os.environ.items(): if k.startswith(prefix): add_env.append(k.replace(prefix, '') + '=' + v) test_cmd = ' '.join(add_env) + ' py.test -vv -s -rs ' + os.getenv('CI_FLAGS', '') result = test_util.cluster.run_integration_tests(cluster, test_cmd=test_cmd) if result == 0: self.log.info("Test successful! Deleting VPC if provided in this run.") vpc.delete() bw.delete_key_pair(stack_name) else: self.log.info("Test failed! VPC cluster will remain available for " "debugging for 2 hour after instantiation.") return result
class VpcClusterUpgradeTest: log = logging.getLogger(__name__) def __init__(self, num_masters: int, num_agents: int, num_public_agents: int, stable_installer_url: str, installer_url: str, aws_region: str, aws_access_key_id: str, aws_secret_access_key: str, default_os_user: str, config_yaml_override_install: str, config_yaml_override_upgrade: str, dcos_api_session_factory_install: VpcClusterUpgradeTestDcosApiSessionFactory, dcos_api_session_factory_upgrade: VpcClusterUpgradeTestDcosApiSessionFactory): self.dcos_api_session_factory_install = dcos_api_session_factory_install self.dcos_api_session_factory_upgrade = dcos_api_session_factory_upgrade self.num_masters = num_masters self.num_agents = num_agents self.num_public_agents = num_public_agents self.stable_installer_url = stable_installer_url self.installer_url = installer_url self.aws_region = aws_region self.aws_access_key_id = aws_access_key_id self.aws_secret_access_key = aws_secret_access_key self.default_os_user = default_os_user self.config_yaml_override_install = config_yaml_override_install self.config_yaml_override_upgrade = config_yaml_override_upgrade self.acs_token = None self.teamcity_msg = TeamcityServiceMessages() # the two following properties are set when running setup_cluster_workload, here we default them to empty # values. self.test_app_ids = [] self.tasks_start = [] @staticmethod def app_task_ids(dcos_api, app_id): """Return a list of Mesos task IDs for app_id's running tasks.""" assert app_id.startswith('/') response = dcos_api.marathon.get('/v2/apps' + app_id + '/tasks') response.raise_for_status() tasks = response.json()['tasks'] return [task['id'] for task in tasks] @staticmethod def get_master_task_state(dcos_api, task_id): """Returns the JSON blob associated with the task from /master/state.""" response = dcos_api.get('/mesos/master/state') response.raise_for_status() master_state = response.json() for framework in master_state['frameworks']: for task in framework['tasks']: if task_id in task['id']: return task def parse_dns_log(self, dns_log_content): """Return a list of (timestamp, status) tuples from dns_log_content.""" dns_log = [line.strip().split(' ') for line in dns_log_content.strip().split('\n')] if any(len(entry) != 2 or entry[1] not in ['SUCCESS', 'FAILURE'] for entry in dns_log): message = 'Malformed DNS log.' self.log.debug(message + ' DNS log content:\n' + dns_log_content) raise Exception(message) return dns_log def log_test(self, test_name: str, call: Callable[[], None]) -> None: try: self.teamcity_msg.testStarted(test_name) call() except Exception: # we want this except to be broad so that we can keep any Exception from taking # everything with it and not asserting the other tests self.teamcity_msg.testFailed(test_name, details=traceback.format_exc()) finally: self.teamcity_msg.testFinished(test_name) @staticmethod @retrying.retry( wait_fixed=(1 * 1000), stop_max_delay=(120 * 1000), retry_on_result=lambda x: not x) def wait_for_dns(dcos_api, hostname): """Return True if Mesos-DNS has at least one entry for hostname.""" hosts = dcos_api.get('/mesos_dns/v1/hosts/' + hostname).json() return any(h['host'] != '' and h['ip'] != '' for h in hosts) def setup_cluster_workload(self, dcos_api: DcosApiSession, healthcheck_app: dict, dns_app: dict, viplisten_app: dict, viptalk_app: dict): # Deploy test apps. # TODO(branden): We ought to be able to deploy these apps concurrently. See # https://mesosphere.atlassian.net/browse/DCOS-13360. with logger.scope("deploy apps"): dcos_api.marathon.deploy_app(viplisten_app) dcos_api.marathon.ensure_deployments_complete() dcos_api.marathon.deploy_app(viptalk_app) dcos_api.marathon.ensure_deployments_complete() dcos_api.marathon.deploy_app(healthcheck_app) dcos_api.marathon.ensure_deployments_complete() # This is a hack to make sure we don't deploy dns_app before the name it's # trying to resolve is available. self.wait_for_dns(dcos_api, dns_app['env']['RESOLVE_NAME']) dcos_api.marathon.deploy_app(dns_app, check_health=False) dcos_api.marathon.ensure_deployments_complete() test_apps = [healthcheck_app, dns_app, viplisten_app, viptalk_app] self.test_app_ids = [app['id'] for app in test_apps] self.tasks_start = {app_id: sorted(self.app_task_ids(dcos_api, app_id)) for app_id in self.test_app_ids} self.log.debug('Test app tasks at start:\n' + pprint.pformat(self.tasks_start)) for app in test_apps: assert app['instances'] == len(self.tasks_start[app['id']]) # Save the master's state of the task to compare with # the master's view after the upgrade. # See this issue for why we check for a difference: # https://issues.apache.org/jira/browse/MESOS-1718 self.task_state_start = self.get_master_task_state(dcos_api, self.tasks_start[self.test_app_ids[0]][0]) def verify_apps_state(self, dcos_api: DcosApiSession, dns_app: dict): with logger.scope("verify apps state"): # nested methods here so we can "close" over external state def marathon_app_tasks_survive_upgrade(): # Verify that the tasks we started are still running. tasks_end = {app_id: sorted(self.app_task_ids(dcos_api, app_id)) for app_id in self.test_app_ids} self.log.debug('Test app tasks at end:\n' + pprint.pformat(tasks_end)) if not self.tasks_start == tasks_end: self.teamcity_msg.testFailed( "test_upgrade_vpc.marathon_app_tasks_survive_upgrade", details="expected: {}\nactual: {}".format(self.tasks_start, tasks_end)) def test_mesos_task_state_remains_consistent(): # Verify that the "state" of the task does not change. task_state_end = self.get_master_task_state(dcos_api, self.tasks_start[self.test_app_ids[0]][0]) if not self.task_state_start == task_state_end: self.teamcity_msg.testFailed( "test_upgrade_vpc.test_mesos_task_state_remains_consistent", details="expected: {}\nactual: {}".format(self.task_state_start, task_state_end)) def test_app_dns_survive_upgrade(): # Verify DNS didn't fail. marathon_framework_id = dcos_api.marathon.get('/v2/info').json()['frameworkId'] dns_app_task = dcos_api.marathon.get('/v2/apps' + dns_app['id'] + '/tasks').json()['tasks'][0] dns_log = self.parse_dns_log(dcos_api.mesos_sandbox_file( dns_app_task['slaveId'], marathon_framework_id, dns_app_task['id'], dns_app['env']['DNS_LOG_FILENAME'], )) dns_failure_times = [entry[0] for entry in dns_log if entry[1] != 'SUCCESS'] if len(dns_failure_times) > 0: message = 'Failed to resolve Marathon app hostname {} at least once.'.format( dns_app['env']['RESOLVE_NAME']) err_msg = message + ' Hostname failed to resolve at these times:\n' + '\n'.join(dns_failure_times) self.log.debug(err_msg) self.teamcity_msg.testFailed("test_upgrade_vpc.test_app_dns_survive_upgrade", details=err_msg) self.log_test("test_upgrade_vpc.marathon_app_tasks_survive_upgrade", marathon_app_tasks_survive_upgrade) self.log_test( "test_upgrade_vpc.test_mesos_task_state_remains_consistent", test_mesos_task_state_remains_consistent ) self.log_test("test_upgrade_vpc.test_app_dns_survive_upgrade", test_app_dns_survive_upgrade) def mesos_metrics_snapshot(self, cluster, host): if host in cluster.masters: port = 5050 else: port = 5051 with cluster.ssher.tunnel(host) as tunnel: return json.loads( tunnel.remote_cmd( curl_cmd + ['{}:{}/metrics/snapshot'.format(host.private_ip, port)] ).decode('utf-8') ) def upgrade_host(self, tunnel, role, bootstrap_url, upgrade_script_path): # Download the upgrade script for the new DC/OS. tunnel.remote_cmd(curl_cmd + ['--remote-name', upgrade_script_path]) # Upgrade to the new DC/OS. tunnel.remote_cmd(['sudo', 'bash', 'dcos_node_upgrade.sh']) @retry( wait_fixed=(1000 * 5), stop_max_delay=(1000 * 60 * 5), # Retry on SSH command error or metric not equal to expected value. retry_on_exception=(lambda exc: isinstance(exc, CalledProcessError)), retry_on_result=(lambda result: not result), ) def wait_for_mesos_metric(self, cluster, host, key, value): """Return True when host's Mesos metric key is equal to value.""" return self.mesos_metrics_snapshot(cluster, host).get(key) == value def upgrade_dcos(self, cluster, installer_url, version, add_config_path=None): assert all(h.public_ip for h in cluster.hosts), ( 'All cluster hosts must be externally reachable. hosts: {}'.formation(cluster.hosts) ) assert cluster.bootstrap_host, 'Upgrade requires a bootstrap host' bootstrap_url = 'http://' + cluster.bootstrap_host.private_ip logging.info('Preparing bootstrap host for upgrade') installer = test_util.installer_api_test.DcosCliInstaller() with cluster.ssher.tunnel(cluster.bootstrap_host) as bootstrap_host_tunnel: installer.setup_remote( tunnel=bootstrap_host_tunnel, installer_path=cluster.ssher.home_dir + '/dcos_generate_config.sh', download_url=installer_url, ) installer.genconf( bootstrap_url=bootstrap_url, zk_host=cluster.bootstrap_host.private_ip + ':2181', master_list=[h.private_ip for h in cluster.masters], agent_list=[h.private_ip for h in cluster.agents], public_agent_list=[h.private_ip for h in cluster.public_agents], ip_detect=cluster.ip_detect, platform=cluster.platform, ssh_user=cluster.ssher.user, ssh_key=cluster.ssher.key, add_config_path=add_config_path, rexray_config_preset=cluster.rexray_config_preset, ) # Generate node upgrade script output = installer.generate_node_upgrade_script(version) upgrade_script_path = output.decode('utf-8').splitlines()[-1].split("Node upgrade script URL: ", 1)[1] # Remove docker (and associated journald) restart from the install # script. This prevents Docker-containerized tasks from being killed # during agent upgrades. bootstrap_host_tunnel.remote_cmd([ 'sudo', 'sed', '-i', '-e', '"s/systemctl restart systemd-journald//g"', '-e', '"s/systemctl restart docker//g"', cluster.ssher.home_dir + '/genconf/serve/dcos_install.sh', ]) run_bootstrap_nginx(bootstrap_host_tunnel, cluster.ssher.home_dir) upgrade_ordering = [ # Upgrade masters in a random order. ('master', 'master', random.sample(cluster.masters, len(cluster.masters))), ('slave', 'agent', cluster.agents), ('slave_public', 'public agent', cluster.public_agents), ] logging.info('\n'.join( ['Upgrade plan:'] + ['{} ({})'.format(host, role_name) for _, role_name, hosts in upgrade_ordering for host in hosts] )) for role, role_name, hosts in upgrade_ordering: logging.info('Upgrading {} nodes: {}'.format(role_name, repr(hosts))) for host in hosts: logging.info('Upgrading {}: {}'.format(role_name, repr(host))) with cluster.ssher.tunnel(host) as tunnel: self.upgrade_host(tunnel, role, bootstrap_url, upgrade_script_path) wait_metric = { 'master': 'registrar/log/recovered', 'slave': 'slave/registered', 'slave_public': 'slave/registered', }[role] logging.info('Waiting for {} to rejoin the cluster...'.format(role_name)) try: self.wait_for_mesos_metric(cluster, host, wait_metric, 1) except RetryError as exc: raise Exception( 'Timed out waiting for {} to rejoin the cluster after upgrade: {}'. format(role_name, repr(host)) ) from exc def run_test(self) -> int: stack_suffix = os.getenv("CF_STACK_NAME_SUFFIX", "open-upgrade") stack_name = "dcos-ci-test-{stack_suffix}-{random_id}".format( stack_suffix=stack_suffix, random_id=random_id(10)) test_id = uuid.uuid4().hex healthcheck_app_id = TEST_APP_NAME_FMT.format('healthcheck-' + test_id) dns_app_id = TEST_APP_NAME_FMT.format('dns-' + test_id) with logger.scope("create vpc cf stack '{}'".format(stack_name)): bw = test_util.aws.BotoWrapper( region=self.aws_region, aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key) ssh_key = bw.create_key_pair(stack_name) write_string('ssh_key', ssh_key) bare_cluster, ssh_info = test_util.aws.BareClusterCfStack.create( stack_name=stack_name, instance_type='m4.xlarge', instance_os='cent-os-7-dcos-prereqs', # An instance for each cluster node plus the bootstrap. instance_count=(self.num_masters + self.num_agents + self.num_public_agents + 1), admin_location='0.0.0.0/0', key_pair_name=stack_name, boto_wrapper=bw ) bare_cluster.wait_for_complete() cluster = test_util.cluster.Cluster.from_bare_cluster( bare_cluster, ssh_info, ssh_key=ssh_key, num_masters=self.num_masters, num_agents=self.num_agents, num_public_agents=self.num_public_agents, ) with logger.scope("install dcos"): # Use the CLI installer to set exhibitor_storage_backend = zookeeper. # Don't install prereqs since stable breaks Docker 1.13. See # https://jira.mesosphere.com/browse/DCOS_OSS-743. test_util.cluster.install_dcos(cluster, self.stable_installer_url, api=False, install_prereqs=False, add_config_path=self.config_yaml_override_install) master_list = [h.private_ip for h in cluster.masters] dcos_api_install = self.dcos_api_session_factory_install.apply( 'http://{ip}'.format(ip=cluster.masters[0].public_ip), master_list, master_list, [h.private_ip for h in cluster.agents], [h.private_ip for h in cluster.public_agents], self.default_os_user) dcos_api_install.wait_for_dcos() installed_version = dcos_api_install.get_version() healthcheck_app = create_marathon_healthcheck_app(healthcheck_app_id) dns_app = create_marathon_dns_app(dns_app_id, healthcheck_app_id) viplisten_app = create_marathon_viplisten_app() viptalk_app = create_marathon_viptalk_app() self.setup_cluster_workload(dcos_api_install, healthcheck_app, dns_app, viplisten_app, viptalk_app) with logger.scope("upgrade cluster"): self.upgrade_dcos(cluster, self.installer_url, installed_version, add_config_path=self.config_yaml_override_upgrade) with cluster.ssher.tunnel(cluster.bootstrap_host) as bootstrap_host_tunnel: bootstrap_host_tunnel.remote_cmd(['sudo', 'rm', '-rf', cluster.ssher.home_dir + '/*']) # this method invocation looks like it is the same as the one above, and that is partially correct. # the arguments to the invocation are the same, but the thing that changes is the lifecycle of the cluster # the client is being created to interact with. This client is specifically for the cluster after the # upgrade has taken place, and can account for any possible settings that may change for the client under # the hood when it probes the cluster. dcos_api_upgrade = self.dcos_api_session_factory_upgrade.apply( 'http://{ip}'.format(ip=cluster.masters[0].public_ip), master_list, master_list, [h.private_ip for h in cluster.agents], [h.private_ip for h in cluster.public_agents], self.default_os_user) dcos_api_upgrade.wait_for_dcos() # here we wait for DC/OS to be "up" so that we can auth this new client self.verify_apps_state(dcos_api_upgrade, dns_app) with logger.scope("run integration tests"): # copied from test_util/test_aws_cf.py:96 add_env = [] prefix = 'TEST_ADD_ENV_' for k, v in os.environ.items(): if k.startswith(prefix): add_env.append(k.replace(prefix, '') + '=' + v) test_cmd = ' '.join(add_env) + ' py.test -vv -s -rs ' + os.getenv('CI_FLAGS', '') result = test_util.cluster.run_integration_tests(cluster, test_cmd=test_cmd) if result == 0: self.log.info("Test successful! Deleting VPC if provided in this run.") bare_cluster.delete() bw.delete_key_pair(stack_name) else: self.log.info("Test failed! VPC cluster will remain available for " "debugging for 2 hour after instantiation.") if os.getenv('CI_FLAGS'): result = 0 return result
class TeamcityFormatter(Formatter): """ Stateful TC reporter. Since we can't fetch all steps from the very beginning (even skipped tests are reported) we store tests and features on each call. To hook into test reporting override _report_suite_started and/or _report_test_started """ def __init__(self, stream_opener, config): super(TeamcityFormatter, self).__init__(stream_opener, config) assert version.LooseVersion(behave_version) >= version.LooseVersion( "1.2.6"), "Only 1.2.6+ is supported" self._messages = TeamcityServiceMessages() self.__feature = None self.__scenario = None self.__steps = deque() self.__scenario_opened = False self.__feature_opened = False self.__test_start_time = None def feature(self, feature): assert isinstance(feature, Feature) assert not self.__feature, "Prev. feature not closed" self.__feature = feature def scenario(self, scenario): assert isinstance(scenario, Scenario) self.__scenario = scenario self.__scenario_opened = False def step(self, step): assert isinstance(step, Step) self.__steps.append(step) def match(self, match): if not self.__feature_opened: self._report_suite_started(self.__feature, _suite_name(self.__feature)) self.__feature_opened = True if not self.__scenario_opened: self._report_suite_started(self.__scenario, _suite_name(self.__scenario)) self.__scenario_opened = True assert self.__steps, "No steps left" step = self.__steps.popleft() self._report_test_started(step, _step_name(step)) self.__test_start_time = datetime.datetime.now() def _report_suite_started(self, suite, suite_name): """ :param suite: behave suite :param suite_name: suite name that must be reported, be sure to use it instead of suite.name """ self._messages.testSuiteStarted(suite_name) def _report_test_started(self, test, test_name): """ Suite name is always stripped, be sure to strip() it too :param test: behave test :param test_name: test name that must be reported, be sure to use it instead of test.name """ self._messages.testStarted(test_name) def result(self, step): assert isinstance(step, Step) step_name = _step_name(step) if step.status == Status.failed: try: error = traceback.format_exc(step.exc_traceback) if error != step.error_message: self._messages.testStdErr(step_name, error) except Exception: pass # exception shall not prevent error message self._messages.testFailed(step_name, message=step.error_message) if step.status == Status.undefined: self._messages.testFailed(step_name, message="Undefined") if step.status == Status.skipped: self._messages.testIgnored(step_name) self._messages.testFinished(step_name, testDuration=datetime.datetime.now() - self.__test_start_time) if not self.__steps: self.__close_scenario() elif step.status in [Status.failed, Status.undefined]: # Broken background/undefined step stops whole scenario reason = "Undefined step" if step.status == Status.undefined else "Prev. step failed" self.__skip_rest_of_scenario(reason) def __skip_rest_of_scenario(self, reason): while self.__steps: step = self.__steps.popleft() self._report_test_started(step, _step_name(step)) self._messages.testIgnored( _step_name(step), message="{0}. Rest part of scenario is skipped".format(reason)) self._messages.testFinished(_step_name(step)) self.__close_scenario() def __close_scenario(self): if self.__scenario: self._messages.testSuiteFinished(_suite_name(self.__scenario)) self.__scenario = None def eof(self): self.__skip_rest_of_scenario("") self._messages.testSuiteFinished(_suite_name(self.__feature)) self.__feature = None self.__feature_opened = False
def testMetadata_number_message(test_name, param_name, param_value): teamcity_messages.message( 'testMetadata', type='number', testName=test_name, name=param_name, value='{:.2f}'.format(param_value), ) for result in json_results.get('results', []): test_name = result['name'] teamcity_messages.testStarted(test_name) testMetadata_number_message( test_name, 'ns/{}'.format(result['unit']), 1e9 * result['median(elapsed)'] / result['batch'], ) testMetadata_number_message( test_name, '{}/s'.format(result['unit']), result['batch'] / result['median(elapsed)'], ) testMetadata_number_message( test_name,
class TeamcityTestResult(TestResult): separator2 = "\n" # noinspection PyUnusedLocal def __init__(self, stream=_real_stdout, descriptions=None, verbosity=None): super(TeamcityTestResult, self).__init__() # Some code may ask for self.failfast, see unittest2.case.TestCase.subTest self.failfast = getattr(self, "failfast", False) self.test_started_datetime_map = {} self.failed_tests = set() self.subtest_failures = {} self.messages = TeamcityServiceMessages(_real_stdout) @staticmethod def get_test_id(test): if is_string(test): return test # Force test_id for doctests if get_class_fullname(test) != "doctest.DocTestCase": desc = test.shortDescription() test_method_name = getattr(test, "_testMethodName", "") if desc and desc != test.id() and desc != test_method_name: return "%s (%s)" % (test.id(), desc.replace('.', '_')) return test.id() def addSuccess(self, test): super(TeamcityTestResult, self).addSuccess(test) def addExpectedFailure(self, test, err): _super = super(TeamcityTestResult, self) if hasattr(_super, "addExpectedFailure"): _super.addExpectedFailure(test, err) err = convert_error_to_string(err) test_id = self.get_test_id(test) self.messages.testIgnored(test_id, message="Expected failure: " + err, flowId=test_id) def get_subtest_block_id(self, test, subtest): test_id = self.get_test_id(test) subtest_id = self.get_test_id(subtest) if subtest_id.startswith(test_id): block_id = subtest_id[len(test_id):].strip() else: block_id = subtest_id if len(block_id) == 0: block_id = test_id return block_id def addSkip(self, test, reason=""): if sys.version_info >= (2, 7): super(TeamcityTestResult, self).addSkip(test, reason) if reason: reason_str = ": " + str(reason) else: reason_str = "" test_class_name = get_class_fullname(test) if test_class_name == "unittest.case._SubTest" or test_class_name == "unittest2.case._SubTest": parent_test = test.test_case parent_test_id = self.get_test_id(parent_test) subtest = test block_id = self.get_subtest_block_id(parent_test, subtest) self.messages.subTestBlockOpened(block_id, subTestResult="Skip", flowId=parent_test_id) self.messages.testStdOut(parent_test_id, out="SubTest skipped" + reason_str + "\n", flowId=parent_test_id) self.messages.blockClosed(block_id, flowId=parent_test_id) else: test_id = self.get_test_id(test) self.messages.testIgnored(test_id, message="Skipped" + reason_str, flowId=test_id) def addUnexpectedSuccess(self, test): _super = super(TeamcityTestResult, self) if hasattr(_super, "addUnexpectedSuccess"): _super.addUnexpectedSuccess(test) test_id = self.get_test_id(test) self.messages.testFailed( test_id, message='Failure', details= "Test should not succeed since it's marked with @unittest.expectedFailure", flowId=test_id) def addError(self, test, err, *k): super(TeamcityTestResult, self).addError(test, err) test_class = get_class_fullname(test) if test_class == "unittest.suite._ErrorHolder" or test_class == "unittest2.suite._ErrorHolder": # This is a standalone error test_name = test.id() # patch setUpModule (__main__) -> __main__.setUpModule test_name = re.sub(r'^(.*) \((.*)\)$', r'\2.\1', test_name) self.messages.testStarted(test_name, flowId=test_name) # noinspection PyTypeChecker self.report_fail(test_name, 'Failure', err) self.messages.testFinished(test_name, flowId=test_name) elif get_class_fullname(err[0]) == "unittest2.case.SkipTest": message = "" if hasattr(err[1], "message"): message = getattr(err[1], "message", "") elif hasattr(err[1], "args"): message = getattr(err[1], "args", [""])[0] self.addSkip(test, message) else: self.report_fail(test, 'Error', err) def addFailure(self, test, err, *k): super(TeamcityTestResult, self).addFailure(test, err) self.report_fail(test, 'Failure', err) def addSubTest(self, test, subtest, err): _super = super(TeamcityTestResult, self) if hasattr(_super, "addSubTest"): _super.addSubTest(test, subtest, err) test_id = self.get_test_id(test) subtest_id = self.get_test_id(subtest) if subtest_id.startswith(test_id): # Replace "." -> "_" since '.' is a test hierarchy separator # See i.e. https://github.com/JetBrains/teamcity-messages/issues/134 (https://youtrack.jetbrains.com/issue/PY-23846) block_id = subtest_id[len(test_id):].strip().replace(".", "_") else: block_id = subtest_id if len(block_id) == 0: block_id = subtest_id if err is not None: self.add_subtest_failure(test_id, block_id) if issubclass(err[0], test.failureException): self.messages.subTestBlockOpened(block_id, subTestResult="Failure", flowId=test_id) self.messages.testStdErr(test_id, out="SubTest failure: %s\n" % convert_error_to_string(err), flowId=test_id) self.messages.blockClosed(block_id, flowId=test_id) else: self.messages.subTestBlockOpened(block_id, subTestResult="Error", flowId=test_id) self.messages.testStdErr(test_id, out="SubTest error: %s\n" % convert_error_to_string(err), flowId=test_id) self.messages.blockClosed(block_id, flowId=test_id) else: self.messages.subTestBlockOpened(block_id, subTestResult="Success", flowId=test_id) self.messages.blockClosed(block_id, flowId=test_id) def add_subtest_failure(self, test_id, subtest_block_id): fail_array = self.subtest_failures.get(test_id, []) fail_array.append(subtest_block_id) self.subtest_failures[test_id] = fail_array def get_subtest_failure(self, test_id): fail_array = self.subtest_failures.get(test_id, []) return ", ".join(fail_array) def report_fail(self, test, fail_type, err): test_id = self.get_test_id(test) if is_string(err): details = err elif get_class_fullname(err) == "twisted.python.failure.Failure": details = err.getTraceback() else: details = convert_error_to_string(err) subtest_failures = self.get_subtest_failure(test_id) if subtest_failures: details = "Failed subtests list: " + subtest_failures + "\n\n" + details.strip( ) details = details.strip() self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id) self.failed_tests.add(test_id) def startTest(self, test): super(TeamcityTestResult, self).startTest(test) test_id = self.get_test_id(test) self.test_started_datetime_map[test_id] = datetime.datetime.now() self.messages.testStarted(test_id, captureStandardOutput='true', flowId=test_id) def stopTest(self, test): test_id = self.get_test_id(test) if getattr(self, 'buffer', None): # Do not allow super() method to print output by itself self._mirrorOutput = False output = sys.stdout.getvalue() if output: for chunk in split_output(limit_output(output)): self.messages.testStdOut(test_id, chunk, flowId=test_id) error = sys.stderr.getvalue() if error: for chunk in split_output(limit_output(error)): self.messages.testStdErr(test_id, chunk, flowId=test_id) super(TeamcityTestResult, self).stopTest(test) if test_id not in self.failed_tests: subtest_failures = self.get_subtest_failure(test_id) if subtest_failures: self.report_fail(test, "One or more subtests failed", "") time_diff = datetime.datetime.now( ) - self.test_started_datetime_map[test_id] self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id) def printErrors(self): pass
class VpcClusterUpgradeTest: log = logging.getLogger(__name__) def __init__(self, num_masters: int, num_agents: int, num_public_agents: int, stable_installer_url: str, installer_url: str, aws_region: str, aws_access_key_id: str, aws_secret_access_key: str, default_os_user: str, config_yaml_override_install: str, config_yaml_override_upgrade: str, dcos_api_session_factory_install: VpcClusterUpgradeTestDcosApiSessionFactory, dcos_api_session_factory_upgrade: VpcClusterUpgradeTestDcosApiSessionFactory): self.dcos_api_session_factory_install = dcos_api_session_factory_install self.dcos_api_session_factory_upgrade = dcos_api_session_factory_upgrade self.num_masters = num_masters self.num_agents = num_agents self.num_public_agents = num_public_agents self.stable_installer_url = stable_installer_url self.installer_url = installer_url self.aws_region = aws_region self.aws_access_key_id = aws_access_key_id self.aws_secret_access_key = aws_secret_access_key self.default_os_user = default_os_user self.config_yaml_override_install = config_yaml_override_install self.config_yaml_override_upgrade = config_yaml_override_upgrade self.teamcity_msg = TeamcityServiceMessages() # the two following properties are set when running setup_cluster_workload, here we default them to empty # values. self.test_app_ids = [] self.tasks_start = [] @staticmethod def app_task_ids(dcos_api, app_id): """Return a list of Mesos task IDs for app_id's running tasks.""" assert app_id.startswith('/') response = dcos_api.marathon.get('/v2/apps' + app_id + '/tasks') response.raise_for_status() tasks = response.json()['tasks'] return [task['id'] for task in tasks] @staticmethod def get_master_task_state(dcos_api, task_id): """Returns the JSON blob associated with the task from /master/state.""" response = dcos_api.get('/mesos/master/state') response.raise_for_status() master_state = response.json() for framework in master_state['frameworks']: for task in framework['tasks']: if task_id in task['id']: return task def parse_dns_log(self, dns_log_content): """Return a list of (timestamp, status) tuples from dns_log_content.""" dns_log = [line.strip().split(' ') for line in dns_log_content.strip().split('\n')] if any(len(entry) != 2 or entry[1] not in ['SUCCESS', 'FAILURE'] for entry in dns_log): message = 'Malformed DNS log.' self.log.debug(message + ' DNS log content:\n' + dns_log_content) raise Exception(message) return dns_log def log_test(self, test_name: str, call: Callable[[], None]) -> None: try: self.teamcity_msg.testStarted(test_name) call() except Exception: # we want this except to be broad so that we can keep any Exception from taking # everything with it and not asserting the other tests self.teamcity_msg.testFailed(test_name, details=traceback.format_exc()) finally: self.teamcity_msg.testFinished(test_name) @staticmethod @retrying.retry( wait_fixed=(1 * 1000), stop_max_delay=(120 * 1000), retry_on_result=lambda x: not x) def wait_for_dns(dcos_api, hostname): """Return True if Mesos-DNS has at least one entry for hostname.""" hosts = dcos_api.get('/mesos_dns/v1/hosts/' + hostname).json() return any(h['host'] != '' and h['ip'] != '' for h in hosts) def setup_cluster_workload(self, dcos_api: DcosApiSession, healthcheck_app: dict, dns_app: dict, viplisten_app: dict, viptalk_app: dict): # Deploy test apps. # TODO(branden): We ought to be able to deploy these apps concurrently. See # https://mesosphere.atlassian.net/browse/DCOS-13360. with logger.scope("deploy apps"): dcos_api.marathon.deploy_app(viplisten_app) dcos_api.marathon.ensure_deployments_complete() dcos_api.marathon.deploy_app(viptalk_app) dcos_api.marathon.ensure_deployments_complete() dcos_api.marathon.deploy_app(healthcheck_app) dcos_api.marathon.ensure_deployments_complete() # This is a hack to make sure we don't deploy dns_app before the name it's # trying to resolve is available. self.wait_for_dns(dcos_api, dns_app['env']['RESOLVE_NAME']) dcos_api.marathon.deploy_app(dns_app, check_health=False) dcos_api.marathon.ensure_deployments_complete() test_apps = [healthcheck_app, dns_app, viplisten_app, viptalk_app] self.test_app_ids = [app['id'] for app in test_apps] self.tasks_start = {app_id: sorted(self.app_task_ids(dcos_api, app_id)) for app_id in self.test_app_ids} self.log.debug('Test app tasks at start:\n' + pprint.pformat(self.tasks_start)) for app in test_apps: assert app['instances'] == len(self.tasks_start[app['id']]) # Save the master's state of the task to compare with # the master's view after the upgrade. # See this issue for why we check for a difference: # https://issues.apache.org/jira/browse/MESOS-1718 self.task_state_start = self.get_master_task_state(dcos_api, self.tasks_start[self.test_app_ids[0]][0]) def verify_apps_state(self, dcos_api: DcosApiSession, dns_app: dict): with logger.scope("verify apps state"): # nested methods here so we can "close" over external state def marathon_app_tasks_survive_upgrade(): # Verify that the tasks we started are still running. tasks_end = {app_id: sorted(self.app_task_ids(dcos_api, app_id)) for app_id in self.test_app_ids} self.log.debug('Test app tasks at end:\n' + pprint.pformat(tasks_end)) if not self.tasks_start == tasks_end: self.teamcity_msg.testFailed( "test_upgrade_vpc.marathon_app_tasks_survive_upgrade", details="expected: {}\nactual: {}".format(self.tasks_start, tasks_end)) def test_mesos_task_state_remains_consistent(): # Verify that the "state" of the task does not change. task_state_end = self.get_master_task_state(dcos_api, self.tasks_start[self.test_app_ids[0]][0]) if not self.task_state_start == task_state_end: self.teamcity_msg.testFailed( "test_upgrade_vpc.test_mesos_task_state_remains_consistent", details="expected: {}\nactual: {}".format(self.task_state_start, task_state_end)) def test_app_dns_survive_upgrade(): # Verify DNS didn't fail. marathon_framework_id = dcos_api.marathon.get('/v2/info').json()['frameworkId'] dns_app_task = dcos_api.marathon.get('/v2/apps' + dns_app['id'] + '/tasks').json()['tasks'][0] dns_log = self.parse_dns_log(dcos_api.mesos_sandbox_file( dns_app_task['slaveId'], marathon_framework_id, dns_app_task['id'], dns_app['env']['DNS_LOG_FILENAME'], )) dns_failure_times = [entry[0] for entry in dns_log if entry[1] != 'SUCCESS'] if len(dns_failure_times) > 0: message = 'Failed to resolve Marathon app hostname {} at least once.'.format( dns_app['env']['RESOLVE_NAME']) err_msg = message + ' Hostname failed to resolve at these times:\n' + '\n'.join(dns_failure_times) self.log.debug(err_msg) self.teamcity_msg.testFailed("test_upgrade_vpc.test_app_dns_survive_upgrade", details=err_msg) self.log_test("test_upgrade_vpc.marathon_app_tasks_survive_upgrade", marathon_app_tasks_survive_upgrade) self.log_test( "test_upgrade_vpc.test_mesos_task_state_remains_consistent", test_mesos_task_state_remains_consistent ) self.log_test("test_upgrade_vpc.test_app_dns_survive_upgrade", test_app_dns_survive_upgrade) def run_test(self) -> int: stack_name = 'dcos-ci-test-upgrade-' + random_id(10) test_id = uuid.uuid4().hex healthcheck_app_id = TEST_APP_NAME_FMT.format('healthcheck-' + test_id) dns_app_id = TEST_APP_NAME_FMT.format('dns-' + test_id) with logger.scope("create vpc cf stack '{}'".format(stack_name)): bw = test_util.aws.BotoWrapper( region=self.aws_region, aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key) ssh_key = bw.create_key_pair(stack_name) write_string('ssh_key', ssh_key) vpc, ssh_info = test_util.aws.VpcCfStack.create( stack_name=stack_name, instance_type='m4.xlarge', instance_os='cent-os-7-dcos-prereqs', # An instance for each cluster node plus the bootstrap. instance_count=(self.num_masters + self.num_agents + self.num_public_agents + 1), admin_location='0.0.0.0/0', key_pair_name=stack_name, boto_wrapper=bw ) vpc.wait_for_complete() cluster = test_util.cluster.Cluster.from_vpc( vpc, ssh_info, ssh_key=ssh_key, num_masters=self.num_masters, num_agents=self.num_agents, num_public_agents=self.num_public_agents, ) with logger.scope("install dcos"): # Use the CLI installer to set exhibitor_storage_backend = zookeeper. # Don't install prereqs since stable breaks Docker 1.13. See # https://jira.mesosphere.com/browse/DCOS_OSS-743. test_util.cluster.install_dcos(cluster, self.stable_installer_url, api=False, install_prereqs=False, add_config_path=self.config_yaml_override_install) master_list = [h.private_ip for h in cluster.masters] dcos_api_install = self.dcos_api_session_factory_install.apply( 'http://{ip}'.format(ip=cluster.masters[0].public_ip), master_list, master_list, [h.private_ip for h in cluster.agents], [h.private_ip for h in cluster.public_agents], self.default_os_user) dcos_api_install.wait_for_dcos() installed_version = dcos_api_install.get_version() healthcheck_app = create_marathon_healthcheck_app(healthcheck_app_id) dns_app = create_marathon_dns_app(dns_app_id, healthcheck_app_id) viplisten_app = create_marathon_viplisten_app() viptalk_app = create_marathon_viptalk_app() self.setup_cluster_workload(dcos_api_install, healthcheck_app, dns_app, viplisten_app, viptalk_app) with logger.scope("upgrade cluster"): test_util.cluster.upgrade_dcos(cluster, self.installer_url, installed_version, add_config_path=self.config_yaml_override_upgrade) with cluster.ssher.tunnel(cluster.bootstrap_host) as bootstrap_host_tunnel: bootstrap_host_tunnel.remote_cmd(['sudo', 'rm', '-rf', cluster.ssher.home_dir + '/*']) # this method invocation looks like it is the same as the one above, and that is partially correct. # the arguments to the invocation are the same, but the thing that changes is the lifecycle of the cluster # the client is being created to interact with. This client is specifically for the cluster after the # upgrade has taken place, and can account for any possible settings that may change for the client under # the hood when it probes the cluster. dcos_api_upgrade = self.dcos_api_session_factory_upgrade.apply( 'http://{ip}'.format(ip=cluster.masters[0].public_ip), master_list, master_list, [h.private_ip for h in cluster.agents], [h.private_ip for h in cluster.public_agents], self.default_os_user) dcos_api_upgrade.wait_for_dcos() # here we wait for DC/OS to be "up" so that we can auth this new client self.verify_apps_state(dcos_api_upgrade, dns_app) with logger.scope("run integration tests"): # copied from test_util/test_aws_cf.py:96 add_env = [] prefix = 'TEST_ADD_ENV_' for k, v in os.environ.items(): if k.startswith(prefix): add_env.append(k.replace(prefix, '') + '=' + v) test_cmd = ' '.join(add_env) + ' py.test -vv -s -rs ' + os.getenv('CI_FLAGS', '') result = test_util.cluster.run_integration_tests(cluster, test_cmd=test_cmd) if result == 0: self.log.info("Test successful! Deleting VPC if provided in this run.") vpc.delete() bw.delete_key_pair(stack_name) else: self.log.info("Test failed! VPC cluster will remain available for " "debugging for 2 hour after instantiation.") return result
class EchoTeamCityMessages(object): def __init__(self, output_capture_enabled, coverage_controller): self.coverage_controller = coverage_controller self.output_capture_enabled = output_capture_enabled self.teamcity = TeamcityServiceMessages() self.test_start_reported_mark = set() self.max_reported_output_size = 1 * 1024 * 1024 self.reported_output_chunk_size = 50000 def format_test_id(self, nodeid): test_id = nodeid if test_id.find("::") < 0: test_id += "::top_level" test_id = test_id.replace("::()::", "::") test_id = re.sub(r"\.pyc?::", r"::", test_id) test_id = test_id.replace(".", "_").replace(os.sep, ".").replace("/", ".").replace('::', '.') return test_id def pytest_runtest_logstart(self, nodeid, location): self.ensure_test_start_reported(self.format_test_id(nodeid)) def ensure_test_start_reported(self, test_id): if test_id not in self.test_start_reported_mark: self.teamcity.testStarted(test_id, flowId=test_id, captureStandardOutput='false' if self.output_capture_enabled else 'true') self.test_start_reported_mark.add(test_id) def report_has_output(self, report): for (secname, data) in report.sections: if report.when in secname and ('stdout' in secname or 'stderr' in secname): return True return False def report_test_output(self, report, test_id, when): for (secname, data) in report.sections: if when not in secname: continue if not data: continue if 'stdout' in secname: for chunk in split_output(limit_output(data)): self.teamcity.testStdOut(test_id, out=chunk, flowId=test_id) elif 'stderr' in secname: for chunk in split_output(limit_output(data)): self.teamcity.testStdErr(test_id, out=chunk, flowId=test_id) def pytest_runtest_logreport(self, report): """ :type report: _pytest.runner.TestReport """ orig_test_id = self.format_test_id(report.nodeid) suffix = '' if report.when == 'call' else ('_' + report.when) test_id = orig_test_id + suffix duration = timedelta(seconds=report.duration) if report.passed: # Do not report passed setup/teardown if no output if report.when == 'call' or self.report_has_output(report): self.ensure_test_start_reported(test_id) self.report_test_output(report, test_id, report.when) self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id) elif report.failed: self.ensure_test_start_reported(test_id) self.report_test_output(report, test_id, report.when) self.teamcity.testFailed(test_id, str(report.location), str(report.longrepr), flowId=test_id) self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id) # If test setup failed, report test failure as well if report.when == 'setup': self.ensure_test_start_reported(orig_test_id) self.teamcity.testFailed(orig_test_id, "test setup failed, see %s test failure" % test_id, flowId=orig_test_id) self.teamcity.testFinished(orig_test_id, flowId=orig_test_id) elif report.skipped: if type(report.longrepr) is tuple and len(report.longrepr) == 3: reason = report.longrepr[2] else: reason = str(report.longrepr) self.ensure_test_start_reported(orig_test_id) self.report_test_output(report, orig_test_id, report.when) self.teamcity.testIgnored(orig_test_id, reason, flowId=orig_test_id) self.teamcity.testFinished(orig_test_id, flowId=orig_test_id) def pytest_collectreport(self, report): if report.failed: test_id = self.format_test_id(report.nodeid) + "_collect" self.teamcity.testStarted(test_id, flowId=test_id) self.teamcity.testFailed(test_id, str(report.location), str(report.longrepr), flowId=test_id) self.teamcity.testFinished(test_id, flowId=test_id) def pytest_terminal_summary(self): if self.coverage_controller is not None: try: self._report_coverage() except: tb = traceback.format_exc() self.teamcity.customMessage("Coverage statistics reporting failed", "ERROR", errorDetails=tb) def _report_coverage(self): from coverage.misc import NotPython from coverage.report import Reporter from coverage.results import Numbers class _CoverageReporter(Reporter): def __init__(self, coverage, config, messages): super(_CoverageReporter, self).__init__(coverage, config) self.branches = coverage.data.has_arcs() self.messages = messages def report(self, morfs, outfile=None): self.find_code_units(morfs) total = Numbers() for cu in self.code_units: try: analysis = self.coverage._analyze(cu) nums = analysis.numbers total += nums except KeyboardInterrupt: raise except: if self.config.ignore_errors: continue err = sys.exc_info() typ, msg = err[:2] if typ is NotPython and not cu.should_be_python(): continue test_id = cu.name details = convert_error_to_string(err) self.messages.testStarted(test_id, flowId=test_id) self.messages.testFailed(test_id, message="Coverage analysis failed", details=details, flowId=test_id) self.messages.testFinished(test_id, flowId=test_id) if total.n_files > 0: covered = total.n_executed + (total.n_executed_branches if self.branches else 0) total_statements = total.n_statements + (total.n_branches if self.branches else 0) self.messages.buildStatisticLinesCovered(covered) self.messages.buildStatisticTotalLines(total_statements) self.messages.buildStatisticLinesUncovered(total_statements - covered) reporter = _CoverageReporter( self.coverage_controller.cov, self.coverage_controller.cov.config, self.teamcity, ) reporter.report(None)
class EchoTeamCityMessages(object): def __init__(self, output_capture_enabled, coverage_controller, skip_passed_output, swap_diff): self.coverage_controller = coverage_controller self.output_capture_enabled = output_capture_enabled self.skip_passed_output = skip_passed_output self.teamcity = TeamcityServiceMessages() self.test_start_reported_mark = set() self.max_reported_output_size = 1 * 1024 * 1024 self.reported_output_chunk_size = 50000 self.swap_diff = swap_diff def get_id_from_location(self, location): if type(location) is not tuple or len(location) != 3 or not hasattr(location[2], "startswith"): return None def convert_file_to_id(filename): filename = re.sub(r"\.pyc?$", "", filename) return filename.replace(os.sep, ".").replace("/", ".") def add_prefix_to_filename_id(filename_id, prefix): dot_location = filename_id.rfind('.') if dot_location <= 0 or dot_location >= len(filename_id) - 1: return None return filename_id[:dot_location + 1] + prefix + filename_id[dot_location + 1:] pylint_prefix = '[pylint] ' if location[2].startswith(pylint_prefix): id_from_file = convert_file_to_id(location[2][len(pylint_prefix):]) return id_from_file + ".Pylint" if location[2] == "PEP8-check": id_from_file = convert_file_to_id(location[0]) return id_from_file + ".PEP8" return None def format_test_id(self, nodeid, location): id_from_location = self.get_id_from_location(location) if id_from_location is not None: return id_from_location test_id = nodeid if test_id: if test_id.find("::") < 0: test_id += "::top_level" else: test_id = "top_level" first_bracket = test_id.find("[") if first_bracket > 0: # [] -> (), make it look like nose parameterized tests params = "(" + test_id[first_bracket + 1:] if params.endswith("]"): params = params[:-1] + ")" test_id = test_id[:first_bracket] if test_id.endswith("::"): test_id = test_id[:-2] else: params = "" test_id = test_id.replace("::()::", "::") test_id = re.sub(r"\.pyc?::", r"::", test_id) test_id = test_id.replace(".", "_").replace(os.sep, ".").replace("/", ".").replace('::', '.') if params: params = params.replace(".", "_") test_id += params return test_id def format_location(self, location): if type(location) is tuple and len(location) == 3: return "%s:%s (%s)" % (str(location[0]), str(location[1]), str(location[2])) return str(location) def pytest_collection_modifyitems(self, session, config, items): self.teamcity.testCount(len(items)) def pytest_runtest_logstart(self, nodeid, location): # test name fetched from location passed as metainfo to PyCharm # it will be used to run specific test using "-k" # See IDEA-176950 # We only need method/function name because only it could be used as -k test_name = location[2] if test_name: test_name = str(test_name).split(".")[-1] self.ensure_test_start_reported(self.format_test_id(nodeid, location), test_name) def ensure_test_start_reported(self, test_id, metainfo=None): if test_id not in self.test_start_reported_mark: if self.output_capture_enabled: capture_standard_output = "false" else: capture_standard_output = "true" self.teamcity.testStarted(test_id, flowId=test_id, captureStandardOutput=capture_standard_output, metainfo=metainfo) self.test_start_reported_mark.add(test_id) def report_has_output(self, report): for (secname, data) in report.sections: if report.when in secname and ('stdout' in secname or 'stderr' in secname): return True return False def report_test_output(self, report, test_id): for (secname, data) in report.sections: # https://github.com/JetBrains/teamcity-messages/issues/112 # CollectReport doesn't have 'when' property if hasattr(report, "when") and report.when not in secname: continue if not data: continue if 'stdout' in secname: dump_test_stdout(self.teamcity, test_id, test_id, data) elif 'stderr' in secname: dump_test_stderr(self.teamcity, test_id, test_id, data) def report_test_finished(self, test_id, duration=None): self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id) self.test_start_reported_mark.remove(test_id) def report_test_failure(self, test_id, report, message=None, report_output=True): if hasattr(report, 'duration'): duration = timedelta(seconds=report.duration) else: duration = None if message is None: message = self.format_location(report.location) self.ensure_test_start_reported(test_id) if report_output: self.report_test_output(report, test_id) diff_error = None try: err_message = str(report.longrepr.reprcrash.message) diff_name = diff_tools.EqualsAssertionError.__name__ # There is a string like "foo.bar.DiffError: [serialized_data]" if diff_name in err_message: serialized_data = err_message[err_message.index(diff_name) + len(diff_name) + 1:] diff_error = diff_tools.deserialize_error(serialized_data) # AssertionError is patched in py.test, we can try to fetch diff from it # In general case message starts with "AssertionError: ", but can also starts with "assert" for top-level # function. To support both cases we unify them if err_message.startswith("assert"): err_message = "AssertionError: " + err_message if err_message.startswith("AssertionError:"): diff_error = fetch_diff_error_from_message(err_message, self.swap_diff) except Exception: pass if diff_error: # Cut everything after postfix: it is internal view of DiffError strace = str(report.longrepr) data_postfix = "_ _ _ _ _" if data_postfix in strace: strace = strace[0:strace.index(data_postfix)] self.teamcity.testFailed(test_id, diff_error.msg if diff_error.msg else message, strace, flowId=test_id, comparison_failure=diff_error ) else: self.teamcity.testFailed(test_id, message, str(report.longrepr), flowId=test_id) self.report_test_finished(test_id, duration) def report_test_skip(self, test_id, report): if type(report.longrepr) is tuple and len(report.longrepr) == 3: reason = report.longrepr[2] else: reason = str(report.longrepr) if hasattr(report, 'duration'): duration = timedelta(seconds=report.duration) else: duration = None self.ensure_test_start_reported(test_id) self.report_test_output(report, test_id) self.teamcity.testIgnored(test_id, reason, flowId=test_id) self.report_test_finished(test_id, duration) def pytest_runtest_logreport(self, report): """ :type report: _pytest.runner.TestReport """ test_id = self.format_test_id(report.nodeid, report.location) duration = timedelta(seconds=report.duration) if report.passed: # Do not report passed setup/teardown if no output if report.when == 'call': self.ensure_test_start_reported(test_id) if not self.skip_passed_output: self.report_test_output(report, test_id) self.report_test_finished(test_id, duration) else: if self.report_has_output(report) and not self.skip_passed_output: block_name = "test " + report.when self.teamcity.blockOpened(block_name, flowId=test_id) self.report_test_output(report, test_id) self.teamcity.blockClosed(block_name, flowId=test_id) elif report.failed: if report.when == 'call': self.report_test_failure(test_id, report) elif report.when == 'setup': if self.report_has_output(report): self.teamcity.blockOpened("test setup", flowId=test_id) self.report_test_output(report, test_id) self.teamcity.blockClosed("test setup", flowId=test_id) self.report_test_failure(test_id, report, message="test setup failed", report_output=False) elif report.when == 'teardown': # Report failed teardown as a separate test as original test is already finished self.report_test_failure(test_id + "_teardown", report) elif report.skipped: self.report_test_skip(test_id, report) def pytest_collectreport(self, report): test_id = self.format_test_id(report.nodeid, report.location) + "_collect" if report.failed: self.report_test_failure(test_id, report) elif report.skipped: self.report_test_skip(test_id, report) def pytest_terminal_summary(self): if self.coverage_controller is not None: try: self._report_coverage() except Exception: tb = traceback.format_exc() self.teamcity.customMessage("Coverage statistics reporting failed", "ERROR", errorDetails=tb) def _report_coverage(self): from coverage.misc import NotPython from coverage.report import Reporter from coverage.results import Numbers class _CoverageReporter(Reporter): def __init__(self, coverage, config, messages): super(_CoverageReporter, self).__init__(coverage, config) self.branches = coverage.data.has_arcs() self.messages = messages def report(self, morfs, outfile=None): if hasattr(self, 'find_code_units'): self.find_code_units(morfs) else: self.find_file_reporters(morfs) total = Numbers() if hasattr(self, 'code_units'): units = self.code_units else: units = self.file_reporters for cu in units: try: analysis = self.coverage._analyze(cu) nums = analysis.numbers total += nums except KeyboardInterrupt: raise except Exception: if self.config.ignore_errors: continue err = sys.exc_info() typ, msg = err[:2] if typ is NotPython and not cu.should_be_python(): continue test_id = cu.name details = convert_error_to_string(err) self.messages.testStarted(test_id, flowId=test_id) self.messages.testFailed(test_id, message="Coverage analysis failed", details=details, flowId=test_id) self.messages.testFinished(test_id, flowId=test_id) if total.n_files > 0: covered = total.n_executed total_statements = total.n_statements if self.branches: covered += total.n_executed_branches total_statements += total.n_branches self.messages.buildStatisticLinesCovered(covered) self.messages.buildStatisticTotalLines(total_statements) self.messages.buildStatisticLinesUncovered(total_statements - covered) reporter = _CoverageReporter( self.coverage_controller.cov, self.coverage_controller.cov.config, self.teamcity, ) reporter.report(None)
class TeamcityTestResult(TestResult): def __init__(self, stream=sys.stdout): TestResult.__init__(self) self.output = stream self.test_started_datetime = None self.test_name = None self.createMessages() def createMessages(self): self.messages = TeamcityServiceMessages(self.output) def formatErr(self, err): try: exctype, value, tb = err return ''.join(traceback.format_exception(exctype, value, tb)) except: tb = traceback.format_exc() return "*FAILED TO GET TRACEBACK*: " + tb def getTestName(self, test): return test.shortDescription() or str(test) def addSuccess(self, test, *k): TestResult.addSuccess(self, test) self.output.write("ok\n") def addError(self, test, err, *k): TestResult.addError(self, test, err) err = self.formatErr(err) if self.getTestName(test) != self.test_name: sys.stderr.write("INTERNAL ERROR: addError(%s) outside of test\n" % self.getTestName(test)) sys.stderr.write("Error: %s\n" % err) return self.messages.testFailed(self.getTestName(test), message='Error', details=err) def addFailure(self, test, err, *k): # workaround nose bug on python 3 if _is_string(err[1]): err = (err[0], Exception(err[1]), err[2]) TestResult.addFailure(self, test, err) err = self.formatErr(err) if self.getTestName(test) != self.test_name: sys.stderr.write( "INTERNAL ERROR: addFailure(%s) outside of test\n" % self.getTestName(test)) sys.stderr.write("Error: %s\n" % err) return self.messages.testFailed(self.getTestName(test), message='Failure', details=err) def startTest(self, test): self.test_started_datetime = datetime.datetime.now() self.test_name = self.getTestName(test) self.messages.testStarted(self.test_name) def stopTest(self, test): time_diff = datetime.datetime.now() - self.test_started_datetime if self.getTestName(test) != self.test_name: sys.stderr.write( "INTERNAL ERROR: stopTest(%s) after startTest(%s)" % (self.getTestName(test), self.test_name)) self.messages.testFinished(self.test_name, time_diff) self.test_name = None
class EchoTeamCityMessages(object): def __init__(self, ): self.tw = py.io.TerminalWriter(py.std.sys.stdout) self.teamcity = TeamcityServiceMessages(self.tw) self.currentSuite = None def format_names(self, name): if name.find("::") > 0: file, testname = name.split("::", 1) else: file, testname = name, "top_level" testname = testname.replace("::()::", ".") testname = testname.replace("::", ".") testname = testname.strip(".") file = file.replace(".", "_").replace(os.sep, ".").replace("/", ".") return file, testname def pytest_runtest_logstart(self, nodeid, location): file, testname = self.format_names(nodeid) if not file == self.currentSuite: if self.currentSuite: self.teamcity.testSuiteFinished(self.currentSuite) self.currentSuite = file self.teamcity.testSuiteStarted(self.currentSuite) self.teamcity.testStarted(testname) def pytest_runtest_logreport(self, report): file, testname = self.format_names(report.nodeid) if report.passed: if report.when == "call": # ignore setup/teardown duration = timedelta(seconds=report.duration) self.teamcity.testFinished(testname, testDuration=duration) elif report.failed: if report.when in ("call", "setup"): self.teamcity.testFailed(testname, str(report.location), str(report.longrepr)) duration = timedelta(seconds=report.duration) self.teamcity.testFinished( testname, testDuration=duration) # report finished after the failure elif report.when == "teardown": name = testname + "_teardown" self.teamcity.testStarted(name) self.teamcity.testFailed(name, str(report.location), str(report.longrepr)) self.teamcity.testFinished(name) elif report.skipped: self.teamcity.testIgnored(testname, str(report.longrepr)) self.teamcity.testFinished( testname) # report finished after the skip def pytest_collectreport(self, report): if report.failed: file, testname = self.format_names(report.nodeid) name = file + "_collect" self.teamcity.testStarted(name) self.teamcity.testFailed(name, str(report.location), str(report.longrepr)) self.teamcity.testFinished(name) def pytest_sessionfinish(self, session, exitstatus, __multicall__): if self.currentSuite: self.teamcity.testSuiteFinished(self.currentSuite)
class TeamcityReport(Plugin): name = 'teamcity-report' score = 10000 def __init__(self): super(TeamcityReport, self).__init__() self.messages = TeamcityServiceMessages(_real_stdout) self.test_started_datetime_map = {} self.config = None self.total_tests = 0 self.enabled = False def get_test_id(self, test): if is_string(test): return test # Handle special "tests" test_class_name = get_class_fullname(test) if test_class_name == CONTEXT_SUITE_FQN: if inspect.ismodule(test.context): module_name = test.context.__name__ return module_name + "." + test.error_context elif inspect.isclass(test.context): class_name = get_class_fullname(test.context) return class_name + "." + test.error_context test_id = test.id() real_test = getattr(test, "test", test) real_test_class_name = get_class_fullname(real_test) test_arg = getattr(real_test, "arg", tuple()) if (type(test_arg) is tuple or type(test_arg) is list) and len(test_arg) > 0: # As written in nose.case.FunctionTestCase#__str__ or nose.case.MethodTestCase#__str__ test_arg_str = "%s" % (test_arg,) if test_id.endswith(test_arg_str): # Replace '.' in test args with '_' to preserve test hierarchy on TeamCity test_id = test_id[:len(test_id) - len(test_arg_str)] + test_arg_str.replace('.', '_') # Force test_id for doctests if real_test_class_name != "doctest.DocTestCase" and real_test_class_name != "nose.plugins.doctests.DocTestCase": desc = test.shortDescription() if desc and desc != test.id(): return "%s (%s)" % (test_id, desc.replace('.', '_')) return test_id def configure(self, options, conf): self.enabled = is_running_under_teamcity() self.config = conf if self._capture_plugin_enabled(): capture_plugin = self._get_capture_plugin() old_before_test = capture_plugin.beforeTest old_after_test = capture_plugin.afterTest old_format_error = capture_plugin.formatError def newCaptureBeforeTest(test): rv = old_before_test(test) test_id = self.get_test_id(test) capture_plugin._buf = FlushingStringIO(lambda data: dump_test_stdout(self.messages, test_id, test_id, data)) sys.stdout = capture_plugin._buf return rv def newCaptureAfterTest(test): if isinstance(capture_plugin._buf, FlushingStringIO): capture_plugin._buf.flush() return old_after_test(test) def newCaptureFormatError(test, err): if isinstance(capture_plugin._buf, FlushingStringIO): capture_plugin._buf.flush() return old_format_error(test, err) capture_plugin.beforeTest = newCaptureBeforeTest capture_plugin.afterTest = newCaptureAfterTest capture_plugin.formatError = newCaptureFormatError def options(self, parser, env=os.environ): pass def _get_capture_plugin(self): """ :rtype: nose.plugins.capture.Capture """ for plugin in self.config.plugins.plugins: if plugin.name == "capture": return plugin return None def _capture_plugin_enabled(self): plugin = self._get_capture_plugin() return plugin is not None and plugin.enabled def _capture_plugin_buffer(self): plugin = self._get_capture_plugin() if plugin is None: return None return getattr(plugin, "buffer", None) def _captureStandardOutput_value(self): if self._capture_plugin_enabled(): return 'false' else: return 'true' def report_started(self, test): test_id = self.get_test_id(test) self.test_started_datetime_map[test_id] = datetime.datetime.now() self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(), flowId=test_id) def report_fail(self, test, fail_type, err): # workaround nose bug on python 3 if is_string(err[1]): err = (err[0], Exception(err[1]), err[2]) test_id = self.get_test_id(test) details = convert_error_to_string(err) start_index = details.find(_captured_output_start_marker) end_index = details.find(_captured_output_end_marker) if 0 <= start_index < end_index: # do not log test output twice, see report_finish for actual output handling details = details[:start_index] + details[end_index + len(_captured_output_end_marker):] try: error = err[1] if isinstance(error, EqualsAssertionError): details = convert_error_to_string(err, 2) self.messages.testFailed(test_id, message=error.msg, details=details, flowId=test_id, comparison_failure=error) return except Exception: pass self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id) def report_finish(self, test): test_id = self.get_test_id(test) if test_id in self.test_started_datetime_map: time_diff = datetime.datetime.now() - self.test_started_datetime_map[test_id] self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id) else: self.messages.testFinished(test_id, flowId=test_id) def prepareTestLoader(self, loader): """Insert ourselves into loader calls to count tests. The top-level loader call often returns lazy results, like a LazySuite. This is a problem, as we would destroy the suite by iterating over it to count the tests. Consequently, we monkey-patch the top-level loader call to do the load twice: once for the actual test running and again to yield something we can iterate over to do the count. from https://github.com/erikrose/nose-progressive/ :type loader: nose.loader.TestLoader """ # TODO: If there's ever a practical need, also patch loader.suiteClass # or even TestProgram.createTests. createTests seems to be main top- # level caller of loader methods, and nose.core.collector() (which # isn't even called in nose) is an alternate one. # # nose 1.3.4 contains required fix: # Another fix for Python 3.4: Call super in LazySuite to access _removed_tests variable if hasattr(loader, 'loadTestsFromNames') and nose.__versioninfo__ >= (1, 3, 4): old_loadTestsFromNames = loader.loadTestsFromNames def _loadTestsFromNames(*args, **kwargs): suite = old_loadTestsFromNames(*args, **kwargs) self.total_tests += suite.countTestCases() # Clear out the loader's cache. Otherwise, it never finds any tests # for the actual test run: loader._visitedPaths = set() return old_loadTestsFromNames(*args, **kwargs) loader.loadTestsFromNames = _loadTestsFromNames # noinspection PyUnusedLocal def prepareTestRunner(self, runner): if self.total_tests: self.messages.testCount(self.total_tests) def addError(self, test, err): test_class_name = get_class_fullname(test) test_id = self.get_test_id(test) if issubclass(err[0], SkipTest): self.messages.testIgnored(test_id, message=("SKIPPED: %s" % str(err[1])), flowId=test_id) self.report_finish(test) elif issubclass(err[0], DeprecatedTest): self.messages.testIgnored(test_id, message="Deprecated", flowId=test_id) self.report_finish(test) elif test_class_name == CONTEXT_SUITE_FQN: self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(), flowId=test_id) self.report_fail(test, 'error in ' + test.error_context + ' context', err) self.messages.testFinished(test_id, flowId=test_id) else: # some test cases may report errors in pre setup when startTest was not called yet # example: https://github.com/JetBrains/teamcity-messages/issues/153 if test_id not in self.test_started_datetime_map: self.report_started(test) self.report_fail(test, 'Error', err) self.report_finish(test) def addFailure(self, test, err): self.report_fail(test, 'Failure', err) self.report_finish(test) def startTest(self, test): test_id = self.get_test_id(test) self.test_started_datetime_map[test_id] = datetime.datetime.now() self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(), flowId=test_id) def addSuccess(self, test): self.report_finish(test)
class TeamcityTestResult(TestResult): separator2 = "\n" def __init__(self, stream=_real_stdout, descriptions=None, verbosity=None): super(TeamcityTestResult, self).__init__() self.test_started_datetime_map = {} self.failed_tests = set() self.subtest_failures = {} self.messages = TeamcityServiceMessages(_real_stdout) def get_test_id(self, test): if is_string(test): return test # Force test_id for doctests if get_class_fullname(test) != "doctest.DocTestCase": desc = test.shortDescription() test_method_name = getattr(test, "_testMethodName", "") if desc and desc != test.id() and desc != test_method_name: return "%s (%s)" % (test.id(), desc.replace('.', '_')) return test.id() def addSuccess(self, test): super(TeamcityTestResult, self).addSuccess(test) def addExpectedFailure(self, test, err): super(TeamcityTestResult, self).addExpectedFailure(test, err) err = convert_error_to_string(err) test_id = self.get_test_id(test) self.messages.testIgnored(test_id, message="Expected failure: " + err, flowId=test_id) def addSkip(self, test, reason=""): if sys.version_info >= (2, 7): super(TeamcityTestResult, self).addSkip(test, reason) test_id = self.get_test_id(test) if reason: reason_str = ": " + str(reason) else: reason_str = "" self.messages.testIgnored(test_id, message="Skipped" + reason_str, flowId=test_id) def addUnexpectedSuccess(self, test): super(TeamcityTestResult, self).addUnexpectedSuccess(test) test_id = self.get_test_id(test) self.messages.testFailed( test_id, message='Failure', details= "Test should not succeed since it's marked with @unittest.expectedFailure", flowId=test_id) def addError(self, test, err, *k): super(TeamcityTestResult, self).addError(test, err) if get_class_fullname(test) == "unittest.suite._ErrorHolder": # This is a standalone error test_name = test.id() # patch setUpModule (__main__) -> __main__.setUpModule test_name = re.sub(r'^(.*) \((.*)\)$', r'\2.\1', test_name) self.messages.testStarted(test_name, flowId=test_name) self.report_fail(test_name, 'Failure', err) self.messages.testFinished(test_name, flowId=test_name) elif get_class_fullname(err[0]) == "unittest2.case.SkipTest": message = "" if hasattr(err[1], "message"): message = getattr(err[1], "message", "") elif hasattr(err[1], "args"): message = getattr(err[1], "args", [""])[0] self.addSkip(test, message) else: self.report_fail(test, 'Error', err) def addFailure(self, test, err, *k): super(TeamcityTestResult, self).addFailure(test, err) self.report_fail(test, 'Failure', err) def addSubTest(self, test, subtest, err): super(TeamcityTestResult, self).addSubTest(test, subtest, err) test_id = self.get_test_id(test) if err is not None: if issubclass(err[0], test.failureException): self.add_subtest_failure(test_id, self.get_test_id(subtest), err) self.messages.testStdErr(test_id, out="%s: failure\n" % self.get_test_id(subtest), flowId=test_id) else: self.add_subtest_failure(test_id, self.get_test_id(subtest), err) self.messages.testStdErr(test_id, out="%s: error\n" % self.get_test_id(subtest), flowId=test_id) else: self.messages.testStdOut(test_id, out="%s: ok\n" % self.get_test_id(subtest), flowId=test_id) def add_subtest_failure(self, test_id, subtest_id, err): fail_array = self.subtest_failures.get(test_id, []) fail_array.append("%s:\n%s" % (subtest_id, convert_error_to_string(err))) self.subtest_failures[test_id] = fail_array def get_subtest_failure(self, test_id): fail_array = self.subtest_failures.get(test_id, []) return "\n".join(fail_array) def report_fail(self, test, fail_type, err): test_id = self.get_test_id(test) if is_string(err): details = err elif get_class_fullname(err) == "twisted.python.failure.Failure": details = err.getTraceback() else: details = convert_error_to_string(err) subtest_failure = self.get_subtest_failure(test_id) if subtest_failure: details = subtest_failure + "\n" + details self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id) self.failed_tests.add(test_id) def startTest(self, test): super(TeamcityTestResult, self).startTest(test) test_id = self.get_test_id(test) self.test_started_datetime_map[test_id] = datetime.datetime.now() self.messages.testStarted(test_id, captureStandardOutput='true', flowId=test_id) def stopTest(self, test): test_id = self.get_test_id(test) if self.buffer: # Do not allow super() method to print output by itself self._mirrorOutput = False output = sys.stdout.getvalue() if output: for chunk in split_output(limit_output(output)): self.messages.testStdOut(test_id, chunk, flowId=test_id) error = sys.stderr.getvalue() if error: for chunk in split_output(limit_output(error)): self.messages.testStdErr(test_id, chunk, flowId=test_id) super(TeamcityTestResult, self).stopTest(test) if test_id not in self.failed_tests and self.subtest_failures.get( test_id, []): self.report_fail(test, "Subtest failed", "") time_diff = datetime.datetime.now( ) - self.test_started_datetime_map[test_id] self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id) def printErrors(self): pass
class TeamcityReport(object): name = 'teamcity-report' score = 10000 def __init__(self): super(TeamcityReport, self).__init__() self.messages = TeamcityServiceMessages(_real_stdout) self.test_started_datetime_map = {} self.enabled = False def get_test_id(self, test): if is_string(test): return test # Handle special "tests" test_class_name = get_class_fullname(test) if test_class_name == CONTEXT_SUITE_FQN: if inspect.ismodule(test.context): module_name = test.context.__name__ return module_name + "." + test.error_context elif inspect.isclass(test.context): class_name = get_class_fullname(test.context) return class_name + "." + test.error_context test_id = test.id() real_test = getattr(test, "test", test) real_test_class_name = get_class_fullname(real_test) test_arg = getattr(real_test, "arg", tuple()) if (type(test_arg) is tuple or type(test_arg) is list) and len(test_arg) > 0: # As written in nose.case.FunctionTestCase#__str__ or nose.case.MethodTestCase#__str__ test_arg_str = "%s" % (test_arg,) if test_id.endswith(test_arg_str): # Replace '.' in test args with '_' to preserve test hierarchy on TeamCity test_id = test_id[:len(test_id) - len(test_arg_str)] + test_arg_str.replace('.', '_') # Force test_id for doctests if real_test_class_name != "doctest.DocTestCase" and real_test_class_name != "nose.plugins.doctests.DocTestCase": desc = test.shortDescription() if desc and desc != test.id(): return "%s (%s)" % (test_id, desc.replace('.', '_')) return test_id def configure(self, options, conf): self.enabled = is_running_under_teamcity() def options(self, parser, env=os.environ): pass def report_fail(self, test, fail_type, err): # workaround nose bug on python 3 if is_string(err[1]): err = (err[0], Exception(err[1]), err[2]) test_id = self.get_test_id(test) details = convert_error_to_string(err) start_index = details.find(_captured_output_start_marker) end_index = details.find(_captured_output_end_marker) if 0 <= start_index < end_index: captured_output = details[start_index + len(_captured_output_start_marker):end_index] details = details[:start_index] + details[end_index + len(_captured_output_end_marker):] for chunk in split_output(limit_output(captured_output)): self.messages.testStdOut(test_id, chunk, flowId=test_id) self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id) def addError(self, test, err): test_class_name = get_class_fullname(test) test_id = self.get_test_id(test) if issubclass(err[0], SkipTest): self.messages.testIgnored(test_id, message=("SKIPPED: %s" % str(err[1])), flowId=test_id) elif issubclass(err[0], DeprecatedTest): self.messages.testIgnored(test_id, message="Deprecated", flowId=test_id) elif test_class_name == CONTEXT_SUITE_FQN: self.messages.testStarted(test_id, captureStandardOutput='true', flowId=test_id) self.report_fail(test, 'error in ' + test.error_context + ' context', err) self.messages.testFinished(test_id, flowId=test_id) else: self.report_fail(test, 'Error', err) def addFailure(self, test, err): self.report_fail(test, 'Failure', err) def startTest(self, test): test_id = self.get_test_id(test) self.test_started_datetime_map[test_id] = datetime.datetime.now() self.messages.testStarted(test_id, captureStandardOutput='true', flowId=test_id) def addSuccess(self, test): test_id = self.get_test_id(test) time_diff = datetime.datetime.now() - self.test_started_datetime_map[test_id] self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id)
class TeamcityTestResult(TestResult): separator2 = "\n" # noinspection PyUnusedLocal def __init__(self, stream=_real_stdout, descriptions=None, verbosity=None): super(TeamcityTestResult, self).__init__() # Some code may ask for self.failfast, see unittest2.case.TestCase.subTest self.failfast = getattr(self, "failfast", False) self.test_started_datetime_map = {} self.failed_tests = set() self.subtest_failures = {} self.messages = TeamcityServiceMessages(_real_stdout) self.current_test_id = None @staticmethod def get_test_id(test): if is_string(test): return test test_class_fullname = get_class_fullname(test) test_id = test.id() if test_class_fullname in _ERROR_HOLDERS_FQN: # patch setUpModule (__main__) -> __main__.setUpModule return re.sub(r'^(.*) \((.*)\)$', r'\2.\1', test_id) # Force test_id for doctests if test_class_fullname != "doctest.DocTestCase": desc = test.shortDescription() test_method_name = getattr(test, "_testMethodName", "") if desc and desc != test_id and desc != test_method_name: return "%s (%s)" % (test_id, desc.replace('.', '_')) return test_id def addSuccess(self, test): super(TeamcityTestResult, self).addSuccess(test) def addExpectedFailure(self, test, err): _super = super(TeamcityTestResult, self) if hasattr(_super, "addExpectedFailure"): _super.addExpectedFailure(test, err) err = convert_error_to_string(err) test_id = self.get_test_id(test) self.messages.testIgnored(test_id, message="Expected failure: " + err, flowId=test_id) def get_subtest_block_id(self, test, subtest): test_id = self.get_test_id(test) subtest_id = self.get_test_id(subtest) if subtest_id.startswith(test_id): block_id = subtest_id[len(test_id):].strip() else: block_id = subtest_id if len(block_id) == 0: block_id = test_id return block_id def addSkip(self, test, reason=""): if sys.version_info >= (2, 7): super(TeamcityTestResult, self).addSkip(test, reason) if reason: if isinstance(reason, Exception): reason_str = ": " + get_exception_message(reason) else: reason_str = ": " + to_unicode(reason) else: reason_str = "" test_class_name = get_class_fullname(test) if test_class_name == "unittest.case._SubTest" or test_class_name == "unittest2.case._SubTest": parent_test = test.test_case parent_test_id = self.get_test_id(parent_test) subtest = test block_id = self.get_subtest_block_id(parent_test, subtest) self.messages.subTestBlockOpened(block_id, subTestResult="Skip", flowId=parent_test_id) self.messages.testStdOut(parent_test_id, out="SubTest skipped" + reason_str + "\n", flowId=parent_test_id) self.messages.blockClosed(block_id, flowId=parent_test_id) else: test_id = self.get_test_id(test) if test_id not in self.test_started_datetime_map: # Test ignored without startTest. Handle start and finish events ourselves self.messages.testStarted(test_id, flowId=test_id) self.messages.testIgnored(test_id, message="Skipped" + reason_str, flowId=test_id) self.messages.testFinished(test_id, flowId=test_id) else: self.messages.testIgnored(test_id, message="Skipped" + reason_str, flowId=test_id) def addUnexpectedSuccess(self, test): _super = super(TeamcityTestResult, self) if hasattr(_super, "addUnexpectedSuccess"): _super.addUnexpectedSuccess(test) test_id = self.get_test_id(test) self.messages.testFailed(test_id, message='Failure', details="Test should not succeed since it's marked with @unittest.expectedFailure", flowId=test_id) def addError(self, test, err, *k): super(TeamcityTestResult, self).addError(test, err) test_class = get_class_fullname(test) if test_class in _ERROR_HOLDERS_FQN: # This is a standalone error test_id = self.get_test_id(test) self.messages.testStarted(test_id, flowId=test_id) self.report_fail(test, 'Failure', err) self.messages.testFinished(test_id, flowId=test_id) elif get_class_fullname(err[0]) == "unittest2.case.SkipTest": message = "" if hasattr(err[1], "message"): message = getattr(err[1], "message", "") elif hasattr(err[1], "args"): message = getattr(err[1], "args", [""])[0] self.addSkip(test, message) else: self.report_fail(test, 'Error', err) def addFailure(self, test, err, *k): super(TeamcityTestResult, self).addFailure(test, err) self.report_fail(test, 'Failure', err) def addSubTest(self, test, subtest, err): _super = super(TeamcityTestResult, self) if hasattr(_super, "addSubTest"): _super.addSubTest(test, subtest, err) test_id = self.get_test_id(test) subtest_id = self.get_test_id(subtest) if subtest_id.startswith(test_id): # Replace "." -> "_" since '.' is a test hierarchy separator # See i.e. https://github.com/JetBrains/teamcity-messages/issues/134 (https://youtrack.jetbrains.com/issue/PY-23846) block_id = subtest_id[len(test_id):].strip().replace(".", "_") else: block_id = subtest_id if len(block_id) == 0: block_id = subtest_id if err is not None: self.add_subtest_failure(test_id, block_id) if issubclass(err[0], test.failureException): self.messages.subTestBlockOpened(block_id, subTestResult="Failure", flowId=test_id) self.messages.testStdErr(test_id, out="SubTest failure: %s\n" % convert_error_to_string(err), flowId=test_id) self.messages.blockClosed(block_id, flowId=test_id) else: self.messages.subTestBlockOpened(block_id, subTestResult="Error", flowId=test_id) self.messages.testStdErr(test_id, out="SubTest error: %s\n" % convert_error_to_string(err), flowId=test_id) self.messages.blockClosed(block_id, flowId=test_id) else: self.messages.subTestBlockOpened(block_id, subTestResult="Success", flowId=test_id) self.messages.blockClosed(block_id, flowId=test_id) def add_subtest_failure(self, test_id, subtest_block_id): fail_array = self.subtest_failures.get(test_id, []) fail_array.append(subtest_block_id) self.subtest_failures[test_id] = fail_array def get_subtest_failure(self, test_id): fail_array = self.subtest_failures.get(test_id, []) return ", ".join(fail_array) def report_fail(self, test, fail_type, err): test_id = self.get_test_id(test) diff_failed = None try: error = err[1] if isinstance(error, EqualsAssertionError): diff_failed = error except Exception: pass if is_string(err): details = err else: try: details = err.getTraceback() except AttributeError: frames_to_skip_from_tail = 2 if diff_failed else 0 details = convert_error_to_string(err, frames_to_skip_from_tail) subtest_failures = self.get_subtest_failure(test_id) if subtest_failures: details = "Failed subtests list: " + subtest_failures + "\n\n" + details.strip() details = details.strip() if diff_failed: self.messages.testFailed(test_id, message=diff_failed.msg, details=details, flowId=test_id, comparison_failure=diff_failed) else: self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id) self.failed_tests.add(test_id) def startTest(self, test): test_id = self.get_test_id(test) self.current_test_id = test_id super(TeamcityTestResult, self).startTest(test) self.test_started_datetime_map[test_id] = datetime.datetime.now() self.messages.testStarted(test_id, captureStandardOutput='true', flowId=test_id) def _dump_test_stderr(self, data): if self.current_test_id is not None: dump_test_stderr(self.messages, self.current_test_id, self.current_test_id, data) else: _real_stderr.write(data) def _dump_test_stdout(self, data): if self.current_test_id is not None: dump_test_stdout(self.messages, self.current_test_id, self.current_test_id, data) else: _real_stdout.write(data) def _setupStdout(self): if getattr(self, 'buffer', None): self._stderr_buffer = FlushingStringIO(self._dump_test_stderr) self._stdout_buffer = FlushingStringIO(self._dump_test_stdout) sys.stdout = self._stdout_buffer sys.stderr = self._stderr_buffer def stopTest(self, test): test_id = self.get_test_id(test) if getattr(self, 'buffer', None): # Do not allow super() method to print output by itself self._mirrorOutput = False output = sys.stdout.getvalue() if output: dump_test_stdout(self.messages, test_id, test_id, output) error = sys.stderr.getvalue() if error: dump_test_stderr(self.messages, test_id, test_id, error) super(TeamcityTestResult, self).stopTest(test) self.current_test_id = None if test_id not in self.failed_tests: subtest_failures = self.get_subtest_failure(test_id) if subtest_failures: self.report_fail(test, "One or more subtests failed", "") try: time_diff = datetime.datetime.now() - self.test_started_datetime_map[test_id] except KeyError: time_diff = None self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id) def printErrors(self): pass
class EchoTeamCityMessages(object): def __init__(self, output_capture_enabled, coverage_controller, skip_passed_output, swap_diff): self.coverage_controller = coverage_controller self.output_capture_enabled = output_capture_enabled self.skip_passed_output = skip_passed_output self.teamcity = TeamcityServiceMessages() self.test_start_reported_mark = set() self.max_reported_output_size = 1 * 1024 * 1024 self.reported_output_chunk_size = 50000 self.swap_diff = swap_diff def get_id_from_location(self, location): if type(location) is not tuple or len(location) != 3 or not hasattr( location[2], "startswith"): return None def convert_file_to_id(filename): filename = re.sub(r"\.pyc?$", "", filename) return filename.replace(os.sep, ".").replace("/", ".") def add_prefix_to_filename_id(filename_id, prefix): dot_location = filename_id.rfind('.') if dot_location <= 0 or dot_location >= len(filename_id) - 1: return None return filename_id[:dot_location + 1] + prefix + filename_id[dot_location + 1:] pylint_prefix = '[pylint] ' if location[2].startswith(pylint_prefix): id_from_file = convert_file_to_id(location[2][len(pylint_prefix):]) return id_from_file + ".Pylint" if location[2] == "PEP8-check": id_from_file = convert_file_to_id(location[0]) return id_from_file + ".PEP8" return None def format_test_id(self, nodeid, location): id_from_location = self.get_id_from_location(location) if id_from_location is not None: return id_from_location test_id = nodeid if test_id: if test_id.find("::") < 0: test_id += "::top_level" else: test_id = "top_level" first_bracket = test_id.find("[") if first_bracket > 0: # [] -> (), make it look like nose parameterized tests params = "(" + test_id[first_bracket + 1:] if params.endswith("]"): params = params[:-1] + ")" test_id = test_id[:first_bracket] if test_id.endswith("::"): test_id = test_id[:-2] else: params = "" test_id = test_id.replace("::()::", "::") test_id = re.sub(r"\.pyc?::", r"::", test_id) test_id = test_id.replace(".", "_").replace(os.sep, ".").replace( "/", ".").replace('::', '.') if params: params = params.replace(".", "_") test_id += params return test_id def format_location(self, location): if type(location) is tuple and len(location) == 3: return "%s:%s (%s)" % (str(location[0]), str( location[1]), str(location[2])) return str(location) def pytest_collection_finish(self, session): self.teamcity.testCount(len(session.items)) def pytest_runtest_logstart(self, nodeid, location): # test name fetched from location passed as metainfo to PyCharm # it will be used to run specific test # See IDEA-176950, PY-31836 test_name = location[2] if test_name: test_name = str(test_name).split(".")[-1] self.ensure_test_start_reported(self.format_test_id(nodeid, location), test_name) def ensure_test_start_reported(self, test_id, metainfo=None): if test_id not in self.test_start_reported_mark: if self.output_capture_enabled: capture_standard_output = "false" else: capture_standard_output = "true" self.teamcity.testStarted( test_id, flowId=test_id, captureStandardOutput=capture_standard_output, metainfo=metainfo) self.test_start_reported_mark.add(test_id) def report_has_output(self, report): for (secname, data) in report.sections: if report.when in secname and ('stdout' in secname or 'stderr' in secname): return True return False def report_test_output(self, report, test_id): for (secname, data) in report.sections: # https://github.com/JetBrains/teamcity-messages/issues/112 # CollectReport didn't have 'when' property, but now it has. # But we still need output on 'collect' state if hasattr( report, "when" ) and report.when not in secname and report.when != 'collect': continue if not data: continue if 'stdout' in secname: dump_test_stdout(self.teamcity, test_id, test_id, data) elif 'stderr' in secname: dump_test_stderr(self.teamcity, test_id, test_id, data) def report_test_finished(self, test_id, duration=None): self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id) self.test_start_reported_mark.remove(test_id) def report_test_failure(self, test_id, report, message=None, report_output=True): if hasattr(report, 'duration'): duration = timedelta(seconds=report.duration) else: duration = None if message is None: message = self.format_location(report.location) self.ensure_test_start_reported(test_id) if report_output: self.report_test_output(report, test_id) diff_error = None try: err_message = str(report.longrepr.reprcrash.message) diff_name = diff_tools.EqualsAssertionError.__name__ # There is a string like "foo.bar.DiffError: [serialized_data]" if diff_name in err_message: serialized_data = err_message[err_message.index(diff_name) + len(diff_name) + 1:] diff_error = diff_tools.deserialize_error(serialized_data) # AssertionError is patched in py.test, we can try to fetch diff from it # In general case message starts with "AssertionError: ", but can also starts with "assert" for top-level # function. To support both cases we unify them if err_message.startswith("assert"): err_message = "AssertionError: " + err_message if err_message.startswith("AssertionError:"): diff_error = fetch_diff_error_from_message( err_message, self.swap_diff) except Exception: pass if not diff_error: from .jb_local_exc_store import get_exception diff_error = get_exception() if diff_error: # Cut everything after postfix: it is internal view of DiffError strace = str(report.longrepr) data_postfix = "_ _ _ _ _" # Error message in pytest must be in "file.py:22 AssertionError" format # This message goes to strace # With custom error we must add real exception class explicitly if data_postfix in strace: strace = strace[0:strace.index(data_postfix)].strip() if strace.endswith(":") and diff_error.real_exception: strace += " " + type(diff_error.real_exception).__name__ self.teamcity.testFailed( test_id, diff_error.msg if diff_error.msg else message, strace, flowId=test_id, comparison_failure=diff_error) else: self.teamcity.testFailed(test_id, message, str(report.longrepr), flowId=test_id) self.report_test_finished(test_id, duration) def report_test_skip(self, test_id, report): if type(report.longrepr) is tuple and len(report.longrepr) == 3: reason = report.longrepr[2] else: reason = str(report.longrepr) if hasattr(report, 'duration'): duration = timedelta(seconds=report.duration) else: duration = None self.ensure_test_start_reported(test_id) self.report_test_output(report, test_id) self.teamcity.testIgnored(test_id, reason, flowId=test_id) self.report_test_finished(test_id, duration) def pytest_assertrepr_compare(self, config, op, left, right): if op in ('==', '!='): return [ '{0} {1} {2}'.format(pprint.pformat(left), op, pprint.pformat(right)) ] def pytest_runtest_logreport(self, report): """ :type report: _pytest.runner.TestReport """ test_id = self.format_test_id(report.nodeid, report.location) duration = timedelta(seconds=report.duration) if report.passed: # Do not report passed setup/teardown if no output if report.when == 'call': self.ensure_test_start_reported(test_id) if not self.skip_passed_output: self.report_test_output(report, test_id) self.report_test_finished(test_id, duration) else: if self.report_has_output( report) and not self.skip_passed_output: block_name = "test " + report.when self.teamcity.blockOpened(block_name, flowId=test_id) self.report_test_output(report, test_id) self.teamcity.blockClosed(block_name, flowId=test_id) elif report.failed: if report.when == 'call': self.report_test_failure(test_id, report) elif report.when == 'setup': if self.report_has_output(report): self.teamcity.blockOpened("test setup", flowId=test_id) self.report_test_output(report, test_id) self.teamcity.blockClosed("test setup", flowId=test_id) self.report_test_failure(test_id, report, message="test setup failed", report_output=False) elif report.when == 'teardown': # Report failed teardown as a separate test as original test is already finished self.report_test_failure(test_id + "_teardown", report) elif report.skipped: self.report_test_skip(test_id, report) def pytest_collectreport(self, report): test_id = self.format_test_id(report.nodeid, report.location) + "_collect" if report.failed: self.report_test_failure(test_id, report) elif report.skipped: self.report_test_skip(test_id, report) def pytest_terminal_summary(self): if self.coverage_controller is not None: try: self._report_coverage() except Exception: tb = traceback.format_exc() self.teamcity.customMessage( "Coverage statistics reporting failed", "ERROR", errorDetails=tb) def _report_coverage(self): from coverage.misc import NotPython from coverage.results import Numbers class _Reporter(object): def __init__(self, coverage, config): try: from coverage.report import Reporter except ImportError: # Support for coverage >= 5.0.1. from coverage.report import get_analysis_to_report class Reporter(object): def __init__(self, coverage, config): self.coverage = coverage self.config = config self._file_reporters = [] def find_file_reporters(self, morfs): return [ fr for fr, _ in get_analysis_to_report( self.coverage, morfs) ] self._reporter = Reporter(coverage, config) def find_file_reporters(self, morfs): self.file_reporters = self._reporter.find_file_reporters(morfs) def __getattr__(self, name): return getattr(self._reporter, name) class _CoverageReporter(_Reporter): def __init__(self, coverage, config, messages): super(_CoverageReporter, self).__init__(coverage, config) if hasattr(coverage, 'data'): self.branches = coverage.data.has_arcs() else: self.branches = coverage.get_data().has_arcs() self.messages = messages def report(self, morfs, outfile=None): if hasattr(self, 'find_code_units'): self.find_code_units(morfs) else: self.find_file_reporters(morfs) total = Numbers() if hasattr(self, 'code_units'): units = self.code_units else: units = self.file_reporters for cu in units: try: analysis = self.coverage._analyze(cu) nums = analysis.numbers total += nums except KeyboardInterrupt: raise except Exception: if self.config.ignore_errors: continue err = sys.exc_info() typ, msg = err[:2] if typ is NotPython and not cu.should_be_python(): continue test_id = cu.name details = convert_error_to_string(err) self.messages.testStarted(test_id, flowId=test_id) self.messages.testFailed( test_id, message="Coverage analysis failed", details=details, flowId=test_id) self.messages.testFinished(test_id, flowId=test_id) if total.n_files > 0: covered = total.n_executed total_statements = total.n_statements if self.branches: covered += total.n_executed_branches total_statements += total.n_branches self.messages.buildStatisticLinesCovered(covered) self.messages.buildStatisticTotalLines(total_statements) self.messages.buildStatisticLinesUncovered( total_statements - covered) reporter = _CoverageReporter( self.coverage_controller.cov, self.coverage_controller.cov.config, self.teamcity, ) reporter.report(None)
class TeamcityFormatter(Formatter): """ Stateful TC reporter. Since we can't fetch all steps from the very beginning (even skipped tests are reported) we store tests and features on each call. To hook into test reporting override _report_suite_started and/or _report_test_started """ def __init__(self, stream_opener, config): super(TeamcityFormatter, self).__init__(stream_opener, config) assert version.LooseVersion(behave_version) >= version.LooseVersion("1.2.6"), "Only 1.2.6+ is supported" self._messages = TeamcityServiceMessages() self.__feature = None self.__scenario = None self.__steps = deque() self.__scenario_opened = False self.__feature_opened = False self.__test_start_time = None def feature(self, feature): assert isinstance(feature, Feature) assert not self.__feature, "Prev. feature not closed" self.__feature = feature def scenario(self, scenario): assert isinstance(scenario, Scenario) self.__scenario = scenario self.__scenario_opened = False self.__steps.clear() def step(self, step): assert isinstance(step, Step) self.__steps.append(step) def match(self, match): if not self.__feature_opened: self._report_suite_started(self.__feature, _suite_name(self.__feature)) self.__feature_opened = True if not self.__scenario_opened: self._report_suite_started(self.__scenario, _suite_name(self.__scenario)) self.__scenario_opened = True assert self.__steps, "No steps left" step = self.__steps.popleft() self._report_test_started(step, _step_name(step)) self.__test_start_time = datetime.datetime.now() def _report_suite_started(self, suite, suite_name): """ :param suite: behave suite :param suite_name: suite name that must be reported, be sure to use it instead of suite.name """ self._messages.testSuiteStarted(suite_name) def _report_test_started(self, test, test_name): """ Suite name is always stripped, be sure to strip() it too :param test: behave test :param test_name: test name that must be reported, be sure to use it instead of test.name """ self._messages.testStarted(test_name) def result(self, step): assert isinstance(step, Step) step_name = _step_name(step) if step.status == Status.failed: try: error = traceback.format_exc(step.exc_traceback) if error != step.error_message: self._messages.testStdErr(step_name, error) except Exception: pass # exception shall not prevent error message self._messages.testFailed(step_name, message=step.error_message) if step.status == Status.undefined: self._messages.testFailed(step_name, message="Undefined") if step.status == Status.skipped: self._messages.testIgnored(step_name) self._messages.testFinished(step_name, testDuration=datetime.datetime.now() - self.__test_start_time) if not self.__steps: self.__close_scenario() elif step.status in [Status.failed, Status.undefined]: # Broken background/undefined step stops whole scenario reason = "Undefined step" if step.status == Status.undefined else "Prev. step failed" self.__skip_rest_of_scenario(reason) def __skip_rest_of_scenario(self, reason): while self.__steps: step = self.__steps.popleft() self._report_test_started(step, _step_name(step)) self._messages.testIgnored(_step_name(step), message="{0}. Rest part of scenario is skipped".format(reason)) self._messages.testFinished(_step_name(step)) self.__close_scenario() def __close_scenario(self): if self.__scenario: self._messages.testSuiteFinished(_suite_name(self.__scenario)) self.__scenario = None def eof(self): self.__skip_rest_of_scenario("") self._messages.testSuiteFinished(_suite_name(self.__feature)) self.__feature = None self.__feature_opened = False
class EchoTeamCityMessages(object): def __init__(self, output_capture_enabled, coverage_controller): self.coverage_controller = coverage_controller self.output_capture_enabled = output_capture_enabled self.teamcity = TeamcityServiceMessages() self.test_start_reported_mark = set() self.max_reported_output_size = 1 * 1024 * 1024 self.reported_output_chunk_size = 50000 def get_id_from_location(self, location): if type(location) is not tuple or len(location) != 3 or not hasattr(location[2], "startswith"): return None def convert_file_to_id(filename): filename = re.sub(r"\.pyc?$", "", filename) return filename.replace(os.sep, ".").replace("/", ".") def add_prefix_to_filename_id(filename_id, prefix): dot_location = filename_id.rfind('.') if dot_location <= 0 or dot_location >= len(filename_id) - 1: return None return filename_id[:dot_location + 1] + prefix + filename_id[dot_location + 1:] pylint_prefix = '[pylint] ' if location[2].startswith(pylint_prefix): id_from_file = convert_file_to_id(location[2][len(pylint_prefix):]) return id_from_file + ".Pylint" if location[2] == "PEP8-check": id_from_file = convert_file_to_id(location[0]) return id_from_file + ".PEP8" return None def format_test_id(self, nodeid, location): id_from_location = self.get_id_from_location(location) if id_from_location is not None: return id_from_location test_id = nodeid if test_id.find("::") < 0: test_id += "::top_level" test_id = test_id.replace("::()::", "::") test_id = re.sub(r"\.pyc?::", r"::", test_id) test_id = test_id.replace(".", "_").replace(os.sep, ".").replace("/", ".").replace('::', '.') return test_id def format_location(self, location): if type(location) is tuple and len(location) == 3: return "%s:%s (%s)" % (str(location[0]), str(location[1]), str(location[2])) return str(location) def pytest_runtest_logstart(self, nodeid, location): self.ensure_test_start_reported(self.format_test_id(nodeid, location)) def ensure_test_start_reported(self, test_id): if test_id not in self.test_start_reported_mark: if self.output_capture_enabled: capture_standard_output = "false" else: capture_standard_output = "true" self.teamcity.testStarted(test_id, flowId=test_id, captureStandardOutput=capture_standard_output) self.test_start_reported_mark.add(test_id) def report_has_output(self, report): for (secname, data) in report.sections: if report.when in secname and ('stdout' in secname or 'stderr' in secname): return True return False def report_test_output(self, report, test_id): for (secname, data) in report.sections: if report.when not in secname: continue if not data: continue if 'stdout' in secname: for chunk in split_output(limit_output(data)): self.teamcity.testStdOut(test_id, out=chunk, flowId=test_id) elif 'stderr' in secname: for chunk in split_output(limit_output(data)): self.teamcity.testStdErr(test_id, out=chunk, flowId=test_id) def report_test_finished(self, test_id, duration=None): self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id) self.test_start_reported_mark.remove(test_id) def report_test_failure(self, test_id, report, message=None, report_output=True): if hasattr(report, 'duration'): duration = timedelta(seconds=report.duration) else: duration = None if message is None: message = self.format_location(report.location) self.ensure_test_start_reported(test_id) if report_output: self.report_test_output(report, test_id) self.teamcity.testFailed(test_id, message, str(report.longrepr), flowId=test_id) self.report_test_finished(test_id, duration) def pytest_runtest_logreport(self, report): """ :type report: _pytest.runner.TestReport """ test_id = self.format_test_id(report.nodeid, report.location) duration = timedelta(seconds=report.duration) if report.passed: # Do not report passed setup/teardown if no output if report.when == 'call': self.ensure_test_start_reported(test_id) self.report_test_output(report, test_id) self.report_test_finished(test_id, duration) else: if self.report_has_output(report): block_name = "test " + report.when self.teamcity.blockOpened(block_name, flowId=test_id) self.report_test_output(report, test_id) self.teamcity.blockClosed(block_name, flowId=test_id) elif report.failed: if report.when == 'call': self.report_test_failure(test_id, report) elif report.when == 'setup': if self.report_has_output(report): self.teamcity.blockOpened("test setup", flowId=test_id) self.report_test_output(report, test_id) self.teamcity.blockClosed("test setup", flowId=test_id) self.report_test_failure(test_id, report, message="test setup failed", report_output=False) elif report.when == 'teardown': # Report failed teardown as a separate test as original test is already finished self.report_test_failure(test_id + "_teardown", report) elif report.skipped: if type(report.longrepr) is tuple and len(report.longrepr) == 3: reason = report.longrepr[2] else: reason = str(report.longrepr) self.ensure_test_start_reported(test_id) self.report_test_output(report, test_id) self.teamcity.testIgnored(test_id, reason, flowId=test_id) self.report_test_finished(test_id, duration) def pytest_collectreport(self, report): if report.failed: test_id = self.format_test_id(report.nodeid, report.location) + "_collect" self.report_test_failure(test_id, report) def pytest_terminal_summary(self): if self.coverage_controller is not None: try: self._report_coverage() except: tb = traceback.format_exc() self.teamcity.customMessage("Coverage statistics reporting failed", "ERROR", errorDetails=tb) def _report_coverage(self): from coverage.misc import NotPython from coverage.report import Reporter from coverage.results import Numbers class _CoverageReporter(Reporter): def __init__(self, coverage, config, messages): super(_CoverageReporter, self).__init__(coverage, config) self.branches = coverage.data.has_arcs() self.messages = messages def report(self, morfs, outfile=None): if hasattr(self, 'find_code_units'): self.find_code_units(morfs) else: self.find_file_reporters(morfs) total = Numbers() if hasattr(self, 'code_units'): units = self.code_units else: units = self.file_reporters for cu in units: try: analysis = self.coverage._analyze(cu) nums = analysis.numbers total += nums except KeyboardInterrupt: raise except: if self.config.ignore_errors: continue err = sys.exc_info() typ, msg = err[:2] if typ is NotPython and not cu.should_be_python(): continue test_id = cu.name details = convert_error_to_string(err) self.messages.testStarted(test_id, flowId=test_id) self.messages.testFailed(test_id, message="Coverage analysis failed", details=details, flowId=test_id) self.messages.testFinished(test_id, flowId=test_id) if total.n_files > 0: covered = total.n_executed total_statements = total.n_statements if self.branches: covered += total.n_executed_branches total_statements += total.n_branches self.messages.buildStatisticLinesCovered(covered) self.messages.buildStatisticTotalLines(total_statements) self.messages.buildStatisticLinesUncovered(total_statements - covered) reporter = _CoverageReporter( self.coverage_controller.cov, self.coverage_controller.cov.config, self.teamcity, ) reporter.report(None)
class EchoTeamCityMessages(object): def __init__(self, ): self.tw = py.io.TerminalWriter(py.std.sys.stdout) self.teamcity = TeamcityServiceMessages(self.tw) self.currentSuite = None def format_names(self, name): if name.find("::") > 0: file, testname = name.split("::", 1) else: file, testname = name, "top_level" testname = testname.replace("::()::", ".") testname = testname.replace("::", ".") testname = testname.strip(".") file = file.replace(".", "_").replace(os.sep, ".").replace("/", ".") return file, testname def pytest_runtest_logstart(self, nodeid, location): file, testname = self.format_names(nodeid) if not file == self.currentSuite: if self.currentSuite: self.teamcity.testSuiteFinished(self.currentSuite) self.currentSuite = file self.teamcity.testSuiteStarted(self.currentSuite) self.teamcity.testStarted(testname) def pytest_runtest_logreport(self, report): file, testname = self.format_names(report.nodeid) if report.passed: if report.when == "call": # ignore setup/teardown duration = timedelta(seconds=report.duration) self.teamcity.testFinished(testname, testDuration=duration) elif report.failed: if report.when in ("call", "setup"): self.teamcity.testFailed(testname, str(report.location), str(report.longrepr)) duration = timedelta(seconds=report.duration) self.teamcity.testFinished(testname, testDuration=duration) # report finished after the failure elif report.when == "teardown": name = testname + "_teardown" self.teamcity.testStarted(name) self.teamcity.testFailed(name, str(report.location), str(report.longrepr)) self.teamcity.testFinished(name) elif report.skipped: self.teamcity.testIgnored(testname, str(report.longrepr)) self.teamcity.testFinished(testname) # report finished after the skip def pytest_collectreport(self, report): if report.failed: file, testname = self.format_names(report.nodeid) name = file + "_collect" self.teamcity.testStarted(name) self.teamcity.testFailed(name, str(report.location), str(report.longrepr)) self.teamcity.testFinished(name) def pytest_sessionfinish(self, session, exitstatus, __multicall__): if self.currentSuite: self.teamcity.testSuiteFinished(self.currentSuite)
class TeamcityTestResult(unittest.TestResult): def __init__(self, stream=sys.stdout): super(TeamcityTestResult, self).__init__() self.output = stream self.test_started_datetime = datetime.datetime.now() self.test_name = None self.messages = None self.createMessages() self.test_succeeded = False self.subtest_errors = [] self.subtest_failures = [] def createMessages(self): self.messages = TeamcityServiceMessages(self.output) def formatErr(self, err): try: exctype, value, tb = err return ''.join(traceback.format_exception(exctype, value, tb)) except Exception: tb = traceback.format_exc() return "*FAILED TO GET TRACEBACK*: " + tb def getTestName(self, test): desc = test.shortDescription() if desc: return "%s (%s)" % (test.id(), desc) return test.id() def addSuccess(self, test, *args): super(TeamcityTestResult, self).addSuccess(test) self.output.write("ok\n") def addError(self, test, err, *k): super(TeamcityTestResult, self).addError(test, err) err = self.formatErr(err) if self.getTestName(test) != self.test_name: sys.stderr.write("INTERNAL ERROR: addError(%s) outside of test\n" % self.getTestName(test)) sys.stderr.write("Error: %s\n" % err) return self.test_succeeded = False self.messages.testFailed(self.getTestName(test), message='Error', details=err) def addFailure(self, test, err, *k): # workaround nose bug on python 3 if _is_string(err[1]): err = (err[0], Exception(err[1]), err[2]) super(TeamcityTestResult, self).addFailure(test, err) err = self.formatErr(err) if self.getTestName(test) != self.test_name: sys.stderr.write("INTERNAL ERROR: addFailure(%s) outside of test\n" % self.getTestName(test)) sys.stderr.write("Error: %s\n" % err) return self.test_succeeded = False self.messages.testFailed(self.getTestName(test), message='Failure', details=err) def addSkip(self, test, reason): super(TeamcityTestResult, self).addSkip(test, reason) if self.getTestName(test) != self.test_name: sys.stderr.write("INTERNAL ERROR: addSkip(%s) outside of test\n" % self.getTestName(test)) sys.stderr.write("Reason: %s\n" % reason) return self.messages.testIgnored(self.getTestName(test), reason) def addExpectedFailure(self, test, err): super(TeamcityTestResult, self).addExpectedFailure(test, err) if self.getTestName(test) != self.test_name: err = self.formatErr(err) sys.stderr.write("INTERNAL ERROR: addExpectedFailure(%s) outside of test\n" % self.getTestName(test)) sys.stderr.write("Error: %s\n" % err) return self.output.write("ok (failure expected)\n") def addUnexpectedSuccess(self, test): super(TeamcityTestResult, self).addUnexpectedSuccess(test) if self.getTestName(test) != self.test_name: sys.stderr.write("INTERNAL ERROR: addUnexpectedSuccess(%s) outside of test\n" % self.getTestName(test)) return self.test_succeeded = False self.messages.testFailed(self.getTestName(test), message='Unexpected success') def addSubTest(self, test, subtest, err): super(TeamcityTestResult, self).addSubTest(test, subtest, err) if err is not None: if issubclass(err[0], test.failureException): self.subtest_failures.append("%s:\n%s" % (self.getTestName(subtest), self.formatErr(err))) self.output.write("%s: failure\n" % self.getTestName(subtest)) else: self.subtest_errors.append("%s:\n%s" % (self.getTestName(subtest), self.formatErr(err))) self.output.write("%s: error\n" % self.getTestName(subtest)) else: self.output.write("%s: ok\n" % self.getTestName(subtest)) def startTest(self, test): self.test_started_datetime = datetime.datetime.now() self.test_name = self.getTestName(test) self.test_succeeded = True # we assume it succeeded, and set it to False when we send a failure message self.subtest_errors = [] self.subtest_failures = [] self.messages.testStarted(self.test_name) def stopTest(self, test): # Record the test as a failure after the entire test was run and any subtest has errors if self.test_succeeded and (self.subtest_errors or self.subtest_failures): self.messages.testFailed(self.getTestName(test), message='Subtest error', details="\n\n".join(self.subtest_failures) + "\n".join(self.subtest_errors)) time_diff = datetime.datetime.now() - self.test_started_datetime if self.getTestName(test) != self.test_name: sys.stderr.write( "INTERNAL ERROR: stopTest(%s) after startTest(%s)" % (self.getTestName(test), self.test_name)) self.messages.testFinished(self.test_name, time_diff) self.test_name = None self.output.flush()
class TeamcityReport(object): name = 'teamcity-report' score = 10000 def __init__(self): super(TeamcityReport, self).__init__() self.messages = TeamcityServiceMessages(_real_stdout) self.test_started_datetime_map = {} self.enabled = False def get_test_id(self, test): if is_string(test): return test # Handle special "tests" test_class_name = get_class_fullname(test) if test_class_name == CONTEXT_SUITE_FQN: if inspect.ismodule(test.context): module_name = test.context.__name__ return module_name + "." + test.error_context elif inspect.isclass(test.context): class_name = get_class_fullname(test.context) return class_name + "." + test.error_context test_id = test.id() real_test = getattr(test, "test", test) real_test_class_name = get_class_fullname(real_test) test_arg = getattr(real_test, "arg", tuple()) if (type(test_arg) is tuple or type(test_arg) is list) and len(test_arg) > 0: # As written in nose.case.FunctionTestCase#__str__ or nose.case.MethodTestCase#__str__ test_arg_str = "%s" % (test_arg,) if test_id.endswith(test_arg_str): # Replace '.' in test args with '_' to preserve test hierarchy on TeamCity test_id = test_id[:len(test_id) - len(test_arg_str)] + test_arg_str.replace('.', '_') # Force test_id for doctests if real_test_class_name != "doctest.DocTestCase" and real_test_class_name != "nose.plugins.doctests.DocTestCase": desc = test.shortDescription() if desc and desc != test.id(): return "%s (%s)" % (test_id, desc.replace('.', '_')) return test_id def configure(self, options, conf): self.enabled = is_running_under_teamcity() def options(self, parser, env=os.environ): pass def _get_capture_plugin(self, test): """ :type test: nose.case.Test :rtype: nose.plugins.base.Plugin """ for plugin in test.config.plugins.plugins: if plugin.name == "capture": return plugin return None def _capture_plugin_enabled(self, test): """ :type test: nose.case.Test """ plugin = self._get_capture_plugin(test) return plugin is not None and plugin.enabled def _capture_plugin_buffer(self, test): """ :type test: nose.case.Test """ plugin = self._get_capture_plugin(test) if plugin is None: return None return getattr(plugin, "buffer", None) def _captureStandardOutput_value(self, test): """ :type test: nose.case.Test """ if self._capture_plugin_enabled(test): return 'false' else: return 'true' def report_fail(self, test, fail_type, err): # workaround nose bug on python 3 if is_string(err[1]): err = (err[0], Exception(err[1]), err[2]) test_id = self.get_test_id(test) details = convert_error_to_string(err) start_index = details.find(_captured_output_start_marker) end_index = details.find(_captured_output_end_marker) if 0 <= start_index < end_index: # do not log test output twice, see report_finish for actual output handling details = details[:start_index] + details[end_index + len(_captured_output_end_marker):] self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id) def report_finish(self, test): test_id = self.get_test_id(test) captured_output = getattr(test, "capturedOutput", None) if captured_output is None and self._capture_plugin_enabled(test): # nose capture does not fill 'capturedOutput' property on successful tests captured_output = self._capture_plugin_buffer(test) if captured_output: for chunk in split_output(limit_output(captured_output)): self.messages.testStdOut(test_id, chunk, flowId=test_id) if test_id in self.test_started_datetime_map: time_diff = datetime.datetime.now() - self.test_started_datetime_map[test_id] self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id) else: self.messages.testFinished(test_id, flowId=test_id) def addError(self, test, err): test_class_name = get_class_fullname(test) test_id = self.get_test_id(test) if issubclass(err[0], SkipTest): self.messages.testIgnored(test_id, message=("SKIPPED: %s" % str(err[1])), flowId=test_id) self.report_finish(test) elif issubclass(err[0], DeprecatedTest): self.messages.testIgnored(test_id, message="Deprecated", flowId=test_id) self.report_finish(test) elif test_class_name == CONTEXT_SUITE_FQN: self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(test), flowId=test_id) self.report_fail(test, 'error in ' + test.error_context + ' context', err) self.messages.testFinished(test_id, flowId=test_id) else: self.report_fail(test, 'Error', err) self.report_finish(test) def addFailure(self, test, err): self.report_fail(test, 'Failure', err) self.report_finish(test) def startTest(self, test): test_id = self.get_test_id(test) self.test_started_datetime_map[test_id] = datetime.datetime.now() self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(test), flowId=test_id) def addSuccess(self, test): self.report_finish(test)
class EchoTeamCityMessages(object): def __init__(self, output_capture_enabled, coverage_controller): self.coverage_controller = coverage_controller self.output_capture_enabled = output_capture_enabled self.teamcity = TeamcityServiceMessages() self.test_start_reported_mark = set() self.max_reported_output_size = 1 * 1024 * 1024 self.reported_output_chunk_size = 50000 def format_test_id(self, nodeid): test_id = nodeid if test_id.find("::") < 0: test_id += "::top_level" test_id = test_id.replace("::()::", "::") test_id = re.sub(r"\.pyc?::", r"::", test_id) test_id = test_id.replace(".", "_").replace(os.sep, ".").replace( "/", ".").replace('::', '.') return test_id def pytest_runtest_logstart(self, nodeid, location): self.ensure_test_start_reported(self.format_test_id(nodeid)) def ensure_test_start_reported(self, test_id): if test_id not in self.test_start_reported_mark: self.teamcity.testStarted(test_id, flowId=test_id, captureStandardOutput='false' if self.output_capture_enabled else 'true') self.test_start_reported_mark.add(test_id) def report_has_output(self, report): for (secname, data) in report.sections: if report.when in secname and ('stdout' in secname or 'stderr' in secname): return True return False def report_test_output(self, report, test_id, when): for (secname, data) in report.sections: if when not in secname: continue if not data: continue if 'stdout' in secname: for chunk in split_output(limit_output(data)): self.teamcity.testStdOut(test_id, out=chunk, flowId=test_id) elif 'stderr' in secname: for chunk in split_output(limit_output(data)): self.teamcity.testStdErr(test_id, out=chunk, flowId=test_id) def pytest_runtest_logreport(self, report): """ :type report: _pytest.runner.TestReport """ orig_test_id = self.format_test_id(report.nodeid) suffix = '' if report.when == 'call' else ('_' + report.when) test_id = orig_test_id + suffix duration = timedelta(seconds=report.duration) if report.passed: # Do not report passed setup/teardown if no output if report.when == 'call' or self.report_has_output(report): self.ensure_test_start_reported(test_id) self.report_test_output(report, test_id, report.when) self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id) elif report.failed: self.ensure_test_start_reported(test_id) self.report_test_output(report, test_id, report.when) self.teamcity.testFailed(test_id, str(report.location), str(report.longrepr), flowId=test_id) self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id) # If test setup failed, report test failure as well if report.when == 'setup': self.ensure_test_start_reported(orig_test_id) self.teamcity.testFailed( orig_test_id, "test setup failed, see %s test failure" % test_id, flowId=orig_test_id) self.teamcity.testFinished(orig_test_id, flowId=orig_test_id) elif report.skipped: if type(report.longrepr) is tuple and len(report.longrepr) == 3: reason = report.longrepr[2] else: reason = str(report.longrepr) self.ensure_test_start_reported(orig_test_id) self.report_test_output(report, orig_test_id, report.when) self.teamcity.testIgnored(orig_test_id, reason, flowId=orig_test_id) self.teamcity.testFinished(orig_test_id, flowId=orig_test_id) def pytest_collectreport(self, report): if report.failed: test_id = self.format_test_id(report.nodeid) + "_collect" self.teamcity.testStarted(test_id, flowId=test_id) self.teamcity.testFailed(test_id, str(report.location), str(report.longrepr), flowId=test_id) self.teamcity.testFinished(test_id, flowId=test_id) def pytest_terminal_summary(self): if self.coverage_controller is not None: try: self._report_coverage() except: tb = traceback.format_exc() self.teamcity.customMessage( "Coverage statistics reporting failed", "ERROR", errorDetails=tb) def _report_coverage(self): from coverage.misc import NotPython from coverage.report import Reporter from coverage.results import Numbers class _CoverageReporter(Reporter): def __init__(self, coverage, config, messages): super(_CoverageReporter, self).__init__(coverage, config) self.branches = coverage.data.has_arcs() self.messages = messages def report(self, morfs, outfile=None): self.find_code_units(morfs) total = Numbers() for cu in self.code_units: try: analysis = self.coverage._analyze(cu) nums = analysis.numbers total += nums except KeyboardInterrupt: raise except: if self.config.ignore_errors: continue err = sys.exc_info() typ, msg = err[:2] if typ is NotPython and not cu.should_be_python(): continue test_id = cu.name details = convert_error_to_string(err) self.messages.testStarted(test_id, flowId=test_id) self.messages.testFailed( test_id, message="Coverage analysis failed", details=details, flowId=test_id) self.messages.testFinished(test_id, flowId=test_id) if total.n_files > 0: covered = total.n_executed + (total.n_executed_branches if self.branches else 0) total_statements = total.n_statements + ( total.n_branches if self.branches else 0) self.messages.buildStatisticLinesCovered(covered) self.messages.buildStatisticTotalLines(total_statements) self.messages.buildStatisticLinesUncovered( total_statements - covered) reporter = _CoverageReporter( self.coverage_controller.cov, self.coverage_controller.cov.config, self.teamcity, ) reporter.report(None)
class TeamcityReport(object): name = 'teamcity-report' score = 10000 def __init__(self): super(TeamcityReport, self).__init__() self.messages = TeamcityServiceMessages(_real_stdout) self.test_started_datetime_map = {} self.enabled = False def get_test_id(self, test): if is_string(test): return test # Handle special "tests" test_class_name = get_class_fullname(test) if test_class_name == CONTEXT_SUITE_FQN: if inspect.ismodule(test.context): module_name = test.context.__name__ return module_name + "." + test.error_context elif inspect.isclass(test.context): class_name = get_class_fullname(test.context) return class_name + "." + test.error_context test_id = test.id() real_test = getattr(test, "test", test) real_test_class_name = get_class_fullname(real_test) test_arg = getattr(real_test, "arg", tuple()) if (type(test_arg) is tuple or type(test_arg) is list) and len(test_arg) > 0: # As written in nose.case.FunctionTestCase#__str__ or nose.case.MethodTestCase#__str__ test_arg_str = "%s" % (test_arg, ) if test_id.endswith(test_arg_str): # Replace '.' in test args with '_' to preserve test hierarchy on TeamCity test_id = test_id[:len(test_id) - len(test_arg_str)] + test_arg_str.replace( '.', '_') # Force test_id for doctests if real_test_class_name != "doctest.DocTestCase" and real_test_class_name != "nose.plugins.doctests.DocTestCase": desc = test.shortDescription() if desc and desc != test.id(): return "%s (%s)" % (test_id, desc.replace('.', '_')) return test_id def configure(self, options, conf): self.enabled = is_running_under_teamcity() def options(self, parser, env=os.environ): pass def report_fail(self, test, fail_type, err): # workaround nose bug on python 3 if is_string(err[1]): err = (err[0], Exception(err[1]), err[2]) test_id = self.get_test_id(test) details = convert_error_to_string(err) start_index = details.find(_captured_output_start_marker) end_index = details.find(_captured_output_end_marker) if 0 <= start_index < end_index: captured_output = details[start_index + len(_captured_output_start_marker ):end_index] details = details[:start_index] + details[ end_index + len(_captured_output_end_marker):] for chunk in split_output(limit_output(captured_output)): self.messages.testStdOut(test_id, chunk, flowId=test_id) self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id) def addError(self, test, err): test_class_name = get_class_fullname(test) test_id = self.get_test_id(test) if issubclass(err[0], SkipTest): self.messages.testIgnored(test_id, message="Skipped", flowId=test_id) elif issubclass(err[0], DeprecatedTest): self.messages.testIgnored(test_id, message="Deprecated", flowId=test_id) elif test_class_name == CONTEXT_SUITE_FQN: self.messages.testStarted(test_id, captureStandardOutput='true', flowId=test_id) self.report_fail(test, 'error in ' + test.error_context + ' context', err) self.messages.testFinished(test_id, flowId=test_id) else: self.report_fail(test, 'Error', err) def addFailure(self, test, err): self.report_fail(test, 'Failure', err) def startTest(self, test): test_id = self.get_test_id(test) self.test_started_datetime_map[test_id] = datetime.datetime.now() self.messages.testStarted(test_id, captureStandardOutput='true', flowId=test_id) def stopTest(self, test): test_id = self.get_test_id(test) time_diff = datetime.datetime.now( ) - self.test_started_datetime_map[test_id] self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id)
class EchoTeamCityMessages(object): def __init__(self, output_capture_enabled, coverage_controller): self.coverage_controller = coverage_controller self.output_capture_enabled = output_capture_enabled self.teamcity = TeamcityServiceMessages() self.test_start_reported_mark = set() self.max_reported_output_size = 1 * 1024 * 1024 self.reported_output_chunk_size = 50000 def get_id_from_location(self, location): if type(location) is not tuple or len(location) != 3 or not hasattr( location[2], "startswith"): return None def convert_file_to_id(filename): filename = re.sub(r"\.pyc?$", "", filename) return filename.replace(os.sep, ".").replace("/", ".") def add_prefix_to_filename_id(filename_id, prefix): dot_location = filename_id.rfind('.') if dot_location <= 0 or dot_location >= len(filename_id) - 1: return None return filename_id[:dot_location + 1] + prefix + filename_id[dot_location + 1:] pylint_prefix = '[pylint] ' if location[2].startswith(pylint_prefix): id_from_file = convert_file_to_id(location[2][len(pylint_prefix):]) return id_from_file + ".Pylint" if location[2] == "PEP8-check": id_from_file = convert_file_to_id(location[0]) return id_from_file + ".PEP8" return None def format_test_id(self, nodeid, location): id_from_location = self.get_id_from_location(location) if id_from_location is not None: return id_from_location test_id = nodeid if test_id: if test_id.find("::") < 0: test_id += "::top_level" else: test_id = "top_level" first_bracket = test_id.find("[") if first_bracket > 0: # [] -> (), make it look like nose parameterized tests params = "(" + test_id[first_bracket + 1:] if params.endswith("]"): params = params[:-1] + ")" test_id = test_id[:first_bracket] if test_id.endswith("::"): test_id = test_id[:-2] else: params = "" test_id = test_id.replace("::()::", "::") test_id = re.sub(r"\.pyc?::", r"::", test_id) test_id = test_id.replace(".", "_").replace(os.sep, ".").replace( "/", ".").replace('::', '.') if params: params = params.replace(".", "_") test_id += params return test_id def format_location(self, location): if type(location) is tuple and len(location) == 3: return "%s:%s (%s)" % (str(location[0]), str( location[1]), str(location[2])) return str(location) def pytest_collection_modifyitems(self, session, config, items): self.teamcity.testCount(len(items)) def pytest_runtest_logstart(self, nodeid, location): self.ensure_test_start_reported(self.format_test_id(nodeid, location)) def ensure_test_start_reported(self, test_id): if test_id not in self.test_start_reported_mark: if self.output_capture_enabled: capture_standard_output = "false" else: capture_standard_output = "true" self.teamcity.testStarted( test_id, flowId=test_id, captureStandardOutput=capture_standard_output) self.test_start_reported_mark.add(test_id) def report_has_output(self, report): for (secname, data) in report.sections: if report.when in secname and ('stdout' in secname or 'stderr' in secname): return True return False def report_test_output(self, report, test_id): for (secname, data) in report.sections: # https://github.com/JetBrains/teamcity-messages/issues/112 # CollectReport doesn't have 'when' property if hasattr(report, "when") and report.when not in secname: continue if not data: continue if 'stdout' in secname: for chunk in split_output(limit_output(data)): self.teamcity.testStdOut(test_id, out=chunk, flowId=test_id) elif 'stderr' in secname: for chunk in split_output(limit_output(data)): self.teamcity.testStdErr(test_id, out=chunk, flowId=test_id) def report_test_finished(self, test_id, duration=None): self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id) self.test_start_reported_mark.remove(test_id) def report_test_failure(self, test_id, report, message=None, report_output=True): if hasattr(report, 'duration'): duration = timedelta(seconds=report.duration) else: duration = None if message is None: message = self.format_location(report.location) self.ensure_test_start_reported(test_id) if report_output: self.report_test_output(report, test_id) self.teamcity.testFailed(test_id, message, str(report.longrepr), flowId=test_id) self.report_test_finished(test_id, duration) def report_test_skip(self, test_id, report): if type(report.longrepr) is tuple and len(report.longrepr) == 3: reason = report.longrepr[2] else: reason = str(report.longrepr) if hasattr(report, 'duration'): duration = timedelta(seconds=report.duration) else: duration = None self.ensure_test_start_reported(test_id) self.report_test_output(report, test_id) self.teamcity.testIgnored(test_id, reason, flowId=test_id) self.report_test_finished(test_id, duration) def pytest_runtest_logreport(self, report): """ :type report: _pytest.runner.TestReport """ test_id = self.format_test_id(report.nodeid, report.location) duration = timedelta(seconds=report.duration) if report.passed: # Do not report passed setup/teardown if no output if report.when == 'call': self.ensure_test_start_reported(test_id) self.report_test_output(report, test_id) self.report_test_finished(test_id, duration) else: if self.report_has_output(report): block_name = "test " + report.when self.teamcity.blockOpened(block_name, flowId=test_id) self.report_test_output(report, test_id) self.teamcity.blockClosed(block_name, flowId=test_id) elif report.failed: if report.when == 'call': self.report_test_failure(test_id, report) elif report.when == 'setup': if self.report_has_output(report): self.teamcity.blockOpened("test setup", flowId=test_id) self.report_test_output(report, test_id) self.teamcity.blockClosed("test setup", flowId=test_id) self.report_test_failure(test_id, report, message="test setup failed", report_output=False) elif report.when == 'teardown': # Report failed teardown as a separate test as original test is already finished self.report_test_failure(test_id + "_teardown", report) elif report.skipped: self.report_test_skip(test_id, report) def pytest_collectreport(self, report): test_id = self.format_test_id(report.nodeid, report.location) + "_collect" if report.failed: self.report_test_failure(test_id, report) elif report.skipped: self.report_test_skip(test_id, report) def pytest_terminal_summary(self): if self.coverage_controller is not None: try: self._report_coverage() except: tb = traceback.format_exc() self.teamcity.customMessage( "Coverage statistics reporting failed", "ERROR", errorDetails=tb) def _report_coverage(self): from coverage.misc import NotPython from coverage.report import Reporter from coverage.results import Numbers class _CoverageReporter(Reporter): def __init__(self, coverage, config, messages): super(_CoverageReporter, self).__init__(coverage, config) self.branches = coverage.data.has_arcs() self.messages = messages def report(self, morfs, outfile=None): if hasattr(self, 'find_code_units'): self.find_code_units(morfs) else: self.find_file_reporters(morfs) total = Numbers() if hasattr(self, 'code_units'): units = self.code_units else: units = self.file_reporters for cu in units: try: analysis = self.coverage._analyze(cu) nums = analysis.numbers total += nums except KeyboardInterrupt: raise except: if self.config.ignore_errors: continue err = sys.exc_info() typ, msg = err[:2] if typ is NotPython and not cu.should_be_python(): continue test_id = cu.name details = convert_error_to_string(err) self.messages.testStarted(test_id, flowId=test_id) self.messages.testFailed( test_id, message="Coverage analysis failed", details=details, flowId=test_id) self.messages.testFinished(test_id, flowId=test_id) if total.n_files > 0: covered = total.n_executed total_statements = total.n_statements if self.branches: covered += total.n_executed_branches total_statements += total.n_branches self.messages.buildStatisticLinesCovered(covered) self.messages.buildStatisticTotalLines(total_statements) self.messages.buildStatisticLinesUncovered( total_statements - covered) reporter = _CoverageReporter( self.coverage_controller.cov, self.coverage_controller.cov.config, self.teamcity, ) reporter.report(None)