def test_test_finished():
stream = StreamStub()
messages = TeamcityServiceMessages(output=stream, now=lambda: fixed_date)
messages.testFinished('only a test')
assert stream.observed_output.strip() == textwrap.dedent("""\
##teamcity[testFinished timestamp='2000-11-02T10:23:01.556' name='only a test']
""").strip().encode('utf-8')
def test_test_finished():
stream = StreamStub()
messages = TeamcityServiceMessages(output=stream, now=lambda: fixed_date)
messages.testFinished('only a test')
assert stream.observed_output.strip() == textwrap.dedent("""\
##teamcity[testFinished timestamp='2000-11-02T10:23:01.556' name='only a test']
""").strip().encode('utf-8')
def _wait_build_finish(
tc: TeamCity, tc_msg: TeamcityServiceMessages,
triggered_builds: dict[str,
int]) -> tuple[dict[str, bool], list[Exception]]:
builds_statuses = {}
errors = []
start_time = datetime.now()
while triggered_builds:
time.sleep(BUILDS_CHECK_DELAY)
for shell_name, build_id in triggered_builds.copy().items():
try:
build = tc.builds.get(f"id:{build_id}")
if is_build_finished(build):
tc_msg.testStarted(shell_name)
is_success = is_build_success(build)
builds_statuses[shell_name] = is_success
triggered_builds.pop(shell_name)
if not is_success:
tc_msg.testFailed(
shell_name,
f"{shell_name} Automation tests is finished"
f" with status {build.status}",
)
tc_msg.testFinished(shell_name,
testDuration=datetime.now() -
start_time)
except Exception as e:
errors.append(e)
click.echo(e, err=True)
return builds_statuses, errors
class TeamcityTestResult(TestResult):
def __init__(self, stream=sys.stdout):
TestResult.__init__(self)
self.output = stream
self.createMessages()
def createMessages(self):
self.messages = TeamcityServiceMessages(self.output)
def formatErr(self, err):
exctype, value, tb = err
return ''.join(traceback.format_exception(exctype, value, tb))
def getTestName(self, test):
return test.shortDescription() or str(test)
def addSuccess(self, test, *k):
TestResult.addSuccess(self, test)
self.output.write("ok\n")
def addError(self, test, err, *k):
TestResult.addError(self, test, err)
err = self.formatErr(err)
self.messages.testFailed(self.getTestName(test),
message='Error',
details=err)
def addFailure(self, test, err, *k):
TestResult.addFailure(self, test, err)
err = self.formatErr(err)
self.messages.testFailed(self.getTestName(test),
message='Failure',
details=err)
def startTest(self, test):
self.messages.testStarted(self.getTestName(test))
def stopTest(self, test):
self.messages.testFinished(self.getTestName(test))
class TeamcityTestResult(TestResult):
def __init__(self, stream=sys.stdout):
TestResult.__init__(self)
self.output = stream
self.createMessages()
def createMessages(self):
self.messages = TeamcityServiceMessages(self.output)
def formatErr(self, err):
exctype, value, tb = err
return ''.join(traceback.format_exception(exctype, value, tb))
def getTestName(self, test):
return test.shortDescription() or str(test)
def addSuccess(self, test, *k):
TestResult.addSuccess(self, test)
self.output.write("ok\n")
def addError(self, test, err, *k):
TestResult.addError(self, test, err)
err = self.formatErr(err)
self.messages.testFailed(self.getTestName(test),
message='Error', details=err)
def addFailure(self, test, err, *k):
TestResult.addFailure(self, test, err)
err = self.formatErr(err)
self.messages.testFailed(self.getTestName(test),
message='Failure', details=err)
def startTest(self, test):
self.messages.testStarted(self.getTestName(test))
def stopTest(self, test):
self.messages.testFinished(self.getTestName(test))
def get_file_results(self):
self._deferred_print.sort()
messages = TeamcityServiceMessages()
normalized_filename = self.filename.replace("\\", "/")
suite_name = 'pep8: %s' % normalized_filename
messages.testSuiteStarted(suite_name)
for line_number, offset, code, text, doc in self._deferred_print:
position = '%(path)s:%(row)d:%(col)d' % {
'path': normalized_filename,
'row': self.line_offset + line_number,
'col': offset + 1,
}
error_message = '%s: %s' % (code, text)
test_name = '%s: %s' % (code, position)
messages.testStarted(test_name)
if line_number > len(self.lines):
line = ''
else:
line = self.lines[line_number - 1]
details = [
line.rstrip(),
re.sub(r'\S', ' ', line[:offset]) + '^',
]
if doc:
details.append(doc.strip())
details = '\n'.join(details)
messages.testFailed(test_name, error_message, details)
messages.testFinished(test_name)
messages.testSuiteFinished(suite_name)
return self.file_errors
def get_file_results(self):
self._deferred_print.sort()
messages = TeamcityServiceMessages()
normalized_filename = self.filename.replace("\\", "/")
suite_name = 'pep8: %s' % normalized_filename
messages.testSuiteStarted(suite_name)
for line_number, offset, code, text, doc in self._deferred_print:
position = '%(path)s:%(row)d:%(col)d' % {
'path': normalized_filename,
'row': self.line_offset + line_number,
'col': offset + 1,
}
error_message = '%s: %s' % (code, text)
test_name = '%s: %s' % (code, position)
messages.testStarted(test_name)
if line_number > len(self.lines):
line = ''
else:
line = self.lines[line_number - 1]
details = [
line.rstrip(),
re.sub(r'\S', ' ', line[:offset]) + '^',
]
if doc:
details.append(doc.strip())
details = '\n'.join(details)
messages.testFailed(test_name, error_message, details)
messages.testFinished(test_name)
messages.testSuiteFinished(suite_name)
return self.file_errors
def format(self, error):
normalized_filename = error.filename.replace("\\", "/")
position = '%s:%d:%d' % (normalized_filename, error.line_number,
error.column_number)
error_message = '%s %s' % (error.code, error.text)
test_name = 'pep8: %s: %s' % (position, error_message)
line = error.physical_line
offset = error.column_number
details = [
line.rstrip(),
re.sub(r'\S', ' ', line[:offset]) + '^',
]
details = '\n'.join(details)
bytesio = BytesIO()
messages = TeamcityServiceMessages(output=bytesio)
messages.testStarted(test_name)
messages.testFailed(test_name, error_message, details)
messages.testFinished(test_name)
return bytesio.getvalue().decode('UTF-8')
def format(self, error):
normalized_filename = error.filename.replace("\\", "/")
position = '%s:%d:%d' % (
normalized_filename, error.line_number, error.column_number)
error_message = '%s %s' % (error.code, error.text)
test_name = 'pep8: %s: %s' % (position, error_message)
line = error.physical_line
offset = error.column_number
details = [
line.rstrip(),
re.sub(r'\S', ' ', line[:offset]) + '^',
]
details = '\n'.join(details)
bytesio = BytesIO()
messages = TeamcityServiceMessages(output=bytesio)
messages.testStarted(test_name)
messages.testFailed(test_name, error_message, details)
messages.testFinished(test_name)
return bytesio.getvalue().decode('UTF-8')
class EchoTeamCityMessages(object):
def __init__(self, ):
self.tw = py.io.TerminalWriter(py.std.sys.stdout)
self.teamcity = TeamcityServiceMessages(self.tw)
self.currentSuite = None
def format_names(self, name):
split = '.py'
file, testname = name.split(split, 1)
if not testname:
testname = file
file = 'NO_TEST_FILE_FOUND'
testname = testname.replace("::()::", ".")
testname = testname.replace("::", ".")
testname = testname.strip(".")
return "".join([file, split]), testname
def pytest_runtest_logstart(self, nodeid, location):
file, testname = self.format_names(nodeid)
if not file == self.currentSuite:
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
self.currentSuite = file
self.teamcity.testSuiteStarted(self.currentSuite)
self.teamcity.testStarted(testname)
def pytest_runtest_logreport(self, report):
file, testname = self.format_names(report.nodeid)
if report.when == "call":
for (secname, data) in report.sections:
if 'stdout' in secname:
self.teamcity.testStdOut(testname, out=data)
elif 'stderr' in secname:
self.teamcity.testStdErr(testname, out=data)
if report.passed:
if report.when == "call": # ignore setup/teardown
duration = timedelta(seconds=report.duration)
self.teamcity.testFinished(testname, testDuration=duration)
elif report.failed:
if report.when == "call":
self.teamcity.testFailed(testname, str(report.location), str(report.longrepr))
duration = timedelta(seconds=report.duration)
self.teamcity.testFinished(testname, testDuration=duration) # report finished after the failure
elif report.skipped:
self.teamcity.testIgnored(testname, str(report.longrepr))
self.teamcity.testFinished(testname) # report finished after the skip
def pytest_sessionfinish(self, session, exitstatus, __multicall__):
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
class EchoTeamCityMessages(object):
def __init__(self, ):
self.tw = py.io.TerminalWriter(py.std.sys.stdout)
self.teamcity = TeamcityServiceMessages(self.tw)
self.currentSuite = None
def format_names(self, name):
split = '.py'
file, testname = name.split(split, 1)
if not testname:
testname = file
file = 'NO_TEST_FILE_FOUND'
testname = testname.replace("::()::", ".")
testname = testname.replace("::", ".")
testname = testname.strip(".")
return ("".join([file, split]), testname)
def pytest_runtest_logstart(self, nodeid, location):
file, testname = self.format_names(nodeid)
if not file == self.currentSuite:
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
self.currentSuite = file
self.teamcity.testSuiteStarted(self.currentSuite)
self.teamcity.testStarted(testname)
def pytest_runtest_logreport(self, report):
file, testname = self.format_names(report.nodeid)
if report.passed:
if report.when == "call": # ignore setup/teardown
self.teamcity.testFinished(testname)
elif report.failed:
if report.when == "call":
self.teamcity.testFailed(testname, str(report.location),
str(report.longrepr))
self.teamcity.testFinished(
testname) # report finished after the failure
elif report.skipped:
self.teamcity.testIgnored(testname, str(report.longrepr))
self.teamcity.testFinished(
testname) # report finished after the skip
def pytest_sessionfinish(self, session, exitstatus, __multicall__):
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
class TeamcityTestResult(TestResult):
separator2 = "\n"
def __init__(self, stream=_real_stdout, descriptions=None, verbosity=None):
super(TeamcityTestResult, self).__init__()
self.test_started_datetime_map = {}
self.failed_tests = set()
self.subtest_failures = {}
self.messages = TeamcityServiceMessages(stream)
def get_test_id(self, test):
if is_string(test):
return test
# Force test_id for doctests
if get_class_fullname(test) != "doctest.DocTestCase":
desc = test.shortDescription()
if desc and desc != test.id():
return "%s (%s)" % (test.id(), desc.replace('.', '_'))
return test.id()
def addSuccess(self, test):
super(TeamcityTestResult, self).addSuccess(test)
def addExpectedFailure(self, test, err):
super(TeamcityTestResult, self).addExpectedFailure(test, err)
err = convert_error_to_string(err)
test_id = self.get_test_id(test)
self.messages.testIgnored(test_id, message="Expected failure: " + err, flowId=test_id)
def addSkip(self, test, reason=""):
if sys.version_info >= (2, 7):
super(TeamcityTestResult, self).addSkip(test, reason)
test_id = self.get_test_id(test)
if reason:
reason_str = ": " + reason
else:
reason_str = ""
self.messages.testIgnored(test_id, message="Skipped" + reason_str, flowId=test_id)
def addUnexpectedSuccess(self, test):
super(TeamcityTestResult, self).addUnexpectedSuccess(test)
test_id = self.get_test_id(test)
self.messages.testFailed(test_id, message='Failure',
details="Test should not succeed since it's marked with @unittest.expectedFailure",
flowId=test_id)
def addError(self, test, err, *k):
super(TeamcityTestResult, self).addError(test, err)
if get_class_fullname(test) == "unittest.suite._ErrorHolder":
# This is a standalone error
test_name = test.id()
# patch setUpModule (__main__) -> __main__.setUpModule
test_name = re.sub(r'^(.*) \((.*)\)$', r'\2.\1', test_name)
self.messages.testStarted(test_name, flowId=test_name)
self.report_fail(test_name, 'Failure', err)
self.messages.testFinished(test_name, flowId=test_name)
elif get_class_fullname(err[0]) == "unittest2.case.SkipTest":
message = ""
if hasattr(err[1], "message"):
message = getattr(err[1], "message", "")
elif hasattr(err[1], "args"):
message = getattr(err[1], "args", [""])[0]
self.addSkip(test, message)
else:
self.report_fail(test, 'Error', err)
def addFailure(self, test, err, *k):
super(TeamcityTestResult, self).addFailure(test, err)
self.report_fail(test, 'Failure', err)
def addSubTest(self, test, subtest, err):
super(TeamcityTestResult, self).addSubTest(test, subtest, err)
test_id = self.get_test_id(test)
if err is not None:
if issubclass(err[0], test.failureException):
self.add_subtest_failure(test_id, self.get_test_id(subtest), err)
self.messages.testStdErr(test_id, out="%s: failure\n" % self.get_test_id(subtest), flowId=test_id)
else:
self.add_subtest_failure(test_id, self.get_test_id(subtest), err)
self.messages.testStdErr(test_id, out="%s: error\n" % self.get_test_id(subtest), flowId=test_id)
else:
self.messages.testStdOut(test_id, out="%s: ok\n" % self.get_test_id(subtest), flowId=test_id)
def add_subtest_failure(self, test_id, subtest_id, err):
fail_array = self.subtest_failures.get(test_id, [])
fail_array.append("%s:\n%s" % (subtest_id, convert_error_to_string(err)))
self.subtest_failures[test_id] = fail_array
def get_subtest_failure(self, test_id):
fail_array = self.subtest_failures.get(test_id, [])
return "\n".join(fail_array)
def report_fail(self, test, fail_type, err):
test_id = self.get_test_id(test)
if is_string(err):
details = err
else:
details = convert_error_to_string(err)
subtest_failure = self.get_subtest_failure(test_id)
if subtest_failure:
details = subtest_failure + "\n" + details
self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id)
self.failed_tests.add(test_id)
def startTest(self, test):
super(TeamcityTestResult, self).startTest(test)
test_id = self.get_test_id(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id, captureStandardOutput='true', flowId=test_id)
def stopTest(self, test):
super(TeamcityTestResult, self).stopTest(test)
test_id = self.get_test_id(test)
if test_id not in self.failed_tests and self.subtest_failures.get(test_id, []):
self.report_fail(test, "Subtest failed", "")
time_diff = datetime.datetime.now() - self.test_started_datetime_map[test_id]
self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id)
def printErrors(self):
pass
class EchoTeamCityMessages(object):
def __init__(self, output_capture_enabled, coverage_controller,
skip_passed_output, swap_diff):
self.coverage_controller = coverage_controller
self.output_capture_enabled = output_capture_enabled
self.skip_passed_output = skip_passed_output
self.teamcity = TeamcityServiceMessages()
self.test_start_reported_mark = set()
self.max_reported_output_size = 1 * 1024 * 1024
self.reported_output_chunk_size = 50000
self.swap_diff = swap_diff
def get_id_from_location(self, location):
if type(location) is not tuple or len(location) != 3 or not hasattr(
location[2], "startswith"):
return None
def convert_file_to_id(filename):
filename = re.sub(r"\.pyc?$", "", filename)
return filename.replace(os.sep, ".").replace("/", ".")
def add_prefix_to_filename_id(filename_id, prefix):
dot_location = filename_id.rfind('.')
if dot_location <= 0 or dot_location >= len(filename_id) - 1:
return None
return filename_id[:dot_location +
1] + prefix + filename_id[dot_location + 1:]
pylint_prefix = '[pylint] '
if location[2].startswith(pylint_prefix):
id_from_file = convert_file_to_id(location[2][len(pylint_prefix):])
return id_from_file + ".Pylint"
if location[2] == "PEP8-check":
id_from_file = convert_file_to_id(location[0])
return id_from_file + ".PEP8"
return None
def format_test_id(self, nodeid, location):
id_from_location = self.get_id_from_location(location)
if id_from_location is not None:
return id_from_location
test_id = nodeid
if test_id:
if test_id.find("::") < 0:
test_id += "::top_level"
else:
test_id = "top_level"
first_bracket = test_id.find("[")
if first_bracket > 0:
# [] -> (), make it look like nose parameterized tests
params = "(" + test_id[first_bracket + 1:]
if params.endswith("]"):
params = params[:-1] + ")"
test_id = test_id[:first_bracket]
if test_id.endswith("::"):
test_id = test_id[:-2]
else:
params = ""
test_id = test_id.replace("::()::", "::")
test_id = re.sub(r"\.pyc?::", r"::", test_id)
test_id = test_id.replace(".", "_").replace(os.sep, ".").replace(
"/", ".").replace('::', '.')
if params:
params = params.replace(".", "_")
test_id += params
return test_id
def format_location(self, location):
if type(location) is tuple and len(location) == 3:
return "%s:%s (%s)" % (str(location[0]), str(
location[1]), str(location[2]))
return str(location)
def pytest_collection_finish(self, session):
self.teamcity.testCount(len(session.items))
def pytest_runtest_logstart(self, nodeid, location):
# test name fetched from location passed as metainfo to PyCharm
# it will be used to run specific test
# See IDEA-176950, PY-31836
test_name = location[2]
if test_name:
test_name = str(test_name).split(".")[-1]
self.ensure_test_start_reported(self.format_test_id(nodeid, location),
test_name)
def ensure_test_start_reported(self, test_id, metainfo=None):
if test_id not in self.test_start_reported_mark:
if self.output_capture_enabled:
capture_standard_output = "false"
else:
capture_standard_output = "true"
self.teamcity.testStarted(
test_id,
flowId=test_id,
captureStandardOutput=capture_standard_output,
metainfo=metainfo)
self.test_start_reported_mark.add(test_id)
def report_has_output(self, report):
for (secname, data) in report.sections:
if report.when in secname and ('stdout' in secname
or 'stderr' in secname):
return True
return False
def report_test_output(self, report, test_id):
for (secname, data) in report.sections:
# https://github.com/JetBrains/teamcity-messages/issues/112
# CollectReport didn't have 'when' property, but now it has.
# But we still need output on 'collect' state
if hasattr(
report, "when"
) and report.when not in secname and report.when != 'collect':
continue
if not data:
continue
if 'stdout' in secname:
dump_test_stdout(self.teamcity, test_id, test_id, data)
elif 'stderr' in secname:
dump_test_stderr(self.teamcity, test_id, test_id, data)
def report_test_finished(self, test_id, duration=None):
self.teamcity.testFinished(test_id,
testDuration=duration,
flowId=test_id)
self.test_start_reported_mark.remove(test_id)
def report_test_failure(self,
test_id,
report,
message=None,
report_output=True):
if hasattr(report, 'duration'):
duration = timedelta(seconds=report.duration)
else:
duration = None
if message is None:
message = self.format_location(report.location)
self.ensure_test_start_reported(test_id)
if report_output:
self.report_test_output(report, test_id)
diff_error = None
try:
err_message = str(report.longrepr.reprcrash.message)
diff_name = diff_tools.EqualsAssertionError.__name__
# There is a string like "foo.bar.DiffError: [serialized_data]"
if diff_name in err_message:
serialized_data = err_message[err_message.index(diff_name) +
len(diff_name) + 1:]
diff_error = diff_tools.deserialize_error(serialized_data)
# AssertionError is patched in py.test, we can try to fetch diff from it
# In general case message starts with "AssertionError: ", but can also starts with "assert" for top-level
# function. To support both cases we unify them
if err_message.startswith("assert"):
err_message = "AssertionError: " + err_message
if err_message.startswith("AssertionError:"):
diff_error = fetch_diff_error_from_message(
err_message, self.swap_diff)
except Exception:
pass
if not diff_error:
from .jb_local_exc_store import get_exception
diff_error = get_exception()
if diff_error:
# Cut everything after postfix: it is internal view of DiffError
strace = str(report.longrepr)
data_postfix = "_ _ _ _ _"
# Error message in pytest must be in "file.py:22 AssertionError" format
# This message goes to strace
# With custom error we must add real exception class explicitly
if data_postfix in strace:
strace = strace[0:strace.index(data_postfix)].strip()
if strace.endswith(":") and diff_error.real_exception:
strace += " " + type(diff_error.real_exception).__name__
self.teamcity.testFailed(
test_id,
diff_error.msg if diff_error.msg else message,
strace,
flowId=test_id,
comparison_failure=diff_error)
else:
self.teamcity.testFailed(test_id,
message,
str(report.longrepr),
flowId=test_id)
self.report_test_finished(test_id, duration)
def report_test_skip(self, test_id, report):
if type(report.longrepr) is tuple and len(report.longrepr) == 3:
reason = report.longrepr[2]
else:
reason = str(report.longrepr)
if hasattr(report, 'duration'):
duration = timedelta(seconds=report.duration)
else:
duration = None
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id)
self.teamcity.testIgnored(test_id, reason, flowId=test_id)
self.report_test_finished(test_id, duration)
def pytest_assertrepr_compare(self, config, op, left, right):
if op in ('==', '!='):
return [
'{0} {1} {2}'.format(pprint.pformat(left), op,
pprint.pformat(right))
]
def pytest_runtest_logreport(self, report):
"""
:type report: _pytest.runner.TestReport
"""
test_id = self.format_test_id(report.nodeid, report.location)
duration = timedelta(seconds=report.duration)
if report.passed:
# Do not report passed setup/teardown if no output
if report.when == 'call':
self.ensure_test_start_reported(test_id)
if not self.skip_passed_output:
self.report_test_output(report, test_id)
self.report_test_finished(test_id, duration)
else:
if self.report_has_output(
report) and not self.skip_passed_output:
block_name = "test " + report.when
self.teamcity.blockOpened(block_name, flowId=test_id)
self.report_test_output(report, test_id)
self.teamcity.blockClosed(block_name, flowId=test_id)
elif report.failed:
if report.when == 'call':
self.report_test_failure(test_id, report)
elif report.when == 'setup':
if self.report_has_output(report):
self.teamcity.blockOpened("test setup", flowId=test_id)
self.report_test_output(report, test_id)
self.teamcity.blockClosed("test setup", flowId=test_id)
self.report_test_failure(test_id,
report,
message="test setup failed",
report_output=False)
elif report.when == 'teardown':
# Report failed teardown as a separate test as original test is already finished
self.report_test_failure(test_id + "_teardown", report)
elif report.skipped:
self.report_test_skip(test_id, report)
def pytest_collectreport(self, report):
test_id = self.format_test_id(report.nodeid,
report.location) + "_collect"
if report.failed:
self.report_test_failure(test_id, report)
elif report.skipped:
self.report_test_skip(test_id, report)
def pytest_terminal_summary(self):
if self.coverage_controller is not None:
try:
self._report_coverage()
except Exception:
tb = traceback.format_exc()
self.teamcity.customMessage(
"Coverage statistics reporting failed",
"ERROR",
errorDetails=tb)
def _report_coverage(self):
from coverage.misc import NotPython
from coverage.results import Numbers
class _Reporter(object):
def __init__(self, coverage, config):
try:
from coverage.report import Reporter
except ImportError:
# Support for coverage >= 5.0.1.
from coverage.report import get_analysis_to_report
class Reporter(object):
def __init__(self, coverage, config):
self.coverage = coverage
self.config = config
self._file_reporters = []
def find_file_reporters(self, morfs):
return [
fr for fr, _ in get_analysis_to_report(
self.coverage, morfs)
]
self._reporter = Reporter(coverage, config)
def find_file_reporters(self, morfs):
self.file_reporters = self._reporter.find_file_reporters(morfs)
def __getattr__(self, name):
return getattr(self._reporter, name)
class _CoverageReporter(_Reporter):
def __init__(self, coverage, config, messages):
super(_CoverageReporter, self).__init__(coverage, config)
if hasattr(coverage, 'data'):
self.branches = coverage.data.has_arcs()
else:
self.branches = coverage.get_data().has_arcs()
self.messages = messages
def report(self, morfs, outfile=None):
if hasattr(self, 'find_code_units'):
self.find_code_units(morfs)
else:
self.find_file_reporters(morfs)
total = Numbers()
if hasattr(self, 'code_units'):
units = self.code_units
else:
units = self.file_reporters
for cu in units:
try:
analysis = self.coverage._analyze(cu)
nums = analysis.numbers
total += nums
except KeyboardInterrupt:
raise
except Exception:
if self.config.ignore_errors:
continue
err = sys.exc_info()
typ, msg = err[:2]
if typ is NotPython and not cu.should_be_python():
continue
test_id = cu.name
details = convert_error_to_string(err)
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testFailed(
test_id,
message="Coverage analysis failed",
details=details,
flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
if total.n_files > 0:
covered = total.n_executed
total_statements = total.n_statements
if self.branches:
covered += total.n_executed_branches
total_statements += total.n_branches
self.messages.buildStatisticLinesCovered(covered)
self.messages.buildStatisticTotalLines(total_statements)
self.messages.buildStatisticLinesUncovered(
total_statements - covered)
reporter = _CoverageReporter(
self.coverage_controller.cov,
self.coverage_controller.cov.config,
self.teamcity,
)
reporter.report(None)
class TeamcityTestResult(TestResult):
separator2 = "\n"
# noinspection PyUnusedLocal
def __init__(self, stream=_real_stdout, descriptions=None, verbosity=None):
super(TeamcityTestResult, self).__init__()
# Some code may ask for self.failfast, see unittest2.case.TestCase.subTest
self.failfast = getattr(self, "failfast", False)
self.test_started_datetime_map = {}
self.failed_tests = set()
self.subtest_failures = {}
self.messages = TeamcityServiceMessages(_real_stdout)
self.current_test_id = None
@staticmethod
def get_test_id(test):
if is_string(test):
return test
test_class_fullname = get_class_fullname(test)
test_id = test.id()
if test_class_fullname in _ERROR_HOLDERS_FQN:
# patch setUpModule (__main__) -> __main__.setUpModule
return re.sub(r'^(.*) \((.*)\)$', r'\2.\1', test_id)
# Force test_id for doctests
if test_class_fullname != "doctest.DocTestCase":
desc = test.shortDescription()
test_method_name = getattr(test, "_testMethodName", "")
if desc and desc != test_id and desc != test_method_name:
return "%s (%s)" % (test_id, desc.replace('.', '_'))
return test_id
def addSuccess(self, test):
super(TeamcityTestResult, self).addSuccess(test)
def addExpectedFailure(self, test, err):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addExpectedFailure"):
_super.addExpectedFailure(test, err)
err = convert_error_to_string(err)
test_id = self.get_test_id(test)
self.messages.testIgnored(test_id, message="Expected failure: " + err, flowId=test_id)
def get_subtest_block_id(self, test, subtest):
test_id = self.get_test_id(test)
subtest_id = self.get_test_id(subtest)
if subtest_id.startswith(test_id):
block_id = subtest_id[len(test_id):].strip()
else:
block_id = subtest_id
if len(block_id) == 0:
block_id = test_id
return block_id
def addSkip(self, test, reason=""):
if sys.version_info >= (2, 7):
super(TeamcityTestResult, self).addSkip(test, reason)
if reason:
if isinstance(reason, Exception):
reason_str = ": " + get_exception_message(reason)
else:
reason_str = ": " + to_unicode(reason)
else:
reason_str = ""
test_class_name = get_class_fullname(test)
if test_class_name == "unittest.case._SubTest" or test_class_name == "unittest2.case._SubTest":
parent_test = test.test_case
parent_test_id = self.get_test_id(parent_test)
subtest = test
block_id = self.get_subtest_block_id(parent_test, subtest)
self.messages.subTestBlockOpened(block_id, subTestResult="Skip", flowId=parent_test_id)
self.messages.testStdOut(parent_test_id, out="SubTest skipped" + reason_str + "\n", flowId=parent_test_id)
self.messages.blockClosed(block_id, flowId=parent_test_id)
else:
test_id = self.get_test_id(test)
if test_id not in self.test_started_datetime_map:
# Test ignored without startTest. Handle start and finish events ourselves
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testIgnored(test_id, message="Skipped" + reason_str, flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
else:
self.messages.testIgnored(test_id, message="Skipped" + reason_str, flowId=test_id)
def addUnexpectedSuccess(self, test):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addUnexpectedSuccess"):
_super.addUnexpectedSuccess(test)
test_id = self.get_test_id(test)
self.messages.testFailed(test_id, message='Failure',
details="Test should not succeed since it's marked with @unittest.expectedFailure",
flowId=test_id)
def addError(self, test, err, *k):
super(TeamcityTestResult, self).addError(test, err)
test_class = get_class_fullname(test)
if test_class in _ERROR_HOLDERS_FQN:
# This is a standalone error
test_id = self.get_test_id(test)
self.messages.testStarted(test_id, flowId=test_id)
self.report_fail(test, 'Failure', err)
self.messages.testFinished(test_id, flowId=test_id)
elif get_class_fullname(err[0]) == "unittest2.case.SkipTest":
message = ""
if hasattr(err[1], "message"):
message = getattr(err[1], "message", "")
elif hasattr(err[1], "args"):
message = getattr(err[1], "args", [""])[0]
self.addSkip(test, message)
else:
self.report_fail(test, 'Error', err)
def addFailure(self, test, err, *k):
super(TeamcityTestResult, self).addFailure(test, err)
self.report_fail(test, 'Failure', err)
def addSubTest(self, test, subtest, err):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addSubTest"):
_super.addSubTest(test, subtest, err)
test_id = self.get_test_id(test)
subtest_id = self.get_test_id(subtest)
if subtest_id.startswith(test_id):
# Replace "." -> "_" since '.' is a test hierarchy separator
# See i.e. https://github.com/JetBrains/teamcity-messages/issues/134 (https://youtrack.jetbrains.com/issue/PY-23846)
block_id = subtest_id[len(test_id):].strip().replace(".", "_")
else:
block_id = subtest_id
if len(block_id) == 0:
block_id = subtest_id
if err is not None:
self.add_subtest_failure(test_id, block_id)
if issubclass(err[0], test.failureException):
self.messages.subTestBlockOpened(block_id, subTestResult="Failure", flowId=test_id)
self.messages.testStdErr(test_id, out="SubTest failure: %s\n" % convert_error_to_string(err), flowId=test_id)
self.messages.blockClosed(block_id, flowId=test_id)
else:
self.messages.subTestBlockOpened(block_id, subTestResult="Error", flowId=test_id)
self.messages.testStdErr(test_id, out="SubTest error: %s\n" % convert_error_to_string(err), flowId=test_id)
self.messages.blockClosed(block_id, flowId=test_id)
else:
self.messages.subTestBlockOpened(block_id, subTestResult="Success", flowId=test_id)
self.messages.blockClosed(block_id, flowId=test_id)
def add_subtest_failure(self, test_id, subtest_block_id):
fail_array = self.subtest_failures.get(test_id, [])
fail_array.append(subtest_block_id)
self.subtest_failures[test_id] = fail_array
def get_subtest_failure(self, test_id):
fail_array = self.subtest_failures.get(test_id, [])
return ", ".join(fail_array)
def report_fail(self, test, fail_type, err):
test_id = self.get_test_id(test)
diff_failed = None
try:
error = err[1]
if isinstance(error, EqualsAssertionError):
diff_failed = error
except Exception:
pass
if is_string(err):
details = err
else:
try:
details = err.getTraceback()
except AttributeError:
frames_to_skip_from_tail = 2 if diff_failed else 0
details = convert_error_to_string(err, frames_to_skip_from_tail)
subtest_failures = self.get_subtest_failure(test_id)
if subtest_failures:
details = "Failed subtests list: " + subtest_failures + "\n\n" + details.strip()
details = details.strip()
if diff_failed:
self.messages.testFailed(test_id,
message=diff_failed.msg,
details=details,
flowId=test_id,
comparison_failure=diff_failed)
else:
self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id)
self.failed_tests.add(test_id)
def startTest(self, test):
test_id = self.get_test_id(test)
self.current_test_id = test_id
super(TeamcityTestResult, self).startTest(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id, captureStandardOutput='true', flowId=test_id)
def _dump_test_stderr(self, data):
if self.current_test_id is not None:
dump_test_stderr(self.messages, self.current_test_id, self.current_test_id, data)
else:
_real_stderr.write(data)
def _dump_test_stdout(self, data):
if self.current_test_id is not None:
dump_test_stdout(self.messages, self.current_test_id, self.current_test_id, data)
else:
_real_stdout.write(data)
def _setupStdout(self):
if getattr(self, 'buffer', None):
self._stderr_buffer = FlushingStringIO(self._dump_test_stderr)
self._stdout_buffer = FlushingStringIO(self._dump_test_stdout)
sys.stdout = self._stdout_buffer
sys.stderr = self._stderr_buffer
def stopTest(self, test):
test_id = self.get_test_id(test)
if getattr(self, 'buffer', None):
# Do not allow super() method to print output by itself
self._mirrorOutput = False
output = sys.stdout.getvalue()
if output:
dump_test_stdout(self.messages, test_id, test_id, output)
error = sys.stderr.getvalue()
if error:
dump_test_stderr(self.messages, test_id, test_id, error)
super(TeamcityTestResult, self).stopTest(test)
self.current_test_id = None
if test_id not in self.failed_tests:
subtest_failures = self.get_subtest_failure(test_id)
if subtest_failures:
self.report_fail(test, "One or more subtests failed", "")
try:
time_diff = datetime.datetime.now() - self.test_started_datetime_map[test_id]
except KeyError:
time_diff = None
self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id)
def printErrors(self):
pass
class TeamcityReport(object):
name = 'teamcity-report'
score = 10000
def __init__(self):
super(TeamcityReport, self).__init__()
self.messages = TeamcityServiceMessages(_real_stdout)
self.test_started_datetime_map = {}
self.enabled = False
def get_test_id(self, test):
if is_string(test):
return test
# Handle special "tests"
test_class_name = get_class_fullname(test)
if test_class_name == CONTEXT_SUITE_FQN:
if inspect.ismodule(test.context):
module_name = test.context.__name__
return module_name + "." + test.error_context
elif inspect.isclass(test.context):
class_name = get_class_fullname(test.context)
return class_name + "." + test.error_context
test_id = test.id()
real_test = getattr(test, "test", test)
real_test_class_name = get_class_fullname(real_test)
test_arg = getattr(real_test, "arg", tuple())
if (type(test_arg) is tuple or type(test_arg) is list) and len(test_arg) > 0:
# As written in nose.case.FunctionTestCase#__str__ or nose.case.MethodTestCase#__str__
test_arg_str = "%s" % (test_arg,)
if test_id.endswith(test_arg_str):
# Replace '.' in test args with '_' to preserve test hierarchy on TeamCity
test_id = test_id[:len(test_id) - len(test_arg_str)] + test_arg_str.replace('.', '_')
# Force test_id for doctests
if real_test_class_name != "doctest.DocTestCase" and real_test_class_name != "nose.plugins.doctests.DocTestCase":
desc = test.shortDescription()
if desc and desc != test.id():
return "%s (%s)" % (test_id, desc.replace('.', '_'))
return test_id
def configure(self, options, conf):
self.enabled = is_running_under_teamcity()
def options(self, parser, env=os.environ):
pass
def report_fail(self, test, fail_type, err):
# workaround nose bug on python 3
if is_string(err[1]):
err = (err[0], Exception(err[1]), err[2])
test_id = self.get_test_id(test)
details = convert_error_to_string(err)
start_index = details.find(_captured_output_start_marker)
end_index = details.find(_captured_output_end_marker)
if 0 <= start_index < end_index:
captured_output = details[start_index + len(_captured_output_start_marker):end_index]
details = details[:start_index] + details[end_index + len(_captured_output_end_marker):]
for chunk in split_output(limit_output(captured_output)):
self.messages.testStdOut(test_id, chunk, flowId=test_id)
self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id)
def addError(self, test, err):
test_class_name = get_class_fullname(test)
test_id = self.get_test_id(test)
if issubclass(err[0], SkipTest):
self.messages.testIgnored(test_id, message=("SKIPPED: %s" % str(err[1])), flowId=test_id)
elif issubclass(err[0], DeprecatedTest):
self.messages.testIgnored(test_id, message="Deprecated", flowId=test_id)
elif test_class_name == CONTEXT_SUITE_FQN:
self.messages.testStarted(test_id, captureStandardOutput='true', flowId=test_id)
self.report_fail(test, 'error in ' + test.error_context + ' context', err)
self.messages.testFinished(test_id, flowId=test_id)
else:
self.report_fail(test, 'Error', err)
def addFailure(self, test, err):
self.report_fail(test, 'Failure', err)
def startTest(self, test):
test_id = self.get_test_id(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id, captureStandardOutput='true', flowId=test_id)
def addSuccess(self, test):
test_id = self.get_test_id(test)
time_diff = datetime.datetime.now() - self.test_started_datetime_map[test_id]
self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id)
class TeamcityTestResult(TestResult):
separator2 = "\n"
# noinspection PyUnusedLocal
def __init__(self, stream=_real_stdout, descriptions=None, verbosity=None):
super(TeamcityTestResult, self).__init__()
# Some code may ask for self.failfast, see unittest2.case.TestCase.subTest
self.failfast = getattr(self, "failfast", False)
self.test_started_datetime_map = {}
self.failed_tests = set()
self.subtest_failures = {}
self.messages = TeamcityServiceMessages(_real_stdout)
@staticmethod
def get_test_id(test):
if is_string(test):
return test
# Force test_id for doctests
if get_class_fullname(test) != "doctest.DocTestCase":
desc = test.shortDescription()
test_method_name = getattr(test, "_testMethodName", "")
if desc and desc != test.id() and desc != test_method_name:
return "%s (%s)" % (test.id(), desc.replace('.', '_'))
return test.id()
def addSuccess(self, test):
super(TeamcityTestResult, self).addSuccess(test)
def addExpectedFailure(self, test, err):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addExpectedFailure"):
_super.addExpectedFailure(test, err)
err = convert_error_to_string(err)
test_id = self.get_test_id(test)
self.messages.testIgnored(test_id,
message="Expected failure: " + err,
flowId=test_id)
def get_subtest_block_id(self, test, subtest):
test_id = self.get_test_id(test)
subtest_id = self.get_test_id(subtest)
if subtest_id.startswith(test_id):
block_id = subtest_id[len(test_id):].strip()
else:
block_id = subtest_id
if len(block_id) == 0:
block_id = test_id
return block_id
def addSkip(self, test, reason=""):
if sys.version_info >= (2, 7):
super(TeamcityTestResult, self).addSkip(test, reason)
if reason:
reason_str = ": " + str(reason)
else:
reason_str = ""
test_class_name = get_class_fullname(test)
if test_class_name == "unittest.case._SubTest" or test_class_name == "unittest2.case._SubTest":
parent_test = test.test_case
parent_test_id = self.get_test_id(parent_test)
subtest = test
block_id = self.get_subtest_block_id(parent_test, subtest)
self.messages.subTestBlockOpened(block_id,
subTestResult="Skip",
flowId=parent_test_id)
self.messages.testStdOut(parent_test_id,
out="SubTest skipped" + reason_str + "\n",
flowId=parent_test_id)
self.messages.blockClosed(block_id, flowId=parent_test_id)
else:
test_id = self.get_test_id(test)
self.messages.testIgnored(test_id,
message="Skipped" + reason_str,
flowId=test_id)
def addUnexpectedSuccess(self, test):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addUnexpectedSuccess"):
_super.addUnexpectedSuccess(test)
test_id = self.get_test_id(test)
self.messages.testFailed(
test_id,
message='Failure',
details=
"Test should not succeed since it's marked with @unittest.expectedFailure",
flowId=test_id)
def addError(self, test, err, *k):
super(TeamcityTestResult, self).addError(test, err)
test_class = get_class_fullname(test)
if test_class == "unittest.suite._ErrorHolder" or test_class == "unittest2.suite._ErrorHolder":
# This is a standalone error
test_name = test.id()
# patch setUpModule (__main__) -> __main__.setUpModule
test_name = re.sub(r'^(.*) \((.*)\)$', r'\2.\1', test_name)
self.messages.testStarted(test_name, flowId=test_name)
# noinspection PyTypeChecker
self.report_fail(test_name, 'Failure', err)
self.messages.testFinished(test_name, flowId=test_name)
elif get_class_fullname(err[0]) == "unittest2.case.SkipTest":
message = ""
if hasattr(err[1], "message"):
message = getattr(err[1], "message", "")
elif hasattr(err[1], "args"):
message = getattr(err[1], "args", [""])[0]
self.addSkip(test, message)
else:
self.report_fail(test, 'Error', err)
def addFailure(self, test, err, *k):
super(TeamcityTestResult, self).addFailure(test, err)
self.report_fail(test, 'Failure', err)
def addSubTest(self, test, subtest, err):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addSubTest"):
_super.addSubTest(test, subtest, err)
test_id = self.get_test_id(test)
subtest_id = self.get_test_id(subtest)
if subtest_id.startswith(test_id):
# Replace "." -> "_" since '.' is a test hierarchy separator
# See i.e. https://github.com/JetBrains/teamcity-messages/issues/134 (https://youtrack.jetbrains.com/issue/PY-23846)
block_id = subtest_id[len(test_id):].strip().replace(".", "_")
else:
block_id = subtest_id
if len(block_id) == 0:
block_id = subtest_id
if err is not None:
self.add_subtest_failure(test_id, block_id)
if issubclass(err[0], test.failureException):
self.messages.subTestBlockOpened(block_id,
subTestResult="Failure",
flowId=test_id)
self.messages.testStdErr(test_id,
out="SubTest failure: %s\n" %
convert_error_to_string(err),
flowId=test_id)
self.messages.blockClosed(block_id, flowId=test_id)
else:
self.messages.subTestBlockOpened(block_id,
subTestResult="Error",
flowId=test_id)
self.messages.testStdErr(test_id,
out="SubTest error: %s\n" %
convert_error_to_string(err),
flowId=test_id)
self.messages.blockClosed(block_id, flowId=test_id)
else:
self.messages.subTestBlockOpened(block_id,
subTestResult="Success",
flowId=test_id)
self.messages.blockClosed(block_id, flowId=test_id)
def add_subtest_failure(self, test_id, subtest_block_id):
fail_array = self.subtest_failures.get(test_id, [])
fail_array.append(subtest_block_id)
self.subtest_failures[test_id] = fail_array
def get_subtest_failure(self, test_id):
fail_array = self.subtest_failures.get(test_id, [])
return ", ".join(fail_array)
def report_fail(self, test, fail_type, err):
test_id = self.get_test_id(test)
if is_string(err):
details = err
elif get_class_fullname(err) == "twisted.python.failure.Failure":
details = err.getTraceback()
else:
details = convert_error_to_string(err)
subtest_failures = self.get_subtest_failure(test_id)
if subtest_failures:
details = "Failed subtests list: " + subtest_failures + "\n\n" + details.strip(
)
details = details.strip()
self.messages.testFailed(test_id,
message=fail_type,
details=details,
flowId=test_id)
self.failed_tests.add(test_id)
def startTest(self, test):
super(TeamcityTestResult, self).startTest(test)
test_id = self.get_test_id(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id,
captureStandardOutput='true',
flowId=test_id)
def stopTest(self, test):
test_id = self.get_test_id(test)
if getattr(self, 'buffer', None):
# Do not allow super() method to print output by itself
self._mirrorOutput = False
output = sys.stdout.getvalue()
if output:
for chunk in split_output(limit_output(output)):
self.messages.testStdOut(test_id, chunk, flowId=test_id)
error = sys.stderr.getvalue()
if error:
for chunk in split_output(limit_output(error)):
self.messages.testStdErr(test_id, chunk, flowId=test_id)
super(TeamcityTestResult, self).stopTest(test)
if test_id not in self.failed_tests:
subtest_failures = self.get_subtest_failure(test_id)
if subtest_failures:
self.report_fail(test, "One or more subtests failed", "")
time_diff = datetime.datetime.now(
) - self.test_started_datetime_map[test_id]
self.messages.testFinished(test_id,
testDuration=time_diff,
flowId=test_id)
def printErrors(self):
pass
class VpcClusterUpgradeTest:
log = logging.getLogger(__name__)
def __init__(self,
num_masters: int, num_agents: int, num_public_agents: int,
stable_installer_url: str, installer_url: str,
aws_region: str, aws_access_key_id: str, aws_secret_access_key: str,
default_os_user: str,
config_yaml_override_install: str, config_yaml_override_upgrade: str,
dcos_api_session_factory_install: VpcClusterUpgradeTestDcosApiSessionFactory,
dcos_api_session_factory_upgrade: VpcClusterUpgradeTestDcosApiSessionFactory):
self.dcos_api_session_factory_install = dcos_api_session_factory_install
self.dcos_api_session_factory_upgrade = dcos_api_session_factory_upgrade
self.num_masters = num_masters
self.num_agents = num_agents
self.num_public_agents = num_public_agents
self.stable_installer_url = stable_installer_url
self.installer_url = installer_url
self.aws_region = aws_region
self.aws_access_key_id = aws_access_key_id
self.aws_secret_access_key = aws_secret_access_key
self.default_os_user = default_os_user
self.config_yaml_override_install = config_yaml_override_install
self.config_yaml_override_upgrade = config_yaml_override_upgrade
self.teamcity_msg = TeamcityServiceMessages()
# the two following properties are set when running setup_cluster_workload, here we default them to empty
# values.
self.test_app_ids = []
self.tasks_start = []
@staticmethod
def app_task_ids(dcos_api, app_id):
"""Return a list of Mesos task IDs for app_id's running tasks."""
assert app_id.startswith('/')
response = dcos_api.marathon.get('/v2/apps' + app_id + '/tasks')
response.raise_for_status()
tasks = response.json()['tasks']
return [task['id'] for task in tasks]
@staticmethod
def get_master_task_state(dcos_api, task_id):
"""Returns the JSON blob associated with the task from /master/state."""
response = dcos_api.get('/mesos/master/state')
response.raise_for_status()
master_state = response.json()
for framework in master_state['frameworks']:
for task in framework['tasks']:
if task_id in task['id']:
return task
def parse_dns_log(self, dns_log_content):
"""Return a list of (timestamp, status) tuples from dns_log_content."""
dns_log = [line.strip().split(' ') for line in dns_log_content.strip().split('\n')]
if any(len(entry) != 2 or entry[1] not in ['SUCCESS', 'FAILURE'] for entry in dns_log):
message = 'Malformed DNS log.'
self.log.debug(message + ' DNS log content:\n' + dns_log_content)
raise Exception(message)
return dns_log
def log_test(self, test_name: str, call: Callable[[], None]) -> None:
try:
self.teamcity_msg.testStarted(test_name)
call()
except Exception:
# we want this except to be broad so that we can keep any Exception from taking
# everything with it and not asserting the other tests
self.teamcity_msg.testFailed(test_name, details=traceback.format_exc())
finally:
self.teamcity_msg.testFinished(test_name)
@staticmethod
@retrying.retry(
wait_fixed=(1 * 1000),
stop_max_delay=(120 * 1000),
retry_on_result=lambda x: not x)
def wait_for_dns(dcos_api, hostname):
"""Return True if Mesos-DNS has at least one entry for hostname."""
hosts = dcos_api.get('/mesos_dns/v1/hosts/' + hostname).json()
return any(h['host'] != '' and h['ip'] != '' for h in hosts)
def setup_cluster_workload(self, dcos_api: DcosApiSession, healthcheck_app: dict, dns_app: dict):
# Deploy test apps.
# TODO(branden): We ought to be able to deploy these apps concurrently. See
# https://mesosphere.atlassian.net/browse/DCOS-13360.
with logger.scope("deploy apps"):
dcos_api.marathon.deploy_app(healthcheck_app)
dcos_api.marathon.ensure_deployments_complete()
# This is a hack to make sure we don't deploy dns_app before the name it's
# trying to resolve is available.
self.wait_for_dns(dcos_api, dns_app['env']['RESOLVE_NAME'])
dcos_api.marathon.deploy_app(dns_app, check_health=False)
dcos_api.marathon.ensure_deployments_complete()
test_apps = [healthcheck_app, dns_app]
self.test_app_ids = [app['id'] for app in test_apps]
self.tasks_start = {app_id: sorted(self.app_task_ids(dcos_api, app_id)) for app_id in self.test_app_ids}
self.log.debug('Test app tasks at start:\n' + pprint.pformat(self.tasks_start))
for app in test_apps:
assert app['instances'] == len(self.tasks_start[app['id']])
# Save the master's state of the task to compare with
# the master's view after the upgrade.
# See this issue for why we check for a difference:
# https://issues.apache.org/jira/browse/MESOS-1718
self.task_state_start = self.get_master_task_state(dcos_api, self.tasks_start[self.test_app_ids[0]][0])
def verify_apps_state(self, dcos_api: DcosApiSession, dns_app: dict):
with logger.scope("verify apps state"):
# nested methods here so we can "close" over external state
def marathon_app_tasks_survive_upgrade():
# Verify that the tasks we started are still running.
tasks_end = {app_id: sorted(self.app_task_ids(dcos_api, app_id)) for app_id in self.test_app_ids}
self.log.debug('Test app tasks at end:\n' + pprint.pformat(tasks_end))
if not self.tasks_start == tasks_end:
self.teamcity_msg.testFailed(
"test_upgrade_vpc.marathon_app_tasks_survive_upgrade",
details="expected: {}\nactual: {}".format(self.tasks_start, tasks_end))
def test_mesos_task_state_remains_consistent():
# Verify that the "state" of the task does not change.
task_state_end = self.get_master_task_state(dcos_api, self.tasks_start[self.test_app_ids[0]][0])
if not self.task_state_start == task_state_end:
self.teamcity_msg.testFailed(
"test_upgrade_vpc.test_mesos_task_state_remains_consistent",
details="expected: {}\nactual: {}".format(self.task_state_start, task_state_end))
def test_app_dns_survive_upgrade():
# Verify DNS didn't fail.
marathon_framework_id = dcos_api.marathon.get('/v2/info').json()['frameworkId']
dns_app_task = dcos_api.marathon.get('/v2/apps' + dns_app['id'] + '/tasks').json()['tasks'][0]
dns_log = self.parse_dns_log(dcos_api.mesos_sandbox_file(
dns_app_task['slaveId'],
marathon_framework_id,
dns_app_task['id'],
dns_app['env']['DNS_LOG_FILENAME'],
))
dns_failure_times = [entry[0] for entry in dns_log if entry[1] != 'SUCCESS']
if len(dns_failure_times) > 0:
message = 'Failed to resolve Marathon app hostname {} at least once.'.format(
dns_app['env']['RESOLVE_NAME'])
err_msg = message + ' Hostname failed to resolve at these times:\n' + '\n'.join(dns_failure_times)
self.log.debug(err_msg)
self.teamcity_msg.testFailed("test_upgrade_vpc.test_app_dns_survive_upgrade", details=err_msg)
self.log_test("test_upgrade_vpc.marathon_app_tasks_survive_upgrade", marathon_app_tasks_survive_upgrade)
self.log_test(
"test_upgrade_vpc.test_mesos_task_state_remains_consistent",
test_mesos_task_state_remains_consistent
)
self.log_test("test_upgrade_vpc.test_app_dns_survive_upgrade", test_app_dns_survive_upgrade)
def run_test(self) -> int:
stack_name = 'dcos-ci-test-upgrade-' + random_id(10)
test_id = uuid.uuid4().hex
healthcheck_app_id = TEST_APP_NAME_FMT.format('healthcheck-' + test_id)
dns_app_id = TEST_APP_NAME_FMT.format('dns-' + test_id)
with logger.scope("create vpc cf stack '{}'".format(stack_name)):
bw = test_util.aws.BotoWrapper(
region=self.aws_region,
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key)
ssh_key = bw.create_key_pair(stack_name)
write_string('ssh_key', ssh_key)
vpc, ssh_info = test_util.aws.VpcCfStack.create(
stack_name=stack_name,
instance_type='m4.xlarge',
instance_os='cent-os-7-dcos-prereqs',
# An instance for each cluster node plus the bootstrap.
instance_count=(self.num_masters + self.num_agents + self.num_public_agents + 1),
admin_location='0.0.0.0/0',
key_pair_name=stack_name,
boto_wrapper=bw
)
vpc.wait_for_complete()
cluster = test_util.cluster.Cluster.from_vpc(
vpc,
ssh_info,
ssh_key=ssh_key,
num_masters=self.num_masters,
num_agents=self.num_agents,
num_public_agents=self.num_public_agents,
)
with logger.scope("install dcos"):
# Use the CLI installer to set exhibitor_storage_backend = zookeeper.
test_util.cluster.install_dcos(cluster, self.stable_installer_url, api=False,
add_config_path=self.config_yaml_override_install)
master_list = [h.private_ip for h in cluster.masters]
dcos_api_install = self.dcos_api_session_factory_install.apply(
'http://{ip}'.format(ip=cluster.masters[0].public_ip),
master_list,
master_list,
[h.private_ip for h in cluster.agents],
[h.private_ip for h in cluster.public_agents],
self.default_os_user)
dcos_api_install.wait_for_dcos()
installed_version = dcos_api_install.get_version()
healthcheck_app = create_marathon_healthcheck_app(healthcheck_app_id)
dns_app = create_marathon_dns_app(dns_app_id, healthcheck_app_id)
self.setup_cluster_workload(dcos_api_install, healthcheck_app, dns_app)
with logger.scope("upgrade cluster"):
test_util.cluster.upgrade_dcos(cluster, self.installer_url,
installed_version, add_config_path=self.config_yaml_override_upgrade)
with cluster.ssher.tunnel(cluster.bootstrap_host) as bootstrap_host_tunnel:
bootstrap_host_tunnel.remote_cmd(['sudo', 'rm', '-rf', cluster.ssher.home_dir + '/*'])
# this method invocation looks like it is the same as the one above, and that is partially correct.
# the arguments to the invocation are the same, but the thing that changes is the lifecycle of the cluster
# the client is being created to interact with. This client is specifically for the cluster after the
# upgrade has taken place, and can account for any possible settings that may change for the client under
# the hood when it probes the cluster.
dcos_api_upgrade = self.dcos_api_session_factory_upgrade.apply(
'http://{ip}'.format(ip=cluster.masters[0].public_ip),
master_list,
master_list,
[h.private_ip for h in cluster.agents],
[h.private_ip for h in cluster.public_agents],
self.default_os_user)
dcos_api_upgrade.wait_for_dcos() # here we wait for DC/OS to be "up" so that we can auth this new client
self.verify_apps_state(dcos_api_upgrade, dns_app)
with logger.scope("run integration tests"):
# copied from test_util/test_aws_cf.py:96
add_env = []
prefix = 'TEST_ADD_ENV_'
for k, v in os.environ.items():
if k.startswith(prefix):
add_env.append(k.replace(prefix, '') + '=' + v)
test_cmd = ' '.join(add_env) + ' py.test -vv -s -rs ' + os.getenv('CI_FLAGS', '')
result = test_util.cluster.run_integration_tests(cluster, test_cmd=test_cmd)
if result == 0:
self.log.info("Test successful! Deleting VPC if provided in this run.")
vpc.delete()
bw.delete_key_pair(stack_name)
else:
self.log.info("Test failed! VPC cluster will remain available for "
"debugging for 2 hour after instantiation.")
return result
class EchoTeamCityMessages(object):
def __init__(self, output_capture_enabled, coverage_controller):
self.coverage_controller = coverage_controller
self.output_capture_enabled = output_capture_enabled
self.teamcity = TeamcityServiceMessages()
self.test_start_reported_mark = set()
self.max_reported_output_size = 1 * 1024 * 1024
self.reported_output_chunk_size = 50000
def format_test_id(self, nodeid):
test_id = nodeid
if test_id.find("::") < 0:
test_id += "::top_level"
test_id = test_id.replace("::()::", "::")
test_id = re.sub(r"\.pyc?::", r"::", test_id)
test_id = test_id.replace(".", "_").replace(os.sep, ".").replace("/", ".").replace('::', '.')
return test_id
def pytest_runtest_logstart(self, nodeid, location):
self.ensure_test_start_reported(self.format_test_id(nodeid))
def ensure_test_start_reported(self, test_id):
if test_id not in self.test_start_reported_mark:
self.teamcity.testStarted(test_id, flowId=test_id, captureStandardOutput='false' if self.output_capture_enabled else 'true')
self.test_start_reported_mark.add(test_id)
def report_has_output(self, report):
for (secname, data) in report.sections:
if report.when in secname and ('stdout' in secname or 'stderr' in secname):
return True
return False
def report_test_output(self, report, test_id, when):
for (secname, data) in report.sections:
if when not in secname:
continue
if not data:
continue
if 'stdout' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdOut(test_id, out=chunk, flowId=test_id)
elif 'stderr' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdErr(test_id, out=chunk, flowId=test_id)
def pytest_runtest_logreport(self, report):
"""
:type report: _pytest.runner.TestReport
"""
orig_test_id = self.format_test_id(report.nodeid)
suffix = '' if report.when == 'call' else ('_' + report.when)
test_id = orig_test_id + suffix
duration = timedelta(seconds=report.duration)
if report.passed:
# Do not report passed setup/teardown if no output
if report.when == 'call' or self.report_has_output(report):
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id, report.when)
self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id)
elif report.failed:
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id, report.when)
self.teamcity.testFailed(test_id, str(report.location), str(report.longrepr), flowId=test_id)
self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id)
# If test setup failed, report test failure as well
if report.when == 'setup':
self.ensure_test_start_reported(orig_test_id)
self.teamcity.testFailed(orig_test_id, "test setup failed, see %s test failure" % test_id, flowId=orig_test_id)
self.teamcity.testFinished(orig_test_id, flowId=orig_test_id)
elif report.skipped:
if type(report.longrepr) is tuple and len(report.longrepr) == 3:
reason = report.longrepr[2]
else:
reason = str(report.longrepr)
self.ensure_test_start_reported(orig_test_id)
self.report_test_output(report, orig_test_id, report.when)
self.teamcity.testIgnored(orig_test_id, reason, flowId=orig_test_id)
self.teamcity.testFinished(orig_test_id, flowId=orig_test_id)
def pytest_collectreport(self, report):
if report.failed:
test_id = self.format_test_id(report.nodeid) + "_collect"
self.teamcity.testStarted(test_id, flowId=test_id)
self.teamcity.testFailed(test_id, str(report.location), str(report.longrepr), flowId=test_id)
self.teamcity.testFinished(test_id, flowId=test_id)
def pytest_terminal_summary(self):
if self.coverage_controller is not None:
try:
self._report_coverage()
except:
tb = traceback.format_exc()
self.teamcity.customMessage("Coverage statistics reporting failed", "ERROR", errorDetails=tb)
def _report_coverage(self):
from coverage.misc import NotPython
from coverage.report import Reporter
from coverage.results import Numbers
class _CoverageReporter(Reporter):
def __init__(self, coverage, config, messages):
super(_CoverageReporter, self).__init__(coverage, config)
self.branches = coverage.data.has_arcs()
self.messages = messages
def report(self, morfs, outfile=None):
self.find_code_units(morfs)
total = Numbers()
for cu in self.code_units:
try:
analysis = self.coverage._analyze(cu)
nums = analysis.numbers
total += nums
except KeyboardInterrupt:
raise
except:
if self.config.ignore_errors:
continue
err = sys.exc_info()
typ, msg = err[:2]
if typ is NotPython and not cu.should_be_python():
continue
test_id = cu.name
details = convert_error_to_string(err)
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testFailed(test_id, message="Coverage analysis failed", details=details, flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
if total.n_files > 0:
covered = total.n_executed + (total.n_executed_branches if self.branches else 0)
total_statements = total.n_statements + (total.n_branches if self.branches else 0)
self.messages.buildStatisticLinesCovered(covered)
self.messages.buildStatisticTotalLines(total_statements)
self.messages.buildStatisticLinesUncovered(total_statements - covered)
reporter = _CoverageReporter(
self.coverage_controller.cov,
self.coverage_controller.cov.config,
self.teamcity,
)
reporter.report(None)
class TeamcityTestResult(TestResult):
separator2 = "\n"
# noinspection PyUnusedLocal
def __init__(self, stream=_real_stdout, descriptions=None, verbosity=None):
super(TeamcityTestResult, self).__init__()
# Some code may ask for self.failfast, see unittest2.case.TestCase.subTest
self.failfast = getattr(self, "failfast", False)
self.test_started_datetime_map = {}
self.failed_tests = set()
self.subtest_failures = {}
self.messages = TeamcityServiceMessages(_real_stdout)
self.current_test_id = None
@staticmethod
def get_test_id_with_description(test):
if is_string(test):
return test
test_class_fullname = get_class_fullname(test)
test_id = test.id()
if test_class_fullname in _ERROR_HOLDERS_FQN:
# patch setUpModule (__main__) -> __main__.setUpModule
return re.sub(r'^(.*) \((.*)\)$', r'\2.\1', test_id)
# Force test_id for doctests
if test_class_fullname != "doctest.DocTestCase":
desc = test.shortDescription()
test_method_name = getattr(test, "_testMethodName", "")
if desc and desc != test_id and desc != test_method_name:
return "%s (%s)" % (test_id, desc.replace('.', '_'))
return test_id
def addSuccess(self, test):
super(TeamcityTestResult, self).addSuccess(test)
def addExpectedFailure(self, test, err):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addExpectedFailure"):
_super.addExpectedFailure(test, err)
err = convert_error_to_string(err)
test_id = self.get_test_id_with_description(test)
self.messages.testIgnored(test_id,
message="Expected failure: " + err,
flowId=test_id)
def get_subtest_block_id(self, test, subtest):
test_id = self.get_test_id_with_description(test)
subtest_id = self.get_test_id_with_description(subtest)
if subtest_id.startswith(test_id):
block_id = subtest_id[len(test_id):].strip()
else:
block_id = subtest_id
if len(block_id) == 0:
block_id = test_id
return block_id
def addSkip(self, test, reason=""):
if sys.version_info >= (2, 7):
super(TeamcityTestResult, self).addSkip(test, reason)
if reason:
if isinstance(reason, Exception):
reason_str = ": " + get_exception_message(reason)
else:
reason_str = ": " + to_unicode(reason)
else:
reason_str = ""
test_class_name = get_class_fullname(test)
if test_class_name == "unittest.case._SubTest" or test_class_name == "unittest2.case._SubTest":
parent_test = test.test_case
parent_test_id = self.get_test_id_with_description(parent_test)
subtest = test
block_id = self.get_subtest_block_id(parent_test, subtest)
self.messages.subTestBlockOpened(block_id,
subTestResult="Skip",
flowId=parent_test_id)
self.messages.testStdOut(parent_test_id,
out="SubTest skipped" + reason_str + "\n",
flowId=parent_test_id)
self.messages.blockClosed(block_id, flowId=parent_test_id)
else:
test_id = self.get_test_id_with_description(test)
if test_id not in self.test_started_datetime_map:
# Test ignored without startTest. Handle start and finish events ourselves
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testIgnored(test_id,
message="Skipped" + reason_str,
flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
else:
self.messages.testIgnored(test_id,
message="Skipped" + reason_str,
flowId=test_id)
def addUnexpectedSuccess(self, test):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addUnexpectedSuccess"):
_super.addUnexpectedSuccess(test)
test_id = self.get_test_id_with_description(test)
self.messages.testFailed(
test_id,
message='Failure',
details=
"Test should not succeed since it's marked with @unittest.expectedFailure",
flowId=test_id)
def addError(self, test, err, *k):
super(TeamcityTestResult, self).addError(test, err)
test_class = get_class_fullname(test)
if test_class in _ERROR_HOLDERS_FQN:
# This is a standalone error
test_id = self.get_test_id_with_description(test)
self.messages.testStarted(test_id, flowId=test_id)
self.report_fail(test, 'Failure', err)
self.messages.testFinished(test_id, flowId=test_id)
elif get_class_fullname(err[0]) == "unittest2.case.SkipTest":
message = ""
if hasattr(err[1], "message"):
message = getattr(err[1], "message", "")
elif hasattr(err[1], "args"):
message = getattr(err[1], "args", [""])[0]
self.addSkip(test, message)
else:
self.report_fail(test, 'Error', err)
def addFailure(self, test, err, *k):
super(TeamcityTestResult, self).addFailure(test, err)
self.report_fail(test, 'Failure', err)
def addSubTest(self, test, subtest, err):
_super = super(TeamcityTestResult, self)
if hasattr(_super, "addSubTest"):
_super.addSubTest(test, subtest, err)
# test_id_with_desc may contain description which breaks process of fetching id from subtest
test_id_with_desc = self.get_test_id_with_description(test)
test_id = test.id()
subtest_id = subtest.id()
if subtest_id.startswith(test_id):
# Replace "." -> "_" since '.' is a test hierarchy separator
# See i.e. https://github.com/JetBrains/teamcity-messages/issues/134 (https://youtrack.jetbrains.com/issue/PY-23846)
block_id = subtest_id[len(test_id):].strip().replace(".", "_")
else:
block_id = subtest_id
if len(block_id) == 0:
block_id = subtest_id
if err is not None:
self.add_subtest_failure(test_id_with_desc, block_id)
if issubclass(err[0], test.failureException):
self.messages.subTestBlockOpened(block_id,
subTestResult="Failure",
flowId=test_id_with_desc)
self.messages.testStdErr(test_id_with_desc,
out="SubTest failure: %s\n" %
convert_error_to_string(err),
flowId=test_id_with_desc)
self.messages.blockClosed(block_id, flowId=test_id_with_desc)
else:
self.messages.subTestBlockOpened(block_id,
subTestResult="Error",
flowId=test_id_with_desc)
self.messages.testStdErr(test_id_with_desc,
out="SubTest error: %s\n" %
convert_error_to_string(err),
flowId=test_id_with_desc)
self.messages.blockClosed(block_id, flowId=test_id_with_desc)
else:
self.messages.subTestBlockOpened(block_id,
subTestResult="Success",
flowId=test_id_with_desc)
self.messages.blockClosed(block_id, flowId=test_id_with_desc)
def add_subtest_failure(self, test_id, subtest_block_id):
fail_array = self.subtest_failures.get(test_id, [])
fail_array.append(subtest_block_id)
self.subtest_failures[test_id] = fail_array
def get_subtest_failure(self, test_id):
fail_array = self.subtest_failures.get(test_id, [])
return ", ".join(fail_array)
def report_fail(self, test, fail_type, err):
test_id = self.get_test_id_with_description(test)
diff_failed = None
try:
from .jb_local_exc_store import get_exception
error = get_exception()
if isinstance(error, EqualsAssertionError):
diff_failed = error
except Exception:
pass
if is_string(err):
details = err
else:
try:
details = err.getTraceback()
except AttributeError:
frames_to_skip_from_tail = 2 if diff_failed else 0
details = convert_error_to_string(err,
frames_to_skip_from_tail)
subtest_failures = self.get_subtest_failure(test_id)
if subtest_failures:
details = "Failed subtests list: " + subtest_failures + "\n\n" + details.strip(
)
details = details.strip()
if diff_failed:
self.messages.testFailed(test_id,
message=diff_failed.msg,
details=details,
flowId=test_id,
comparison_failure=diff_failed)
else:
self.messages.testFailed(test_id,
message=fail_type,
details=details,
flowId=test_id)
self.failed_tests.add(test_id)
def startTest(self, test):
test_id = self.get_test_id_with_description(test)
self.current_test_id = test_id
super(TeamcityTestResult, self).startTest(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id,
captureStandardOutput='true',
flowId=test_id)
def _dump_test_stderr(self, data):
if self.current_test_id is not None:
dump_test_stderr(self.messages, self.current_test_id,
self.current_test_id, data)
else:
_real_stderr.write(data)
def _dump_test_stdout(self, data):
if self.current_test_id is not None:
dump_test_stdout(self.messages, self.current_test_id,
self.current_test_id, data)
else:
_real_stdout.write(data)
def _setupStdout(self):
if getattr(self, 'buffer', None):
self._stderr_buffer = FlushingStringIO(self._dump_test_stderr)
self._stdout_buffer = FlushingStringIO(self._dump_test_stdout)
sys.stdout = self._stdout_buffer
sys.stderr = self._stderr_buffer
def stopTest(self, test):
test_id = self.get_test_id_with_description(test)
if getattr(self, 'buffer', None):
# Do not allow super() method to print output by itself
self._mirrorOutput = False
output = sys.stdout.getvalue()
if output:
dump_test_stdout(self.messages, test_id, test_id, output)
error = sys.stderr.getvalue()
if error:
dump_test_stderr(self.messages, test_id, test_id, error)
super(TeamcityTestResult, self).stopTest(test)
self.current_test_id = None
if test_id not in self.failed_tests:
subtest_failures = self.get_subtest_failure(test_id)
if subtest_failures:
self.report_fail(test, "One or more subtests failed", "")
try:
time_diff = datetime.datetime.now(
) - self.test_started_datetime_map[test_id]
except KeyError:
time_diff = None
self.messages.testFinished(test_id,
testDuration=time_diff,
flowId=test_id)
def printErrors(self):
pass
class TeamcityTestResult(TestResult):
separator2 = "\n"
def __init__(self, stream=_real_stdout, descriptions=None, verbosity=None):
super(TeamcityTestResult, self).__init__()
self.test_started_datetime_map = {}
self.failed_tests = set()
self.subtest_failures = {}
self.messages = TeamcityServiceMessages(_real_stdout)
def get_test_id(self, test):
if is_string(test):
return test
# Force test_id for doctests
if get_class_fullname(test) != "doctest.DocTestCase":
desc = test.shortDescription()
test_method_name = getattr(test, "_testMethodName", "")
if desc and desc != test.id() and desc != test_method_name:
return "%s (%s)" % (test.id(), desc.replace('.', '_'))
return test.id()
def addSuccess(self, test):
super(TeamcityTestResult, self).addSuccess(test)
def addExpectedFailure(self, test, err):
super(TeamcityTestResult, self).addExpectedFailure(test, err)
err = convert_error_to_string(err)
test_id = self.get_test_id(test)
self.messages.testIgnored(test_id,
message="Expected failure: " + err,
flowId=test_id)
def addSkip(self, test, reason=""):
if sys.version_info >= (2, 7):
super(TeamcityTestResult, self).addSkip(test, reason)
test_id = self.get_test_id(test)
if reason:
reason_str = ": " + str(reason)
else:
reason_str = ""
self.messages.testIgnored(test_id,
message="Skipped" + reason_str,
flowId=test_id)
def addUnexpectedSuccess(self, test):
super(TeamcityTestResult, self).addUnexpectedSuccess(test)
test_id = self.get_test_id(test)
self.messages.testFailed(
test_id,
message='Failure',
details=
"Test should not succeed since it's marked with @unittest.expectedFailure",
flowId=test_id)
def addError(self, test, err, *k):
super(TeamcityTestResult, self).addError(test, err)
if get_class_fullname(test) == "unittest.suite._ErrorHolder":
# This is a standalone error
test_name = test.id()
# patch setUpModule (__main__) -> __main__.setUpModule
test_name = re.sub(r'^(.*) \((.*)\)$', r'\2.\1', test_name)
self.messages.testStarted(test_name, flowId=test_name)
self.report_fail(test_name, 'Failure', err)
self.messages.testFinished(test_name, flowId=test_name)
elif get_class_fullname(err[0]) == "unittest2.case.SkipTest":
message = ""
if hasattr(err[1], "message"):
message = getattr(err[1], "message", "")
elif hasattr(err[1], "args"):
message = getattr(err[1], "args", [""])[0]
self.addSkip(test, message)
else:
self.report_fail(test, 'Error', err)
def addFailure(self, test, err, *k):
super(TeamcityTestResult, self).addFailure(test, err)
self.report_fail(test, 'Failure', err)
def addSubTest(self, test, subtest, err):
super(TeamcityTestResult, self).addSubTest(test, subtest, err)
test_id = self.get_test_id(test)
if err is not None:
if issubclass(err[0], test.failureException):
self.add_subtest_failure(test_id, self.get_test_id(subtest),
err)
self.messages.testStdErr(test_id,
out="%s: failure\n" %
self.get_test_id(subtest),
flowId=test_id)
else:
self.add_subtest_failure(test_id, self.get_test_id(subtest),
err)
self.messages.testStdErr(test_id,
out="%s: error\n" %
self.get_test_id(subtest),
flowId=test_id)
else:
self.messages.testStdOut(test_id,
out="%s: ok\n" %
self.get_test_id(subtest),
flowId=test_id)
def add_subtest_failure(self, test_id, subtest_id, err):
fail_array = self.subtest_failures.get(test_id, [])
fail_array.append("%s:\n%s" %
(subtest_id, convert_error_to_string(err)))
self.subtest_failures[test_id] = fail_array
def get_subtest_failure(self, test_id):
fail_array = self.subtest_failures.get(test_id, [])
return "\n".join(fail_array)
def report_fail(self, test, fail_type, err):
test_id = self.get_test_id(test)
if is_string(err):
details = err
elif get_class_fullname(err) == "twisted.python.failure.Failure":
details = err.getTraceback()
else:
details = convert_error_to_string(err)
subtest_failure = self.get_subtest_failure(test_id)
if subtest_failure:
details = subtest_failure + "\n" + details
self.messages.testFailed(test_id,
message=fail_type,
details=details,
flowId=test_id)
self.failed_tests.add(test_id)
def startTest(self, test):
super(TeamcityTestResult, self).startTest(test)
test_id = self.get_test_id(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id,
captureStandardOutput='true',
flowId=test_id)
def stopTest(self, test):
test_id = self.get_test_id(test)
if self.buffer:
# Do not allow super() method to print output by itself
self._mirrorOutput = False
output = sys.stdout.getvalue()
if output:
for chunk in split_output(limit_output(output)):
self.messages.testStdOut(test_id, chunk, flowId=test_id)
error = sys.stderr.getvalue()
if error:
for chunk in split_output(limit_output(error)):
self.messages.testStdErr(test_id, chunk, flowId=test_id)
super(TeamcityTestResult, self).stopTest(test)
if test_id not in self.failed_tests and self.subtest_failures.get(
test_id, []):
self.report_fail(test, "Subtest failed", "")
time_diff = datetime.datetime.now(
) - self.test_started_datetime_map[test_id]
self.messages.testFinished(test_id,
testDuration=time_diff,
flowId=test_id)
def printErrors(self):
pass
class EchoTeamCityMessages(object):
def __init__(self, ):
self.tw = py.io.TerminalWriter(py.std.sys.stdout)
self.teamcity = TeamcityServiceMessages(self.tw)
self.currentSuite = None
def format_names(self, name):
if name.find("::") > 0:
file, testname = name.split("::", 1)
else:
file, testname = name, "top_level"
testname = testname.replace("::()::", ".")
testname = testname.replace("::", ".")
testname = testname.strip(".")
file = file.replace(".", "_").replace(os.sep, ".").replace("/", ".")
return file, testname
def pytest_runtest_logstart(self, nodeid, location):
file, testname = self.format_names(nodeid)
if not file == self.currentSuite:
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
self.currentSuite = file
self.teamcity.testSuiteStarted(self.currentSuite)
self.teamcity.testStarted(testname)
def pytest_runtest_logreport(self, report):
file, testname = self.format_names(report.nodeid)
if report.passed:
if report.when == "call": # ignore setup/teardown
duration = timedelta(seconds=report.duration)
self.teamcity.testFinished(testname, testDuration=duration)
elif report.failed:
if report.when in ("call", "setup"):
self.teamcity.testFailed(testname, str(report.location),
str(report.longrepr))
duration = timedelta(seconds=report.duration)
self.teamcity.testFinished(
testname,
testDuration=duration) # report finished after the failure
elif report.when == "teardown":
name = testname + "_teardown"
self.teamcity.testStarted(name)
self.teamcity.testFailed(name, str(report.location),
str(report.longrepr))
self.teamcity.testFinished(name)
elif report.skipped:
self.teamcity.testIgnored(testname, str(report.longrepr))
self.teamcity.testFinished(
testname) # report finished after the skip
def pytest_collectreport(self, report):
if report.failed:
file, testname = self.format_names(report.nodeid)
name = file + "_collect"
self.teamcity.testStarted(name)
self.teamcity.testFailed(name, str(report.location),
str(report.longrepr))
self.teamcity.testFinished(name)
def pytest_sessionfinish(self, session, exitstatus, __multicall__):
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
class EchoTeamCityMessages(object):
def __init__(self, output_capture_enabled, coverage_controller):
self.coverage_controller = coverage_controller
self.output_capture_enabled = output_capture_enabled
self.teamcity = TeamcityServiceMessages()
self.test_start_reported_mark = set()
self.max_reported_output_size = 1 * 1024 * 1024
self.reported_output_chunk_size = 50000
def get_id_from_location(self, location):
if type(location) is not tuple or len(location) != 3 or not hasattr(location[2], "startswith"):
return None
def convert_file_to_id(filename):
filename = re.sub(r"\.pyc?$", "", filename)
return filename.replace(os.sep, ".").replace("/", ".")
def add_prefix_to_filename_id(filename_id, prefix):
dot_location = filename_id.rfind('.')
if dot_location <= 0 or dot_location >= len(filename_id) - 1:
return None
return filename_id[:dot_location + 1] + prefix + filename_id[dot_location + 1:]
pylint_prefix = '[pylint] '
if location[2].startswith(pylint_prefix):
id_from_file = convert_file_to_id(location[2][len(pylint_prefix):])
return id_from_file + ".Pylint"
if location[2] == "PEP8-check":
id_from_file = convert_file_to_id(location[0])
return id_from_file + ".PEP8"
return None
def format_test_id(self, nodeid, location):
id_from_location = self.get_id_from_location(location)
if id_from_location is not None:
return id_from_location
test_id = nodeid
if test_id.find("::") < 0:
test_id += "::top_level"
test_id = test_id.replace("::()::", "::")
test_id = re.sub(r"\.pyc?::", r"::", test_id)
test_id = test_id.replace(".", "_").replace(os.sep, ".").replace("/", ".").replace('::', '.')
return test_id
def format_location(self, location):
if type(location) is tuple and len(location) == 3:
return "%s:%s (%s)" % (str(location[0]), str(location[1]), str(location[2]))
return str(location)
def pytest_runtest_logstart(self, nodeid, location):
self.ensure_test_start_reported(self.format_test_id(nodeid, location))
def ensure_test_start_reported(self, test_id):
if test_id not in self.test_start_reported_mark:
if self.output_capture_enabled:
capture_standard_output = "false"
else:
capture_standard_output = "true"
self.teamcity.testStarted(test_id, flowId=test_id, captureStandardOutput=capture_standard_output)
self.test_start_reported_mark.add(test_id)
def report_has_output(self, report):
for (secname, data) in report.sections:
if report.when in secname and ('stdout' in secname or 'stderr' in secname):
return True
return False
def report_test_output(self, report, test_id):
for (secname, data) in report.sections:
if report.when not in secname:
continue
if not data:
continue
if 'stdout' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdOut(test_id, out=chunk, flowId=test_id)
elif 'stderr' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdErr(test_id, out=chunk, flowId=test_id)
def report_test_finished(self, test_id, duration=None):
self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id)
self.test_start_reported_mark.remove(test_id)
def report_test_failure(self, test_id, report, message=None, report_output=True):
if hasattr(report, 'duration'):
duration = timedelta(seconds=report.duration)
else:
duration = None
if message is None:
message = self.format_location(report.location)
self.ensure_test_start_reported(test_id)
if report_output:
self.report_test_output(report, test_id)
self.teamcity.testFailed(test_id, message, str(report.longrepr), flowId=test_id)
self.report_test_finished(test_id, duration)
def pytest_runtest_logreport(self, report):
"""
:type report: _pytest.runner.TestReport
"""
test_id = self.format_test_id(report.nodeid, report.location)
duration = timedelta(seconds=report.duration)
if report.passed:
# Do not report passed setup/teardown if no output
if report.when == 'call':
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id)
self.report_test_finished(test_id, duration)
else:
if self.report_has_output(report):
block_name = "test " + report.when
self.teamcity.blockOpened(block_name, flowId=test_id)
self.report_test_output(report, test_id)
self.teamcity.blockClosed(block_name, flowId=test_id)
elif report.failed:
if report.when == 'call':
self.report_test_failure(test_id, report)
elif report.when == 'setup':
if self.report_has_output(report):
self.teamcity.blockOpened("test setup", flowId=test_id)
self.report_test_output(report, test_id)
self.teamcity.blockClosed("test setup", flowId=test_id)
self.report_test_failure(test_id, report, message="test setup failed", report_output=False)
elif report.when == 'teardown':
# Report failed teardown as a separate test as original test is already finished
self.report_test_failure(test_id + "_teardown", report)
elif report.skipped:
if type(report.longrepr) is tuple and len(report.longrepr) == 3:
reason = report.longrepr[2]
else:
reason = str(report.longrepr)
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id)
self.teamcity.testIgnored(test_id, reason, flowId=test_id)
self.report_test_finished(test_id, duration)
def pytest_collectreport(self, report):
if report.failed:
test_id = self.format_test_id(report.nodeid, report.location) + "_collect"
self.report_test_failure(test_id, report)
def pytest_terminal_summary(self):
if self.coverage_controller is not None:
try:
self._report_coverage()
except:
tb = traceback.format_exc()
self.teamcity.customMessage("Coverage statistics reporting failed", "ERROR", errorDetails=tb)
def _report_coverage(self):
from coverage.misc import NotPython
from coverage.report import Reporter
from coverage.results import Numbers
class _CoverageReporter(Reporter):
def __init__(self, coverage, config, messages):
super(_CoverageReporter, self).__init__(coverage, config)
self.branches = coverage.data.has_arcs()
self.messages = messages
def report(self, morfs, outfile=None):
if hasattr(self, 'find_code_units'):
self.find_code_units(morfs)
else:
self.find_file_reporters(morfs)
total = Numbers()
if hasattr(self, 'code_units'):
units = self.code_units
else:
units = self.file_reporters
for cu in units:
try:
analysis = self.coverage._analyze(cu)
nums = analysis.numbers
total += nums
except KeyboardInterrupt:
raise
except:
if self.config.ignore_errors:
continue
err = sys.exc_info()
typ, msg = err[:2]
if typ is NotPython and not cu.should_be_python():
continue
test_id = cu.name
details = convert_error_to_string(err)
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testFailed(test_id, message="Coverage analysis failed", details=details, flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
if total.n_files > 0:
covered = total.n_executed
total_statements = total.n_statements
if self.branches:
covered += total.n_executed_branches
total_statements += total.n_branches
self.messages.buildStatisticLinesCovered(covered)
self.messages.buildStatisticTotalLines(total_statements)
self.messages.buildStatisticLinesUncovered(total_statements - covered)
reporter = _CoverageReporter(
self.coverage_controller.cov,
self.coverage_controller.cov.config,
self.teamcity,
)
reporter.report(None)
class TeamcityTestResult(TestResult):
def __init__(self, stream=sys.stdout):
TestResult.__init__(self)
self.output = stream
self.test_started_datetime = None
self.test_name = None
self.createMessages()
def createMessages(self):
self.messages = TeamcityServiceMessages(self.output)
def formatErr(self, err):
try:
exctype, value, tb = err
return ''.join(traceback.format_exception(exctype, value, tb))
except:
tb = traceback.format_exc()
return "*FAILED TO GET TRACEBACK*: " + tb
def getTestName(self, test):
return test.shortDescription() or str(test)
def addSuccess(self, test, *k):
TestResult.addSuccess(self, test)
self.output.write("ok\n")
def addError(self, test, err, *k):
TestResult.addError(self, test, err)
err = self.formatErr(err)
if self.getTestName(test) != self.test_name:
sys.stderr.write("INTERNAL ERROR: addError(%s) outside of test\n" %
self.getTestName(test))
sys.stderr.write("Error: %s\n" % err)
return
self.messages.testFailed(self.getTestName(test),
message='Error',
details=err)
def addFailure(self, test, err, *k):
# workaround nose bug on python 3
if _is_string(err[1]):
err = (err[0], Exception(err[1]), err[2])
TestResult.addFailure(self, test, err)
err = self.formatErr(err)
if self.getTestName(test) != self.test_name:
sys.stderr.write(
"INTERNAL ERROR: addFailure(%s) outside of test\n" %
self.getTestName(test))
sys.stderr.write("Error: %s\n" % err)
return
self.messages.testFailed(self.getTestName(test),
message='Failure',
details=err)
def startTest(self, test):
self.test_started_datetime = datetime.datetime.now()
self.test_name = self.getTestName(test)
self.messages.testStarted(self.test_name)
def stopTest(self, test):
time_diff = datetime.datetime.now() - self.test_started_datetime
if self.getTestName(test) != self.test_name:
sys.stderr.write(
"INTERNAL ERROR: stopTest(%s) after startTest(%s)" %
(self.getTestName(test), self.test_name))
self.messages.testFinished(self.test_name, time_diff)
self.test_name = None
class VpcClusterUpgradeTest:
log = logging.getLogger(__name__)
def __init__(self,
num_masters: int, num_agents: int, num_public_agents: int,
stable_installer_url: str, installer_url: str,
aws_region: str, aws_access_key_id: str, aws_secret_access_key: str,
default_os_user: str,
config_yaml_override_install: str, config_yaml_override_upgrade: str,
dcos_api_session_factory_install: VpcClusterUpgradeTestDcosApiSessionFactory,
dcos_api_session_factory_upgrade: VpcClusterUpgradeTestDcosApiSessionFactory):
self.dcos_api_session_factory_install = dcos_api_session_factory_install
self.dcos_api_session_factory_upgrade = dcos_api_session_factory_upgrade
self.num_masters = num_masters
self.num_agents = num_agents
self.num_public_agents = num_public_agents
self.stable_installer_url = stable_installer_url
self.installer_url = installer_url
self.aws_region = aws_region
self.aws_access_key_id = aws_access_key_id
self.aws_secret_access_key = aws_secret_access_key
self.default_os_user = default_os_user
self.config_yaml_override_install = config_yaml_override_install
self.config_yaml_override_upgrade = config_yaml_override_upgrade
self.teamcity_msg = TeamcityServiceMessages()
# the two following properties are set when running setup_cluster_workload, here we default them to empty
# values.
self.test_app_ids = []
self.tasks_start = []
@staticmethod
def app_task_ids(dcos_api, app_id):
"""Return a list of Mesos task IDs for app_id's running tasks."""
assert app_id.startswith('/')
response = dcos_api.marathon.get('/v2/apps' + app_id + '/tasks')
response.raise_for_status()
tasks = response.json()['tasks']
return [task['id'] for task in tasks]
@staticmethod
def get_master_task_state(dcos_api, task_id):
"""Returns the JSON blob associated with the task from /master/state."""
response = dcos_api.get('/mesos/master/state')
response.raise_for_status()
master_state = response.json()
for framework in master_state['frameworks']:
for task in framework['tasks']:
if task_id in task['id']:
return task
def parse_dns_log(self, dns_log_content):
"""Return a list of (timestamp, status) tuples from dns_log_content."""
dns_log = [line.strip().split(' ') for line in dns_log_content.strip().split('\n')]
if any(len(entry) != 2 or entry[1] not in ['SUCCESS', 'FAILURE'] for entry in dns_log):
message = 'Malformed DNS log.'
self.log.debug(message + ' DNS log content:\n' + dns_log_content)
raise Exception(message)
return dns_log
def log_test(self, test_name: str, call: Callable[[], None]) -> None:
try:
self.teamcity_msg.testStarted(test_name)
call()
except Exception:
# we want this except to be broad so that we can keep any Exception from taking
# everything with it and not asserting the other tests
self.teamcity_msg.testFailed(test_name, details=traceback.format_exc())
finally:
self.teamcity_msg.testFinished(test_name)
@staticmethod
@retrying.retry(
wait_fixed=(1 * 1000),
stop_max_delay=(120 * 1000),
retry_on_result=lambda x: not x)
def wait_for_dns(dcos_api, hostname):
"""Return True if Mesos-DNS has at least one entry for hostname."""
hosts = dcos_api.get('/mesos_dns/v1/hosts/' + hostname).json()
return any(h['host'] != '' and h['ip'] != '' for h in hosts)
def setup_cluster_workload(self, dcos_api: DcosApiSession, healthcheck_app: dict, dns_app: dict,
viplisten_app: dict, viptalk_app: dict):
# Deploy test apps.
# TODO(branden): We ought to be able to deploy these apps concurrently. See
# https://mesosphere.atlassian.net/browse/DCOS-13360.
with logger.scope("deploy apps"):
dcos_api.marathon.deploy_app(viplisten_app)
dcos_api.marathon.ensure_deployments_complete()
dcos_api.marathon.deploy_app(viptalk_app)
dcos_api.marathon.ensure_deployments_complete()
dcos_api.marathon.deploy_app(healthcheck_app)
dcos_api.marathon.ensure_deployments_complete()
# This is a hack to make sure we don't deploy dns_app before the name it's
# trying to resolve is available.
self.wait_for_dns(dcos_api, dns_app['env']['RESOLVE_NAME'])
dcos_api.marathon.deploy_app(dns_app, check_health=False)
dcos_api.marathon.ensure_deployments_complete()
test_apps = [healthcheck_app, dns_app, viplisten_app, viptalk_app]
self.test_app_ids = [app['id'] for app in test_apps]
self.tasks_start = {app_id: sorted(self.app_task_ids(dcos_api, app_id)) for app_id in self.test_app_ids}
self.log.debug('Test app tasks at start:\n' + pprint.pformat(self.tasks_start))
for app in test_apps:
assert app['instances'] == len(self.tasks_start[app['id']])
# Save the master's state of the task to compare with
# the master's view after the upgrade.
# See this issue for why we check for a difference:
# https://issues.apache.org/jira/browse/MESOS-1718
self.task_state_start = self.get_master_task_state(dcos_api, self.tasks_start[self.test_app_ids[0]][0])
def verify_apps_state(self, dcos_api: DcosApiSession, dns_app: dict):
with logger.scope("verify apps state"):
# nested methods here so we can "close" over external state
def marathon_app_tasks_survive_upgrade():
# Verify that the tasks we started are still running.
tasks_end = {app_id: sorted(self.app_task_ids(dcos_api, app_id)) for app_id in self.test_app_ids}
self.log.debug('Test app tasks at end:\n' + pprint.pformat(tasks_end))
if not self.tasks_start == tasks_end:
self.teamcity_msg.testFailed(
"test_upgrade_vpc.marathon_app_tasks_survive_upgrade",
details="expected: {}\nactual: {}".format(self.tasks_start, tasks_end))
def test_mesos_task_state_remains_consistent():
# Verify that the "state" of the task does not change.
task_state_end = self.get_master_task_state(dcos_api, self.tasks_start[self.test_app_ids[0]][0])
if not self.task_state_start == task_state_end:
self.teamcity_msg.testFailed(
"test_upgrade_vpc.test_mesos_task_state_remains_consistent",
details="expected: {}\nactual: {}".format(self.task_state_start, task_state_end))
def test_app_dns_survive_upgrade():
# Verify DNS didn't fail.
marathon_framework_id = dcos_api.marathon.get('/v2/info').json()['frameworkId']
dns_app_task = dcos_api.marathon.get('/v2/apps' + dns_app['id'] + '/tasks').json()['tasks'][0]
dns_log = self.parse_dns_log(dcos_api.mesos_sandbox_file(
dns_app_task['slaveId'],
marathon_framework_id,
dns_app_task['id'],
dns_app['env']['DNS_LOG_FILENAME'],
))
dns_failure_times = [entry[0] for entry in dns_log if entry[1] != 'SUCCESS']
if len(dns_failure_times) > 0:
message = 'Failed to resolve Marathon app hostname {} at least once.'.format(
dns_app['env']['RESOLVE_NAME'])
err_msg = message + ' Hostname failed to resolve at these times:\n' + '\n'.join(dns_failure_times)
self.log.debug(err_msg)
self.teamcity_msg.testFailed("test_upgrade_vpc.test_app_dns_survive_upgrade", details=err_msg)
self.log_test("test_upgrade_vpc.marathon_app_tasks_survive_upgrade", marathon_app_tasks_survive_upgrade)
self.log_test(
"test_upgrade_vpc.test_mesos_task_state_remains_consistent",
test_mesos_task_state_remains_consistent
)
self.log_test("test_upgrade_vpc.test_app_dns_survive_upgrade", test_app_dns_survive_upgrade)
def run_test(self) -> int:
stack_name = 'dcos-ci-test-upgrade-' + random_id(10)
test_id = uuid.uuid4().hex
healthcheck_app_id = TEST_APP_NAME_FMT.format('healthcheck-' + test_id)
dns_app_id = TEST_APP_NAME_FMT.format('dns-' + test_id)
with logger.scope("create vpc cf stack '{}'".format(stack_name)):
bw = test_util.aws.BotoWrapper(
region=self.aws_region,
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key)
ssh_key = bw.create_key_pair(stack_name)
write_string('ssh_key', ssh_key)
vpc, ssh_info = test_util.aws.VpcCfStack.create(
stack_name=stack_name,
instance_type='m4.xlarge',
instance_os='cent-os-7-dcos-prereqs',
# An instance for each cluster node plus the bootstrap.
instance_count=(self.num_masters + self.num_agents + self.num_public_agents + 1),
admin_location='0.0.0.0/0',
key_pair_name=stack_name,
boto_wrapper=bw
)
vpc.wait_for_complete()
cluster = test_util.cluster.Cluster.from_vpc(
vpc,
ssh_info,
ssh_key=ssh_key,
num_masters=self.num_masters,
num_agents=self.num_agents,
num_public_agents=self.num_public_agents,
)
with logger.scope("install dcos"):
# Use the CLI installer to set exhibitor_storage_backend = zookeeper.
# Don't install prereqs since stable breaks Docker 1.13. See
# https://jira.mesosphere.com/browse/DCOS_OSS-743.
test_util.cluster.install_dcos(cluster, self.stable_installer_url, api=False, install_prereqs=False,
add_config_path=self.config_yaml_override_install)
master_list = [h.private_ip for h in cluster.masters]
dcos_api_install = self.dcos_api_session_factory_install.apply(
'http://{ip}'.format(ip=cluster.masters[0].public_ip),
master_list,
master_list,
[h.private_ip for h in cluster.agents],
[h.private_ip for h in cluster.public_agents],
self.default_os_user)
dcos_api_install.wait_for_dcos()
installed_version = dcos_api_install.get_version()
healthcheck_app = create_marathon_healthcheck_app(healthcheck_app_id)
dns_app = create_marathon_dns_app(dns_app_id, healthcheck_app_id)
viplisten_app = create_marathon_viplisten_app()
viptalk_app = create_marathon_viptalk_app()
self.setup_cluster_workload(dcos_api_install, healthcheck_app, dns_app, viplisten_app, viptalk_app)
with logger.scope("upgrade cluster"):
test_util.cluster.upgrade_dcos(cluster, self.installer_url,
installed_version, add_config_path=self.config_yaml_override_upgrade)
with cluster.ssher.tunnel(cluster.bootstrap_host) as bootstrap_host_tunnel:
bootstrap_host_tunnel.remote_cmd(['sudo', 'rm', '-rf', cluster.ssher.home_dir + '/*'])
# this method invocation looks like it is the same as the one above, and that is partially correct.
# the arguments to the invocation are the same, but the thing that changes is the lifecycle of the cluster
# the client is being created to interact with. This client is specifically for the cluster after the
# upgrade has taken place, and can account for any possible settings that may change for the client under
# the hood when it probes the cluster.
dcos_api_upgrade = self.dcos_api_session_factory_upgrade.apply(
'http://{ip}'.format(ip=cluster.masters[0].public_ip),
master_list,
master_list,
[h.private_ip for h in cluster.agents],
[h.private_ip for h in cluster.public_agents],
self.default_os_user)
dcos_api_upgrade.wait_for_dcos() # here we wait for DC/OS to be "up" so that we can auth this new client
self.verify_apps_state(dcos_api_upgrade, dns_app)
with logger.scope("run integration tests"):
# copied from test_util/test_aws_cf.py:96
add_env = []
prefix = 'TEST_ADD_ENV_'
for k, v in os.environ.items():
if k.startswith(prefix):
add_env.append(k.replace(prefix, '') + '=' + v)
test_cmd = ' '.join(add_env) + ' py.test -vv -s -rs ' + os.getenv('CI_FLAGS', '')
result = test_util.cluster.run_integration_tests(cluster, test_cmd=test_cmd)
if result == 0:
self.log.info("Test successful! Deleting VPC if provided in this run.")
vpc.delete()
bw.delete_key_pair(stack_name)
else:
self.log.info("Test failed! VPC cluster will remain available for "
"debugging for 2 hour after instantiation.")
return result
class TeamcityReport(Plugin):
name = 'teamcity-report'
score = 10000
def __init__(self):
super(TeamcityReport, self).__init__()
self.messages = TeamcityServiceMessages(_real_stdout)
self.test_started_datetime_map = {}
self.config = None
self.total_tests = 0
self.enabled = False
def get_test_id(self, test):
if is_string(test):
return test
# Handle special "tests"
test_class_name = get_class_fullname(test)
if test_class_name == CONTEXT_SUITE_FQN:
if inspect.ismodule(test.context):
module_name = test.context.__name__
return module_name + "." + test.error_context
elif inspect.isclass(test.context):
class_name = get_class_fullname(test.context)
return class_name + "." + test.error_context
test_id = test.id()
real_test = getattr(test, "test", test)
real_test_class_name = get_class_fullname(real_test)
test_arg = getattr(real_test, "arg", tuple())
if (type(test_arg) is tuple or type(test_arg) is list) and len(test_arg) > 0:
# As written in nose.case.FunctionTestCase#__str__ or nose.case.MethodTestCase#__str__
test_arg_str = "%s" % (test_arg,)
if test_id.endswith(test_arg_str):
# Replace '.' in test args with '_' to preserve test hierarchy on TeamCity
test_id = test_id[:len(test_id) - len(test_arg_str)] + test_arg_str.replace('.', '_')
# Force test_id for doctests
if real_test_class_name != "doctest.DocTestCase" and real_test_class_name != "nose.plugins.doctests.DocTestCase":
desc = test.shortDescription()
if desc and desc != test.id():
return "%s (%s)" % (test_id, desc.replace('.', '_'))
return test_id
def configure(self, options, conf):
self.enabled = is_running_under_teamcity()
self.config = conf
if self._capture_plugin_enabled():
capture_plugin = self._get_capture_plugin()
old_before_test = capture_plugin.beforeTest
old_after_test = capture_plugin.afterTest
old_format_error = capture_plugin.formatError
def newCaptureBeforeTest(test):
old_before_test(test)
test_id = self.get_test_id(test)
capture_plugin._buf = FlushingStringIO(lambda data: dump_test_stdout(self.messages, test_id, test_id, data))
sys.stdout = capture_plugin._buf
def newCaptureAfterTest(test):
if isinstance(capture_plugin._buf, FlushingStringIO):
capture_plugin._buf.flush()
old_after_test(test)
def newCaptureFormatError(test, err):
if isinstance(capture_plugin._buf, FlushingStringIO):
capture_plugin._buf.flush()
old_format_error(test, err)
capture_plugin.beforeTest = newCaptureBeforeTest
capture_plugin.afterTest = newCaptureAfterTest
capture_plugin.formatError = newCaptureFormatError
def options(self, parser, env=os.environ):
pass
def _get_capture_plugin(self):
"""
:rtype: nose.plugins.capture.Capture
"""
for plugin in self.config.plugins.plugins:
if plugin.name == "capture":
return plugin
return None
def _capture_plugin_enabled(self):
plugin = self._get_capture_plugin()
return plugin is not None and plugin.enabled
def _capture_plugin_buffer(self):
plugin = self._get_capture_plugin()
if plugin is None:
return None
return getattr(plugin, "buffer", None)
def _captureStandardOutput_value(self):
if self._capture_plugin_enabled():
return 'false'
else:
return 'true'
def report_fail(self, test, fail_type, err):
# workaround nose bug on python 3
if is_string(err[1]):
err = (err[0], Exception(err[1]), err[2])
test_id = self.get_test_id(test)
details = convert_error_to_string(err)
start_index = details.find(_captured_output_start_marker)
end_index = details.find(_captured_output_end_marker)
if 0 <= start_index < end_index:
# do not log test output twice, see report_finish for actual output handling
details = details[:start_index] + details[end_index + len(_captured_output_end_marker):]
try:
error = err[1]
if isinstance(error, EqualsAssertionError):
details = convert_error_to_string(err, 2)
self.messages.testFailed(test_id, message=error.msg, details=details, flowId=test_id, comparison_failure=error)
return
except Exception:
pass
self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id)
def report_finish(self, test):
test_id = self.get_test_id(test)
if test_id in self.test_started_datetime_map:
time_diff = datetime.datetime.now() - self.test_started_datetime_map[test_id]
self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id)
else:
self.messages.testFinished(test_id, flowId=test_id)
def prepareTestLoader(self, loader):
"""Insert ourselves into loader calls to count tests.
The top-level loader call often returns lazy results, like a LazySuite.
This is a problem, as we would destroy the suite by iterating over it
to count the tests. Consequently, we monkey-patch the top-level loader
call to do the load twice: once for the actual test running and again
to yield something we can iterate over to do the count.
from https://github.com/erikrose/nose-progressive/
:type loader: nose.loader.TestLoader
"""
# TODO: If there's ever a practical need, also patch loader.suiteClass
# or even TestProgram.createTests. createTests seems to be main top-
# level caller of loader methods, and nose.core.collector() (which
# isn't even called in nose) is an alternate one.
#
# nose 1.3.4 contains required fix:
# Another fix for Python 3.4: Call super in LazySuite to access _removed_tests variable
if hasattr(loader, 'loadTestsFromNames') and nose.__versioninfo__ >= (1, 3, 4):
old_loadTestsFromNames = loader.loadTestsFromNames
def _loadTestsFromNames(*args, **kwargs):
suite = old_loadTestsFromNames(*args, **kwargs)
self.total_tests += suite.countTestCases()
# Clear out the loader's cache. Otherwise, it never finds any tests
# for the actual test run:
loader._visitedPaths = set()
return old_loadTestsFromNames(*args, **kwargs)
loader.loadTestsFromNames = _loadTestsFromNames
# noinspection PyUnusedLocal
def prepareTestRunner(self, runner):
if self.total_tests:
self.messages.testCount(self.total_tests)
def addError(self, test, err):
test_class_name = get_class_fullname(test)
test_id = self.get_test_id(test)
if issubclass(err[0], SkipTest):
self.messages.testIgnored(test_id, message=("SKIPPED: %s" % str(err[1])), flowId=test_id)
self.report_finish(test)
elif issubclass(err[0], DeprecatedTest):
self.messages.testIgnored(test_id, message="Deprecated", flowId=test_id)
self.report_finish(test)
elif test_class_name == CONTEXT_SUITE_FQN:
self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(), flowId=test_id)
self.report_fail(test, 'error in ' + test.error_context + ' context', err)
self.messages.testFinished(test_id, flowId=test_id)
else:
self.report_fail(test, 'Error', err)
self.report_finish(test)
def addFailure(self, test, err):
self.report_fail(test, 'Failure', err)
self.report_finish(test)
def startTest(self, test):
test_id = self.get_test_id(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(), flowId=test_id)
def addSuccess(self, test):
self.report_finish(test)
class TeamcityTestResult(TestResult):
def __init__(self, stream=sys.stdout):
TestResult.__init__(self)
self.output = stream
self.test_started_datetime = None
self.test_name = None
self.createMessages()
def createMessages(self):
self.messages = TeamcityServiceMessages(self.output)
def formatErr(self, err):
try:
exctype, value, tb = err
return ''.join(traceback.format_exception(exctype, value, tb))
except:
tb = traceback.format_exc()
return "*FAILED TO GET TRACEBACK*: " + tb
def getTestName(self, test):
return test.shortDescription() or str(test)
def addSuccess(self, test, *k):
TestResult.addSuccess(self, test)
self.output.write("ok\n")
def addError(self, test, err, *k):
TestResult.addError(self, test, err)
err = self.formatErr(err)
if self.getTestName(test) != self.test_name:
sys.stderr.write("INTERNAL ERROR: addError(%s) outside of test\n" % self.getTestName(test))
sys.stderr.write("Error: %s\n" % err)
return
self.messages.testFailed(self.getTestName(test),
message='Error', details=err)
def addSkip(self, test, reason):
TestResult.addSkip(self, test, reason)
self.output.write("skipped %s - %s\n" % (self.getTestName(test), reason))
#TODO: "testIgnored" should be replaced by "testSkipped" when implemented
self.messages.testIgnored(self.getTestName(test), reason)
def addFailure(self, test, err, *k):
# workaround nose bug on python 3
if _is_string(err[1]):
err = (err[0], Exception(err[1]), err[2])
TestResult.addFailure(self, test, err)
err = self.formatErr(err)
if self.getTestName(test) != self.test_name:
sys.stderr.write("INTERNAL ERROR: addFailure(%s) outside of test\n" % self.getTestName(test))
sys.stderr.write("Error: %s\n" % err)
return
self.messages.testFailed(self.getTestName(test),
message='Failure', details=err)
def startTest(self, test):
self.test_started_datetime = datetime.datetime.now()
self.test_name = self.getTestName(test)
self.messages.testStarted(self.test_name)
def stopTest(self, test):
time_diff = datetime.datetime.now() - self.test_started_datetime
if self.getTestName(test) != self.test_name:
sys.stderr.write("INTERNAL ERROR: stopTest(%s) after startTest(%s)" % (self.getTestName(test), self.test_name))
self.messages.testFinished(self.test_name, time_diff)
self.test_name = None
class TeamcityReport(Plugin):
name = 'teamcity-report'
score = 10000
def __init__(self):
super(TeamcityReport, self).__init__()
self.messages = TeamcityServiceMessages(_real_stdout)
self.test_started_datetime_map = {}
self.config = None
self.total_tests = 0
self.enabled = False
def get_test_id(self, test):
if is_string(test):
return test
# Handle special "tests"
test_class_name = get_class_fullname(test)
if test_class_name == CONTEXT_SUITE_FQN:
if inspect.ismodule(test.context):
module_name = test.context.__name__
return module_name + "." + test.error_context
elif inspect.isclass(test.context):
class_name = get_class_fullname(test.context)
return class_name + "." + test.error_context
test_id = test.id()
real_test = getattr(test, "test", test)
real_test_class_name = get_class_fullname(real_test)
test_arg = getattr(real_test, "arg", tuple())
if (type(test_arg) is tuple or type(test_arg) is list) and len(test_arg) > 0:
# As written in nose.case.FunctionTestCase#__str__ or nose.case.MethodTestCase#__str__
test_arg_str = "%s" % (test_arg,)
if test_id.endswith(test_arg_str):
# Replace '.' in test args with '_' to preserve test hierarchy on TeamCity
test_id = test_id[:len(test_id) - len(test_arg_str)] + test_arg_str.replace('.', '_')
# Force test_id for doctests
if real_test_class_name != "doctest.DocTestCase" and real_test_class_name != "nose.plugins.doctests.DocTestCase":
desc = test.shortDescription()
if desc and desc != test.id():
return "%s (%s)" % (test_id, desc.replace('.', '_'))
return test_id
def configure(self, options, conf):
self.enabled = is_running_under_teamcity()
self.config = conf
if self._capture_plugin_enabled():
capture_plugin = self._get_capture_plugin()
old_before_test = capture_plugin.beforeTest
old_after_test = capture_plugin.afterTest
old_format_error = capture_plugin.formatError
def newCaptureBeforeTest(test):
rv = old_before_test(test)
test_id = self.get_test_id(test)
capture_plugin._buf = FlushingStringIO(lambda data: dump_test_stdout(self.messages, test_id, test_id, data))
sys.stdout = capture_plugin._buf
return rv
def newCaptureAfterTest(test):
if isinstance(capture_plugin._buf, FlushingStringIO):
capture_plugin._buf.flush()
return old_after_test(test)
def newCaptureFormatError(test, err):
if isinstance(capture_plugin._buf, FlushingStringIO):
capture_plugin._buf.flush()
return old_format_error(test, err)
capture_plugin.beforeTest = newCaptureBeforeTest
capture_plugin.afterTest = newCaptureAfterTest
capture_plugin.formatError = newCaptureFormatError
def options(self, parser, env=os.environ):
pass
def _get_capture_plugin(self):
"""
:rtype: nose.plugins.capture.Capture
"""
for plugin in self.config.plugins.plugins:
if plugin.name == "capture":
return plugin
return None
def _capture_plugin_enabled(self):
plugin = self._get_capture_plugin()
return plugin is not None and plugin.enabled
def _capture_plugin_buffer(self):
plugin = self._get_capture_plugin()
if plugin is None:
return None
return getattr(plugin, "buffer", None)
def _captureStandardOutput_value(self):
if self._capture_plugin_enabled():
return 'false'
else:
return 'true'
def report_started(self, test):
test_id = self.get_test_id(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(), flowId=test_id)
def report_fail(self, test, fail_type, err):
# workaround nose bug on python 3
if is_string(err[1]):
err = (err[0], Exception(err[1]), err[2])
test_id = self.get_test_id(test)
details = convert_error_to_string(err)
start_index = details.find(_captured_output_start_marker)
end_index = details.find(_captured_output_end_marker)
if 0 <= start_index < end_index:
# do not log test output twice, see report_finish for actual output handling
details = details[:start_index] + details[end_index + len(_captured_output_end_marker):]
try:
error = err[1]
if isinstance(error, EqualsAssertionError):
details = convert_error_to_string(err, 2)
self.messages.testFailed(test_id, message=error.msg, details=details, flowId=test_id, comparison_failure=error)
return
except Exception:
pass
self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id)
def report_finish(self, test):
test_id = self.get_test_id(test)
if test_id in self.test_started_datetime_map:
time_diff = datetime.datetime.now() - self.test_started_datetime_map[test_id]
self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id)
else:
self.messages.testFinished(test_id, flowId=test_id)
def prepareTestLoader(self, loader):
"""Insert ourselves into loader calls to count tests.
The top-level loader call often returns lazy results, like a LazySuite.
This is a problem, as we would destroy the suite by iterating over it
to count the tests. Consequently, we monkey-patch the top-level loader
call to do the load twice: once for the actual test running and again
to yield something we can iterate over to do the count.
from https://github.com/erikrose/nose-progressive/
:type loader: nose.loader.TestLoader
"""
# TODO: If there's ever a practical need, also patch loader.suiteClass
# or even TestProgram.createTests. createTests seems to be main top-
# level caller of loader methods, and nose.core.collector() (which
# isn't even called in nose) is an alternate one.
#
# nose 1.3.4 contains required fix:
# Another fix for Python 3.4: Call super in LazySuite to access _removed_tests variable
if hasattr(loader, 'loadTestsFromNames') and nose.__versioninfo__ >= (1, 3, 4):
old_loadTestsFromNames = loader.loadTestsFromNames
def _loadTestsFromNames(*args, **kwargs):
suite = old_loadTestsFromNames(*args, **kwargs)
self.total_tests += suite.countTestCases()
# Clear out the loader's cache. Otherwise, it never finds any tests
# for the actual test run:
loader._visitedPaths = set()
return old_loadTestsFromNames(*args, **kwargs)
loader.loadTestsFromNames = _loadTestsFromNames
# noinspection PyUnusedLocal
def prepareTestRunner(self, runner):
if self.total_tests:
self.messages.testCount(self.total_tests)
def addError(self, test, err):
test_class_name = get_class_fullname(test)
test_id = self.get_test_id(test)
if issubclass(err[0], SkipTest):
self.messages.testIgnored(test_id, message=("SKIPPED: %s" % str(err[1])), flowId=test_id)
self.report_finish(test)
elif issubclass(err[0], DeprecatedTest):
self.messages.testIgnored(test_id, message="Deprecated", flowId=test_id)
self.report_finish(test)
elif test_class_name == CONTEXT_SUITE_FQN:
self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(), flowId=test_id)
self.report_fail(test, 'error in ' + test.error_context + ' context', err)
self.messages.testFinished(test_id, flowId=test_id)
else:
# some test cases may report errors in pre setup when startTest was not called yet
# example: https://github.com/JetBrains/teamcity-messages/issues/153
if test_id not in self.test_started_datetime_map:
self.report_started(test)
self.report_fail(test, 'Error', err)
self.report_finish(test)
def addFailure(self, test, err):
self.report_fail(test, 'Failure', err)
self.report_finish(test)
def startTest(self, test):
test_id = self.get_test_id(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(), flowId=test_id)
def addSuccess(self, test):
self.report_finish(test)
testMetadata_number_message(
test_name,
'ns/{}'.format(result['unit']),
1e9 * result['median(elapsed)'] / result['batch'],
)
testMetadata_number_message(
test_name,
'{}/s'.format(result['unit']),
result['batch'] / result['median(elapsed)'],
)
testMetadata_number_message(
test_name,
'err%',
100 * result['medianAbsolutePercentError(elapsed)'],
)
testMetadata_number_message(
test_name,
'ins/{}'.format(result['unit']),
result['median(instructions)'] / result['batch'],
)
teamcity_messages.testFinished(
test_name,
testDuration=timedelta(seconds=result['totalTime']),
)
teamcity_messages.testSuiteFinished(suite_name)
class TeamcityFormatter(Formatter):
"""
Stateful TC reporter.
Since we can't fetch all steps from the very beginning (even skipped tests are reported)
we store tests and features on each call.
To hook into test reporting override _report_suite_started and/or _report_test_started
"""
def __init__(self, stream_opener, config):
super(TeamcityFormatter, self).__init__(stream_opener, config)
assert version.LooseVersion(behave_version) >= version.LooseVersion(
"1.2.6"), "Only 1.2.6+ is supported"
self._messages = TeamcityServiceMessages()
self.__feature = None
self.__scenario = None
self.__steps = deque()
self.__scenario_opened = False
self.__feature_opened = False
self.__test_start_time = None
def feature(self, feature):
assert isinstance(feature, Feature)
assert not self.__feature, "Prev. feature not closed"
self.__feature = feature
def scenario(self, scenario):
assert isinstance(scenario, Scenario)
self.__scenario = scenario
self.__scenario_opened = False
def step(self, step):
assert isinstance(step, Step)
self.__steps.append(step)
def match(self, match):
if not self.__feature_opened:
self._report_suite_started(self.__feature,
_suite_name(self.__feature))
self.__feature_opened = True
if not self.__scenario_opened:
self._report_suite_started(self.__scenario,
_suite_name(self.__scenario))
self.__scenario_opened = True
assert self.__steps, "No steps left"
step = self.__steps.popleft()
self._report_test_started(step, _step_name(step))
self.__test_start_time = datetime.datetime.now()
def _report_suite_started(self, suite, suite_name):
"""
:param suite: behave suite
:param suite_name: suite name that must be reported, be sure to use it instead of suite.name
"""
self._messages.testSuiteStarted(suite_name)
def _report_test_started(self, test, test_name):
"""
Suite name is always stripped, be sure to strip() it too
:param test: behave test
:param test_name: test name that must be reported, be sure to use it instead of test.name
"""
self._messages.testStarted(test_name)
def result(self, step):
assert isinstance(step, Step)
step_name = _step_name(step)
if step.status == Status.failed:
try:
error = traceback.format_exc(step.exc_traceback)
if error != step.error_message:
self._messages.testStdErr(step_name, error)
except Exception:
pass # exception shall not prevent error message
self._messages.testFailed(step_name, message=step.error_message)
if step.status == Status.undefined:
self._messages.testFailed(step_name, message="Undefined")
if step.status == Status.skipped:
self._messages.testIgnored(step_name)
self._messages.testFinished(step_name,
testDuration=datetime.datetime.now() -
self.__test_start_time)
if not self.__steps:
self.__close_scenario()
elif step.status in [Status.failed, Status.undefined]:
# Broken background/undefined step stops whole scenario
reason = "Undefined step" if step.status == Status.undefined else "Prev. step failed"
self.__skip_rest_of_scenario(reason)
def __skip_rest_of_scenario(self, reason):
while self.__steps:
step = self.__steps.popleft()
self._report_test_started(step, _step_name(step))
self._messages.testIgnored(
_step_name(step),
message="{0}. Rest part of scenario is skipped".format(reason))
self._messages.testFinished(_step_name(step))
self.__close_scenario()
def __close_scenario(self):
if self.__scenario:
self._messages.testSuiteFinished(_suite_name(self.__scenario))
self.__scenario = None
def eof(self):
self.__skip_rest_of_scenario("")
self._messages.testSuiteFinished(_suite_name(self.__feature))
self.__feature = None
self.__feature_opened = False
class EchoTeamCityMessages(object):
def __init__(self, output_capture_enabled, coverage_controller, skip_passed_output, swap_diff):
self.coverage_controller = coverage_controller
self.output_capture_enabled = output_capture_enabled
self.skip_passed_output = skip_passed_output
self.teamcity = TeamcityServiceMessages()
self.test_start_reported_mark = set()
self.max_reported_output_size = 1 * 1024 * 1024
self.reported_output_chunk_size = 50000
self.swap_diff = swap_diff
def get_id_from_location(self, location):
if type(location) is not tuple or len(location) != 3 or not hasattr(location[2], "startswith"):
return None
def convert_file_to_id(filename):
filename = re.sub(r"\.pyc?$", "", filename)
return filename.replace(os.sep, ".").replace("/", ".")
def add_prefix_to_filename_id(filename_id, prefix):
dot_location = filename_id.rfind('.')
if dot_location <= 0 or dot_location >= len(filename_id) - 1:
return None
return filename_id[:dot_location + 1] + prefix + filename_id[dot_location + 1:]
pylint_prefix = '[pylint] '
if location[2].startswith(pylint_prefix):
id_from_file = convert_file_to_id(location[2][len(pylint_prefix):])
return id_from_file + ".Pylint"
if location[2] == "PEP8-check":
id_from_file = convert_file_to_id(location[0])
return id_from_file + ".PEP8"
return None
def format_test_id(self, nodeid, location):
id_from_location = self.get_id_from_location(location)
if id_from_location is not None:
return id_from_location
test_id = nodeid
if test_id:
if test_id.find("::") < 0:
test_id += "::top_level"
else:
test_id = "top_level"
first_bracket = test_id.find("[")
if first_bracket > 0:
# [] -> (), make it look like nose parameterized tests
params = "(" + test_id[first_bracket + 1:]
if params.endswith("]"):
params = params[:-1] + ")"
test_id = test_id[:first_bracket]
if test_id.endswith("::"):
test_id = test_id[:-2]
else:
params = ""
test_id = test_id.replace("::()::", "::")
test_id = re.sub(r"\.pyc?::", r"::", test_id)
test_id = test_id.replace(".", "_").replace(os.sep, ".").replace("/", ".").replace('::', '.')
if params:
params = params.replace(".", "_")
test_id += params
return test_id
def format_location(self, location):
if type(location) is tuple and len(location) == 3:
return "%s:%s (%s)" % (str(location[0]), str(location[1]), str(location[2]))
return str(location)
def pytest_collection_modifyitems(self, session, config, items):
self.teamcity.testCount(len(items))
def pytest_runtest_logstart(self, nodeid, location):
# test name fetched from location passed as metainfo to PyCharm
# it will be used to run specific test using "-k"
# See IDEA-176950
# We only need method/function name because only it could be used as -k
test_name = location[2]
if test_name:
test_name = str(test_name).split(".")[-1]
self.ensure_test_start_reported(self.format_test_id(nodeid, location), test_name)
def ensure_test_start_reported(self, test_id, metainfo=None):
if test_id not in self.test_start_reported_mark:
if self.output_capture_enabled:
capture_standard_output = "false"
else:
capture_standard_output = "true"
self.teamcity.testStarted(test_id, flowId=test_id, captureStandardOutput=capture_standard_output, metainfo=metainfo)
self.test_start_reported_mark.add(test_id)
def report_has_output(self, report):
for (secname, data) in report.sections:
if report.when in secname and ('stdout' in secname or 'stderr' in secname):
return True
return False
def report_test_output(self, report, test_id):
for (secname, data) in report.sections:
# https://github.com/JetBrains/teamcity-messages/issues/112
# CollectReport doesn't have 'when' property
if hasattr(report, "when") and report.when not in secname:
continue
if not data:
continue
if 'stdout' in secname:
dump_test_stdout(self.teamcity, test_id, test_id, data)
elif 'stderr' in secname:
dump_test_stderr(self.teamcity, test_id, test_id, data)
def report_test_finished(self, test_id, duration=None):
self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id)
self.test_start_reported_mark.remove(test_id)
def report_test_failure(self, test_id, report, message=None, report_output=True):
if hasattr(report, 'duration'):
duration = timedelta(seconds=report.duration)
else:
duration = None
if message is None:
message = self.format_location(report.location)
self.ensure_test_start_reported(test_id)
if report_output:
self.report_test_output(report, test_id)
diff_error = None
try:
err_message = str(report.longrepr.reprcrash.message)
diff_name = diff_tools.EqualsAssertionError.__name__
# There is a string like "foo.bar.DiffError: [serialized_data]"
if diff_name in err_message:
serialized_data = err_message[err_message.index(diff_name) + len(diff_name) + 1:]
diff_error = diff_tools.deserialize_error(serialized_data)
# AssertionError is patched in py.test, we can try to fetch diff from it
# In general case message starts with "AssertionError: ", but can also starts with "assert" for top-level
# function. To support both cases we unify them
if err_message.startswith("assert"):
err_message = "AssertionError: " + err_message
if err_message.startswith("AssertionError:"):
diff_error = fetch_diff_error_from_message(err_message, self.swap_diff)
except Exception:
pass
if diff_error:
# Cut everything after postfix: it is internal view of DiffError
strace = str(report.longrepr)
data_postfix = "_ _ _ _ _"
if data_postfix in strace:
strace = strace[0:strace.index(data_postfix)]
self.teamcity.testFailed(test_id, diff_error.msg if diff_error.msg else message, strace,
flowId=test_id,
comparison_failure=diff_error
)
else:
self.teamcity.testFailed(test_id, message, str(report.longrepr), flowId=test_id)
self.report_test_finished(test_id, duration)
def report_test_skip(self, test_id, report):
if type(report.longrepr) is tuple and len(report.longrepr) == 3:
reason = report.longrepr[2]
else:
reason = str(report.longrepr)
if hasattr(report, 'duration'):
duration = timedelta(seconds=report.duration)
else:
duration = None
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id)
self.teamcity.testIgnored(test_id, reason, flowId=test_id)
self.report_test_finished(test_id, duration)
def pytest_runtest_logreport(self, report):
"""
:type report: _pytest.runner.TestReport
"""
test_id = self.format_test_id(report.nodeid, report.location)
duration = timedelta(seconds=report.duration)
if report.passed:
# Do not report passed setup/teardown if no output
if report.when == 'call':
self.ensure_test_start_reported(test_id)
if not self.skip_passed_output:
self.report_test_output(report, test_id)
self.report_test_finished(test_id, duration)
else:
if self.report_has_output(report) and not self.skip_passed_output:
block_name = "test " + report.when
self.teamcity.blockOpened(block_name, flowId=test_id)
self.report_test_output(report, test_id)
self.teamcity.blockClosed(block_name, flowId=test_id)
elif report.failed:
if report.when == 'call':
self.report_test_failure(test_id, report)
elif report.when == 'setup':
if self.report_has_output(report):
self.teamcity.blockOpened("test setup", flowId=test_id)
self.report_test_output(report, test_id)
self.teamcity.blockClosed("test setup", flowId=test_id)
self.report_test_failure(test_id, report, message="test setup failed", report_output=False)
elif report.when == 'teardown':
# Report failed teardown as a separate test as original test is already finished
self.report_test_failure(test_id + "_teardown", report)
elif report.skipped:
self.report_test_skip(test_id, report)
def pytest_collectreport(self, report):
test_id = self.format_test_id(report.nodeid, report.location) + "_collect"
if report.failed:
self.report_test_failure(test_id, report)
elif report.skipped:
self.report_test_skip(test_id, report)
def pytest_terminal_summary(self):
if self.coverage_controller is not None:
try:
self._report_coverage()
except Exception:
tb = traceback.format_exc()
self.teamcity.customMessage("Coverage statistics reporting failed", "ERROR", errorDetails=tb)
def _report_coverage(self):
from coverage.misc import NotPython
from coverage.report import Reporter
from coverage.results import Numbers
class _CoverageReporter(Reporter):
def __init__(self, coverage, config, messages):
super(_CoverageReporter, self).__init__(coverage, config)
self.branches = coverage.data.has_arcs()
self.messages = messages
def report(self, morfs, outfile=None):
if hasattr(self, 'find_code_units'):
self.find_code_units(morfs)
else:
self.find_file_reporters(morfs)
total = Numbers()
if hasattr(self, 'code_units'):
units = self.code_units
else:
units = self.file_reporters
for cu in units:
try:
analysis = self.coverage._analyze(cu)
nums = analysis.numbers
total += nums
except KeyboardInterrupt:
raise
except Exception:
if self.config.ignore_errors:
continue
err = sys.exc_info()
typ, msg = err[:2]
if typ is NotPython and not cu.should_be_python():
continue
test_id = cu.name
details = convert_error_to_string(err)
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testFailed(test_id, message="Coverage analysis failed", details=details, flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
if total.n_files > 0:
covered = total.n_executed
total_statements = total.n_statements
if self.branches:
covered += total.n_executed_branches
total_statements += total.n_branches
self.messages.buildStatisticLinesCovered(covered)
self.messages.buildStatisticTotalLines(total_statements)
self.messages.buildStatisticLinesUncovered(total_statements - covered)
reporter = _CoverageReporter(
self.coverage_controller.cov,
self.coverage_controller.cov.config,
self.teamcity,
)
reporter.report(None)
class VpcClusterUpgradeTest:
log = logging.getLogger(__name__)
def __init__(self,
num_masters: int, num_agents: int, num_public_agents: int,
stable_installer_url: str, installer_url: str,
aws_region: str, aws_access_key_id: str, aws_secret_access_key: str,
default_os_user: str,
config_yaml_override_install: str, config_yaml_override_upgrade: str,
dcos_api_session_factory_install: VpcClusterUpgradeTestDcosApiSessionFactory,
dcos_api_session_factory_upgrade: VpcClusterUpgradeTestDcosApiSessionFactory):
self.dcos_api_session_factory_install = dcos_api_session_factory_install
self.dcos_api_session_factory_upgrade = dcos_api_session_factory_upgrade
self.num_masters = num_masters
self.num_agents = num_agents
self.num_public_agents = num_public_agents
self.stable_installer_url = stable_installer_url
self.installer_url = installer_url
self.aws_region = aws_region
self.aws_access_key_id = aws_access_key_id
self.aws_secret_access_key = aws_secret_access_key
self.default_os_user = default_os_user
self.config_yaml_override_install = config_yaml_override_install
self.config_yaml_override_upgrade = config_yaml_override_upgrade
self.acs_token = None
self.teamcity_msg = TeamcityServiceMessages()
# the two following properties are set when running setup_cluster_workload, here we default them to empty
# values.
self.test_app_ids = []
self.tasks_start = []
@staticmethod
def app_task_ids(dcos_api, app_id):
"""Return a list of Mesos task IDs for app_id's running tasks."""
assert app_id.startswith('/')
response = dcos_api.marathon.get('/v2/apps' + app_id + '/tasks')
response.raise_for_status()
tasks = response.json()['tasks']
return [task['id'] for task in tasks]
@staticmethod
def get_master_task_state(dcos_api, task_id):
"""Returns the JSON blob associated with the task from /master/state."""
response = dcos_api.get('/mesos/master/state')
response.raise_for_status()
master_state = response.json()
for framework in master_state['frameworks']:
for task in framework['tasks']:
if task_id in task['id']:
return task
def parse_dns_log(self, dns_log_content):
"""Return a list of (timestamp, status) tuples from dns_log_content."""
dns_log = [line.strip().split(' ') for line in dns_log_content.strip().split('\n')]
if any(len(entry) != 2 or entry[1] not in ['SUCCESS', 'FAILURE'] for entry in dns_log):
message = 'Malformed DNS log.'
self.log.debug(message + ' DNS log content:\n' + dns_log_content)
raise Exception(message)
return dns_log
def log_test(self, test_name: str, call: Callable[[], None]) -> None:
try:
self.teamcity_msg.testStarted(test_name)
call()
except Exception:
# we want this except to be broad so that we can keep any Exception from taking
# everything with it and not asserting the other tests
self.teamcity_msg.testFailed(test_name, details=traceback.format_exc())
finally:
self.teamcity_msg.testFinished(test_name)
@staticmethod
@retrying.retry(
wait_fixed=(1 * 1000),
stop_max_delay=(120 * 1000),
retry_on_result=lambda x: not x)
def wait_for_dns(dcos_api, hostname):
"""Return True if Mesos-DNS has at least one entry for hostname."""
hosts = dcos_api.get('/mesos_dns/v1/hosts/' + hostname).json()
return any(h['host'] != '' and h['ip'] != '' for h in hosts)
def setup_cluster_workload(self, dcos_api: DcosApiSession, healthcheck_app: dict, dns_app: dict,
viplisten_app: dict, viptalk_app: dict):
# Deploy test apps.
# TODO(branden): We ought to be able to deploy these apps concurrently. See
# https://mesosphere.atlassian.net/browse/DCOS-13360.
with logger.scope("deploy apps"):
dcos_api.marathon.deploy_app(viplisten_app)
dcos_api.marathon.ensure_deployments_complete()
dcos_api.marathon.deploy_app(viptalk_app)
dcos_api.marathon.ensure_deployments_complete()
dcos_api.marathon.deploy_app(healthcheck_app)
dcos_api.marathon.ensure_deployments_complete()
# This is a hack to make sure we don't deploy dns_app before the name it's
# trying to resolve is available.
self.wait_for_dns(dcos_api, dns_app['env']['RESOLVE_NAME'])
dcos_api.marathon.deploy_app(dns_app, check_health=False)
dcos_api.marathon.ensure_deployments_complete()
test_apps = [healthcheck_app, dns_app, viplisten_app, viptalk_app]
self.test_app_ids = [app['id'] for app in test_apps]
self.tasks_start = {app_id: sorted(self.app_task_ids(dcos_api, app_id)) for app_id in self.test_app_ids}
self.log.debug('Test app tasks at start:\n' + pprint.pformat(self.tasks_start))
for app in test_apps:
assert app['instances'] == len(self.tasks_start[app['id']])
# Save the master's state of the task to compare with
# the master's view after the upgrade.
# See this issue for why we check for a difference:
# https://issues.apache.org/jira/browse/MESOS-1718
self.task_state_start = self.get_master_task_state(dcos_api, self.tasks_start[self.test_app_ids[0]][0])
def verify_apps_state(self, dcos_api: DcosApiSession, dns_app: dict):
with logger.scope("verify apps state"):
# nested methods here so we can "close" over external state
def marathon_app_tasks_survive_upgrade():
# Verify that the tasks we started are still running.
tasks_end = {app_id: sorted(self.app_task_ids(dcos_api, app_id)) for app_id in self.test_app_ids}
self.log.debug('Test app tasks at end:\n' + pprint.pformat(tasks_end))
if not self.tasks_start == tasks_end:
self.teamcity_msg.testFailed(
"test_upgrade_vpc.marathon_app_tasks_survive_upgrade",
details="expected: {}\nactual: {}".format(self.tasks_start, tasks_end))
def test_mesos_task_state_remains_consistent():
# Verify that the "state" of the task does not change.
task_state_end = self.get_master_task_state(dcos_api, self.tasks_start[self.test_app_ids[0]][0])
if not self.task_state_start == task_state_end:
self.teamcity_msg.testFailed(
"test_upgrade_vpc.test_mesos_task_state_remains_consistent",
details="expected: {}\nactual: {}".format(self.task_state_start, task_state_end))
def test_app_dns_survive_upgrade():
# Verify DNS didn't fail.
marathon_framework_id = dcos_api.marathon.get('/v2/info').json()['frameworkId']
dns_app_task = dcos_api.marathon.get('/v2/apps' + dns_app['id'] + '/tasks').json()['tasks'][0]
dns_log = self.parse_dns_log(dcos_api.mesos_sandbox_file(
dns_app_task['slaveId'],
marathon_framework_id,
dns_app_task['id'],
dns_app['env']['DNS_LOG_FILENAME'],
))
dns_failure_times = [entry[0] for entry in dns_log if entry[1] != 'SUCCESS']
if len(dns_failure_times) > 0:
message = 'Failed to resolve Marathon app hostname {} at least once.'.format(
dns_app['env']['RESOLVE_NAME'])
err_msg = message + ' Hostname failed to resolve at these times:\n' + '\n'.join(dns_failure_times)
self.log.debug(err_msg)
self.teamcity_msg.testFailed("test_upgrade_vpc.test_app_dns_survive_upgrade", details=err_msg)
self.log_test("test_upgrade_vpc.marathon_app_tasks_survive_upgrade", marathon_app_tasks_survive_upgrade)
self.log_test(
"test_upgrade_vpc.test_mesos_task_state_remains_consistent",
test_mesos_task_state_remains_consistent
)
self.log_test("test_upgrade_vpc.test_app_dns_survive_upgrade", test_app_dns_survive_upgrade)
def mesos_metrics_snapshot(self, cluster, host):
if host in cluster.masters:
port = 5050
else:
port = 5051
with cluster.ssher.tunnel(host) as tunnel:
return json.loads(
tunnel.remote_cmd(
curl_cmd + ['{}:{}/metrics/snapshot'.format(host.private_ip, port)]
).decode('utf-8')
)
def upgrade_host(self, tunnel, role, bootstrap_url, upgrade_script_path):
# Download the upgrade script for the new DC/OS.
tunnel.remote_cmd(curl_cmd + ['--remote-name', upgrade_script_path])
# Upgrade to the new DC/OS.
tunnel.remote_cmd(['sudo', 'bash', 'dcos_node_upgrade.sh'])
@retry(
wait_fixed=(1000 * 5),
stop_max_delay=(1000 * 60 * 5),
# Retry on SSH command error or metric not equal to expected value.
retry_on_exception=(lambda exc: isinstance(exc, CalledProcessError)),
retry_on_result=(lambda result: not result),
)
def wait_for_mesos_metric(self, cluster, host, key, value):
"""Return True when host's Mesos metric key is equal to value."""
return self.mesos_metrics_snapshot(cluster, host).get(key) == value
def upgrade_dcos(self, cluster, installer_url, version, add_config_path=None):
assert all(h.public_ip for h in cluster.hosts), (
'All cluster hosts must be externally reachable. hosts: {}'.formation(cluster.hosts)
)
assert cluster.bootstrap_host, 'Upgrade requires a bootstrap host'
bootstrap_url = 'http://' + cluster.bootstrap_host.private_ip
logging.info('Preparing bootstrap host for upgrade')
installer = test_util.installer_api_test.DcosCliInstaller()
with cluster.ssher.tunnel(cluster.bootstrap_host) as bootstrap_host_tunnel:
installer.setup_remote(
tunnel=bootstrap_host_tunnel,
installer_path=cluster.ssher.home_dir + '/dcos_generate_config.sh',
download_url=installer_url,
)
installer.genconf(
bootstrap_url=bootstrap_url,
zk_host=cluster.bootstrap_host.private_ip + ':2181',
master_list=[h.private_ip for h in cluster.masters],
agent_list=[h.private_ip for h in cluster.agents],
public_agent_list=[h.private_ip for h in cluster.public_agents],
ip_detect=cluster.ip_detect,
platform=cluster.platform,
ssh_user=cluster.ssher.user,
ssh_key=cluster.ssher.key,
add_config_path=add_config_path,
rexray_config_preset=cluster.rexray_config_preset,
)
# Generate node upgrade script
output = installer.generate_node_upgrade_script(version)
upgrade_script_path = output.decode('utf-8').splitlines()[-1].split("Node upgrade script URL: ", 1)[1]
# Remove docker (and associated journald) restart from the install
# script. This prevents Docker-containerized tasks from being killed
# during agent upgrades.
bootstrap_host_tunnel.remote_cmd([
'sudo', 'sed',
'-i',
'-e', '"s/systemctl restart systemd-journald//g"',
'-e', '"s/systemctl restart docker//g"',
cluster.ssher.home_dir + '/genconf/serve/dcos_install.sh',
])
run_bootstrap_nginx(bootstrap_host_tunnel, cluster.ssher.home_dir)
upgrade_ordering = [
# Upgrade masters in a random order.
('master', 'master', random.sample(cluster.masters, len(cluster.masters))),
('slave', 'agent', cluster.agents),
('slave_public', 'public agent', cluster.public_agents),
]
logging.info('\n'.join(
['Upgrade plan:'] +
['{} ({})'.format(host, role_name) for _, role_name, hosts in upgrade_ordering for host in hosts]
))
for role, role_name, hosts in upgrade_ordering:
logging.info('Upgrading {} nodes: {}'.format(role_name, repr(hosts)))
for host in hosts:
logging.info('Upgrading {}: {}'.format(role_name, repr(host)))
with cluster.ssher.tunnel(host) as tunnel:
self.upgrade_host(tunnel, role, bootstrap_url, upgrade_script_path)
wait_metric = {
'master': 'registrar/log/recovered',
'slave': 'slave/registered',
'slave_public': 'slave/registered',
}[role]
logging.info('Waiting for {} to rejoin the cluster...'.format(role_name))
try:
self.wait_for_mesos_metric(cluster, host, wait_metric, 1)
except RetryError as exc:
raise Exception(
'Timed out waiting for {} to rejoin the cluster after upgrade: {}'.
format(role_name, repr(host))
) from exc
def run_test(self) -> int:
stack_suffix = os.getenv("CF_STACK_NAME_SUFFIX", "open-upgrade")
stack_name = "dcos-ci-test-{stack_suffix}-{random_id}".format(
stack_suffix=stack_suffix, random_id=random_id(10))
test_id = uuid.uuid4().hex
healthcheck_app_id = TEST_APP_NAME_FMT.format('healthcheck-' + test_id)
dns_app_id = TEST_APP_NAME_FMT.format('dns-' + test_id)
with logger.scope("create vpc cf stack '{}'".format(stack_name)):
bw = test_util.aws.BotoWrapper(
region=self.aws_region,
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key)
ssh_key = bw.create_key_pair(stack_name)
write_string('ssh_key', ssh_key)
bare_cluster, ssh_info = test_util.aws.BareClusterCfStack.create(
stack_name=stack_name,
instance_type='m4.xlarge',
instance_os='cent-os-7-dcos-prereqs',
# An instance for each cluster node plus the bootstrap.
instance_count=(self.num_masters + self.num_agents + self.num_public_agents + 1),
admin_location='0.0.0.0/0',
key_pair_name=stack_name,
boto_wrapper=bw
)
bare_cluster.wait_for_complete()
cluster = test_util.cluster.Cluster.from_bare_cluster(
bare_cluster,
ssh_info,
ssh_key=ssh_key,
num_masters=self.num_masters,
num_agents=self.num_agents,
num_public_agents=self.num_public_agents,
)
with logger.scope("install dcos"):
# Use the CLI installer to set exhibitor_storage_backend = zookeeper.
# Don't install prereqs since stable breaks Docker 1.13. See
# https://jira.mesosphere.com/browse/DCOS_OSS-743.
test_util.cluster.install_dcos(cluster, self.stable_installer_url, api=False, install_prereqs=False,
add_config_path=self.config_yaml_override_install)
master_list = [h.private_ip for h in cluster.masters]
dcos_api_install = self.dcos_api_session_factory_install.apply(
'http://{ip}'.format(ip=cluster.masters[0].public_ip),
master_list,
master_list,
[h.private_ip for h in cluster.agents],
[h.private_ip for h in cluster.public_agents],
self.default_os_user)
dcos_api_install.wait_for_dcos()
installed_version = dcos_api_install.get_version()
healthcheck_app = create_marathon_healthcheck_app(healthcheck_app_id)
dns_app = create_marathon_dns_app(dns_app_id, healthcheck_app_id)
viplisten_app = create_marathon_viplisten_app()
viptalk_app = create_marathon_viptalk_app()
self.setup_cluster_workload(dcos_api_install, healthcheck_app, dns_app, viplisten_app, viptalk_app)
with logger.scope("upgrade cluster"):
self.upgrade_dcos(cluster, self.installer_url,
installed_version, add_config_path=self.config_yaml_override_upgrade)
with cluster.ssher.tunnel(cluster.bootstrap_host) as bootstrap_host_tunnel:
bootstrap_host_tunnel.remote_cmd(['sudo', 'rm', '-rf', cluster.ssher.home_dir + '/*'])
# this method invocation looks like it is the same as the one above, and that is partially correct.
# the arguments to the invocation are the same, but the thing that changes is the lifecycle of the cluster
# the client is being created to interact with. This client is specifically for the cluster after the
# upgrade has taken place, and can account for any possible settings that may change for the client under
# the hood when it probes the cluster.
dcos_api_upgrade = self.dcos_api_session_factory_upgrade.apply(
'http://{ip}'.format(ip=cluster.masters[0].public_ip),
master_list,
master_list,
[h.private_ip for h in cluster.agents],
[h.private_ip for h in cluster.public_agents],
self.default_os_user)
dcos_api_upgrade.wait_for_dcos() # here we wait for DC/OS to be "up" so that we can auth this new client
self.verify_apps_state(dcos_api_upgrade, dns_app)
with logger.scope("run integration tests"):
# copied from test_util/test_aws_cf.py:96
add_env = []
prefix = 'TEST_ADD_ENV_'
for k, v in os.environ.items():
if k.startswith(prefix):
add_env.append(k.replace(prefix, '') + '=' + v)
test_cmd = ' '.join(add_env) + ' py.test -vv -s -rs ' + os.getenv('CI_FLAGS', '')
result = test_util.cluster.run_integration_tests(cluster, test_cmd=test_cmd)
if result == 0:
self.log.info("Test successful! Deleting VPC if provided in this run.")
bare_cluster.delete()
bw.delete_key_pair(stack_name)
else:
self.log.info("Test failed! VPC cluster will remain available for "
"debugging for 2 hour after instantiation.")
if os.getenv('CI_FLAGS'):
result = 0
return result
class TeamcityFormatter(Formatter):
"""
Stateful TC reporter.
Since we can't fetch all steps from the very beginning (even skipped tests are reported)
we store tests and features on each call.
To hook into test reporting override _report_suite_started and/or _report_test_started
"""
def __init__(self, stream_opener, config):
super(TeamcityFormatter, self).__init__(stream_opener, config)
assert version.LooseVersion(behave_version) >= version.LooseVersion("1.2.6"), "Only 1.2.6+ is supported"
self._messages = TeamcityServiceMessages()
self.__feature = None
self.__scenario = None
self.__steps = deque()
self.__scenario_opened = False
self.__feature_opened = False
self.__test_start_time = None
def feature(self, feature):
assert isinstance(feature, Feature)
assert not self.__feature, "Prev. feature not closed"
self.__feature = feature
def scenario(self, scenario):
assert isinstance(scenario, Scenario)
self.__scenario = scenario
self.__scenario_opened = False
self.__steps.clear()
def step(self, step):
assert isinstance(step, Step)
self.__steps.append(step)
def match(self, match):
if not self.__feature_opened:
self._report_suite_started(self.__feature, _suite_name(self.__feature))
self.__feature_opened = True
if not self.__scenario_opened:
self._report_suite_started(self.__scenario, _suite_name(self.__scenario))
self.__scenario_opened = True
assert self.__steps, "No steps left"
step = self.__steps.popleft()
self._report_test_started(step, _step_name(step))
self.__test_start_time = datetime.datetime.now()
def _report_suite_started(self, suite, suite_name):
"""
:param suite: behave suite
:param suite_name: suite name that must be reported, be sure to use it instead of suite.name
"""
self._messages.testSuiteStarted(suite_name)
def _report_test_started(self, test, test_name):
"""
Suite name is always stripped, be sure to strip() it too
:param test: behave test
:param test_name: test name that must be reported, be sure to use it instead of test.name
"""
self._messages.testStarted(test_name)
def result(self, step):
assert isinstance(step, Step)
step_name = _step_name(step)
if step.status == Status.failed:
try:
error = traceback.format_exc(step.exc_traceback)
if error != step.error_message:
self._messages.testStdErr(step_name, error)
except Exception:
pass # exception shall not prevent error message
self._messages.testFailed(step_name, message=step.error_message)
if step.status == Status.undefined:
self._messages.testFailed(step_name, message="Undefined")
if step.status == Status.skipped:
self._messages.testIgnored(step_name)
self._messages.testFinished(step_name, testDuration=datetime.datetime.now() - self.__test_start_time)
if not self.__steps:
self.__close_scenario()
elif step.status in [Status.failed, Status.undefined]:
# Broken background/undefined step stops whole scenario
reason = "Undefined step" if step.status == Status.undefined else "Prev. step failed"
self.__skip_rest_of_scenario(reason)
def __skip_rest_of_scenario(self, reason):
while self.__steps:
step = self.__steps.popleft()
self._report_test_started(step, _step_name(step))
self._messages.testIgnored(_step_name(step),
message="{0}. Rest part of scenario is skipped".format(reason))
self._messages.testFinished(_step_name(step))
self.__close_scenario()
def __close_scenario(self):
if self.__scenario:
self._messages.testSuiteFinished(_suite_name(self.__scenario))
self.__scenario = None
def eof(self):
self.__skip_rest_of_scenario("")
self._messages.testSuiteFinished(_suite_name(self.__feature))
self.__feature = None
self.__feature_opened = False
class EchoTeamCityMessages(object):
def __init__(self, ):
self.tw = py.io.TerminalWriter(py.std.sys.stdout)
self.teamcity = TeamcityServiceMessages(self.tw)
self.currentSuite = None
def format_names(self, name):
if name.find("::") > 0:
file, testname = name.split("::", 1)
else:
file, testname = name, "top_level"
testname = testname.replace("::()::", ".")
testname = testname.replace("::", ".")
testname = testname.strip(".")
file = file.replace(".", "_").replace(os.sep, ".").replace("/", ".")
return file, testname
def pytest_runtest_logstart(self, nodeid, location):
file, testname = self.format_names(nodeid)
if not file == self.currentSuite:
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
self.currentSuite = file
self.teamcity.testSuiteStarted(self.currentSuite)
self.teamcity.testStarted(testname)
def pytest_runtest_logreport(self, report):
file, testname = self.format_names(report.nodeid)
if report.passed:
if report.when == "call": # ignore setup/teardown
duration = timedelta(seconds=report.duration)
self.teamcity.testFinished(testname, testDuration=duration)
elif report.failed:
if report.when in ("call", "setup"):
self.teamcity.testFailed(testname, str(report.location), str(report.longrepr))
duration = timedelta(seconds=report.duration)
self.teamcity.testFinished(testname, testDuration=duration) # report finished after the failure
elif report.when == "teardown":
name = testname + "_teardown"
self.teamcity.testStarted(name)
self.teamcity.testFailed(name, str(report.location), str(report.longrepr))
self.teamcity.testFinished(name)
elif report.skipped:
self.teamcity.testIgnored(testname, str(report.longrepr))
self.teamcity.testFinished(testname) # report finished after the skip
def pytest_collectreport(self, report):
if report.failed:
file, testname = self.format_names(report.nodeid)
name = file + "_collect"
self.teamcity.testStarted(name)
self.teamcity.testFailed(name, str(report.location), str(report.longrepr))
self.teamcity.testFinished(name)
def pytest_sessionfinish(self, session, exitstatus, __multicall__):
if self.currentSuite:
self.teamcity.testSuiteFinished(self.currentSuite)
class TeamcityTestResult(unittest.TestResult):
def __init__(self, stream=sys.stdout):
super(TeamcityTestResult, self).__init__()
self.output = stream
self.test_started_datetime = datetime.datetime.now()
self.test_name = None
self.messages = None
self.createMessages()
self.test_succeeded = False
self.subtest_errors = []
self.subtest_failures = []
def createMessages(self):
self.messages = TeamcityServiceMessages(self.output)
def formatErr(self, err):
try:
exctype, value, tb = err
return ''.join(traceback.format_exception(exctype, value, tb))
except Exception:
tb = traceback.format_exc()
return "*FAILED TO GET TRACEBACK*: " + tb
def getTestName(self, test):
desc = test.shortDescription()
if desc:
return "%s (%s)" % (test.id(), desc)
return test.id()
def addSuccess(self, test, *args):
super(TeamcityTestResult, self).addSuccess(test)
self.output.write("ok\n")
def addError(self, test, err, *k):
super(TeamcityTestResult, self).addError(test, err)
err = self.formatErr(err)
if self.getTestName(test) != self.test_name:
sys.stderr.write("INTERNAL ERROR: addError(%s) outside of test\n" % self.getTestName(test))
sys.stderr.write("Error: %s\n" % err)
return
self.test_succeeded = False
self.messages.testFailed(self.getTestName(test), message='Error', details=err)
def addFailure(self, test, err, *k):
# workaround nose bug on python 3
if _is_string(err[1]):
err = (err[0], Exception(err[1]), err[2])
super(TeamcityTestResult, self).addFailure(test, err)
err = self.formatErr(err)
if self.getTestName(test) != self.test_name:
sys.stderr.write("INTERNAL ERROR: addFailure(%s) outside of test\n" % self.getTestName(test))
sys.stderr.write("Error: %s\n" % err)
return
self.test_succeeded = False
self.messages.testFailed(self.getTestName(test), message='Failure', details=err)
def addSkip(self, test, reason):
super(TeamcityTestResult, self).addSkip(test, reason)
if self.getTestName(test) != self.test_name:
sys.stderr.write("INTERNAL ERROR: addSkip(%s) outside of test\n" % self.getTestName(test))
sys.stderr.write("Reason: %s\n" % reason)
return
self.messages.testIgnored(self.getTestName(test), reason)
def addExpectedFailure(self, test, err):
super(TeamcityTestResult, self).addExpectedFailure(test, err)
if self.getTestName(test) != self.test_name:
err = self.formatErr(err)
sys.stderr.write("INTERNAL ERROR: addExpectedFailure(%s) outside of test\n" % self.getTestName(test))
sys.stderr.write("Error: %s\n" % err)
return
self.output.write("ok (failure expected)\n")
def addUnexpectedSuccess(self, test):
super(TeamcityTestResult, self).addUnexpectedSuccess(test)
if self.getTestName(test) != self.test_name:
sys.stderr.write("INTERNAL ERROR: addUnexpectedSuccess(%s) outside of test\n" % self.getTestName(test))
return
self.test_succeeded = False
self.messages.testFailed(self.getTestName(test), message='Unexpected success')
def addSubTest(self, test, subtest, err):
super(TeamcityTestResult, self).addSubTest(test, subtest, err)
if err is not None:
if issubclass(err[0], test.failureException):
self.subtest_failures.append("%s:\n%s" % (self.getTestName(subtest), self.formatErr(err)))
self.output.write("%s: failure\n" % self.getTestName(subtest))
else:
self.subtest_errors.append("%s:\n%s" % (self.getTestName(subtest), self.formatErr(err)))
self.output.write("%s: error\n" % self.getTestName(subtest))
else:
self.output.write("%s: ok\n" % self.getTestName(subtest))
def startTest(self, test):
self.test_started_datetime = datetime.datetime.now()
self.test_name = self.getTestName(test)
self.test_succeeded = True # we assume it succeeded, and set it to False when we send a failure message
self.subtest_errors = []
self.subtest_failures = []
self.messages.testStarted(self.test_name)
def stopTest(self, test):
# Record the test as a failure after the entire test was run and any subtest has errors
if self.test_succeeded and (self.subtest_errors or self.subtest_failures):
self.messages.testFailed(self.getTestName(test), message='Subtest error',
details="\n\n".join(self.subtest_failures) + "\n".join(self.subtest_errors))
time_diff = datetime.datetime.now() - self.test_started_datetime
if self.getTestName(test) != self.test_name:
sys.stderr.write(
"INTERNAL ERROR: stopTest(%s) after startTest(%s)" % (self.getTestName(test), self.test_name))
self.messages.testFinished(self.test_name, time_diff)
self.test_name = None
self.output.flush()
class TeamcityReport(object):
name = 'teamcity-report'
score = 10000
def __init__(self):
super(TeamcityReport, self).__init__()
self.messages = TeamcityServiceMessages(_real_stdout)
self.test_started_datetime_map = {}
self.enabled = False
def get_test_id(self, test):
if is_string(test):
return test
# Handle special "tests"
test_class_name = get_class_fullname(test)
if test_class_name == CONTEXT_SUITE_FQN:
if inspect.ismodule(test.context):
module_name = test.context.__name__
return module_name + "." + test.error_context
elif inspect.isclass(test.context):
class_name = get_class_fullname(test.context)
return class_name + "." + test.error_context
test_id = test.id()
real_test = getattr(test, "test", test)
real_test_class_name = get_class_fullname(real_test)
test_arg = getattr(real_test, "arg", tuple())
if (type(test_arg) is tuple or type(test_arg) is list) and len(test_arg) > 0:
# As written in nose.case.FunctionTestCase#__str__ or nose.case.MethodTestCase#__str__
test_arg_str = "%s" % (test_arg,)
if test_id.endswith(test_arg_str):
# Replace '.' in test args with '_' to preserve test hierarchy on TeamCity
test_id = test_id[:len(test_id) - len(test_arg_str)] + test_arg_str.replace('.', '_')
# Force test_id for doctests
if real_test_class_name != "doctest.DocTestCase" and real_test_class_name != "nose.plugins.doctests.DocTestCase":
desc = test.shortDescription()
if desc and desc != test.id():
return "%s (%s)" % (test_id, desc.replace('.', '_'))
return test_id
def configure(self, options, conf):
self.enabled = is_running_under_teamcity()
def options(self, parser, env=os.environ):
pass
def _get_capture_plugin(self, test):
"""
:type test: nose.case.Test
:rtype: nose.plugins.base.Plugin
"""
for plugin in test.config.plugins.plugins:
if plugin.name == "capture":
return plugin
return None
def _capture_plugin_enabled(self, test):
"""
:type test: nose.case.Test
"""
plugin = self._get_capture_plugin(test)
return plugin is not None and plugin.enabled
def _capture_plugin_buffer(self, test):
"""
:type test: nose.case.Test
"""
plugin = self._get_capture_plugin(test)
if plugin is None:
return None
return getattr(plugin, "buffer", None)
def _captureStandardOutput_value(self, test):
"""
:type test: nose.case.Test
"""
if self._capture_plugin_enabled(test):
return 'false'
else:
return 'true'
def report_fail(self, test, fail_type, err):
# workaround nose bug on python 3
if is_string(err[1]):
err = (err[0], Exception(err[1]), err[2])
test_id = self.get_test_id(test)
details = convert_error_to_string(err)
start_index = details.find(_captured_output_start_marker)
end_index = details.find(_captured_output_end_marker)
if 0 <= start_index < end_index:
# do not log test output twice, see report_finish for actual output handling
details = details[:start_index] + details[end_index + len(_captured_output_end_marker):]
self.messages.testFailed(test_id, message=fail_type, details=details, flowId=test_id)
def report_finish(self, test):
test_id = self.get_test_id(test)
captured_output = getattr(test, "capturedOutput", None)
if captured_output is None and self._capture_plugin_enabled(test):
# nose capture does not fill 'capturedOutput' property on successful tests
captured_output = self._capture_plugin_buffer(test)
if captured_output:
for chunk in split_output(limit_output(captured_output)):
self.messages.testStdOut(test_id, chunk, flowId=test_id)
if test_id in self.test_started_datetime_map:
time_diff = datetime.datetime.now() - self.test_started_datetime_map[test_id]
self.messages.testFinished(test_id, testDuration=time_diff, flowId=test_id)
else:
self.messages.testFinished(test_id, flowId=test_id)
def addError(self, test, err):
test_class_name = get_class_fullname(test)
test_id = self.get_test_id(test)
if issubclass(err[0], SkipTest):
self.messages.testIgnored(test_id, message=("SKIPPED: %s" % str(err[1])), flowId=test_id)
self.report_finish(test)
elif issubclass(err[0], DeprecatedTest):
self.messages.testIgnored(test_id, message="Deprecated", flowId=test_id)
self.report_finish(test)
elif test_class_name == CONTEXT_SUITE_FQN:
self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(test), flowId=test_id)
self.report_fail(test, 'error in ' + test.error_context + ' context', err)
self.messages.testFinished(test_id, flowId=test_id)
else:
self.report_fail(test, 'Error', err)
self.report_finish(test)
def addFailure(self, test, err):
self.report_fail(test, 'Failure', err)
self.report_finish(test)
def startTest(self, test):
test_id = self.get_test_id(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id, captureStandardOutput=self._captureStandardOutput_value(test), flowId=test_id)
def addSuccess(self, test):
self.report_finish(test)
class EchoTeamCityMessages(object):
def __init__(self, output_capture_enabled, coverage_controller):
self.coverage_controller = coverage_controller
self.output_capture_enabled = output_capture_enabled
self.teamcity = TeamcityServiceMessages()
self.test_start_reported_mark = set()
self.max_reported_output_size = 1 * 1024 * 1024
self.reported_output_chunk_size = 50000
def format_test_id(self, nodeid):
test_id = nodeid
if test_id.find("::") < 0:
test_id += "::top_level"
test_id = test_id.replace("::()::", "::")
test_id = re.sub(r"\.pyc?::", r"::", test_id)
test_id = test_id.replace(".", "_").replace(os.sep, ".").replace(
"/", ".").replace('::', '.')
return test_id
def pytest_runtest_logstart(self, nodeid, location):
self.ensure_test_start_reported(self.format_test_id(nodeid))
def ensure_test_start_reported(self, test_id):
if test_id not in self.test_start_reported_mark:
self.teamcity.testStarted(test_id,
flowId=test_id,
captureStandardOutput='false' if
self.output_capture_enabled else 'true')
self.test_start_reported_mark.add(test_id)
def report_has_output(self, report):
for (secname, data) in report.sections:
if report.when in secname and ('stdout' in secname
or 'stderr' in secname):
return True
return False
def report_test_output(self, report, test_id, when):
for (secname, data) in report.sections:
if when not in secname:
continue
if not data:
continue
if 'stdout' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdOut(test_id,
out=chunk,
flowId=test_id)
elif 'stderr' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdErr(test_id,
out=chunk,
flowId=test_id)
def pytest_runtest_logreport(self, report):
"""
:type report: _pytest.runner.TestReport
"""
orig_test_id = self.format_test_id(report.nodeid)
suffix = '' if report.when == 'call' else ('_' + report.when)
test_id = orig_test_id + suffix
duration = timedelta(seconds=report.duration)
if report.passed:
# Do not report passed setup/teardown if no output
if report.when == 'call' or self.report_has_output(report):
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id, report.when)
self.teamcity.testFinished(test_id,
testDuration=duration,
flowId=test_id)
elif report.failed:
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id, report.when)
self.teamcity.testFailed(test_id,
str(report.location),
str(report.longrepr),
flowId=test_id)
self.teamcity.testFinished(test_id,
testDuration=duration,
flowId=test_id)
# If test setup failed, report test failure as well
if report.when == 'setup':
self.ensure_test_start_reported(orig_test_id)
self.teamcity.testFailed(
orig_test_id,
"test setup failed, see %s test failure" % test_id,
flowId=orig_test_id)
self.teamcity.testFinished(orig_test_id, flowId=orig_test_id)
elif report.skipped:
if type(report.longrepr) is tuple and len(report.longrepr) == 3:
reason = report.longrepr[2]
else:
reason = str(report.longrepr)
self.ensure_test_start_reported(orig_test_id)
self.report_test_output(report, orig_test_id, report.when)
self.teamcity.testIgnored(orig_test_id,
reason,
flowId=orig_test_id)
self.teamcity.testFinished(orig_test_id, flowId=orig_test_id)
def pytest_collectreport(self, report):
if report.failed:
test_id = self.format_test_id(report.nodeid) + "_collect"
self.teamcity.testStarted(test_id, flowId=test_id)
self.teamcity.testFailed(test_id,
str(report.location),
str(report.longrepr),
flowId=test_id)
self.teamcity.testFinished(test_id, flowId=test_id)
def pytest_terminal_summary(self):
if self.coverage_controller is not None:
try:
self._report_coverage()
except:
tb = traceback.format_exc()
self.teamcity.customMessage(
"Coverage statistics reporting failed",
"ERROR",
errorDetails=tb)
def _report_coverage(self):
from coverage.misc import NotPython
from coverage.report import Reporter
from coverage.results import Numbers
class _CoverageReporter(Reporter):
def __init__(self, coverage, config, messages):
super(_CoverageReporter, self).__init__(coverage, config)
self.branches = coverage.data.has_arcs()
self.messages = messages
def report(self, morfs, outfile=None):
self.find_code_units(morfs)
total = Numbers()
for cu in self.code_units:
try:
analysis = self.coverage._analyze(cu)
nums = analysis.numbers
total += nums
except KeyboardInterrupt:
raise
except:
if self.config.ignore_errors:
continue
err = sys.exc_info()
typ, msg = err[:2]
if typ is NotPython and not cu.should_be_python():
continue
test_id = cu.name
details = convert_error_to_string(err)
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testFailed(
test_id,
message="Coverage analysis failed",
details=details,
flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
if total.n_files > 0:
covered = total.n_executed + (total.n_executed_branches
if self.branches else 0)
total_statements = total.n_statements + (
total.n_branches if self.branches else 0)
self.messages.buildStatisticLinesCovered(covered)
self.messages.buildStatisticTotalLines(total_statements)
self.messages.buildStatisticLinesUncovered(
total_statements - covered)
reporter = _CoverageReporter(
self.coverage_controller.cov,
self.coverage_controller.cov.config,
self.teamcity,
)
reporter.report(None)
class TeamcityReport(object):
name = 'teamcity-report'
score = 10000
def __init__(self):
super(TeamcityReport, self).__init__()
self.messages = TeamcityServiceMessages(_real_stdout)
self.test_started_datetime_map = {}
self.enabled = False
def get_test_id(self, test):
if is_string(test):
return test
# Handle special "tests"
test_class_name = get_class_fullname(test)
if test_class_name == CONTEXT_SUITE_FQN:
if inspect.ismodule(test.context):
module_name = test.context.__name__
return module_name + "." + test.error_context
elif inspect.isclass(test.context):
class_name = get_class_fullname(test.context)
return class_name + "." + test.error_context
test_id = test.id()
real_test = getattr(test, "test", test)
real_test_class_name = get_class_fullname(real_test)
test_arg = getattr(real_test, "arg", tuple())
if (type(test_arg) is tuple
or type(test_arg) is list) and len(test_arg) > 0:
# As written in nose.case.FunctionTestCase#__str__ or nose.case.MethodTestCase#__str__
test_arg_str = "%s" % (test_arg, )
if test_id.endswith(test_arg_str):
# Replace '.' in test args with '_' to preserve test hierarchy on TeamCity
test_id = test_id[:len(test_id) -
len(test_arg_str)] + test_arg_str.replace(
'.', '_')
# Force test_id for doctests
if real_test_class_name != "doctest.DocTestCase" and real_test_class_name != "nose.plugins.doctests.DocTestCase":
desc = test.shortDescription()
if desc and desc != test.id():
return "%s (%s)" % (test_id, desc.replace('.', '_'))
return test_id
def configure(self, options, conf):
self.enabled = is_running_under_teamcity()
def options(self, parser, env=os.environ):
pass
def report_fail(self, test, fail_type, err):
# workaround nose bug on python 3
if is_string(err[1]):
err = (err[0], Exception(err[1]), err[2])
test_id = self.get_test_id(test)
details = convert_error_to_string(err)
start_index = details.find(_captured_output_start_marker)
end_index = details.find(_captured_output_end_marker)
if 0 <= start_index < end_index:
captured_output = details[start_index +
len(_captured_output_start_marker
):end_index]
details = details[:start_index] + details[
end_index + len(_captured_output_end_marker):]
for chunk in split_output(limit_output(captured_output)):
self.messages.testStdOut(test_id, chunk, flowId=test_id)
self.messages.testFailed(test_id,
message=fail_type,
details=details,
flowId=test_id)
def addError(self, test, err):
test_class_name = get_class_fullname(test)
test_id = self.get_test_id(test)
if issubclass(err[0], SkipTest):
self.messages.testIgnored(test_id,
message="Skipped",
flowId=test_id)
elif issubclass(err[0], DeprecatedTest):
self.messages.testIgnored(test_id,
message="Deprecated",
flowId=test_id)
elif test_class_name == CONTEXT_SUITE_FQN:
self.messages.testStarted(test_id,
captureStandardOutput='true',
flowId=test_id)
self.report_fail(test,
'error in ' + test.error_context + ' context',
err)
self.messages.testFinished(test_id, flowId=test_id)
else:
self.report_fail(test, 'Error', err)
def addFailure(self, test, err):
self.report_fail(test, 'Failure', err)
def startTest(self, test):
test_id = self.get_test_id(test)
self.test_started_datetime_map[test_id] = datetime.datetime.now()
self.messages.testStarted(test_id,
captureStandardOutput='true',
flowId=test_id)
def stopTest(self, test):
test_id = self.get_test_id(test)
time_diff = datetime.datetime.now(
) - self.test_started_datetime_map[test_id]
self.messages.testFinished(test_id,
testDuration=time_diff,
flowId=test_id)
class EchoTeamCityMessages(object):
def __init__(self, output_capture_enabled, coverage_controller):
self.coverage_controller = coverage_controller
self.output_capture_enabled = output_capture_enabled
self.teamcity = TeamcityServiceMessages()
self.test_start_reported_mark = set()
self.max_reported_output_size = 1 * 1024 * 1024
self.reported_output_chunk_size = 50000
def get_id_from_location(self, location):
if type(location) is not tuple or len(location) != 3 or not hasattr(
location[2], "startswith"):
return None
def convert_file_to_id(filename):
filename = re.sub(r"\.pyc?$", "", filename)
return filename.replace(os.sep, ".").replace("/", ".")
def add_prefix_to_filename_id(filename_id, prefix):
dot_location = filename_id.rfind('.')
if dot_location <= 0 or dot_location >= len(filename_id) - 1:
return None
return filename_id[:dot_location +
1] + prefix + filename_id[dot_location + 1:]
pylint_prefix = '[pylint] '
if location[2].startswith(pylint_prefix):
id_from_file = convert_file_to_id(location[2][len(pylint_prefix):])
return id_from_file + ".Pylint"
if location[2] == "PEP8-check":
id_from_file = convert_file_to_id(location[0])
return id_from_file + ".PEP8"
return None
def format_test_id(self, nodeid, location):
id_from_location = self.get_id_from_location(location)
if id_from_location is not None:
return id_from_location
test_id = nodeid
if test_id:
if test_id.find("::") < 0:
test_id += "::top_level"
else:
test_id = "top_level"
first_bracket = test_id.find("[")
if first_bracket > 0:
# [] -> (), make it look like nose parameterized tests
params = "(" + test_id[first_bracket + 1:]
if params.endswith("]"):
params = params[:-1] + ")"
test_id = test_id[:first_bracket]
if test_id.endswith("::"):
test_id = test_id[:-2]
else:
params = ""
test_id = test_id.replace("::()::", "::")
test_id = re.sub(r"\.pyc?::", r"::", test_id)
test_id = test_id.replace(".", "_").replace(os.sep, ".").replace(
"/", ".").replace('::', '.')
if params:
params = params.replace(".", "_")
test_id += params
return test_id
def format_location(self, location):
if type(location) is tuple and len(location) == 3:
return "%s:%s (%s)" % (str(location[0]), str(
location[1]), str(location[2]))
return str(location)
def pytest_collection_modifyitems(self, session, config, items):
self.teamcity.testCount(len(items))
def pytest_runtest_logstart(self, nodeid, location):
self.ensure_test_start_reported(self.format_test_id(nodeid, location))
def ensure_test_start_reported(self, test_id):
if test_id not in self.test_start_reported_mark:
if self.output_capture_enabled:
capture_standard_output = "false"
else:
capture_standard_output = "true"
self.teamcity.testStarted(
test_id,
flowId=test_id,
captureStandardOutput=capture_standard_output)
self.test_start_reported_mark.add(test_id)
def report_has_output(self, report):
for (secname, data) in report.sections:
if report.when in secname and ('stdout' in secname
or 'stderr' in secname):
return True
return False
def report_test_output(self, report, test_id):
for (secname, data) in report.sections:
# https://github.com/JetBrains/teamcity-messages/issues/112
# CollectReport doesn't have 'when' property
if hasattr(report, "when") and report.when not in secname:
continue
if not data:
continue
if 'stdout' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdOut(test_id,
out=chunk,
flowId=test_id)
elif 'stderr' in secname:
for chunk in split_output(limit_output(data)):
self.teamcity.testStdErr(test_id,
out=chunk,
flowId=test_id)
def report_test_finished(self, test_id, duration=None):
self.teamcity.testFinished(test_id,
testDuration=duration,
flowId=test_id)
self.test_start_reported_mark.remove(test_id)
def report_test_failure(self,
test_id,
report,
message=None,
report_output=True):
if hasattr(report, 'duration'):
duration = timedelta(seconds=report.duration)
else:
duration = None
if message is None:
message = self.format_location(report.location)
self.ensure_test_start_reported(test_id)
if report_output:
self.report_test_output(report, test_id)
self.teamcity.testFailed(test_id,
message,
str(report.longrepr),
flowId=test_id)
self.report_test_finished(test_id, duration)
def report_test_skip(self, test_id, report):
if type(report.longrepr) is tuple and len(report.longrepr) == 3:
reason = report.longrepr[2]
else:
reason = str(report.longrepr)
if hasattr(report, 'duration'):
duration = timedelta(seconds=report.duration)
else:
duration = None
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id)
self.teamcity.testIgnored(test_id, reason, flowId=test_id)
self.report_test_finished(test_id, duration)
def pytest_runtest_logreport(self, report):
"""
:type report: _pytest.runner.TestReport
"""
test_id = self.format_test_id(report.nodeid, report.location)
duration = timedelta(seconds=report.duration)
if report.passed:
# Do not report passed setup/teardown if no output
if report.when == 'call':
self.ensure_test_start_reported(test_id)
self.report_test_output(report, test_id)
self.report_test_finished(test_id, duration)
else:
if self.report_has_output(report):
block_name = "test " + report.when
self.teamcity.blockOpened(block_name, flowId=test_id)
self.report_test_output(report, test_id)
self.teamcity.blockClosed(block_name, flowId=test_id)
elif report.failed:
if report.when == 'call':
self.report_test_failure(test_id, report)
elif report.when == 'setup':
if self.report_has_output(report):
self.teamcity.blockOpened("test setup", flowId=test_id)
self.report_test_output(report, test_id)
self.teamcity.blockClosed("test setup", flowId=test_id)
self.report_test_failure(test_id,
report,
message="test setup failed",
report_output=False)
elif report.when == 'teardown':
# Report failed teardown as a separate test as original test is already finished
self.report_test_failure(test_id + "_teardown", report)
elif report.skipped:
self.report_test_skip(test_id, report)
def pytest_collectreport(self, report):
test_id = self.format_test_id(report.nodeid,
report.location) + "_collect"
if report.failed:
self.report_test_failure(test_id, report)
elif report.skipped:
self.report_test_skip(test_id, report)
def pytest_terminal_summary(self):
if self.coverage_controller is not None:
try:
self._report_coverage()
except:
tb = traceback.format_exc()
self.teamcity.customMessage(
"Coverage statistics reporting failed",
"ERROR",
errorDetails=tb)
def _report_coverage(self):
from coverage.misc import NotPython
from coverage.report import Reporter
from coverage.results import Numbers
class _CoverageReporter(Reporter):
def __init__(self, coverage, config, messages):
super(_CoverageReporter, self).__init__(coverage, config)
self.branches = coverage.data.has_arcs()
self.messages = messages
def report(self, morfs, outfile=None):
if hasattr(self, 'find_code_units'):
self.find_code_units(morfs)
else:
self.find_file_reporters(morfs)
total = Numbers()
if hasattr(self, 'code_units'):
units = self.code_units
else:
units = self.file_reporters
for cu in units:
try:
analysis = self.coverage._analyze(cu)
nums = analysis.numbers
total += nums
except KeyboardInterrupt:
raise
except:
if self.config.ignore_errors:
continue
err = sys.exc_info()
typ, msg = err[:2]
if typ is NotPython and not cu.should_be_python():
continue
test_id = cu.name
details = convert_error_to_string(err)
self.messages.testStarted(test_id, flowId=test_id)
self.messages.testFailed(
test_id,
message="Coverage analysis failed",
details=details,
flowId=test_id)
self.messages.testFinished(test_id, flowId=test_id)
if total.n_files > 0:
covered = total.n_executed
total_statements = total.n_statements
if self.branches:
covered += total.n_executed_branches
total_statements += total.n_branches
self.messages.buildStatisticLinesCovered(covered)
self.messages.buildStatisticTotalLines(total_statements)
self.messages.buildStatisticLinesUncovered(
total_statements - covered)
reporter = _CoverageReporter(
self.coverage_controller.cov,
self.coverage_controller.cov.config,
self.teamcity,
)
reporter.report(None)
