def test_artifacts_merged(self):
test_names = [ 'foo_test.FooTest.foobar' ]
result_set = json_results.ResultSet()
ar = FakeArtifacts()
ar.artifacts = {'artifact_name': ['a/b/c.txt']}
result_set.add(json_results.Result(
'foo_test.FooTest.foobar', json_results.ResultType.Failure,
0, 0.2, 0, artifacts=ar.artifacts))
ar2 = FakeArtifacts()
ar2.artifacts = {'artifact_name': ['d/e/f.txt']}
result_set.add(json_results.Result(
'foo_test.FooTest.foobar', json_results.ResultType.Failure,
0, 0.2, 0, artifacts=ar2.artifacts))
full_results = json_results.make_full_results(
{'foo': 'bar'}, 0, test_names, result_set)
tests = full_results['tests']['foo_test']['FooTest']
self.assertIn('artifacts', tests['foobar'])
# Order is not guaranteed for the list of paths, so don't use
# assertEqual if there are multiple entries
artifacts = tests['foobar']['artifacts']
self.assertIn('artifact_name', artifacts)
self.assertEqual(len(artifacts['artifact_name']), 2)
self.assertIn('a/b/c.txt', artifacts['artifact_name'])
self.assertIn('d/e/f.txt', artifacts['artifact_name'])
def _run_tests(self, result_set, test_set, all_tests):
h = self.host
self.last_runs_retry_on_failure_tests = set()
def get_tests_to_retry(results):
# If the --retry-only-retry-on-failure-tests command line argument
# is passed , then a set of test failures with the RetryOnFailure
# expectation from the last run of tests will be returned. The
# self.last_runs_retry_on_failure_tests will be set to an empty set
# for the next run of tests. Otherwise all regressions from the
# last run will be returned.
if self.args.retry_only_retry_on_failure_tests:
ret = self.last_runs_retry_on_failure_tests.copy()
self.last_runs_retry_on_failure_tests = set()
return ret
else:
return json_results.regressions(results)
self._run_one_set(self.stats, result_set, test_set)
tests_to_retry = sorted(get_tests_to_retry(result_set))
retry_limit = self.args.retry_limit
while retry_limit and tests_to_retry:
if retry_limit == self.args.retry_limit:
self.flush()
self.args.overwrite = False
self.printer.should_overwrite = False
self.args.verbose = min(self.args.verbose, 1)
self.print_('')
self.print_('Retrying failed tests (attempt #%d of %d)...' %
(self.args.retry_limit - retry_limit + 1,
self.args.retry_limit))
self.print_('')
stats = Stats(self.args.status_format, h.time, 1)
stats.total = len(tests_to_retry)
test_set = TestSet(self.args.test_name_prefix)
for name in tests_to_retry:
test_set.add_test_to_run_isolated(
list(
self.loader.loadTestsFromName(
self.args.test_name_prefix + name))[0])
tests_to_retry = test_set
retry_set = ResultSet()
self._run_one_set(stats, retry_set, tests_to_retry)
result_set.results.extend(retry_set.results)
tests_to_retry = get_tests_to_retry(retry_set)
retry_limit -= 1
if retry_limit != self.args.retry_limit:
self.print_('')
full_results = json_results.make_full_results(self.args.metadata,
int(h.time()), all_tests,
result_set)
return (json_results.exit_code_from_full_results(full_results),
full_results)
def _run_tests(self, result_set, test_set, all_tests):
h = self.host
self.last_runs_retry_on_failure_tests = set()
def get_tests_to_retry(results):
# If the --retry-only-retry-on-failure-tests command line argument
# is passed , then a set of test failures with the RetryOnFailure
# expectation from the last run of tests will be returned. The
# self.last_runs_retry_on_failure_tests will be set to an empty set
# for the next run of tests. Otherwise all regressions from the
# last run will be returned.
if self.args.retry_only_retry_on_failure_tests:
ret = self.last_runs_retry_on_failure_tests.copy()
self.last_runs_retry_on_failure_tests = set()
return ret
else:
return json_results.regressions(results)
self._run_one_set(self.stats, result_set, test_set)
tests_to_retry = sorted(get_tests_to_retry(result_set))
retry_limit = self.args.retry_limit
while retry_limit and tests_to_retry:
if retry_limit == self.args.retry_limit:
self.flush()
self.args.overwrite = False
self.printer.should_overwrite = False
self.args.verbose = min(self.args.verbose, 1)
self.print_('')
self.print_('Retrying failed tests (attempt #%d of %d)...' %
(self.args.retry_limit - retry_limit + 1,
self.args.retry_limit))
self.print_('')
stats = Stats(self.args.status_format, h.time, 1)
stats.total = len(tests_to_retry)
test_set = TestSet(self.args.test_name_prefix)
test_set.isolated_tests = [
TestInput(name) for name in tests_to_retry]
tests_to_retry = test_set
retry_set = ResultSet()
self._run_one_set(stats, retry_set, tests_to_retry)
result_set.results.extend(retry_set.results)
tests_to_retry = get_tests_to_retry(retry_set)
retry_limit -= 1
if retry_limit != self.args.retry_limit:
self.print_('')
full_results = json_results.make_full_results(self.metadata,
int(h.time()),
all_tests, result_set,
self.path_delimiter)
return (json_results.exit_code_from_full_results(full_results),
full_results)
def test_basic(self):
test_names = ['foo_test.FooTest.test_fail',
'foo_test.FooTest.test_pass',
'foo_test.FooTest.test_skip']
result_set = json_results.ResultSet()
result_set.add(
json_results.Result('foo_test.FooTest.test_fail',
json_results.ResultType.Failure, 0, 0.1, 0,
unexpected=True))
result_set.add(json_results.Result('foo_test.FooTest.test_pass',
json_results.ResultType.Pass,
0, 0.2, 0))
result_set.add(json_results.Result('foo_test.FooTest.test_skip',
json_results.ResultType.Skip,
0, 0.3, 0,
expected=[json_results.ResultType.Skip],
unexpected=False))
full_results = json_results.make_full_results(
{'foo': 'bar'}, 0, test_names, result_set)
expected_full_results = {
'foo': 'bar',
'metadata': {
'foo': 'bar'},
'interrupted': False,
'num_failures_by_type': {
'FAIL': 1,
'PASS': 1,
'SKIP': 1},
'num_regressions': 1,
'path_delimiter': '.',
'seconds_since_epoch': 0,
'tests': {
'foo_test': {
'FooTest': {
'test_fail': {
'expected': 'PASS',
'actual': 'FAIL',
'times': [0.1],
'is_unexpected': True,
'is_regression': True,
},
'test_pass': {
'expected': 'PASS',
'actual': 'PASS',
'times': [0.2],
},
'test_skip': {
'expected': 'SKIP',
'actual': 'SKIP',
'times': [0.3],
}}}},
'version': 3}
self.assertEqual(full_results, expected_full_results)
def test_basic(self):
test_names = ['foo_test.FooTest.test_fail',
'foo_test.FooTest.test_pass',
'foo_test.FooTest.test_skip']
result_set = json_results.ResultSet()
result_set.add(
json_results.Result('foo_test.FooTest.test_fail',
json_results.ResultType.Failure, 0, 0.1, 0,
unexpected=True))
result_set.add(json_results.Result('foo_test.FooTest.test_pass',
json_results.ResultType.Pass,
0, 0.2, 0))
result_set.add(json_results.Result('foo_test.FooTest.test_skip',
json_results.ResultType.Skip,
0, 0.3, 0,
expected=[json_results.ResultType.Skip],
unexpected=False))
full_results = json_results.make_full_results(
{'foo': 'bar'}, 0, test_names, result_set)
expected_full_results = {
'foo': 'bar',
'metadata': {
'foo': 'bar'},
'interrupted': False,
'num_failures_by_type': {
'FAIL': 1,
'PASS': 1,
'SKIP': 1},
'num_regressions': 1,
'path_delimiter': '.',
'seconds_since_epoch': 0,
'tests': {
'foo_test': {
'FooTest': {
'test_fail': {
'expected': 'PASS',
'actual': 'FAIL',
'times': [0.1],
'is_unexpected': True,
'is_regression': True,
},
'test_pass': {
'expected': 'PASS',
'actual': 'PASS',
'times': [0.2],
},
'test_skip': {
'expected': 'SKIP',
'actual': 'SKIP',
'times': [0.3],
}}}},
'version': 3}
self.assertEqual(full_results, expected_full_results)
def test_unknown_result(self):
test_names = ['foo_test.FooTest.test_unknown_result']
result_set = json_results.ResultSet()
result_set.add(
json_results.Result('foo_test.FooTest.test_unknown_result',
'UNKNOWN', 0, 0.2, 0))
with self.assertRaises(ValueError) as ctx:
full_results = json_results.make_full_results({'foo': 'bar'}, 0,
test_names,
result_set)
self.assertIn('UNKNOWN', str(ctx.exception))
def _run_tests(self, result_set, test_set):
h = self.host
if not test_set.parallel_tests and not test_set.isolated_tests:
self.print_('No tests to run.')
return 1, None
all_tests = [
ti.name for ti in _sort_inputs(test_set.parallel_tests +
test_set.isolated_tests +
test_set.tests_to_skip)
]
if self.args.list_only:
self.print_('\n'.join(all_tests))
return 0, None
self._run_one_set(self.stats, result_set, test_set)
failed_tests = sorted(json_results.failed_test_names(result_set))
retry_limit = self.args.retry_limit
while retry_limit and failed_tests:
if retry_limit == self.args.retry_limit:
self.flush()
self.args.overwrite = False
self.printer.should_overwrite = False
self.args.verbose = min(self.args.verbose, 1)
self.print_('')
self.print_('Retrying failed tests (attempt #%d of %d)...' %
(self.args.retry_limit - retry_limit + 1,
self.args.retry_limit))
self.print_('')
stats = Stats(self.args.status_format, h.time, 1)
stats.total = len(failed_tests)
tests_to_retry = TestSet(isolated_tests=list(failed_tests))
retry_set = ResultSet()
self._run_one_set(stats, retry_set, tests_to_retry)
result_set.results.extend(retry_set.results)
failed_tests = json_results.failed_test_names(retry_set)
retry_limit -= 1
if retry_limit != self.args.retry_limit:
self.print_('')
full_results = json_results.make_full_results(self.args.metadata,
int(h.time()), all_tests,
result_set)
return (json_results.exit_code_from_full_results(full_results),
full_results)
def _run_tests(self, result_set, test_set):
h = self.host
if not test_set.parallel_tests and not test_set.isolated_tests:
self.print_('No tests to run.')
return 1, None
all_tests = [ti.name for ti in
_sort_inputs(test_set.parallel_tests +
test_set.isolated_tests +
test_set.tests_to_skip)]
if self.args.list_only:
self.print_('\n'.join(all_tests))
return 0, None
self._run_one_set(self.stats, result_set, test_set)
failed_tests = sorted(json_results.failed_test_names(result_set))
retry_limit = self.args.retry_limit
while retry_limit and failed_tests:
if retry_limit == self.args.retry_limit:
self.flush()
self.args.overwrite = False
self.printer.should_overwrite = False
self.args.verbose = min(self.args.verbose, 1)
self.print_('')
self.print_('Retrying failed tests (attempt #%d of %d)...' %
(self.args.retry_limit - retry_limit + 1,
self.args.retry_limit))
self.print_('')
stats = Stats(self.args.status_format, h.time, 1)
stats.total = len(failed_tests)
tests_to_retry = TestSet(isolated_tests=list(failed_tests))
retry_set = ResultSet()
self._run_one_set(stats, retry_set, tests_to_retry)
result_set.results.extend(retry_set.results)
failed_tests = json_results.failed_test_names(retry_set)
retry_limit -= 1
if retry_limit != self.args.retry_limit:
self.print_('')
full_results = json_results.make_full_results(self.args.metadata,
int(h.time()),
all_tests, result_set)
return (json_results.exit_code_from_full_results(full_results),
full_results)
def test_artifacts_and_types_added(self):
ar = FakeArtifacts()
ar.artifacts = {'artifact_name': ['a/b/c.txt']}
test_names = [ 'foo_test.FooTest.foobar' ]
result_set = json_results.ResultSet()
result_set.add(json_results.Result(
'foo_test.FooTest.foobar', json_results.ResultType.Pass,
0, 0.2, 0, artifacts=ar.artifacts))
full_results = json_results.make_full_results(
{'foo': 'bar'}, 0, test_names, result_set)
tests = full_results['tests']['foo_test']['FooTest']
self.assertIn('artifacts', tests['foobar'])
self.assertEqual(tests['foobar']['artifacts'],
{'artifact_name': ['a/b/c.txt']})
def _run_tests(self, result_set, test_set, all_tests):
h = self.host
self._run_one_set(self.stats, result_set, test_set)
failed_tests = sorted(json_results.failed_test_names(result_set))
retry_limit = self.args.retry_limit
while retry_limit and failed_tests:
if retry_limit == self.args.retry_limit:
self.flush()
self.args.overwrite = False
self.printer.should_overwrite = False
self.args.verbose = min(self.args.verbose, 1)
self.print_('')
self.print_('Retrying failed tests (attempt #%d of %d)...' %
(self.args.retry_limit - retry_limit + 1,
self.args.retry_limit))
self.print_('')
stats = Stats(self.args.status_format, h.time, 1)
stats.total = len(failed_tests)
tests_to_retry = TestSet(isolated_tests=list(failed_tests))
retry_set = ResultSet()
self._run_one_set(stats, retry_set, tests_to_retry)
result_set.results.extend(retry_set.results)
failed_tests = json_results.failed_test_names(retry_set)
retry_limit -= 1
if retry_limit != self.args.retry_limit:
self.print_('')
full_results = json_results.make_full_results(self.args.metadata,
int(h.time()), all_tests,
result_set)
return (json_results.exit_code_from_full_results(full_results),
full_results)
def test_basic_upload(self):
results = json_results.ResultSet()
full_results = json_results.make_full_results({}, 0, [], results)
url, content_type, data = json_results.make_upload_request(
'localhost', 'fake_builder_name', 'fake_master', 'fake_test_type',
full_results)
self.assertEqual(
content_type, 'multipart/form-data; '
'boundary=-J-S-O-N-R-E-S-U-L-T-S---B-O-U-N-D-A-R-Y-')
self.assertEqual(url, 'https://localhost/testfile/upload')
self.assertMultiLineEqual(
data,
('---J-S-O-N-R-E-S-U-L-T-S---B-O-U-N-D-A-R-Y-\r\n'
'Content-Disposition: form-data; name="builder"\r\n'
'\r\n'
'fake_builder_name\r\n'
'---J-S-O-N-R-E-S-U-L-T-S---B-O-U-N-D-A-R-Y-\r\n'
'Content-Disposition: form-data; name="master"\r\n'
'\r\n'
'fake_master\r\n'
'---J-S-O-N-R-E-S-U-L-T-S---B-O-U-N-D-A-R-Y-\r\n'
'Content-Disposition: form-data; name="testtype"\r\n'
'\r\n'
'fake_test_type\r\n'
'---J-S-O-N-R-E-S-U-L-T-S---B-O-U-N-D-A-R-Y-\r\n'
'Content-Disposition: form-data; name="file"; '
'filename="full_results.json"\r\n'
'Content-Type: application/json\r\n'
'\r\n'
'{"version": 3, "interrupted": false, "path_delimiter": ".", '
'"seconds_since_epoch": 0, '
'"num_failures_by_type": {"FAIL": 0, "TIMEOUT": 0, "CRASH": 0,'
' "PASS": 0, "SKIP": 0}, '
'"num_regressions": 0, '
'"tests": {}}\r\n'
'---J-S-O-N-R-E-S-U-L-T-S---B-O-U-N-D-A-R-Y---\r\n'))
def test_basic_upload(self):
results = json_results.ResultSet()
full_results = json_results.make_full_results({}, 0, [], results)
url, content_type, data = json_results.make_upload_request(
'localhost', 'fake_builder_name', 'fake_master', 'fake_test_type',
full_results)
self.assertEqual(
content_type,
'multipart/form-data; '
'boundary=-J-S-O-N-R-E-S-U-L-T-S---B-O-U-N-D-A-R-Y-')
self.assertEqual(url, 'https://localhost/testfile/upload')
self.assertMultiLineEqual(
data,
('---J-S-O-N-R-E-S-U-L-T-S---B-O-U-N-D-A-R-Y-\r\n'
'Content-Disposition: form-data; name="builder"\r\n'
'\r\n'
'fake_builder_name\r\n'
'---J-S-O-N-R-E-S-U-L-T-S---B-O-U-N-D-A-R-Y-\r\n'
'Content-Disposition: form-data; name="master"\r\n'
'\r\n'
'fake_master\r\n'
'---J-S-O-N-R-E-S-U-L-T-S---B-O-U-N-D-A-R-Y-\r\n'
'Content-Disposition: form-data; name="testtype"\r\n'
'\r\n'
'fake_test_type\r\n'
'---J-S-O-N-R-E-S-U-L-T-S---B-O-U-N-D-A-R-Y-\r\n'
'Content-Disposition: form-data; name="file"; '
'filename="full_results.json"\r\n'
'Content-Type: application/json\r\n'
'\r\n'
'{"version": 3, "interrupted": false, "path_delimiter": ".", '
'"seconds_since_epoch": 0, '
'"num_failures_by_type": {"FAIL": 0, "PASS": 0, "SKIP": 0}, '
'"num_regressions": 0, '
'"tests": {}}\r\n'
'---J-S-O-N-R-E-S-U-L-T-S---B-O-U-N-D-A-R-Y---\r\n'))
def _run_tests(self, result_set, test_set, all_tests):
h = self.host
self.last_runs_retry_on_failure_tests = set()
def get_tests_to_retry(results):
# If the --retry-only-retry-on-failure-tests command line argument
# is passed , then a set of test failures with the RetryOnFailure
# expectation from the last run of tests will be returned. The
# self.last_runs_retry_on_failure_tests will be set to an empty set
# for the next run of tests. Otherwise all regressions from the
# last run will be returned.
if self.args.retry_only_retry_on_failure_tests:
ret = self.last_runs_retry_on_failure_tests.copy()
self.last_runs_retry_on_failure_tests = set()
return ret
else:
return json_results.regressions(results)
if len(test_set.parallel_tests):
jobs = min(
len(test_set.parallel_tests), self.args.jobs)
else:
jobs = 1
child = _Child(self)
pool = make_pool(h, jobs, _run_one_test, child,
_setup_process, _teardown_process)
self._run_one_set(self.stats, result_set, test_set, jobs, pool)
tests_to_retry = sorted(get_tests_to_retry(result_set))
retry_limit = self.args.retry_limit
try:
# Start at 1 since we already did iteration 0 above.
for iteration in range(1, self.args.retry_limit + 1):
if not tests_to_retry:
break
if retry_limit == self.args.retry_limit:
self.flush()
self.args.overwrite = False
self.printer.should_overwrite = False
self.args.verbose = min(self.args.verbose, 1)
self.print_('')
self.print_('Retrying failed tests (attempt #%d of %d)...' %
(iteration, self.args.retry_limit))
self.print_('')
stats = Stats(self.args.status_format, h.time, 1)
stats.total = len(tests_to_retry)
test_set = TestSet(self.args.test_name_prefix)
test_set.isolated_tests = [
TestInput(name,
iteration=iteration) for name in tests_to_retry]
tests_to_retry = test_set
retry_set = ResultSet()
self._run_one_set(stats, retry_set, tests_to_retry, 1, pool)
result_set.results.extend(retry_set.results)
tests_to_retry = get_tests_to_retry(retry_set)
retry_limit -= 1
pool.close()
finally:
self.final_responses.extend(pool.join())
if retry_limit != self.args.retry_limit:
self.print_('')
full_results = json_results.make_full_results(self.metadata,
int(h.time()),
all_tests, result_set,
self.path_delimiter)
retcode = (json_results.exit_code_from_full_results(full_results)
| result_sink.result_sink_retcode_from_result_set(result_set))
return (retcode, full_results)
