def testEmptyInput(self):
"""Tests that nothing blows up with empty input."""
stale_dict, semi_stale_dict, active_dict =\
expectations.SplitExpectationsByStaleness({})
self.assertEqual(stale_dict, {})
self.assertEqual(semi_stale_dict, {})
self.assertEqual(active_dict, {})
def testActiveExpectations(self):
"""Tests output when only active expectations are provided."""
expectation_map = {
'foo': {
data_types.Expectation('foo', ['win'], ['Failure']): {
'foo_builder': {
'step1': uu.CreateStatsWithPassFails(0, 1),
'step2': uu.CreateStatsWithPassFails(0, 2),
},
'bar_builder': {
'step1': uu.CreateStatsWithPassFails(0, 3),
'step2': uu.CreateStatsWithPassFails(0, 4)
},
},
data_types.Expectation('foo', ['linux'], ['RetryOnFailure']): {
'foo_builder': {
'step1': uu.CreateStatsWithPassFails(0, 5),
'step2': uu.CreateStatsWithPassFails(0, 6),
},
},
},
'bar': {
data_types.Expectation('bar', ['win'], ['Failure']): {
'foo_builder': {
'step1': uu.CreateStatsWithPassFails(0, 7),
},
},
},
}
expected_active_dict = copy.deepcopy(expectation_map)
stale_dict, semi_stale_dict, active_dict =\
expectations.SplitExpectationsByStaleness(expectation_map)
self.assertEqual(stale_dict, {})
self.assertEqual(semi_stale_dict, {})
self.assertEqual(active_dict, expected_active_dict)
def testOutputResultsSmoketest(self):
"""Test that nothing blows up when outputting."""
expectation_map = {
'foo': {
data_types.Expectation('foo', ['win', 'intel'], 'RetryOnFailure'):
{
'stale': {
'all_pass': uu.CreateStatsWithPassFails(2, 0),
},
},
data_types.Expectation('foo', ['linux'], 'Failure'): {
'semi_stale': {
'all_pass': uu.CreateStatsWithPassFails(2, 0),
'some_pass': uu.CreateStatsWithPassFails(1, 1),
'none_pass': uu.CreateStatsWithPassFails(0, 2),
},
},
data_types.Expectation('foo', ['mac'], 'Failure'): {
'active': {
'none_pass': uu.CreateStatsWithPassFails(0, 2),
},
},
},
}
unmatched_results = {
'builder': [
data_types.Result('foo', ['win', 'intel'], 'Failure',
'step_name', 'build_id'),
],
}
unmatched_expectations = [
data_types.Expectation('foo', ['linux'], 'RetryOnFailure')
]
stale, semi_stale, active = expectations.SplitExpectationsByStaleness(
expectation_map)
result_output.OutputResults(stale, semi_stale, active, {}, [], 'print',
self._file_handle)
result_output.OutputResults(stale, semi_stale, active,
unmatched_results, [], 'print',
self._file_handle)
result_output.OutputResults(stale, semi_stale, active, {},
unmatched_expectations, 'print',
self._file_handle)
result_output.OutputResults(stale, semi_stale, active,
unmatched_results, unmatched_expectations,
'print', self._file_handle)
result_output.OutputResults(stale, semi_stale, active, {}, [], 'html',
self._file_handle)
result_output.OutputResults(stale, semi_stale, active,
unmatched_results, [], 'html',
self._file_handle)
result_output.OutputResults(stale, semi_stale, active, {},
unmatched_expectations, 'html',
self._file_handle)
result_output.OutputResults(stale, semi_stale, active,
unmatched_results, unmatched_expectations,
'html', self._file_handle)
def testSemiStaleExpectations(self):
"""Tests output when only semi-stale expectations are provided."""
expectation_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['win'], ['Failure']):
data_types.BuilderStepMap({
'foo_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(1, 0),
'step2':
uu.CreateStatsWithPassFails(2, 2),
}),
'bar_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(3, 0),
'step2':
uu.CreateStatsWithPassFails(0, 4)
}),
}),
data_types.Expectation('foo', ['linux'], ['RetryOnFailure']):
data_types.BuilderStepMap({
'foo_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(5, 0),
'step2':
uu.CreateStatsWithPassFails(6, 6),
}),
}),
}),
'bar':
data_types.ExpectationBuilderMap({
data_types.Expectation('bar', ['win'], ['Failure']):
data_types.BuilderStepMap({
'foo_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(7, 0),
}),
'bar_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(0, 8),
}),
}),
}),
})
expected_semi_stale_dict = copy.deepcopy(expectation_map)
stale_dict, semi_stale_dict, active_dict =\
expectations.SplitExpectationsByStaleness(expectation_map)
self.assertEqual(stale_dict, {})
self.assertEqual(semi_stale_dict, expected_semi_stale_dict)
self.assertEqual(active_dict, {})
def testEmptyInput(self):
"""Tests that nothing blows up with empty input."""
stale_dict, semi_stale_dict, active_dict =\
expectations.SplitExpectationsByStaleness(
data_types.TestExpectationMap())
self.assertEqual(stale_dict, {})
self.assertEqual(semi_stale_dict, {})
self.assertEqual(active_dict, {})
self.assertIsInstance(stale_dict, data_types.TestExpectationMap)
self.assertIsInstance(semi_stale_dict, data_types.TestExpectationMap)
self.assertIsInstance(active_dict, data_types.TestExpectationMap)
def main():
args = ParseArgs()
test_expectation_map = expectations.CreateTestExpectationMap(
args.expectation_file, args.tests)
ci_builders = builders.GetCiBuilders(
SUITE_TO_TELEMETRY_SUITE_MAP.get(args.suite, args.suite))
querier = queries.BigQueryQuerier(args.suite, args.project,
args.num_samples, args.large_query_mode)
# Unmatched results are mainly useful for script maintainers, as they don't
# provide any additional information for the purposes of finding unexpectedly
# passing tests or unused expectations.
unmatched = querier.FillExpectationMapForCiBuilders(
test_expectation_map, ci_builders)
try_builders = builders.GetTryBuilders(ci_builders)
unmatched.update(
querier.FillExpectationMapForTryBuilders(test_expectation_map,
try_builders))
unused_expectations = expectations.FilterOutUnusedExpectations(
test_expectation_map)
stale, semi_stale, active = expectations.SplitExpectationsByStaleness(
test_expectation_map)
result_output.OutputResults(stale, semi_stale, active, unmatched,
unused_expectations, args.output_format)
affected_urls = set()
stale_message = ''
if args.remove_stale_expectations:
stale_expectations = []
for _, expectation_map in stale.iteritems():
stale_expectations.extend(expectation_map.keys())
stale_expectations.extend(unused_expectations)
affected_urls |= expectations.RemoveExpectationsFromFile(
stale_expectations, args.expectation_file)
stale_message += (
'Stale expectations removed from %s. Stale comments, '
'etc. may still need to be removed.\n' % args.expectation_file)
if args.modify_semi_stale_expectations:
affected_urls |= expectations.ModifySemiStaleExpectations(
semi_stale, args.expectation_file)
stale_message += ('Semi-stale expectations modified in %s. Stale '
'comments, etc. may still need to be removed.\n' %
args.expectation_file)
if stale_message:
print(stale_message)
if affected_urls:
result_output.OutputAffectedUrls(affected_urls)
def main():
args = ParseArgs()
# TODO(crbug.com/1108016): Remove this warning once ResultDB is enabled on all
# builders and there is enough data for the results to be trusted.
WarnUserOfIncompleteRollout()
test_expectation_map = expectations.CreateTestExpectationMap(
args.expectation_file, args.tests)
ci_builders = builders.GetCiBuilders(
SUITE_TO_TELEMETRY_SUITE_MAP.get(args.suite, args.suite))
# Unmatched results are mainly useful for script maintainers, as they don't
# provide any additional information for the purposes of finding unexpectedly
# passing tests or unused expectations.
unmatched = queries.FillExpectationMapForCiBuilders(
test_expectation_map, ci_builders, args.suite, args.project,
args.num_samples)
try_builders = builders.GetTryBuilders(ci_builders)
unmatched.update(
queries.FillExpectationMapForTryBuilders(test_expectation_map,
try_builders, args.suite,
args.project,
args.num_samples))
unused_expectations = expectations.FilterOutUnusedExpectations(
test_expectation_map)
stale, semi_stale, active = expectations.SplitExpectationsByStaleness(
test_expectation_map)
result_output.OutputResults(stale, semi_stale, active, unmatched,
unused_expectations, args.output_format)
if args.remove_stale_expectations:
stale_expectations = []
for _, expectation_map in stale.iteritems():
stale_expectations.extend(expectation_map.keys())
stale_expectations.extend(unused_expectations)
removed_urls = expectations.RemoveExpectationsFromFile(
stale_expectations, args.expectation_file)
print(
'Stale expectations removed from %s. Stale comments, etc. may still '
'need to be removed.' % args.expectation_file)
result_output.OutputRemovedUrls(removed_urls)
def testAllExpectations(self):
"""Tests output when all three types of expectations are provided."""
expectation_map = {
'foo': {
data_types.Expectation('foo', ['stale'], 'Failure'): {
'foo_builder': {
'step1': uu.CreateStatsWithPassFails(1, 0),
'step2': uu.CreateStatsWithPassFails(2, 0),
},
'bar_builder': {
'step1': uu.CreateStatsWithPassFails(3, 0),
'step2': uu.CreateStatsWithPassFails(4, 0)
},
},
data_types.Expectation('foo', ['semistale'], 'Failure'): {
'foo_builder': {
'step1': uu.CreateStatsWithPassFails(1, 0),
'step2': uu.CreateStatsWithPassFails(2, 2),
},
'bar_builder': {
'step1': uu.CreateStatsWithPassFails(3, 0),
'step2': uu.CreateStatsWithPassFails(0, 4)
},
},
data_types.Expectation('foo', ['active'], 'Failure'): {
'foo_builder': {
'step1': uu.CreateStatsWithPassFails(1, 1),
'step2': uu.CreateStatsWithPassFails(2, 2),
},
'bar_builder': {
'step1': uu.CreateStatsWithPassFails(3, 3),
'step2': uu.CreateStatsWithPassFails(0, 4)
},
},
},
}
expected_stale = {
'foo': {
data_types.Expectation('foo', ['stale'], 'Failure'): {
'foo_builder': {
'step1': uu.CreateStatsWithPassFails(1, 0),
'step2': uu.CreateStatsWithPassFails(2, 0),
},
'bar_builder': {
'step1': uu.CreateStatsWithPassFails(3, 0),
'step2': uu.CreateStatsWithPassFails(4, 0)
},
},
},
}
expected_semi_stale = {
'foo': {
data_types.Expectation('foo', ['semistale'], 'Failure'): {
'foo_builder': {
'step1': uu.CreateStatsWithPassFails(1, 0),
'step2': uu.CreateStatsWithPassFails(2, 2),
},
'bar_builder': {
'step1': uu.CreateStatsWithPassFails(3, 0),
'step2': uu.CreateStatsWithPassFails(0, 4)
},
},
},
}
expected_active = {
'foo': {
data_types.Expectation('foo', ['active'], 'Failure'): {
'foo_builder': {
'step1': uu.CreateStatsWithPassFails(1, 1),
'step2': uu.CreateStatsWithPassFails(2, 2),
},
'bar_builder': {
'step1': uu.CreateStatsWithPassFails(3, 3),
'step2': uu.CreateStatsWithPassFails(0, 4)
},
},
},
}
stale_dict, semi_stale_dict, active_dict =\
expectations.SplitExpectationsByStaleness(expectation_map)
self.assertEqual(stale_dict, expected_stale)
self.assertEqual(semi_stale_dict, expected_semi_stale)
self.assertEqual(active_dict, expected_active)
