def test_all_results_by_path(self):
test_data = {
'tests': {
'foo': {
'multiple_pass.html': {
'results': [[4, 'P'], [1, 'P'], [2, 'P']]
},
'fail.html': {
'results': [[2, 'F']]
},
'all_types.html': {
'results': [[1, 'C'], [2, 'F'], [1, 'N'], [1, 'P'],
[1, 'T'], [1, 'Y'], [10, 'X']]
},
'not_run.html': {
'results': []
},
}
}
}
results_json = self._results_json_from_test_data(test_data)
expectations = bot_test_expectations.BotTestExpectations(
results_json, BuilderList({}), set('test'))
results_by_path = expectations.all_results_by_path()
expected_output = {
'foo/multiple_pass.html': ['PASS'],
'foo/fail.html': ['FAIL'],
'foo/all_types.html': ['CRASH', 'FAIL', 'PASS', 'TIMEOUT']
}
self.assertEqual(results_by_path, expected_output)
def _assert_expectations(self, test_data, expectations_string,
only_ignore_very_flaky):
results_json = self._results_json_from_test_data(test_data)
expectations = bot_test_expectations.BotTestExpectations(
results_json, BuilderList({}), set('test'))
self.assertEqual(expectations.flakes_by_path(only_ignore_very_flaky),
expectations_string)
def test_filtered_all_results_by_path(self):
test_data = {
'buildNumbers': [1, 2 , 3, 4, 5, 6, 7],
'tests': {
'foo': {
'fail_filtered.html': {'results': [[4, 'P'], [1, 'F'], [1, 'C'], [1, 'P']]},
'fail_not_filtered.html': {'results': [[3, 'P'], [2, 'F'], [2, 'P']]},
}
}
}
filter_data = {
'buildNumbers': [3, 4, 5, 6, 8],
'num_failures_by_type' : {
'PASS' : [0, 0, 1, 1, 1, 0, 1]
}
}
results_json = self._results_json_from_test_data(test_data)
results_filter = self._results_filter_from_test_data(filter_data)
expectations = bot_test_expectations.BotTestExpectations(results_json, BuilderList({}), set('test'), results_filter)
results_by_path = expectations.all_results_by_path()
expected_output = {
'foo/fail_filtered.html': ['PASS'],
'foo/fail_not_filtered.html': ['FAIL', 'PASS'],
}
self.assertEqual(results_by_path, expected_output)
def _expectations_from_test_data(self, builder, test_data):
test_data[bot_test_expectations.ResultsJSON.FAILURE_MAP_KEY] = self.FAILURE_MAP
json_dict = {
builder: test_data,
}
results = bot_test_expectations.ResultsJSON(builder, json_dict)
return bot_test_expectations.BotTestExpectations(
results, self.builders, self.builders.specifiers_for_builder(builder))
def _assert_is_flaky(self, results_string, should_be_flaky, only_ignore_very_flaky, expected=None):
results_json = self._results_json_from_test_data({})
expectations = bot_test_expectations.BotTestExpectations(results_json, BuilderList({}), set('test'))
results_entry = self._results_from_string(results_string)
if expected:
results_entry[bot_test_expectations.ResultsJSON.EXPECTATIONS_KEY] = expected
num_actual_results = len(expectations._flaky_types_in_results(results_entry, only_ignore_very_flaky))
if should_be_flaky:
self.assertGreater(num_actual_results, 1)
else:
self.assertLessEqual(num_actual_results, 1)
def _assert_unexpected_results(self, test_data, expectations_string):
results_json = self._results_json_from_test_data(test_data)
expectations = bot_test_expectations.BotTestExpectations(
results_json, BuilderList({}), set('test'))
self.assertEqual(expectations.unexpected_results_by_path(),
expectations_string)
