def test_all_results_by_path(self):
test_data = {
'tests': {
'foo': {
'multiple_pass.html': {'results': [[4, 'P'], [1, 'P'], [2, 'P']]},
'fail.html': {'results': [[2, 'Z']]},
'all_types.html': {
'results': [[2, 'A'], [1, 'C'], [2, 'F'], [1, 'I'], [1, 'O'], [1, 'N'], [1, 'P'], [1, 'T'],
[1, 'Y'], [10, 'X'], [1, 'Z'], [1, 'K']]
},
'not_run.html': {'results': []},
}
}
}
results_json = self._results_json_from_test_data(test_data)
expectations = bot_test_expectations.BotTestExpectations(results_json, BuilderList({}), set('test'))
results_by_path = expectations.all_results_by_path()
expected_output = {
'foo/multiple_pass.html': ['PASS'],
'foo/fail.html': ['IMAGE+TEXT'],
'foo/all_types.html': ['AUDIO', 'CRASH', 'IMAGE', 'IMAGE+TEXT', 'LEAK', 'MISSING', 'PASS', 'TEXT', 'TIMEOUT']
}
self.assertEqual(results_by_path, expected_output)
def _assert_expectations(self, test_data, expectations_string,
only_ignore_very_flaky):
results_json = self._results_json_from_test_data(test_data)
expectations = bot_test_expectations.BotTestExpectations(
results_json, BuilderList({}), set('test'))
self.assertEqual(expectations.flakes_by_path(only_ignore_very_flaky),
expectations_string)
def _expectations_from_test_data(self, builder, test_data):
test_data[bot_test_expectations.ResultsJSON.FAILURE_MAP_KEY] = self.FAILURE_MAP
json_dict = {
builder: test_data,
}
results = bot_test_expectations.ResultsJSON(builder, json_dict)
return bot_test_expectations.BotTestExpectations(
results, self.builders, self.builders.specifiers_for_builder(builder))
def _assert_is_flaky(self, results_string, should_be_flaky):
results_json = self._results_json_from_test_data({})
expectations = bot_test_expectations.BotTestExpectations(results_json, set('test'))
length_encoded = self._results_from_string(results_string)['results']
num_actual_results = len(expectations._flaky_types_in_results(length_encoded, only_ignore_very_flaky=True))
if should_be_flaky:
self.assertGreater(num_actual_results, 1)
else:
self.assertEqual(num_actual_results, 1)
def _expectations_from_test_data(self, builder, test_data):
test_data[bot_test_expectations.ResultsJSON.
FAILURE_MAP_KEY] = self.FAILURE_MAP
json_dict = {
builder: test_data,
}
results = bot_test_expectations.ResultsJSON(builder, json_dict)
return bot_test_expectations.BotTestExpectations(
results, builders._exact_matches[builder]["specifiers"])
def _assert_is_flaky(self, results_string, should_be_flaky, only_ignore_very_flaky, expected=None):
results_json = self._results_json_from_test_data({})
expectations = bot_test_expectations.BotTestExpectations(results_json, set('test'))
results_entry = self._results_from_string(results_string)
if expected:
results_entry[bot_test_expectations.ResultsJSON.EXPECTATIONS_KEY] = expected
num_actual_results = len(expectations._flaky_types_in_results(results_entry, only_ignore_very_flaky))
if should_be_flaky:
self.assertGreater(num_actual_results, 1)
else:
self.assertLessEqual(num_actual_results, 1)
def _assert_unexpected_results(self, test_data, expectations_string):
results_json = self._results_json_from_test_data(test_data)
expectations = bot_test_expectations.BotTestExpectations(results_json)
self.assertEqual(expectations.unexpected_results_by_path(),
expectations_string)
def _assert_unexpected_results(self, test_data, expectations_string):
results_json = self._results_json_from_test_data(test_data)
expectations = bot_test_expectations.BotTestExpectations(
results_json, BuilderList(BUILDERS), set('test'))
self.assertEqual(expectations.unexpected_results_by_path(),
expectations_string)
