def testCreate(self):
luci_project = 'chromium'
step_ui_name = 'step'
test_name = 'test'
normalized_step_name = 'normalized_step_name'
normalized_test_name = 'normalized_test_name'
test_label_name = 'test_label'
flake = Flake.Create(luci_project=luci_project,
normalized_step_name=normalized_step_name,
normalized_test_name=normalized_test_name,
test_label_name=test_label_name)
flake.put()
build_id = 123
luci_bucket = 'try'
luci_builder = 'luci builder'
legacy_master_name = 'buildbot master'
legacy_build_number = 999
time_happened = datetime.datetime(2018, 1, 1)
gerrit_cl_id = 98765
cq_false_rejection_occurrence = FlakeOccurrence.Create(
FlakeType.CQ_FALSE_REJECTION,
build_id=build_id,
step_ui_name=step_ui_name,
test_name=test_name,
luci_project=luci_project,
luci_bucket=luci_bucket,
luci_builder=luci_builder,
legacy_master_name=legacy_master_name,
legacy_build_number=legacy_build_number,
time_happened=time_happened,
gerrit_cl_id=gerrit_cl_id,
parent_flake_key=flake.key)
cq_false_rejection_occurrence.put()
retry_with_patch_occurrence = FlakeOccurrence.Create(
FlakeType.RETRY_WITH_PATCH,
build_id=build_id,
step_ui_name=step_ui_name,
test_name=test_name,
luci_project=luci_project,
luci_bucket=luci_bucket,
luci_builder=luci_builder,
legacy_master_name=legacy_master_name,
legacy_build_number=legacy_build_number,
time_happened=time_happened,
gerrit_cl_id=gerrit_cl_id,
parent_flake_key=flake.key)
retry_with_patch_occurrence.put()
fetched_flake_occurrences = FlakeOccurrence.query().fetch()
self.assertEqual(2, len(fetched_flake_occurrences))
self.assertIn(cq_false_rejection_occurrence, fetched_flake_occurrences)
self.assertIn(retry_with_patch_occurrence, fetched_flake_occurrences)
self.assertIsNotNone(fetched_flake_occurrences[0].time_detected)
self.assertIsNotNone(fetched_flake_occurrences[1].time_detected)
def _FetchFlakeOccurrences(flake, flake_type, max_occurrence_count):
"""Fetches flake occurrences of a certain type within a time range.
Args:
flake(Flake): Flake object for a flaky test.
flake_type(FlakeType): Type of the occurrences.
max_occurrence_count(int): Maximum number of occurrences to fetch.
Returns:
(list): A list of occurrences.
"""
start_date = time_util.GetDateDaysBeforeNow(days=constants.DAYS_IN_A_WEEK)
occurrences_query = FlakeOccurrence.query(ancestor=flake.key).filter(
ndb.AND(FlakeOccurrence.flake_type == flake_type,
FlakeOccurrence.time_happened >
start_date)).order(-FlakeOccurrence.time_happened)
occurrences_query_old_flakes = FlakeOccurrence.query(
ancestor=flake.key).filter(FlakeOccurrence.flake_type == flake_type
).order(-FlakeOccurrence.time_happened)
if max_occurrence_count:
flake_occurrences = occurrences_query.fetch(max_occurrence_count)
# No occurrences in the past 7 days.
if not flake_occurrences:
flake_occurrences = occurrences_query_old_flakes.fetch(
max_occurrence_count)
else:
flake_occurrences = occurrences_query.fetch()
# No occurrences in the past 7 days.
if not flake_occurrences:
flake_occurrences = occurrences_query_old_flakes.fetch()
return flake_occurrences
def testStoreDetectedCIFlakes(self, *_):
master_name = 'm'
builder_name = 'b'
build_number = 123
build = WfBuild.Create(master_name, builder_name, build_number)
build.build_id = '87654321'
build.put()
flaky_tests = {'s': ['t1', 't2']}
detect_flake_occurrences.StoreDetectedCIFlakes(master_name, builder_name,
build_number, flaky_tests)
flake = Flake.Get('chromium', 'normalized_step_name', 't1')
self.assertIsNotNone(flake)
occurrences = FlakeOccurrence.query(ancestor=flake.key).fetch()
self.assertEqual(1, len(occurrences))
self.assertEqual(FlakeType.CI_FAILED_STEP, occurrences[0].flake_type)
def testDetectCQHiddenFlakes(self, mocked_get_client, *_):
query_response = self._GetEmptyFlakeQueryResponse()
query_response['pageToken'] = 'token'
test_name1 = 'suite.test'
test_name2 = 'suite.test_1'
self._AddRowToFlakeQueryResponse(query_response=query_response,
step_ui_name='step_ui_name',
test_name=test_name1,
gerrit_cl_id='10000')
self._AddRowToFlakeQueryResponse(query_response=query_response,
step_ui_name='step_ui_name',
test_name=test_name1,
gerrit_cl_id='10001',
build_id='124',
test_start_msec='1')
self._AddRowToFlakeQueryResponse(query_response=query_response,
luci_builder='another_builder',
step_ui_name='step_ui_name',
test_name=test_name1,
gerrit_cl_id='10001',
build_id='125')
self._AddRowToFlakeQueryResponse(query_response=query_response,
step_ui_name='step_ui_name',
test_name=test_name2,
gerrit_cl_id='10000')
mocked_client = mock.Mock()
mocked_get_client.return_value = mocked_client
mocked_client.jobs().query().execute.return_value = query_response
query_response_2 = self._GetEmptyFlakeQueryResponse()
mocked_client.jobs().getQueryResults().execute.side_effect = [
query_response, query_response_2
]
detect_flake_occurrences.QueryAndStoreHiddenFlakes()
all_flake_occurrences = FlakeOccurrence.query().fetch()
self.assertEqual(4, len(all_flake_occurrences))
def _AddDistinctCLsToCounters(counters, flake_info_dict, start, end,
save_test_report):
occurrences_query = FlakeOccurrence.query(
projection=[FlakeOccurrence.flake_type, FlakeOccurrence.gerrit_cl_id
]).filter(
ndb.AND(FlakeOccurrence.time_happened >= start,
FlakeOccurrence.time_happened < end))
cursor = None
more = True
while more:
occurrences, cursor, more = occurrences_query.fetch_page(
500, start_cursor=cursor)
for occurrence in occurrences:
flake_key = occurrence.key.parent()
luci_project = flake_info_dict.get(flake_key,
{}).get('luci_project')
component = flake_info_dict.get(flake_key, {}).get('component')
test = flake_info_dict.get(flake_key, {}).get('test')
# Broken data, bails out on this occurrence.
if not luci_project or not counters.get(luci_project):
continue
if not component or not counters[luci_project].get(component):
continue
if save_test_report and (
not test
or not counters[luci_project][component].get(test)):
continue
flake_type = occurrence.flake_type
cl = occurrence.gerrit_cl_id
counters[luci_project]['_impacted_cls'][flake_type].add(cl)
counters[luci_project][component]['_impacted_cls'][flake_type].add(
cl)
if save_test_report:
counters[luci_project][component][test]['_impacted_cls'][
flake_type].add(cl)
_UpdateDuplicatedClsBetweenTypes(counters)
def GetFlakeInformation(flake, max_occurrence_count, with_occurrences=True):
"""Gets information for a detected flakes.
Gets occurrences of the flake and the attached monorail issue.
Args:
flake(Flake): Flake object for a flaky test.
max_occurrence_count(int): Maximum number of occurrences to fetch.
with_occurrences(bool): If the flake must be with occurrences or not.
For flakes reported by Flake detection, there should always be
occurrences, but it's not always true for flakes reported by
Flake Analyzer, ignore those flakes for now.
Returns:
flake_dict(dict): A dict of information for the test. Including data from
its Flake entity, its flake issue information and information of all its
flake occurrences.
"""
occurrences = []
if (max_occurrence_count and not flake.archived
and flake.flake_score_last_week > 0):
# On-going flakes, queries the ones in the past-week.
for flake_type in [
FlakeType.CQ_FALSE_REJECTION, FlakeType.RETRY_WITH_PATCH,
FlakeType.CI_FAILED_STEP, FlakeType.CQ_HIDDEN_FLAKE
]:
occurrences.extend(
_FetchFlakeOccurrences(flake, flake_type,
max_occurrence_count -
len(occurrences)))
if len(occurrences) >= max_occurrence_count:
# Bails out if the number of occurrences with higher impact has hit
# the cap.
break
# Makes sure occurrences are sorted by time_happened in descending order,
# regardless of types.
occurrences.sort(key=lambda x: x.time_happened, reverse=True)
# Falls back to query all recent occurrences.
occurrences = occurrences or FlakeOccurrence.query(
ancestor=flake.key).order(-FlakeOccurrence.time_happened).fetch(
max_occurrence_count)
if not occurrences and with_occurrences:
# Flake must be with occurrences, but there is no occurrence, bail out.
return None
flake_dict = flake.to_dict()
flake_dict['occurrences'] = _GetGroupedOccurrencesByBuilder(occurrences)
flake_dict['flake_counts_last_week'] = _GetFlakeCountsList(
flake.flake_counts_last_week)
flake_issue = GetFlakeIssue(flake)
if flake_issue and flake_issue.status and flake_issue.status in OPEN_STATUSES:
flake_dict['flake_issue'] = flake_issue.to_dict()
flake_dict['flake_issue']['issue_link'] = FlakeIssue.GetLinkForIssue(
flake_issue.monorail_project, flake_issue.issue_id)
flake_dict['flake_issue'][
'last_updated_time_in_monorail'] = _GetLastUpdatedTimeDelta(
flake_issue)
flake_dict['culprits'], flake_dict['sample_analysis'] = (
_GetFlakeAnalysesResults(flake_issue.issue_id))
return flake_dict
def testProcessBuildForFlakes(self, mock_metadata, mock_build,
mock_normalized_test_name, mock_lable_name, *_):
flake_type_enum = FlakeType.CQ_FALSE_REJECTION
build_id = 123
luci_project = 'luci_project'
luci_bucket = 'luci_bucket'
luci_builder = 'luci_builder'
legacy_master_name = 'legacy_master_name'
start_time = datetime(2019, 3, 6)
end_time = datetime(2019, 3, 6, 0, 0, 10)
findit_step = Step()
findit_step.name = 'FindIt Flakiness'
step1 = Step()
step1.name = 'step1 (with patch)'
step1.start_time.FromDatetime(start_time)
step1.end_time.FromDatetime(end_time)
builder = BuilderID(
project=luci_project,
bucket=luci_bucket,
builder=luci_builder,
)
build = Build(id=build_id, builder=builder, number=build_id)
build.steps.extend([findit_step, step1])
build.input.properties['mastername'] = legacy_master_name
build.input.properties['patch_project'] = 'chromium/src'
mock_change = build.input.gerrit_changes.add()
mock_change.host = 'mock.gerrit.host'
mock_change.change = 12345
mock_change.patchset = 1
mock_build.return_value = build
def _MockTestName(test_name, _step_ui_name): # pylint: disable=unused-argument
return test_name
mock_normalized_test_name.side_effect = _MockTestName
mock_lable_name.side_effect = _MockTestName
flakiness_metadata = {
'Failing With Patch Tests That Caused Build Failure': {
'step1 (with patch)': ['s1_t1', 's1_t2']
},
'Step Layer Flakiness': {}
}
mock_metadata.return_value = flakiness_metadata
# Flake object for s2_t1 exists.
flake1 = Flake.Create(
luci_project=luci_project,
normalized_step_name='step1',
normalized_test_name='s1_t1',
test_label_name='s1_t1')
flake1.put()
detect_flake_occurrences.ProcessBuildForFlakes(
detect_flake_occurrences.DetectFlakesFromFlakyCQBuildParam(
build_id=build_id,
flake_type_desc=FLAKE_TYPE_DESCRIPTIONS[flake_type_enum]))
flake1_occurrence_num = FlakeOccurrence.query(ancestor=flake1.key).count()
self.assertEqual(1, flake1_occurrence_num)
flake2 = Flake.Get(luci_project, 'step1', 's1_t2')
self.assertIsNotNone(flake2)
def _QueryTypedFlakeOccurrences(flake, start_date, flake_type):
return FlakeOccurrence.query(ancestor=flake.key).filter(
ndb.AND(FlakeOccurrence.flake_type == flake_type,
FlakeOccurrence.time_happened > start_date)).fetch()
def GetFlakesWithEnoughOccurrences():
"""Queries Datastore and returns flakes that has enough occurrences.
The most intuitive algorithm is to fetch all flakes first, and then for each
flake, fetch its recent and unreported flake occurrences, but it has
performance implications when there are a lot of flakes (too many RPC calls)
because a large number of calls are wasted on flakes that don't even have any
recent and unreported flake occurrence.
So, instead, this algorithm first fetches all recent and unreported flake
occurrences, and then by looking up their parents to figure out the subset of
flakes that need to be fetched.
Returns:
A list of tuples whose first element is a flake entity, second element is
number of corresponding recent and unreported occurrences, third element is
the flake issue the flake links to if exist. And this list is sorted by each
flake's flake_score_last_week in descending order.
"""
utc_one_day_ago = time_util.GetUTCNow() - datetime.timedelta(days=1)
occurrences = FlakeOccurrence.query(
ndb.AND(
FlakeOccurrence.flake_type.IN(
[FlakeType.CQ_FALSE_REJECTION, FlakeType.RETRY_WITH_PATCH]),
FlakeOccurrence.time_happened > utc_one_day_ago)).fetch()
logging.info(
'There are %d non cq hidden occurrences within the past 24h.' %
len(occurrences))
flake_key_to_occurrences = defaultdict(list)
for occurrence in occurrences:
flake_key_to_occurrences[occurrence.key.parent()].append(occurrence)
unique_flake_keys = flake_key_to_occurrences.keys()
flakes = ndb.get_multi(unique_flake_keys)
key_to_flake = dict(zip(unique_flake_keys, flakes))
# Filter out occurrences that have already been reported according to the
# last update time of the associated flake issue.
flake_key_to_enough_unreported_occurrences = {}
for flake_key, occurrences in flake_key_to_occurrences.iteritems():
if not key_to_flake[flake_key]:
logging.error('Flake not found for key %s', flake_key.urlsafe())
continue
if not _FlakeHasEnoughOccurrences(occurrences):
continue
flake_issue = GetFlakeIssue(flake_key.get())
last_updated_time_by_flake_detection = (
flake_issue.last_updated_time_by_flake_detection
if flake_issue else None)
if (last_updated_time_by_flake_detection
and last_updated_time_by_flake_detection > utc_one_day_ago):
# An issue can be updated at most once in any 24h window avoid noises.
continue
flake_key_to_enough_unreported_occurrences[flake_key] = {
'occurrences': occurrences,
'flake_issue': flake_issue
}
flake_tuples_to_report = [
(key_to_flake[flake_key], info['occurrences'], info['flake_issue'])
for flake_key, info in
flake_key_to_enough_unreported_occurrences.iteritems()
]
return sorted(flake_tuples_to_report,
key=lambda tup: tup[0].flake_score_last_week,
reverse=True)