def testUpdateTestLocationAndTagsGPUTest(self, *_):
flake = Flake(
normalized_test_name='suite.test',
normalized_step_name='telemetry_gpu_integration_test',
tags=['gerrit_project::chromium/src'],
)
occurrences = [
FlakeOccurrence(step_ui_name='context_lost_tests',
build_configuration=BuildConfiguration(
legacy_master_name='master',
luci_builder='builder',
legacy_build_number=123,
)),
FlakeOccurrence(
step_ui_name='webgl_conformance_vulkan_passthrough_tests',
build_configuration=BuildConfiguration(
legacy_master_name='master',
luci_builder='builder',
legacy_build_number=124,
)),
]
component_mapping = {
'base/feature/': 'root>a>b',
'base/feature/url': 'root>a>b>c',
}
watchlist = {
'feature': 'base/feature',
'url': r'base/feature/url_test\.cc',
'other': 'a/b/c',
}
expected_tags = sorted([
'gerrit_project::chromium/src', 'component::Internals>GPU>Testing',
'component::Blink>WebGL'
])
for tag in flake.tags:
name = tag.split(TAG_DELIMITER)[0]
self.assertTrue(name in detect_flake_occurrences.SUPPORTED_TAGS)
self.assertTrue(
detect_flake_occurrences._UpdateTestLocationAndTags(
flake, occurrences, component_mapping, watchlist))
self.assertEqual(expected_tags, flake.tags)
def testGetTestLocation(self, *_):
occurrence = FlakeOccurrence(
build_configuration=BuildConfiguration(
legacy_master_name='master',
luci_builder='builder',
legacy_build_number=123,
),
step_ui_name='test on Mac',
)
self.assertEqual(
'path/a.cc',
detect_flake_occurrences._GetTestLocation(occurrence).file_path)
def _CreateFlake(flake_data, with_component=True):
"""
Args:
with_component (bool): Sets flake.component if True, otherwise sets tags.
"""
luci_project = 'chromium'
normalized_step_name = 'normalized_step_name'
flake_issue = _GetOrCreateFlakeIssue(flake_data['bug_id'])
flake = Flake.Create(normalized_test_name=flake_data['test'],
luci_project=luci_project,
normalized_step_name=normalized_step_name,
test_label_name='test_label')
if with_component:
flake.component = flake_data['component']
else:
flake.tags = ['component::{}'.format(flake_data['component'])]
flake.flake_issue_key = flake_issue.key
flake.flake_counts_last_week = []
for flake_type, counts in flake_data['counts'].iteritems():
flake.flake_counts_last_week.append(
FlakeCountsByType(flake_type=flake_type,
occurrence_count=counts[0],
impacted_cl_count=counts[1]))
flake.last_occurred_time = datetime.strptime(
flake_data['last_occurred_time'], '%Y-W%W-%w')
flake.put()
for occurrence_data in flake_data['occurrences']:
time_happened = datetime.strptime('2018-%d-4' % occurrence_data[2],
'%Y-%W-%w')
hour = occurrence_data[3]
time_happened += timedelta(hours=hour)
occurrence = FlakeOccurrence.Create(
flake_type=occurrence_data[0],
build_id=123 + hour,
step_ui_name='step',
test_name=flake.normalized_test_name,
luci_project='chromium',
luci_bucket='try',
luci_builder='builder',
legacy_master_name='master',
legacy_build_number=42,
time_happened=time_happened,
gerrit_cl_id=occurrence_data[1],
parent_flake_key=flake.key)
occurrence.put()
def _FetchFlakeOccurrences(flake, flake_type, max_occurrence_count):
"""Fetches flake occurrences of a certain type within a time range.
Args:
flake(Flake): Flake object for a flaky test.
flake_type(FlakeType): Type of the occurrences.
max_occurrence_count(int): Maximum number of occurrences to fetch.
Returns:
(list): A list of occurrences.
"""
start_date = time_util.GetDateDaysBeforeNow(days=constants.DAYS_IN_A_WEEK)
occurrences_query = FlakeOccurrence.query(ancestor=flake.key).filter(
ndb.AND(FlakeOccurrence.flake_type == flake_type,
FlakeOccurrence.time_happened >
start_date)).order(-FlakeOccurrence.time_happened)
occurrences_query_old_flakes = FlakeOccurrence.query(
ancestor=flake.key).filter(FlakeOccurrence.flake_type == flake_type
).order(-FlakeOccurrence.time_happened)
if max_occurrence_count:
flake_occurrences = occurrences_query.fetch(max_occurrence_count)
# No occurrences in the past 7 days.
if not flake_occurrences:
flake_occurrences = occurrences_query_old_flakes.fetch(
max_occurrence_count)
else:
flake_occurrences = occurrences_query.fetch()
# No occurrences in the past 7 days.
if not flake_occurrences:
flake_occurrences = occurrences_query_old_flakes.fetch()
return flake_occurrences
def testStoreDetectedCIFlakes(self, *_):
master_name = 'm'
builder_name = 'b'
build_number = 123
build = WfBuild.Create(master_name, builder_name, build_number)
build.build_id = '87654321'
build.put()
flaky_tests = {'s': ['t1', 't2']}
detect_flake_occurrences.StoreDetectedCIFlakes(master_name, builder_name,
build_number, flaky_tests)
flake = Flake.Get('chromium', 'normalized_step_name', 't1')
self.assertIsNotNone(flake)
occurrences = FlakeOccurrence.query(ancestor=flake.key).fetch()
self.assertEqual(1, len(occurrences))
self.assertEqual(FlakeType.CI_FAILED_STEP, occurrences[0].flake_type)
def testUpdateTestLocationAndTagsFromLocation(self, *_):
flake = Flake(
normalized_test_name='suite.test',
tags=[
'gerrit_project::chromium/src', 'watchlist::old',
'directory::old', 'component::old', 'parent_component::old',
'source::old'
],
)
occurrences = [
FlakeOccurrence(step_ui_name='browser_tests'),
]
component_mapping = {
'base/feature/': 'root>a>b',
'base/feature/url': 'root>a>b>c',
}
watchlist = {
'feature': 'base/feature',
'url': r'base/feature/url_test\.cc',
'other': 'a/b/c',
}
expected_tags = sorted([
'gerrit_project::chromium/src',
'watchlist::feature',
'watchlist::url',
'directory::base/feature/',
'directory::base/',
'source::base/feature/url_test.cc',
'component::root>a>b',
'parent_component::root>a>b',
'parent_component::root>a',
'parent_component::root',
])
for tag in flake.tags:
name = tag.split(TAG_DELIMITER)[0]
self.assertTrue(name in detect_flake_occurrences.SUPPORTED_TAGS)
self.assertTrue(
detect_flake_occurrences._UpdateTestLocationAndTags(
flake, occurrences, component_mapping, watchlist))
self.assertEqual(expected_tags, flake.tags)
def testDetectCQHiddenFlakes(self, mocked_get_client, *_):
query_response = self._GetEmptyFlakeQueryResponse()
query_response['pageToken'] = 'token'
test_name1 = 'suite.test'
test_name2 = 'suite.test_1'
self._AddRowToFlakeQueryResponse(query_response=query_response,
step_ui_name='step_ui_name',
test_name=test_name1,
gerrit_cl_id='10000')
self._AddRowToFlakeQueryResponse(query_response=query_response,
step_ui_name='step_ui_name',
test_name=test_name1,
gerrit_cl_id='10001',
build_id='124',
test_start_msec='1')
self._AddRowToFlakeQueryResponse(query_response=query_response,
luci_builder='another_builder',
step_ui_name='step_ui_name',
test_name=test_name1,
gerrit_cl_id='10001',
build_id='125')
self._AddRowToFlakeQueryResponse(query_response=query_response,
step_ui_name='step_ui_name',
test_name=test_name2,
gerrit_cl_id='10000')
mocked_client = mock.Mock()
mocked_get_client.return_value = mocked_client
mocked_client.jobs().query().execute.return_value = query_response
query_response_2 = self._GetEmptyFlakeQueryResponse()
mocked_client.jobs().getQueryResults().execute.side_effect = [
query_response, query_response_2
]
detect_flake_occurrences.QueryAndStoreHiddenFlakes()
all_flake_occurrences = FlakeOccurrence.query().fetch()
self.assertEqual(4, len(all_flake_occurrences))
def _AddDistinctCLsToCounters(counters, flake_info_dict, start, end,
save_test_report):
occurrences_query = FlakeOccurrence.query(
projection=[FlakeOccurrence.flake_type, FlakeOccurrence.gerrit_cl_id
]).filter(
ndb.AND(FlakeOccurrence.time_happened >= start,
FlakeOccurrence.time_happened < end))
cursor = None
more = True
while more:
occurrences, cursor, more = occurrences_query.fetch_page(
500, start_cursor=cursor)
for occurrence in occurrences:
flake_key = occurrence.key.parent()
luci_project = flake_info_dict.get(flake_key,
{}).get('luci_project')
component = flake_info_dict.get(flake_key, {}).get('component')
test = flake_info_dict.get(flake_key, {}).get('test')
# Broken data, bails out on this occurrence.
if not luci_project or not counters.get(luci_project):
continue
if not component or not counters[luci_project].get(component):
continue
if save_test_report and (
not test
or not counters[luci_project][component].get(test)):
continue
flake_type = occurrence.flake_type
cl = occurrence.gerrit_cl_id
counters[luci_project]['_impacted_cls'][flake_type].add(cl)
counters[luci_project][component]['_impacted_cls'][flake_type].add(
cl)
if save_test_report:
counters[luci_project][component][test]['_impacted_cls'][
flake_type].add(cl)
_UpdateDuplicatedClsBetweenTypes(counters)
def testAnalyzeDetectedFlakeOccurrence(self):
step = 'step1'
test = 'test1'
luci_project = 'chromium'
bug_id = 12345
flake = Flake.Create(luci_project, step, test, 'l')
flake_issue = FlakeIssue.Create(luci_project, bug_id)
flake.flake_issue_key = flake_issue.key
occurrence = FlakeOccurrence.Create(
flake_type=FlakeType.CQ_FALSE_REJECTION,
build_id=111,
step_ui_name=step,
test_name=test,
luci_project=luci_project,
luci_bucket='try',
luci_builder='tryserver.chromium.linux',
legacy_master_name='linux_chromium_rel_ng',
legacy_build_number=999,
time_happened=None,
gerrit_cl_id=98765,
parent_flake_key=None)
apis.AnalyzeDetectedFlakeOccurrence(flake, occurrence, bug_id)
self.assertEqual(1, len(self.taskqueue_requests))
def testUpdateMetadataForFlakes(self, *_):
luci_project = 'chromium'
normalized_step_name = 'normalized_step_name'
normalized_test_name = 'normalized_test_name'
test_label_name = 'test_label'
flake = Flake.Create(
luci_project=luci_project,
normalized_step_name=normalized_step_name,
normalized_test_name=normalized_test_name,
test_label_name=test_label_name)
flake.archived = True
flake.put()
flake_key = flake.key
step_ui_name = 'step'
test_name = 'test'
luci_bucket = 'try'
luci_builder = 'luci builder'
legacy_master_name = 'buildbot master'
legacy_build_number = 999
gerrit_cl_id = 98765
# Flake's last_occurred_time and tags are empty, updated.
occurrence_1 = FlakeOccurrence.Create(
flake_type=FlakeType.CQ_FALSE_REJECTION,
build_id=123,
step_ui_name=step_ui_name,
test_name=test_name,
luci_project=luci_project,
luci_bucket=luci_bucket,
luci_builder=luci_builder,
legacy_master_name=legacy_master_name,
legacy_build_number=legacy_build_number,
time_happened=datetime(2018, 1, 1, 1),
gerrit_cl_id=gerrit_cl_id,
parent_flake_key=flake_key,
tags=['tag1::v1'])
occurrence_1.put()
_UpdateFlakeMetadata([occurrence_1])
flake = flake_key.get()
self.assertEqual(flake.last_occurred_time, datetime(2018, 1, 1, 1))
self.assertEqual(flake.tags, ['tag1::v1'])
self.assertFalse(flake.archived)
# Flake's last_occurred_time is earlier and tags are different, updated.
occurrence_2 = FlakeOccurrence.Create(
flake_type=FlakeType.CQ_FALSE_REJECTION,
build_id=124,
step_ui_name=step_ui_name,
test_name=test_name,
luci_project=luci_project,
luci_bucket=luci_bucket,
luci_builder=luci_builder,
legacy_master_name=legacy_master_name,
legacy_build_number=legacy_build_number,
time_happened=datetime(2018, 1, 1, 2),
gerrit_cl_id=gerrit_cl_id,
parent_flake_key=flake_key,
tags=['tag2::v2'])
occurrence_2.put()
_UpdateFlakeMetadata([occurrence_2])
flake = flake_key.get()
self.assertEqual(flake.last_occurred_time, datetime(2018, 1, 1, 2))
self.assertEqual(flake.tags, ['tag1::v1', 'tag2::v2'])
# Flake's last_occurred_time is later and tags are the same, not updated.
occurrence_3 = FlakeOccurrence.Create(
flake_type=FlakeType.CQ_FALSE_REJECTION,
build_id=125,
step_ui_name=step_ui_name,
test_name=test_name,
luci_project=luci_project,
luci_bucket=luci_bucket,
luci_builder=luci_builder,
legacy_master_name=legacy_master_name,
legacy_build_number=legacy_build_number,
time_happened=datetime(2018, 1, 1),
gerrit_cl_id=gerrit_cl_id,
parent_flake_key=flake_key,
tags=['tag2::v2'])
occurrence_3.put()
_UpdateFlakeMetadata([occurrence_3])
flake = flake_key.get()
self.assertEqual(flake.last_occurred_time, datetime(2018, 1, 1, 2))
self.assertEqual(flake.tags, ['tag1::v1', 'tag2::v2'])
def testShowFlake(self, *_):
flake_issue = FlakeIssue.Create(monorail_project='chromium',
issue_id=900)
flake_issue.last_updated_time_by_flake_detection = datetime(2018, 1, 1)
flake_issue.status = 'Assigned'
flake_issue.last_updated_time_in_monorail = datetime(2018, 1, 1)
flake_issue.put()
luci_project = 'chromium'
step_ui_name = 'step'
test_name = 'test'
normalized_step_name = 'normalized_step_name'
normalized_test_name = 'normalized_test_name'
test_label_name = 'test_label'
flake = Flake.Create(
luci_project=luci_project,
normalized_step_name=normalized_step_name,
normalized_test_name=normalized_test_name,
test_label_name=test_label_name,
)
flake.component = 'Mock>Component'
flake.test_location = TestLocation()
flake.test_location.file_path = '../../some/test/path/a.cc'
flake.test_location.line_number = 42
flake.flake_issue_key = flake_issue.key
flake.put()
build_id = 123
luci_bucket = 'try'
luci_builder = 'luci builder'
legacy_master_name = 'buildbot master'
legacy_build_number = 999
time_happened = datetime(2018, 1, 1)
gerrit_cl_id = 98765
occurrence = FlakeOccurrence.Create(
flake_type=FlakeType.CQ_FALSE_REJECTION,
build_id=build_id,
step_ui_name=step_ui_name,
test_name=test_name,
luci_project=luci_project,
luci_bucket=luci_bucket,
luci_builder=luci_builder,
legacy_master_name=legacy_master_name,
legacy_build_number=legacy_build_number,
time_happened=time_happened,
gerrit_cl_id=gerrit_cl_id,
parent_flake_key=flake.key)
occurrence.time_detected = datetime(2018, 1, 1)
occurrence.put()
response = self.test_app.get('/flake/detection/ui/show-flake',
params={
'key': flake.key.urlsafe(),
'format': 'json',
},
status=200)
flake_dict = {
'flake_issue': {
'flake_culprit_key':
None,
'issue_id':
900,
'issue_link':
('https://monorail-prod.appspot.com/p/chromium/issues'
'/detail?id=900'),
'last_updated_time_by_flake_detection':
'2018-01-01 00:00:00',
'monorail_project':
'chromium',
'merge_destination_key':
None,
'last_updated_time_in_monorail':
'2 days, 00:00:00',
'last_updated_time_with_analysis_results':
None,
'create_time_in_monorail':
None,
'labels': [],
'status':
'Assigned',
},
'flake_issue_key':
flake_issue.key,
'luci_project':
'chromium',
'last_occurred_time':
None,
'last_test_location_based_tag_update_time':
None,
'normalized_step_name':
'normalized_step_name',
'normalized_test_name':
'normalized_test_name',
'test_label_name':
'test_label',
'false_rejection_count_last_week':
0,
'impacted_cl_count_last_week':
0,
'archived':
False,
'flake_counts_last_week': [
{
'flake_type': 'cq false rejection',
'impacted_cl_count': 0,
'occurrence_count': 0
},
{
'flake_type': 'cq step level retry',
'impacted_cl_count': 0,
'occurrence_count': 0
},
{
'flake_type': 'cq hidden flake',
'impacted_cl_count': 0,
'occurrence_count': 0
},
{
'flake_type': 'ci failed step',
'impacted_cl_count': 0,
'occurrence_count': 0
},
],
'flake_score_last_week':
0,
'component':
'Mock>Component',
'tags': [],
'test_location': {
'file_path': '../../some/test/path/a.cc',
'line_number': 42,
},
'culprits': [],
'sample_analysis':
None,
'occurrences': [{
'group_by_field':
'luci builder',
'occurrences': [{
'build_configuration': {
'legacy_build_number': 999,
'legacy_master_name': 'buildbot master',
'luci_bucket': 'try',
'luci_builder': 'luci builder',
'luci_project': 'chromium'
},
'tags': [],
'build_id': '123',
'flake_type': 'cq false rejection',
'gerrit_cl_id': 98765,
'step_ui_name': 'step',
'test_name': 'test',
'time_detected': '2018-01-01 00:00:00 UTC',
'time_happened': '2018-01-01 00:00:00 UTC'
}]
}]
}
self.assertEqual(
json.dumps(
{
'flake_json':
flake_dict,
'key':
flake.key.urlsafe(),
'show_all_occurrences':
'',
'weights': [('cq false rejection', 100),
('cq step level retry', 10),
('cq hidden flake', 1), ('ci failed step', 10)]
},
default=str,
sort_keys=True,
indent=2),
json.dumps(json.loads(response.body), sort_keys=True, indent=2))
def testFlakeUpdates(self, _):
luci_project = 'chromium'
step_ui_name = 'step'
normalized_step_name = 'normalized_step_name'
flake1 = Flake.Create(luci_project=luci_project,
normalized_step_name=normalized_step_name,
normalized_test_name='normalized_test_name_1',
test_label_name='test_label1')
flake1.last_occurred_time = datetime(2018, 9, 1)
flake1.put()
flake1_key = flake1.key
luci_bucket = 'try'
luci_builder = 'luci builder'
legacy_master_name = 'buildbot master'
legacy_build_number = 999
occurrence1 = FlakeOccurrence.Create(
flake_type=FlakeType.CQ_FALSE_REJECTION,
build_id=1,
step_ui_name=step_ui_name,
test_name='t1',
luci_project=luci_project,
luci_bucket=luci_bucket,
luci_builder=luci_builder,
legacy_master_name=legacy_master_name,
legacy_build_number=legacy_build_number,
time_happened=datetime(2018, 9, 1),
gerrit_cl_id=98761,
parent_flake_key=flake1_key)
occurrence1.put()
occurrence2 = FlakeOccurrence.Create(
flake_type=FlakeType.CQ_FALSE_REJECTION,
build_id=2,
step_ui_name=step_ui_name,
test_name='t1',
luci_project=luci_project,
luci_bucket=luci_bucket,
luci_builder=luci_builder,
legacy_master_name=legacy_master_name,
legacy_build_number=legacy_build_number,
time_happened=datetime(2018, 8, 31),
gerrit_cl_id=98761,
parent_flake_key=flake1_key)
occurrence2.put()
occurrence3 = FlakeOccurrence.Create(
flake_type=FlakeType.CQ_FALSE_REJECTION,
build_id=3,
step_ui_name=step_ui_name,
test_name='t2',
luci_project=luci_project,
luci_bucket=luci_bucket,
luci_builder=luci_builder,
legacy_master_name=legacy_master_name,
legacy_build_number=legacy_build_number,
time_happened=datetime(2018, 7, 31),
gerrit_cl_id=98763,
parent_flake_key=flake1_key)
occurrence3.put()
occurrence5 = FlakeOccurrence.Create(
flake_type=FlakeType.RETRY_WITH_PATCH,
build_id=5,
step_ui_name=step_ui_name,
test_name='t2',
luci_project=luci_project,
luci_bucket=luci_bucket,
luci_builder=luci_builder,
legacy_master_name=legacy_master_name,
legacy_build_number=legacy_build_number,
time_happened=datetime(2018, 8, 31),
gerrit_cl_id=98761,
parent_flake_key=flake1_key)
occurrence5.put()
occurrence6 = FlakeOccurrence.Create(
flake_type=FlakeType.RETRY_WITH_PATCH,
build_id=6,
step_ui_name=step_ui_name,
test_name='t2',
luci_project=luci_project,
luci_bucket=luci_bucket,
luci_builder=luci_builder,
legacy_master_name=legacy_master_name,
legacy_build_number=legacy_build_number,
time_happened=datetime(2018, 8, 31),
gerrit_cl_id=98766,
parent_flake_key=flake1_key)
occurrence6.put()
occurrence7 = FlakeOccurrence.Create(
flake_type=FlakeType.RETRY_WITH_PATCH,
build_id=7,
step_ui_name=step_ui_name,
test_name='t2',
luci_project=luci_project,
luci_bucket=luci_bucket,
luci_builder=luci_builder,
legacy_master_name=legacy_master_name,
legacy_build_number=legacy_build_number,
time_happened=datetime(2018, 8, 31),
gerrit_cl_id=98767,
parent_flake_key=flake1_key)
occurrence7.put()
UpdateFlakeCounts()
flake1 = flake1_key.get()
self.assertEqual(5, flake1.false_rejection_count_last_week)
self.assertEqual(3, flake1.impacted_cl_count_last_week)
self.assertEqual([
FlakeCountsByType(flake_type=FlakeType.CQ_FALSE_REJECTION,
impacted_cl_count=1,
occurrence_count=2),
FlakeCountsByType(flake_type=FlakeType.RETRY_WITH_PATCH,
impacted_cl_count=2,
occurrence_count=3)
], flake1.flake_counts_last_week)
self.assertEqual(120, flake1.flake_score_last_week)
def _CreateFlakeOccurrenceFromRow(row, flake_type_enum):
"""Creates a FlakeOccurrence from a row fetched from BigQuery."""
luci_project = row['luci_project']
luci_builder = row['luci_builder']
step_ui_name = row['step_ui_name']
test_name = row['test_name']
legacy_master_name = row['legacy_master_name']
legacy_build_number = row['legacy_build_number']
normalized_step_name = Flake.NormalizeStepName(
step_name=step_ui_name,
master_name=legacy_master_name,
builder_name=luci_builder,
build_number=legacy_build_number)
normalized_test_name = Flake.NormalizeTestName(test_name, step_ui_name)
flake_id = Flake.GetId(luci_project=luci_project,
normalized_step_name=normalized_step_name,
normalized_test_name=normalized_test_name)
flake_key = ndb.Key(Flake, flake_id)
gerrit_project = row['gerrit_project']
build_id = row['build_id']
luci_bucket = row['luci_bucket']
time_happened = row['test_start_msec']
gerrit_cl_id = row['gerrit_cl_id']
# Not add the original test name as a tag here, because all the tags will be
# merged into Flake model, and there might be 100s of parameterized tests
# which might lead to too large data for a single Flake entity.
tags = [
'gerrit_project::%s' % gerrit_project,
'luci_project::%s' % luci_project,
'bucket::%s' % luci_bucket,
'master::%s' % legacy_master_name,
'builder::%s' % luci_builder,
'binary::%s' % normalized_step_name, # e.g. "tests"
'test_type::%s' %
step_ui_name.split(' ', 1)[0], # e.g. "flavored_tests"
'step::%s' % step_ui_name, # e.g. "flavored_tests on Mac 10.13"
'flake::%s' % normalized_test_name,
]
suite = _GetTestSuiteForOccurrence(row, normalized_test_name,
normalized_step_name)
if suite:
tags.append('suite::%s' % suite)
tags.sort()
flake_occurrence = FlakeOccurrence.Create(
flake_type=flake_type_enum,
build_id=build_id,
step_ui_name=step_ui_name,
test_name=test_name,
luci_project=luci_project,
luci_bucket=luci_bucket,
luci_builder=luci_builder,
legacy_master_name=legacy_master_name,
legacy_build_number=legacy_build_number,
time_happened=time_happened,
gerrit_cl_id=gerrit_cl_id,
parent_flake_key=flake_key,
tags=tags)
return flake_occurrence
def testGetFlakeInformationNoIssue(self, *_):
luci_project = 'chromium'
step_ui_name = 'step'
test_name = 'test'
normalized_step_name = 'normalized_step_name'
normalized_test_name = 'normalized_test_name_3'
test_label_name = 'test_label'
flake = Flake.Create(luci_project=luci_project,
normalized_step_name=normalized_step_name,
normalized_test_name=normalized_test_name,
test_label_name=test_label_name)
flake.put()
build_id = 123
luci_bucket = 'try'
luci_builder = 'luci builder'
legacy_master_name = 'buildbot master'
legacy_build_number = 999
time_happened = datetime(2018, 1, 1)
gerrit_cl_id = 98765
occurrence = FlakeOccurrence.Create(
flake_type=FlakeType.CQ_FALSE_REJECTION,
build_id=build_id,
step_ui_name=step_ui_name,
test_name=test_name,
luci_project=luci_project,
luci_bucket=luci_bucket,
luci_builder=luci_builder,
legacy_master_name=legacy_master_name,
legacy_build_number=legacy_build_number,
time_happened=time_happened,
gerrit_cl_id=gerrit_cl_id,
parent_flake_key=flake.key)
occurrence.time_detected = datetime(2018, 1, 1)
occurrence.put()
expected_flake_dict = {
'luci_project':
'chromium',
'normalized_step_name':
normalized_step_name,
'normalized_test_name':
normalized_test_name,
'test_label_name':
test_label_name,
'flake_issue_key':
None,
'last_occurred_time':
None,
'last_test_location_based_tag_update_time':
None,
'false_rejection_count_last_week':
0,
'impacted_cl_count_last_week':
0,
'archived':
False,
'flake_counts_last_week': [
{
'flake_type': 'cq false rejection',
'impacted_cl_count': 0,
'occurrence_count': 0
},
{
'flake_type': 'cq step level retry',
'impacted_cl_count': 0,
'occurrence_count': 0
},
{
'flake_type': 'cq hidden flake',
'impacted_cl_count': 0,
'occurrence_count': 0
},
{
'flake_type': 'ci failed step',
'impacted_cl_count': 0,
'occurrence_count': 0
},
],
'flake_score_last_week':
0,
'component':
None,
'test_location':
None,
'tags': [],
'occurrences': [{
'group_by_field':
'luci builder',
'occurrences': [{
'flake_type': 'cq false rejection',
'build_id': '123',
'step_ui_name': step_ui_name,
'test_name': test_name,
'tags': [],
'build_configuration': {
'luci_project': 'chromium',
'luci_bucket': 'try',
'luci_builder': 'luci builder',
'legacy_master_name': 'buildbot master',
'legacy_build_number': 999
},
'time_happened': '2018-01-01 00:00:00 UTC',
'time_detected': '2018-01-01 00:00:00 UTC',
'gerrit_cl_id': gerrit_cl_id
}]
}],
}
self.assertEqual(expected_flake_dict,
flake_detection_utils.GetFlakeInformation(flake, 1))
def testGetFlakeInformationOldFlakes(self, *_):
flake_issue = FlakeIssue.Create(monorail_project='chromium',
issue_id=900)
flake_issue.last_updated_time_by_flake_detection = datetime(2018, 1, 1)
flake_issue.last_updated_time_in_monorail = datetime(2018, 1, 2)
flake_issue.status = 'Started'
flake_issue.put()
luci_project = 'chromium'
step_ui_name = 'step'
test_name = 'test'
normalized_step_name = 'normalized_step_name'
normalized_test_name = 'normalized_test_name'
test_label_name = 'test_label'
flake = Flake.Create(
luci_project=luci_project,
normalized_step_name=normalized_step_name,
normalized_test_name=normalized_test_name,
test_label_name=test_label_name,
)
flake.component = 'Mock>Component'
flake.test_location = TestLocation()
flake.test_location.file_path = '../../some/test/path/a.cc'
flake.test_location.line_number = 42
flake.flake_issue_key = flake_issue.key
flake.put()
build_id = 123
luci_bucket = 'try'
luci_builder = 'luci builder'
legacy_master_name = 'buildbot master'
legacy_build_number = 999
time_happened = datetime(2018, 1, 1)
gerrit_cl_id = 98765
occurrence = FlakeOccurrence.Create(
flake_type=FlakeType.CQ_FALSE_REJECTION,
build_id=build_id,
step_ui_name=step_ui_name,
test_name=test_name,
luci_project=luci_project,
luci_bucket=luci_bucket,
luci_builder=luci_builder,
legacy_master_name=legacy_master_name,
legacy_build_number=legacy_build_number,
time_happened=time_happened,
gerrit_cl_id=gerrit_cl_id,
parent_flake_key=flake.key)
occurrence.time_detected = datetime(2018, 1, 1)
occurrence.put()
occurrence2 = FlakeOccurrence.Create(
flake_type=FlakeType.RETRY_WITH_PATCH,
build_id=124,
step_ui_name=step_ui_name,
test_name=test_name,
luci_project=luci_project,
luci_bucket=luci_bucket,
luci_builder='luci builder 2',
legacy_master_name=legacy_master_name,
legacy_build_number=legacy_build_number,
time_happened=datetime(2018, 1, 2, 3),
gerrit_cl_id=gerrit_cl_id,
parent_flake_key=flake.key)
occurrence2.time_detected = datetime(2018, 1, 2, 3)
occurrence2.put()
occurrence3 = FlakeOccurrence.Create(
flake_type=FlakeType.CQ_FALSE_REJECTION,
build_id=125,
step_ui_name=step_ui_name,
test_name=test_name,
luci_project=luci_project,
luci_bucket=luci_bucket,
luci_builder='luci builder 2',
legacy_master_name=legacy_master_name,
legacy_build_number=legacy_build_number,
time_happened=datetime(2018, 1, 2, 2),
gerrit_cl_id=gerrit_cl_id,
parent_flake_key=flake.key)
occurrence3.time_detected = datetime(2018, 1, 2, 2)
occurrence3.put()
occurrence4 = FlakeOccurrence.Create(
flake_type=FlakeType.CQ_HIDDEN_FLAKE,
build_id=126,
step_ui_name=step_ui_name,
test_name=test_name,
luci_project=luci_project,
luci_bucket=luci_bucket,
luci_builder='luci builder 2',
legacy_master_name=legacy_master_name,
legacy_build_number=legacy_build_number,
time_happened=datetime(2018, 1, 2, 2),
gerrit_cl_id=gerrit_cl_id,
parent_flake_key=flake.key)
occurrence4.time_detected = datetime(2018, 1, 2, 2)
occurrence4.put()
occurrence5 = FlakeOccurrence.Create(
flake_type=FlakeType.CI_FAILED_STEP,
build_id=127,
step_ui_name=step_ui_name,
test_name=test_name,
luci_project=luci_project,
luci_bucket=luci_bucket,
luci_builder='luci builder 2',
legacy_master_name=legacy_master_name,
legacy_build_number=legacy_build_number,
time_happened=datetime(2018, 1, 2, 2),
gerrit_cl_id=-1,
parent_flake_key=flake.key)
occurrence5.time_detected = datetime(2018, 1, 2, 2)
occurrence5.put()
culprit1 = FlakeCulprit.Create('chromium', 'rev1', 123456,
'culprit_url')
culprit1.put()
analysis = MasterFlakeAnalysis.Create(legacy_master_name, luci_builder,
legacy_build_number,
step_ui_name, test_name)
analysis.bug_id = 900
analysis.culprit_urlsafe_key = culprit1.key.urlsafe()
analysis.confidence_in_culprit = 0.98
analysis.put()
culprit2 = FlakeCulprit.Create('chromium', 'rev2', 123457,
'culprit_url')
culprit2.put()
analysis_1 = MasterFlakeAnalysis.Create(legacy_master_name,
luci_builder,
legacy_build_number - 1,
step_ui_name, test_name)
analysis_1.bug_id = 900
analysis_1.culprit_urlsafe_key = culprit2.key.urlsafe()
analysis_1.put()
expected_flake_dict = {
'luci_project':
'chromium',
'normalized_step_name':
'normalized_step_name',
'normalized_test_name':
'normalized_test_name',
'test_label_name':
'test_label',
'flake_issue_key':
flake_issue.key,
'last_occurred_time':
None,
'last_test_location_based_tag_update_time':
None,
'false_rejection_count_last_week':
0,
'impacted_cl_count_last_week':
0,
'archived':
False,
'flake_counts_last_week': [
{
'flake_type': 'cq false rejection',
'impacted_cl_count': 0,
'occurrence_count': 0
},
{
'flake_type': 'cq step level retry',
'impacted_cl_count': 0,
'occurrence_count': 0
},
{
'flake_type': 'cq hidden flake',
'impacted_cl_count': 0,
'occurrence_count': 0
},
{
'flake_type': 'ci failed step',
'impacted_cl_count': 0,
'occurrence_count': 0
},
],
'flake_score_last_week':
0,
'flake_issue': {
'flake_culprit_key':
None,
'monorail_project':
'chromium',
'issue_id':
900,
'last_updated_time_by_flake_detection':
datetime(2018, 1, 1),
'issue_link': ('https://monorail-prod.appspot.com/p/chromium/'
'issues/detail?id=900'),
'merge_destination_key':
None,
'last_updated_time_in_monorail':
'366 days, 00:00:00',
'last_updated_time_with_analysis_results':
None,
'create_time_in_monorail':
None,
'labels': [],
'status':
'Started',
},
'component':
'Mock>Component',
'test_location': {
'file_path': '../../some/test/path/a.cc',
'line_number': 42,
},
'tags': [],
'culprits': [{
'revision': 'rev1',
'commit_position': culprit1.commit_position,
'culprit_key': culprit1.key.urlsafe()
}],
'sample_analysis':
None,
'occurrences': [{
'group_by_field':
'luci builder 2',
'occurrences': [
{
'flake_type': 'cq step level retry',
'build_id': '124',
'step_ui_name': step_ui_name,
'test_name': test_name,
'tags': [],
'build_configuration': {
'luci_project': 'chromium',
'luci_bucket': 'try',
'luci_builder': 'luci builder 2',
'legacy_master_name': 'buildbot master',
'legacy_build_number': 999
},
'time_happened': '2018-01-02 03:00:00 UTC',
'time_detected': '2018-01-02 03:00:00 UTC',
'gerrit_cl_id': gerrit_cl_id
},
{
'flake_type': 'cq false rejection',
'build_id': '125',
'step_ui_name': step_ui_name,
'test_name': test_name,
'tags': [],
'build_configuration': {
'luci_project': 'chromium',
'luci_bucket': 'try',
'luci_builder': 'luci builder 2',
'legacy_master_name': 'buildbot master',
'legacy_build_number': 999
},
'time_happened': '2018-01-02 02:00:00 UTC',
'time_detected': '2018-01-02 02:00:00 UTC',
'gerrit_cl_id': gerrit_cl_id
},
{
'flake_type': 'ci failed step',
'build_id': '127',
'step_ui_name': step_ui_name,
'test_name': test_name,
'tags': [],
'build_configuration': {
'luci_project': 'chromium',
'luci_bucket': 'try',
'luci_builder': 'luci builder 2',
'legacy_master_name': 'buildbot master',
'legacy_build_number': 999
},
'time_happened': '2018-01-02 02:00:00 UTC',
'time_detected': '2018-01-02 02:00:00 UTC',
'gerrit_cl_id': -1,
},
{
'flake_type': 'cq hidden flake',
'build_id': '126',
'step_ui_name': step_ui_name,
'test_name': test_name,
'tags': [],
'build_configuration': {
'luci_project': 'chromium',
'luci_bucket': 'try',
'luci_builder': 'luci builder 2',
'legacy_master_name': 'buildbot master',
'legacy_build_number': 999
},
'time_happened': '2018-01-02 02:00:00 UTC',
'time_detected': '2018-01-02 02:00:00 UTC',
'gerrit_cl_id': gerrit_cl_id,
},
]
}, {
'group_by_field':
'luci builder',
'occurrences': [{
'flake_type': 'cq false rejection',
'build_id': '123',
'step_ui_name': step_ui_name,
'test_name': test_name,
'tags': [],
'build_configuration': {
'luci_project': 'chromium',
'luci_bucket': 'try',
'luci_builder': 'luci builder',
'legacy_master_name': 'buildbot master',
'legacy_build_number': 999
},
'time_happened': '2018-01-01 00:00:00 UTC',
'time_detected': '2018-01-01 00:00:00 UTC',
'gerrit_cl_id': gerrit_cl_id
}]
}],
}
self.assertEqual(expected_flake_dict,
flake_detection_utils.GetFlakeInformation(flake, 5))
def testProcessBuildForFlakes(self, mock_metadata, mock_build,
mock_normalized_test_name, mock_lable_name, *_):
flake_type_enum = FlakeType.CQ_FALSE_REJECTION
build_id = 123
luci_project = 'luci_project'
luci_bucket = 'luci_bucket'
luci_builder = 'luci_builder'
legacy_master_name = 'legacy_master_name'
start_time = datetime(2019, 3, 6)
end_time = datetime(2019, 3, 6, 0, 0, 10)
findit_step = Step()
findit_step.name = 'FindIt Flakiness'
step1 = Step()
step1.name = 'step1 (with patch)'
step1.start_time.FromDatetime(start_time)
step1.end_time.FromDatetime(end_time)
builder = BuilderID(
project=luci_project,
bucket=luci_bucket,
builder=luci_builder,
)
build = Build(id=build_id, builder=builder, number=build_id)
build.steps.extend([findit_step, step1])
build.input.properties['mastername'] = legacy_master_name
build.input.properties['patch_project'] = 'chromium/src'
mock_change = build.input.gerrit_changes.add()
mock_change.host = 'mock.gerrit.host'
mock_change.change = 12345
mock_change.patchset = 1
mock_build.return_value = build
def _MockTestName(test_name, _step_ui_name): # pylint: disable=unused-argument
return test_name
mock_normalized_test_name.side_effect = _MockTestName
mock_lable_name.side_effect = _MockTestName
flakiness_metadata = {
'Failing With Patch Tests That Caused Build Failure': {
'step1 (with patch)': ['s1_t1', 's1_t2']
},
'Step Layer Flakiness': {}
}
mock_metadata.return_value = flakiness_metadata
# Flake object for s2_t1 exists.
flake1 = Flake.Create(
luci_project=luci_project,
normalized_step_name='step1',
normalized_test_name='s1_t1',
test_label_name='s1_t1')
flake1.put()
detect_flake_occurrences.ProcessBuildForFlakes(
detect_flake_occurrences.DetectFlakesFromFlakyCQBuildParam(
build_id=build_id,
flake_type_desc=FLAKE_TYPE_DESCRIPTIONS[flake_type_enum]))
flake1_occurrence_num = FlakeOccurrence.query(ancestor=flake1.key).count()
self.assertEqual(1, flake1_occurrence_num)
flake2 = Flake.Get(luci_project, 'step1', 's1_t2')
self.assertIsNotNone(flake2)
def testFlakesWithCQHiddenFlakes(self, _):
luci_project = 'chromium'
normalized_step_name = 'normalized_step_name'
flake3 = Flake.Create(luci_project=luci_project,
normalized_step_name=normalized_step_name,
normalized_test_name='normalized_test_name_3',
test_label_name='test_label3')
flake3.last_occurred_time = datetime(2018, 9, 1)
flake3.false_rejection_count_last_week = 5
flake3.impacted_cl_count_last_week = 3
flake3.put()
flake3_key = flake3.key
step_ui_name = 'step'
luci_bucket = 'try'
luci_builder = 'luci builder'
legacy_master_name = 'buildbot master'
legacy_build_number = 999
occurrence4 = FlakeOccurrence.Create(
flake_type=FlakeType.CQ_FALSE_REJECTION,
build_id=4,
step_ui_name=step_ui_name,
test_name='t1',
luci_project=luci_project,
luci_bucket=luci_bucket,
luci_builder=luci_builder,
legacy_master_name=legacy_master_name,
legacy_build_number=legacy_build_number,
time_happened=datetime(2018, 8, 31),
gerrit_cl_id=98764,
parent_flake_key=flake3_key)
occurrence4.put()
for i in xrange(98760, 98790):
occurrence = FlakeOccurrence.Create(
flake_type=FlakeType.CQ_HIDDEN_FLAKE,
build_id=i,
step_ui_name=step_ui_name,
test_name='t1',
luci_project=luci_project,
luci_bucket=luci_bucket,
luci_builder=luci_builder,
legacy_master_name=legacy_master_name,
legacy_build_number=i,
time_happened=datetime(2018, 8, 31),
gerrit_cl_id=i,
parent_flake_key=flake3_key)
occurrence.put()
UpdateFlakeCounts()
flake3 = flake3_key.get()
self.assertEqual([
FlakeCountsByType(flake_type=FlakeType.CQ_FALSE_REJECTION,
impacted_cl_count=1,
occurrence_count=1),
FlakeCountsByType(flake_type=FlakeType.CQ_HIDDEN_FLAKE,
impacted_cl_count=29,
occurrence_count=30)
], flake3.flake_counts_last_week)
self.assertEqual(129, flake3.flake_score_last_week)
def _QueryTypedFlakeOccurrences(flake, start_date, flake_type):
return FlakeOccurrence.query(ancestor=flake.key).filter(
ndb.AND(FlakeOccurrence.flake_type == flake_type,
FlakeOccurrence.time_happened > start_date)).fetch()
def GetFlakeInformation(flake, max_occurrence_count, with_occurrences=True):
"""Gets information for a detected flakes.
Gets occurrences of the flake and the attached monorail issue.
Args:
flake(Flake): Flake object for a flaky test.
max_occurrence_count(int): Maximum number of occurrences to fetch.
with_occurrences(bool): If the flake must be with occurrences or not.
For flakes reported by Flake detection, there should always be
occurrences, but it's not always true for flakes reported by
Flake Analyzer, ignore those flakes for now.
Returns:
flake_dict(dict): A dict of information for the test. Including data from
its Flake entity, its flake issue information and information of all its
flake occurrences.
"""
occurrences = []
if (max_occurrence_count and not flake.archived
and flake.flake_score_last_week > 0):
# On-going flakes, queries the ones in the past-week.
for flake_type in [
FlakeType.CQ_FALSE_REJECTION, FlakeType.RETRY_WITH_PATCH,
FlakeType.CI_FAILED_STEP, FlakeType.CQ_HIDDEN_FLAKE
]:
occurrences.extend(
_FetchFlakeOccurrences(flake, flake_type,
max_occurrence_count -
len(occurrences)))
if len(occurrences) >= max_occurrence_count:
# Bails out if the number of occurrences with higher impact has hit
# the cap.
break
# Makes sure occurrences are sorted by time_happened in descending order,
# regardless of types.
occurrences.sort(key=lambda x: x.time_happened, reverse=True)
# Falls back to query all recent occurrences.
occurrences = occurrences or FlakeOccurrence.query(
ancestor=flake.key).order(-FlakeOccurrence.time_happened).fetch(
max_occurrence_count)
if not occurrences and with_occurrences:
# Flake must be with occurrences, but there is no occurrence, bail out.
return None
flake_dict = flake.to_dict()
flake_dict['occurrences'] = _GetGroupedOccurrencesByBuilder(occurrences)
flake_dict['flake_counts_last_week'] = _GetFlakeCountsList(
flake.flake_counts_last_week)
flake_issue = GetFlakeIssue(flake)
if flake_issue and flake_issue.status and flake_issue.status in OPEN_STATUSES:
flake_dict['flake_issue'] = flake_issue.to_dict()
flake_dict['flake_issue']['issue_link'] = FlakeIssue.GetLinkForIssue(
flake_issue.monorail_project, flake_issue.issue_id)
flake_dict['flake_issue'][
'last_updated_time_in_monorail'] = _GetLastUpdatedTimeDelta(
flake_issue)
flake_dict['culprits'], flake_dict['sample_analysis'] = (
_GetFlakeAnalysesResults(flake_issue.issue_id))
return flake_dict
def GetFlakesWithEnoughOccurrences():
"""Queries Datastore and returns flakes that has enough occurrences.
The most intuitive algorithm is to fetch all flakes first, and then for each
flake, fetch its recent and unreported flake occurrences, but it has
performance implications when there are a lot of flakes (too many RPC calls)
because a large number of calls are wasted on flakes that don't even have any
recent and unreported flake occurrence.
So, instead, this algorithm first fetches all recent and unreported flake
occurrences, and then by looking up their parents to figure out the subset of
flakes that need to be fetched.
Returns:
A list of tuples whose first element is a flake entity, second element is
number of corresponding recent and unreported occurrences, third element is
the flake issue the flake links to if exist. And this list is sorted by each
flake's flake_score_last_week in descending order.
"""
utc_one_day_ago = time_util.GetUTCNow() - datetime.timedelta(days=1)
occurrences = FlakeOccurrence.query(
ndb.AND(
FlakeOccurrence.flake_type.IN(
[FlakeType.CQ_FALSE_REJECTION, FlakeType.RETRY_WITH_PATCH]),
FlakeOccurrence.time_happened > utc_one_day_ago)).fetch()
logging.info(
'There are %d non cq hidden occurrences within the past 24h.' %
len(occurrences))
flake_key_to_occurrences = defaultdict(list)
for occurrence in occurrences:
flake_key_to_occurrences[occurrence.key.parent()].append(occurrence)
unique_flake_keys = flake_key_to_occurrences.keys()
flakes = ndb.get_multi(unique_flake_keys)
key_to_flake = dict(zip(unique_flake_keys, flakes))
# Filter out occurrences that have already been reported according to the
# last update time of the associated flake issue.
flake_key_to_enough_unreported_occurrences = {}
for flake_key, occurrences in flake_key_to_occurrences.iteritems():
if not key_to_flake[flake_key]:
logging.error('Flake not found for key %s', flake_key.urlsafe())
continue
if not _FlakeHasEnoughOccurrences(occurrences):
continue
flake_issue = GetFlakeIssue(flake_key.get())
last_updated_time_by_flake_detection = (
flake_issue.last_updated_time_by_flake_detection
if flake_issue else None)
if (last_updated_time_by_flake_detection
and last_updated_time_by_flake_detection > utc_one_day_ago):
# An issue can be updated at most once in any 24h window avoid noises.
continue
flake_key_to_enough_unreported_occurrences[flake_key] = {
'occurrences': occurrences,
'flake_issue': flake_issue
}
flake_tuples_to_report = [
(key_to_flake[flake_key], info['occurrences'], info['flake_issue'])
for flake_key, info in
flake_key_to_enough_unreported_occurrences.iteritems()
]
return sorted(flake_tuples_to_report,
key=lambda tup: tup[0].flake_score_last_week,
reverse=True)