def testNeedNewAnalysisWhenPreviousOneWasForAnotherBug(self):
existing_request = FlakeAnalysisRequest.Create('flake', False, 123)
existing_request.user_emails = ['*****@*****.**']
existing_request.Save()
request = FlakeAnalysisRequest.Create('flake', False, 456)
step1 = BuildStep.Create('m', 'b1', 10, 's', datetime(2016, 10, 01))
step1.swarmed = False
step1.supported = False
step2 = BuildStep.Create('m', 'b2', 10, 's', datetime(2016, 10, 01))
step2.swarmed = True
step2.supported = True
request.build_steps = [step1, step2]
request.user_emails = ['*****@*****.**']
mocked_now = datetime(2017, 05, 01, 10, 10, 10)
self.MockUTCNow(mocked_now)
version, step = flake_analysis_service._CheckForNewAnalysis(request)
self.assertEqual(2, version)
new_request = FlakeAnalysisRequest.GetVersion(key='flake', version=version)
self.assertEqual(['*****@*****.**', '*****@*****.**'],
new_request.user_emails)
self.assertFalse(new_request.user_emails_obscured)
self.assertEqual(mocked_now, new_request.user_emails_last_edit)
self.assertIsNotNone(step)
self.assertTrue(step.scheduled)
self.assertTrue(step.swarmed)
self.assertTrue(step.supported)
def testNeedNewAnalysisWithFreshEnoughPreviousAnalysisWithRerunFlag(self):
existing_request = FlakeAnalysisRequest.Create('flake', False, 123)
step1 = BuildStep.Create('m', 'b1', 11, 's', datetime(2016, 10, 01))
step1.swarmed = True
step1.supported = True
step1.scheduled = True
step2 = BuildStep.Create('m', 'b2', 12, 's', datetime(2016, 10, 01))
step2.swarmed = True
step2.supported = True
step2.scheduled = True
existing_request.supported = True
existing_request.swarmed = True
existing_request.build_steps = [step1, step2]
existing_request.Save()
request = FlakeAnalysisRequest.Create('flake', False, 123)
step3 = BuildStep.Create('m', 'b2', 20, 's', datetime(2016, 10, 01))
step3.swarmed = True
step3.supported = True
request.build_steps = [step3]
request.user_emails = ['*****@*****.**']
mocked_now = datetime(2016, 10, 01)
self.MockUTCNow(mocked_now)
version, step = flake_analysis_service._CheckForNewAnalysis(request, True)
self.assertEqual(1, version)
new_request = FlakeAnalysisRequest.GetVersion(key='flake', version=version)
self.assertEqual(['*****@*****.**'], new_request.user_emails)
self.assertFalse(new_request.user_emails_obscured)
self.assertEqual(datetime(2016, 10, 01), new_request.user_emails_last_edit)
self.assertIsNotNone(step)
self.assertTrue(step.scheduled)
def testNeedNewAnalysisWithTooOldPreviousAnalysis(self):
existing_request = FlakeAnalysisRequest.Create('flake', False, None)
step1 = BuildStep.Create('m', 'b1', 11, 's', datetime(2016, 10, 01))
step1.swarmed = True
step1.supported = True
step1.scheduled = True
step2 = BuildStep.Create('m', 'b2', 12, 's', datetime(2016, 10, 01))
step2.swarmed = True
step2.supported = True
step2.scheduled = True
existing_request.supported = True
existing_request.swarmed = True
existing_request.user_emails = ['*****@*****.**']
existing_request.build_steps = [step1, step2]
existing_request.Save()
request = FlakeAnalysisRequest.Create('flake', False, 123)
step3 = BuildStep.Create('m', 'b2', 80, 's', datetime(2016, 10, 20))
step3.swarmed = True
step3.supported = True
request.build_steps = [step3]
request.user_emails = ['*****@*****.**']
version, step = flake_analysis_service._CheckForNewAnalysis(request)
self.assertEqual(1, version)
self.assertIsNotNone(step)
self.assertEqual(80, step.build_number)
request = FlakeAnalysisRequest.GetVersion(key='flake')
self.assertEqual(['*****@*****.**', '*****@*****.**'],
request.user_emails)
def testNotNeedNewAnalysisWithFreshEnoughPreviousAnalysis(self):
existing_request = FlakeAnalysisRequest.Create('flake', False, 123)
step1 = BuildStep.Create('m', 'b1', 11, 's', datetime(2016, 10, 01))
step1.swarmed = True
step1.supported = True
step1.scheduled = True
step2 = BuildStep.Create('m', 'b2', 12, 's', datetime(2016, 10, 01))
step2.swarmed = True
step2.supported = True
step2.scheduled = True
existing_request.supported = True
existing_request.swarmed = True
existing_request.build_steps = [step1, step2]
existing_request.Save()
request = FlakeAnalysisRequest.Create('flake', False, 123)
step3 = BuildStep.Create('m', 'b2', 20, 's', datetime(2016, 10, 03))
step3.swarmed = True
step3.supported = True
request.build_steps = [step3]
version, step = flake_analysis_service._CheckForNewAnalysis(request)
self.assertEqual(0, version)
self.assertIsNone(step)
def testNeedNewAnalysisWithADifferentFormerReportedStep(self):
existing_request = FlakeAnalysisRequest.Create('flake', False, 123)
step1 = BuildStep.Create('m', 'b1', 11, 's', datetime(2016, 10, 01))
step1.swarmed = True
step1.supported = True
step1.scheduled = True
step2 = BuildStep.Create('m', 'b2', 12, 's', datetime(2016, 10, 01))
step2.swarmed = True
step2.supported = True
step2.scheduled = False
existing_request.supported = True
existing_request.swarmed = True
existing_request.build_steps = [step1, step2]
existing_request.Save()
request = FlakeAnalysisRequest.Create('flake', False, 123)
step3 = BuildStep.Create('m', 'b3', 13, 's', datetime(2016, 10, 01))
step3.swarmed = False
step3.supported = False
request.build_steps = [step3]
version, step = flake_analysis_service._CheckForNewAnalysis(request)
self.assertEqual(1, version)
self.assertIsNotNone(step)
self.assertTrue(step.scheduled)
self.assertEqual('b2', step.builder_name)
def AnalyzeDetectedFlakeOccurrence(flake, flake_occurrence, bug_id):
"""Analyze detected flake occurrence by Flake Detection.
Args:
flake (Flake): The Flake triggering this analysis.
flake_occurrece (FlakeOccurrence): A FlakeOccurrence model entity.
bug_id (int): Id of the bug to update after the analysis finishes.
"""
test_name = flake_occurrence.test_name
analysis_request = FlakeAnalysisRequest.Create(test_name, False, bug_id)
analysis_request.flake_key = flake.key
master_name = flake_occurrence.build_configuration.legacy_master_name
builder_name = flake_occurrence.build_configuration.luci_builder
build_number = flake_occurrence.build_configuration.legacy_build_number
step_ui_name = flake_occurrence.step_ui_name
analysis_request.AddBuildStep(master_name, builder_name, build_number,
step_ui_name, time_util.GetUTCNow())
analysis_request.Save()
logging.info('flake report for detected flake occurrence: %r',
analysis_request)
AsyncProcessFlakeReport(analysis_request,
user_email=constants.DEFAULT_SERVICE_ACCOUNT,
is_admin=False)
def testCopyFrom(self):
request1 = FlakeAnalysisRequest.Create('flaky_test', False, 123)
request2 = FlakeAnalysisRequest.Create('flaky_test', True, 456)
request2.AddBuildStep('m', 'b1', 1, 's', datetime(2016, 10, 1))
request2.user_emails = ['email']
analysis = MasterFlakeAnalysis.Create('m', 'b', 100, 's', 't')
analysis.Save()
request2.analyses.append(analysis.key)
request1.CopyFrom(request2)
self.assertEqual(request2.is_step, request1.is_step)
self.assertEqual(request2.bug_id, request1.bug_id)
self.assertEqual(request2.user_emails, request1.user_emails)
self.assertEqual(request2.build_steps, request1.build_steps)
self.assertEqual(request2.analyses, request1.analyses)
def testUnauthorizedAccess(self):
request = FlakeAnalysisRequest.Create('flake', False, 123)
step = BuildStep.Create('m', 'b2', 80, 's', datetime(2016, 10, 20))
request.build_steps = [step]
self.assertIsNone(
flake_analysis_service.ScheduleAnalysisForFlake(
request, '*****@*****.**', False, triggering_sources.FINDIT_UI))
def RunImpl(self, build_key):
"""Triggers flake analyses for flaky tests found by CI failure analysis."""
master_name, builder_name, build_number = build_key.GetParts()
flake_settings = waterfall_config.GetCheckFlakeSettings()
throttled = flake_settings.get('throttle_flake_analyses', True)
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
if not analysis or not analysis.flaky_tests:
return
analysis_counts = defaultdict(lambda: defaultdict(int))
for step, flaky_tests in analysis.flaky_tests.iteritems():
logging.info('%s/%s/%s/%s has %s flaky tests.', master_name,
builder_name, build_number, step, len(flaky_tests))
for test_name in flaky_tests:
# TODO(crbug.com/904050): Deprecate FlakeAnalysisRequest in favor of
# Flake.
flake = flake_util.GetFlake(_LUCI_PROJECT, step, test_name,
master_name, builder_name,
build_number)
request = FlakeAnalysisRequest.Create(test_name, False, None)
request.AddBuildStep(master_name, builder_name, build_number,
step, time_util.GetUTCNow())
request.flake_key = flake.key
scheduled = flake_analysis_service.ScheduleAnalysisForFlake(
request, '*****@*****.**',
False, triggering_sources.FINDIT_PIPELINE)
if scheduled: # pragma: no branch
analysis_counts[step]['analyzed'] += 1
logging.info(
'A flake analysis has been triggered for %s/%s', step,
test_name)
if throttled and len(flaky_tests) > 1:
logging.info(
'Throttling is enabled, skipping %d tests.',
len(flaky_tests) - 1)
analysis_counts[step]['throttled'] = len(
flaky_tests) - 1
break # If we're throttled, stop after the first.
else:
analysis_counts[step]['error'] += 1
for step, step_counts in analysis_counts.iteritems():
# Collects metrics.
step_metadata = step_util.GetStepMetadata(master_name,
builder_name,
build_number, step)
canonical_step_name = step_metadata.get(
'canonical_step_name') or 'Unknown'
isolate_target_name = step_metadata.get(
'isolate_target_name') or 'Unknown'
for operation, count in step_counts.iteritems():
monitoring.OnFlakeIdentified(canonical_step_name,
isolate_target_name, operation,
count)
def testAuthorizedAccessAndNewAnalysisNeededButNotTriggered(
self, mock_mon, _):
step = BuildStep.Create('m', 'b', 80, 's', datetime(2016, 10, 20))
step.step_metadata = {
'isolate_target_name': 'wf_s',
'canonical_step_name': 'wf_s'
}
flake = Flake.Create('chromium', 's', 't', 'l')
request = FlakeAnalysisRequest.Create('flake', False, 123)
request.flake_key = flake.key
request.build_steps = [step]
user_email = '*****@*****.**'
triggering_source = triggering_sources.FINDIT_UI
def CheckForNewAnalysis(*_):
step.wf_master_name = 'wf_m'
step.wf_builder_name = 'wf_b'
step.wf_build_number = 100
step.wf_step_name = 'wf_s'
return 1, step
normalized_test = TestInfo('wf_m', 'wf_b', 100, 'wf_s', 'flake')
original_test = TestInfo('m', 'b', 80, 's', 'flake')
with mock.patch.object(
flake_analysis_service,
'_CheckForNewAnalysis',
side_effect=CheckForNewAnalysis) as (
mocked_CheckForNewAnalysis), mock.patch.object(
flake_analysis_service.initialize_flake_pipeline,
'ScheduleAnalysisIfNeeded',
return_value=None) as (
mocked_ScheduleAnalysisIfNeeded), mock.patch.object(
flake_analysis_service.FlakeAnalysisRequest,
'GetVersion',
return_value=None) as mocked_GetVersion:
self.assertFalse(
flake_analysis_service.ScheduleAnalysisForFlake(
request, user_email, True, triggering_sources.FINDIT_UI))
mocked_CheckForNewAnalysis.assert_called_once_with(request, False)
mocked_ScheduleAnalysisIfNeeded.assert_called_once_with(
normalized_test,
original_test,
flake.key,
bug_id=123,
allow_new_analysis=True,
manually_triggered=False,
user_email=user_email,
triggering_source=triggering_source,
queue_name=constants.WATERFALL_ANALYSIS_QUEUE,
force=False)
self.assertFalse(mocked_GetVersion.called)
mock_mon.assert_called_once_with(
source='waterfall',
operation='error',
trigger='auto',
canonical_step_name='wf_s',
isolate_target_name='wf_s')
def testFindMatchingAnalysisForConfiguration(self, _):
request = FlakeAnalysisRequest.Create('test', False, 123)
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 'test')
analysis.Save()
request.analyses.append(analysis.key)
request.Save()
self.assertEqual(
analysis, request.FindMatchingAnalysisForConfiguration('m', 'b'))
def CreateFlakeAnalysisRequest(flake):
analysis_request = FlakeAnalysisRequest.Create(
flake.name, flake.is_step, flake.bug_id)
for step in flake.build_steps:
analysis_request.AddBuildStep(step.master_name,
step.builder_name,
step.build_number,
step.step_name,
time_util.GetUTCNow())
return analysis_request
def testNotNeedNewAnalysisForStepLevelFlake(self):
request = FlakeAnalysisRequest.Create('flake', True, 123)
step1 = BuildStep.Create('m', 'b1', 10, 's', datetime(2016, 10, 01))
step1.swarmed = True
step1.supported = True
request.build_steps = [step1]
version, step = flake_analysis_service._CheckForNewAnalysis(request)
self.assertEqual(0, version)
self.assertIsNone(step)
def testCheckFlakeSwarmedAndSupportedWhenNotSupported(self):
request = FlakeAnalysisRequest.Create('flake', False, 123)
step1 = BuildStep.Create('m', 'b1', 10, 's', datetime(2016, 10, 01))
step1.swarmed = False
step1.supported = False
step2 = BuildStep.Create('m', 'b2', 10, 's', datetime(2016, 10, 01))
step2.swarmed = False
step2.supported = False
request.build_steps = [step1, step2]
self.assertEqual(
(False, False, None),
flake_analysis_service._CheckFlakeSwarmedAndSupported(request))
def testObscureFlakeAnalysisRequest(self):
mocked_utcnow = datetime(2017, 05, 05, 22, 50, 10)
self.MockUTCNow(mocked_utcnow)
valid_request_time = obscure_emails._TimeBeforeNow(days=5)
invalid_request_time = obscure_emails._TimeBeforeNow(
days=obscure_emails._REQUEST_RECORD_RENTENSION_DAYS + 10)
old_request = FlakeAnalysisRequest.Create('flake1', False, 123)
old_request.user_emails.append('*****@*****.**')
old_request.user_emails_obscured = False
old_request.user_emails_last_edit = invalid_request_time
old_request.Save()
recent_request = FlakeAnalysisRequest.Create('flake2', False, 321)
recent_request.user_emails.append('*****@*****.**')
recent_request.user_emails_obscured = False
recent_request.user_emails_last_edit = valid_request_time
recent_request.Save()
response = self.test_app.get(
'/obscure-emails',
params={'format': 'json'},
headers={'X-AppEngine-Cron': 'true'},
)
expected_response = {
'failure_triage_count': 0,
'flake_triage_count': 0,
'flake_request_aggregated_count': 1,
'flake_request_count': 0,
}
self.assertEqual(expected_response, response.json_body)
old_request = FlakeAnalysisRequest.GetVersion(key='flake1', version=1)
self.assertTrue(old_request.user_emails_obscured)
self.assertEqual(['*****@*****.**'], old_request.user_emails)
recent_request = FlakeAnalysisRequest.GetVersion(key='flake2', version=1)
self.assertFalse(recent_request.user_emails_obscured)
self.assertEqual(['*****@*****.**'], recent_request.user_emails)
def testAuthorizedAccessButNoNewAnalysisNeeded(self, mock_mon, *_):
request = FlakeAnalysisRequest.Create('flake', False, 123)
step = BuildStep.Create('m', 'b2', 80, 's', datetime(2016, 10, 20))
request.build_steps = [step]
self.assertFalse(
flake_analysis_service.ScheduleAnalysisForFlake(
request, '*****@*****.**', True, triggering_sources.FINDIT_UI))
mock_mon.assert_called_once_with(
source='waterfall',
operation='skip',
trigger='auto',
canonical_step_name='unknown',
isolate_target_name='unknown')
def testAddBuildStep(self):
t1 = datetime(2016, 10, 1, 0, 0, 0)
t2 = datetime(2016, 10, 2, 0, 0, 0)
t3 = datetime(2016, 10, 2, 1, 0, 0)
t4 = datetime(2016, 10, 2, 0, 30, 0)
request = FlakeAnalysisRequest.Create('flaky_test', False, 123)
self.assertTrue(request.AddBuildStep('m', 'b1', 1, 's', t1))
self.assertTrue(request.AddBuildStep('m', 'b2', 10, 's', t2))
self.assertFalse(request.AddBuildStep('m', 'b2', 11, 's', t3))
self.assertTrue(request.AddBuildStep('m', 'b2', 9, 's', t4))
self.assertEqual(2, len(request.build_steps), request.build_steps)
self.assertEqual(BuildStep.Create('m', 'b1', 1, 's', t1),
request.build_steps[0])
self.assertEqual(BuildStep.Create('m', 'b2', 9, 's', t4),
request.build_steps[1])
def testGetNormalizedConfigurationNames(self):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 's'
test_name = 't'
reported_time = datetime(2016, 11, 16)
request = FlakeAnalysisRequest.Create(test_name, False, 123)
build_step = BuildStep.Create(master_name, builder_name, build_number,
step_name, reported_time)
build_step.wf_master_name = master_name
build_step.wf_builder_name = builder_name
build_step.wf_build_number = build_number
build_step.wf_step_name = step_name
request.build_steps.append(build_step)
self.assertEqual((None, None),
request._GetNormalizedConfigurationNames('m2', 'b2'))
self.assertEqual(
(master_name, builder_name),
request._GetNormalizedConfigurationNames(master_name,
builder_name))
def testBailoutForAndroidAndFuchsia(self, *_):
step1 = BuildStep.Create('m', 'Win10 Tests x64 (dbg)', 80, 's',
datetime(2016, 10, 20))
step2 = BuildStep.Create('m', 'Fuchsia t', 80, 's', datetime(2016, 10, 20))
request = FlakeAnalysisRequest.Create('flake', False, 123)
request.build_steps = [step1, step2]
def FindMatchingWaterfallStep(step, _):
step.wf_master_name = step.master_name
step.wf_builder_name = step.builder_name
step.wf_build_number = 100
step.wf_step_name = 'step'
with mock.patch.object(
flake_analysis_service.step_mapper,
'FindMatchingWaterfallStep',
side_effect=FindMatchingWaterfallStep):
self.assertIsNone(
flake_analysis_service.ScheduleAnalysisForFlake(
request, '*****@*****.**', True,
triggering_sources.FINDIT_API))
def _HandleRerunAnalysis(self):
"""Rerun an analysis as a response to a user request."""
# If the key has been specified, we can derive the above information
# from the analysis itself.
if not auth_util.IsCurrentUserAdmin():
return self.CreateError('Only admin is allowed to rerun.', 403)
key = self.request.get('key')
if not key:
return self.CreateError('No key was provided.', 404)
analysis = ndb.Key(urlsafe=key).get()
if not analysis:
return self.CreateError('Analysis of flake is not found.', 404)
if not self._AnalysisCompleted(analysis):
return self.CreateError(
'Cannot rerun analysis if one is currently running or pending.', 400)
logging.info(
'Rerun button pushed, analysis will be reset and triggered.\n'
'Analysis key: %s', key)
request = FlakeAnalysisRequest.Create(analysis.original_test_name, False,
analysis.bug_id)
request.AddBuildStep(
analysis.original_master_name, analysis.original_builder_name,
analysis.original_build_number, analysis.original_step_name,
time_util.GetUTCNow())
analysis, _ = self._CreateAndScheduleFlakeAnalysis(
request, analysis.master_name, analysis.builder_name,
analysis.build_number, analysis.step_name, analysis.test_name,
analysis.bug_id, True)
return self.CreateRedirect(
'/p/chromium/flake-portal/analysis/analyze?redirect=1&key=%s' %
analysis.key.urlsafe())
def HandlePost(self):
# Information needed to execute this endpoint, will be populated
# by the branches below.
rerun = self.request.get('rerun', '0').strip() == '1'
cancel = self.request.get('cancel', '0').strip() == '1'
analyze_recent_commit = (
self.request.get('analyze_recent_commit', '0').strip() == '1')
if rerun: # Rerun an analysis.
return self._HandleRerunAnalysis()
elif cancel: # Force an analysis to be cancelled.
return self._HandleCancelAnalysis()
elif analyze_recent_commit:
return self._HandleAnalyzeRecentCommit()
else: # Regular POST requests to start an analysis.
# If the key hasn't been specified, then we get the information from
# other URL parameters.
build_url = self.request.get('url', '').strip()
build_info = buildbot.ParseBuildUrl(build_url)
if not build_info:
return self.CreateError('Unknown build info!', 400)
master_name, builder_name, build_number = build_info
step_name = self.request.get('step_name', '').strip()
test_name = self.request.get('test_name', '').strip()
bug_id = self.request.get('bug_id', '').strip()
error = self._ValidateInput(step_name, test_name, bug_id)
if error:
return error
build_number = int(build_number)
bug_id = int(bug_id) if bug_id else None
request = FlakeAnalysisRequest.Create(test_name, False, bug_id)
request.AddBuildStep(master_name, builder_name, build_number, step_name,
time_util.GetUTCNow())
analysis, scheduled = self._CreateAndScheduleFlakeAnalysis(
request, master_name, builder_name, build_number, step_name,
test_name, bug_id, False)
if not analysis:
if scheduled is None:
# User does not have permission to trigger, nor was any previous
# analysis triggered to view.
return {
'template': 'error.html',
'data': {
'error_message': (
'No permission to schedule an analysis for flaky test. '
'Please log in with your @google.com account first.'),
},
'return_code': 403,
}
# Check if a previous request has already covered this analysis so use
# the results from that analysis.
request = FlakeAnalysisRequest.GetVersion(key=test_name)
if not (request and request.analyses):
return {
'template': 'error.html',
'data': {
'error_message': (
'Flake analysis is not supported for "%s/%s". Either '
'the test type is not supported or the test is not '
'swarmed yet.' % (step_name, test_name)),
},
'return_code': 400,
}
analysis = request.FindMatchingAnalysisForConfiguration(
master_name, builder_name)
if not analysis:
logging.error('Flake analysis was deleted unexpectedly!')
return {
'template': 'error.html',
'data': {
'error_message': 'Flake analysis was deleted unexpectedly!',
},
'return_code': 404,
}
logging.info('Analysis: %s has a scheduled status of: %r', analysis.key,
scheduled)
return self.CreateRedirect(
'/p/chromium/flake-portal/analysis/analyze?redirect=1&key=%s' %
analysis.key.urlsafe())
def testWaterfallFlake(self):
request = FlakeAnalysisRequest.Create('flaky_test', False, 123)
request.AddBuildStep('chromium.linux', 'b1', 1, 's',
datetime(2016, 11, 14))
self.assertFalse(request.on_cq)
def testUpdateNoChanges(self):
request = FlakeAnalysisRequest.Create('test', False, 123)
request.Update(name='test')
self.assertEqual('test', request.name)
def testAsyncProcessFlakeReportOnStaging(self, _):
analysis_request = FlakeAnalysisRequest.Create('t', False, 12345)
apis.AsyncProcessFlakeReport(analysis_request, '*****@*****.**', False)
self.assertEqual(0, len(self.taskqueue_requests))