def testCopyFrom(self):
request1 = FlakeAnalysisRequest.Create('flaky_test', False, 123)
request2 = FlakeAnalysisRequest.Create('flaky_test', True, 456)
request2.AddBuildStep('m', 'b1', 1, 's', datetime(2016, 10, 1))
request2.user_emails = ['email']
analysis = MasterFlakeAnalysis.Create('m', 'b', 100, 's', 't')
analysis.Save()
request2.analyses.append(analysis.key)
request1.CopyFrom(request2)
self.assertEqual(request2.is_step, request1.is_step)
self.assertEqual(request2.bug_id, request1.bug_id)
self.assertEqual(request2.user_emails, request1.user_emails)
self.assertEqual(request2.build_steps, request1.build_steps)
self.assertEqual(request2.analyses, request1.analyses)
def CreateFlakeAnalysisRequest(flake):
analysis_request = FlakeAnalysisRequest.Create(
flake.name, flake.is_step, flake.bug_id)
for step in flake.build_steps:
analysis_request.AddBuildStep(step.master_name, step.builder_name,
step.build_number, step.step_name,
time_util.GetUTCNow())
return analysis_request
def testFindMatchingAnalysisForConfiguration(self, _):
request = FlakeAnalysisRequest.Create('test', False, 123)
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 'test')
analysis.Save()
request.analyses.append(analysis.key)
request.Save()
self.assertEqual(
analysis, request.FindMatchingAnalysisForConfiguration('m', 'b'))
def _CheckForNewAnalysis(request):
"""Checks if a new analysis is needed for the requested flake.
Args:
request (FlakeAnalysisRequest): The request to analyze a flake.
Returns:
(version_number, build_step)
version_number (int): The version of the FlakeAnalysisRequest if a new
analysis is needed; otherwise 0.
build_step (BuildStep): a BuildStep instance if a new analysis is needed;
otherwise None.
"""
existing_request = FlakeAnalysisRequest.GetVersion(key=request.name)
if not existing_request or (existing_request.bug_id and request.bug_id
and existing_request.bug_id != request.bug_id):
# If no existing analysis or last analysis was for a different bug, randomly
# pick one configuration for a new analysis.
if existing_request:
# Make a copy to preserve the version number of existing analysis and
# prevent concurrent analyses of the same flake.
user_emails = (email_util.ObscureEmails(
existing_request.user_emails, ['google.com']) +
list(set(request.user_emails)))
existing_request.CopyFrom(request)
request = existing_request
request.user_emails = user_emails
request.user_emails_obscured = False
request.user_emails_last_edit = time_util.GetUTCNow()
swarmed, supported, supported_build_step = _CheckFlakeSwarmedAndSupported(
request)
request.swarmed = swarmed
request.supported = supported
if supported_build_step and not request.is_step:
supported_build_step.scheduled = True # This step will be analyzed.
# For unsupported or step-level flakes, still save them for monitoring.
_, saved = request.Save(
retry_on_conflict=False) # Create a new version.
if not saved or not supported_build_step or request.is_step:
# No new analysis if:
# 1. Another analysis was just triggered.
# 2. No representative step is Swarmed Gtest.
# 3. The flake is a step-level one.
return 0, None
return request.version_number, supported_build_step
else:
# If no bug is attached to the existing analysis or the new request, or both
# are attached to the same bug, start a new analysis with a different
# configuration. For a configuration that was analyzed 7 days ago, reset it
# to use the new reported step of the same configuration.
# TODO: move this setting to config.
return _MergeNewRequestIntoExistingOne(request, existing_request)
def testObscureFlakeAnalysisRequest(self):
self.mock_current_user(user_email='*****@*****.**', is_admin=True)
mocked_utcnow = datetime(2017, 05, 05, 22, 50, 10)
self.MockUTCNow(mocked_utcnow)
valid_request_time = obscure_emails._TimeBeforeNow(days=5)
invalid_request_time = obscure_emails._TimeBeforeNow(
days=obscure_emails._REQUEST_RECORD_RENTENSION_DAYS + 10)
old_request = FlakeAnalysisRequest.Create('flake1', False, 123)
old_request.user_emails.append('*****@*****.**')
old_request.user_emails_obscured = False
old_request.user_emails_last_edit = invalid_request_time
old_request.Save()
recent_request = FlakeAnalysisRequest.Create('flake2', False, 321)
recent_request.user_emails.append('*****@*****.**')
recent_request.user_emails_obscured = False
recent_request.user_emails_last_edit = valid_request_time
recent_request.Save()
response = self.test_app.get('/obscure-emails',
params={'format': 'json'})
expected_response = {
'failure_triage_count': 0,
'flake_triage_count': 0,
'flake_request_aggregated_count': 1,
'flake_request_count': 0,
}
self.assertEqual(expected_response, response.json_body)
old_request = FlakeAnalysisRequest.GetVersion(key='flake1', version=1)
self.assertTrue(old_request.user_emails_obscured)
self.assertEqual(['*****@*****.**'], old_request.user_emails)
recent_request = FlakeAnalysisRequest.GetVersion(key='flake2',
version=1)
self.assertFalse(recent_request.user_emails_obscured)
self.assertEqual(['*****@*****.**'], recent_request.user_emails)
def testAddBuildStep(self):
t1 = datetime(2016, 10, 1, 0, 0, 0)
t2 = datetime(2016, 10, 2, 0, 0, 0)
t3 = datetime(2016, 10, 2, 1, 0, 0)
t4 = datetime(2016, 10, 2, 0, 30, 0)
request = FlakeAnalysisRequest.Create('flaky_test', False, 123)
self.assertTrue(request.AddBuildStep('m', 'b1', 1, 's', t1))
self.assertTrue(request.AddBuildStep('m', 'b2', 10, 's', t2))
self.assertFalse(request.AddBuildStep('m', 'b2', 11, 's', t3))
self.assertTrue(request.AddBuildStep('m', 'b2', 9, 's', t4))
self.assertEqual(2, len(request.build_steps), request.build_steps)
self.assertEqual(BuildStep.Create('m', 'b1', 1, 's', t1),
request.build_steps[0])
self.assertEqual(BuildStep.Create('m', 'b2', 9, 's', t4),
request.build_steps[1])
def run(self, master_name, builder_name, build_number):
"""Triggers flake analyses for flaky tests found by build failure analysis.
Args:
master_name (str): The master name.
builder_name (str): The builder name.
build_number (str): The build number.
"""
analysis = WfAnalysis.Get(master_name, builder_name, build_number)
if not analysis or not analysis.failure_result_map: # pragma: no cover
return
for step in analysis.failure_result_map.iterkeys():
task = WfSwarmingTask.Get(
master_name, builder_name, build_number, step)
if not task: # pragma: no cover
continue
flaky_tests = task.classified_tests.get('flaky_tests', [])
if not flaky_tests: # pragma: no cover
continue
# Trigger a master flake analysis on each detected flaky test.
# TODO lijeffrey): rerun all tests once typical load is determined to be
# within reasonable limits. For experimentation with automatic flakiness
# checking, only run 1 test per anaysis to avoid excessive load on the
# swarming server in case there are too many flaky tests per analysis for
# now.
test_name = flaky_tests[0]
request = FlakeAnalysisRequest.Create(test_name, False, None)
request.AddBuildStep(
master_name, builder_name, build_number, step,
time_util.GetUTCNow())
scheduled = flake_analysis_service.ScheduleAnalysisForFlake(
request, '*****@*****.**', False,
triggering_sources.FINDIT_PIPELINE)
if scheduled: # pragma: no branch
logging.info('%s/%s/%s has %s flaky tests.',
master_name, builder_name, build_number, len(flaky_tests))
logging.info('A flake analysis has been triggered for %s', test_name)
def _ObscureFlakeAnalysisRequest():
"""Obscures the user emails in FlakeAnalysisRequest."""
count = 0
time_limit = _TimeBeforeNow(days=_REQUEST_RECORD_RENTENSION_DAYS)
query = FlakeAnalysisRequest.query(
FlakeAnalysisRequest.user_emails_obscured == False,
FlakeAnalysisRequest.user_emails_last_edit < time_limit)
more = True
cursor = None
while more:
entities, cursor, more = query.fetch_page(_PAGE_SIZE,
start_cursor=cursor)
for entity in entities:
entity.user_emails = email_util.ObscureEmails(
entity.user_emails, ['google.com'])
entity.user_emails_obscured = True
ndb.put_multi(entities)
count += len(entities)
return count
def testGetNormalizedConfigurationNames(self):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 's'
test_name = 't'
reported_time = datetime(2016, 11, 16)
request = FlakeAnalysisRequest.Create(test_name, False, 123)
build_step = BuildStep.Create(master_name, builder_name, build_number,
step_name, reported_time)
build_step.wf_master_name = master_name
build_step.wf_builder_name = builder_name
build_step.wf_build_number = build_number
build_step.wf_step_name = step_name
request.build_steps.append(build_step)
self.assertEqual((None, None),
request._GetNormalizedConfigurationNames('m2', 'b2'))
self.assertEqual(
(master_name, builder_name),
request._GetNormalizedConfigurationNames(master_name,
builder_name))
def testRequestUnsupportedAnalysis(self, _):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 's'
test_name = 't'
previous_request = FlakeAnalysisRequest.Create(test_name, False, None)
previous_request.AddBuildStep(
master_name, builder_name, build_number, step_name, None)
previous_request.swarmed = False
previous_request.supported = False
self.assertRaisesRegexp(
webtest.app.AppError,
re.compile('.*not supported.*', re.MULTILINE | re.DOTALL),
self.test_app.get,
'/waterfall/flake',
params={
'url': buildbot.CreateBuildUrl(
master_name, builder_name, build_number),
'step_name': step_name,
'test_name': test_name,
'format': 'json'})
def testWaterfallFlake(self):
request = FlakeAnalysisRequest.Create('flaky_test', False, 123)
request.AddBuildStep('chromium.linux', 'b1', 1, 's',
datetime(2016, 11, 14))
self.assertFalse(request.on_cq)
def ScheduleAnalysisForFlake(request, user_email, is_admin, triggering_source):
"""Schedules an analysis on the flake in the given request if needed.
Args:
request (FlakeAnalysisRequest): The request to analyze a flake.
user_email (str): The email of the requester.
is_admin (bool): Whether the requester is an admin.
triggering_source (int): Where the request is coming from, either Findit
UI (check flake page), pipeline (from analysis) or Findit API.
Returns:
True if an analysis was scheduled; False if a new analysis is not needed;
None if the user has no permission to.
"""
assert len(request.build_steps), 'At least 1 build step is needed!'
if not IsAuthorizedUser(user_email, is_admin):
logging.info('user:%s, admin:%s', user_email, is_admin)
return None
request.user_emails = [user_email]
manually_triggered = user_email.endswith('@google.com')
trigger_action = 'manual' if manually_triggered else 'auto'
flake_source = 'cq' if request.on_cq else 'waterfall'
for build_step in request.build_steps:
step_mapper.FindMatchingWaterfallStep(build_step, request.name)
version_number, build_step = _CheckForNewAnalysis(request)
if version_number and build_step:
# A new analysis is needed.
# TODO(lijeffrey): Add support for the force flag to trigger a rerun.
logging.info('A new analysis is needed for: %s', build_step)
normalized_test = TestInfo(build_step.wf_master_name,
build_step.wf_builder_name,
build_step.wf_build_number,
build_step.wf_step_name, request.name)
original_test = TestInfo(build_step.master_name,
build_step.builder_name,
build_step.build_number, build_step.step_name,
request.name)
analysis = initialize_flake_pipeline.ScheduleAnalysisIfNeeded(
normalized_test,
original_test,
bug_id=request.bug_id,
allow_new_analysis=True,
manually_triggered=manually_triggered,
user_email=user_email,
triggering_source=triggering_source,
queue_name=constants.WATERFALL_ANALYSIS_QUEUE)
if analysis:
# TODO: put this in a transaction.
request = FlakeAnalysisRequest.GetVersion(key=request.name,
version=version_number)
request.analyses.append(analysis.key)
request.put()
logging.info('A new analysis was triggered successfully: %s',
analysis.key)
monitoring.flakes.increment({
'operation': 'analyze',
'trigger': trigger_action,
'source': flake_source,
})
return True
else:
logging.error('But new analysis was not triggered!')
else:
logging.info('No new analysis is needed: %s', request)
monitoring.flakes.increment({
'operation': 'skip',
'trigger': trigger_action,
'source': flake_source,
})
return False
def testRequestExistingAnalysis(self, *_):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 's'
test_name = 't'
success_rate = 0.9
previous_analysis = MasterFlakeAnalysis.Create(
master_name, builder_name, build_number - 1, step_name, test_name)
data_point = DataPoint()
data_point.build_number = build_number - 1
data_point.pass_rate = success_rate
previous_analysis.data_points.append(data_point)
previous_analysis.status = analysis_status.COMPLETED
previous_analysis.suspected_flake_build_number = 100
previous_analysis.request_time = datetime.datetime(2016, 10, 01, 12, 10, 00)
previous_analysis.start_time = datetime.datetime(2016, 10, 01, 12, 10, 05)
previous_analysis.end_time = datetime.datetime(2016, 10, 01, 13, 10, 00)
previous_analysis.algorithm_parameters = {'iterations_to_rerun': 100}
previous_analysis.Save()
previous_request = FlakeAnalysisRequest.Create(test_name, False, None)
build_step = BuildStep.Create(
master_name, builder_name, build_number, step_name, None)
build_step.wf_master_name = build_step.master_name
build_step.wf_builder_name = build_step.builder_name
build_step.wf_build_number = build_step.build_number
build_step.wf_step_name = build_step.step_name
previous_request.build_steps.append(build_step)
previous_request.analyses.append(previous_analysis.key)
previous_request.Save()
self.mock_current_user(user_email='*****@*****.**')
response = self.test_app.get('/waterfall/flake', params={
'url': buildbot.CreateBuildUrl(master_name, builder_name, build_number),
'step_name': step_name,
'test_name': test_name,
'format': 'json'})
expected_check_flake_result = {
'key': previous_analysis.key.urlsafe(),
'pass_rates': [[12345, 0.9, '1', 100, 'git_hash_2', 12344,
'git_hash_1']],
'analysis_status': STATUS_TO_DESCRIPTION.get(previous_analysis.status),
'master_name': master_name,
'builder_name': builder_name,
'build_number': build_number - 1,
'step_name': step_name,
'test_name': test_name,
'request_time': '2016-10-01 12:10:00 UTC',
'build_level_number': 1,
'revision_level_number': 0,
'error': None,
'iterations_to_rerun': 100,
'pending_time': '00:00:05',
'duration': '00:59:55',
'suspected_flake': {
'build_number': 100,
'commit_position': 12345,
'git_hash': 'a_git_hash',
'triage_result': 0
},
'version_number': 1,
'show_input_ui': False,
'culprit': {},
'try_job_status': None,
'last_attempted_swarming_task': {
'task_id': None,
'build_number': None
},
'last_attempted_try_job': {},
'user_email': '*****@*****.**'
}
self.assertEqual(200, response.status_int)
self.assertEqual(expected_check_flake_result, response.json_body)
def HandleGet(self):
key = self.request.get('key')
if key:
analysis = ndb.Key(urlsafe=key).get()
if not analysis: # pragma: no cover
return self.CreateError('Analysis of flake is not found', 404)
else:
build_url = self.request.get('url', '').strip()
build_info = buildbot.ParseBuildUrl(build_url)
if not build_info: # pragma: no cover
return self.CreateError('Unknown build info!', 400)
master_name, builder_name, build_number = build_info
step_name = self.request.get('step_name', '').strip()
test_name = self.request.get('test_name', '').strip()
bug_id = self.request.get('bug_id', '').strip()
# TODO(lijeffrey): Add support for force flag to trigger a rerun.
error = self._ValidateInput(step_name, test_name, bug_id)
if error: # pragma: no cover
return error
build_number = int(build_number)
bug_id = int(bug_id) if bug_id else None
user_email = auth_util.GetUserEmail()
is_admin = auth_util.IsCurrentUserAdmin()
request = FlakeAnalysisRequest.Create(test_name, False, bug_id)
request.AddBuildStep(master_name, builder_name, build_number,
step_name, time_util.GetUTCNow())
scheduled = flake_analysis_service.ScheduleAnalysisForFlake(
request, user_email, is_admin, triggering_sources.FINDIT_UI)
analysis = MasterFlakeAnalysis.GetVersion(master_name,
builder_name,
build_number, step_name,
test_name)
if not analysis:
if scheduled is None:
# User does not have permission to trigger, nor was any previous
# analysis triggered to view.
return {
'template': 'error.html',
'data': {
'error_message':
('You could schedule an analysis for flaky test only '
'after you login with google.com account.'),
'login_url':
self.GetLoginUrl(),
},
'return_code': 401,
}
# Check if a previous request has already covered this analysis so use
# the results from that analysis.
request = FlakeAnalysisRequest.GetVersion(key=test_name)
if not (request and request.analyses):
return {
'template': 'error.html',
'data': {
'error_message':
('Flake analysis is not supported for this request. Either'
' the build step may not be supported or the test is not '
'swarmed.'),
},
'return_code': 400,
}
analysis = request.FindMatchingAnalysisForConfiguration(
master_name, builder_name)
if not analysis: # pragma: no cover
logging.error('Flake analysis was deleted unexpectedly!')
return {
'template': 'error.html',
'data': {
'error_message':
'Flake analysis was deleted unexpectedly!',
},
'return_code': 400
}
suspected_flake = _GetSuspectedFlakeInfo(analysis)
culprit = _GetCulpritInfo(analysis)
build_level_number, revision_level_number = _GetNumbersOfDataPointGroups(
analysis.data_points)
data = {
'key':
analysis.key.urlsafe(),
'master_name':
analysis.master_name,
'builder_name':
analysis.builder_name,
'build_number':
analysis.build_number,
'step_name':
analysis.step_name,
'test_name':
analysis.test_name,
'pass_rates': [],
'analysis_status':
analysis.status_description,
'try_job_status':
analysis_status.STATUS_TO_DESCRIPTION.get(analysis.try_job_status),
'last_attempted_swarming_task':
_GetLastAttemptedSwarmingTaskDetails(analysis),
'last_attempted_try_job':
_GetLastAttemptedTryJobDetails(analysis),
'version_number':
analysis.version_number,
'suspected_flake':
suspected_flake,
'culprit':
culprit,
'request_time':
time_util.FormatDatetime(analysis.request_time),
'build_level_number':
build_level_number,
'revision_level_number':
revision_level_number,
'error':
analysis.error_message,
'iterations_to_rerun':
analysis.iterations_to_rerun,
'show_input_ui':
self._ShowInputUI(analysis)
}
if (users.is_current_user_admin() and analysis.completed
and analysis.triage_history):
data['triage_history'] = analysis.GetTriageHistory()
data['pending_time'] = time_util.FormatDuration(
analysis.request_time, analysis.start_time
or time_util.GetUTCNow())
if analysis.status != analysis_status.PENDING:
data['duration'] = time_util.FormatDuration(
analysis.start_time, analysis.end_time
or time_util.GetUTCNow())
data['pass_rates'] = _GetCoordinatesData(analysis)
return {'template': 'flake/result.html', 'data': data}