def test_ignores_null_flaky_runs(self):
last_updated = datetime.datetime.now()
fake_build_key = BuildRun(buildnumber=1,
result=1,
time_finished=last_updated).put()
flake_run_key = FlakyRun(failure_run=fake_build_key,
success_run=fake_build_key,
failure_run_time_finished=last_updated,
flakes=[
FlakeOccurrence(name='fake_step',
failure='fake_test_name'),
FlakeOccurrence(name='fake_step2',
failure='fake_test_name')
]).put()
null_flake_run_key = ndb.Key('FlakyRun', 'fake-key')
Flake(issue_id=1,
is_step=False,
name='fake_test_name',
issue_last_updated=last_updated,
occurrences=[
flake_run_key,
null_flake_run_key,
]).put()
self.test_app.get('/migrate')
flake_types = FlakeType.query().fetch()
self.assertEqual(len(flake_types), 2)
flake_type_1 = flake_types[0]
self.assertEqual(flake_type_1.project, 'chromium')
self.assertEqual(flake_type_1.step_name, 'fake_step')
self.assertEqual(flake_type_1.test_name, 'fake_test_name')
self.assertIsNone(flake_type_1.config)
self.assertEqual(flake_type_1.last_updated, last_updated)
flake_type_2 = flake_types[1]
self.assertEqual(flake_type_2.project, 'chromium')
self.assertEqual(flake_type_2.step_name, 'fake_step2')
self.assertEqual(flake_type_2.test_name, 'fake_test_name')
self.assertIsNone(flake_type_2.config)
self.assertEqual(flake_type_2.last_updated, last_updated)
issues = Issue.query().fetch()
self.assertEqual(len(issues), 1)
issue = issues[0]
self.assertEqual(issue.issue_id, 1)
self.assertEqual(issue.project, 'chromium')
self.assertEqual(sorted(issue.flake_type_keys),
sorted(flake_type.key for flake_type in flake_types))
def _create_flake():
tf = datetime.datetime.utcnow()
ts = tf - datetime.timedelta(hours=1)
p = PatchsetBuilderRuns(issue=1,
patchset=1,
master='tryserver.bar',
builder='baz').put()
br_f0 = BuildRun(parent=p,
buildnumber=10,
result=2,
time_started=ts,
time_finished=tf).put()
br_f1 = BuildRun(parent=p,
buildnumber=20,
result=2,
time_started=ts,
time_finished=tf).put()
br_s0 = BuildRun(parent=p,
buildnumber=30,
result=0,
time_started=ts,
time_finished=tf).put()
occ1 = FlakyRun(failure_run=br_f0,
success_run=br_s0,
failure_run_time_started=ts,
failure_run_time_finished=tf,
flakes=[
FlakeOccurrence(name='step1', failure='testX'),
])
occ2 = FlakyRun(failure_run=br_f1,
success_run=br_s0,
failure_run_time_started=ts,
failure_run_time_finished=tf,
flakes=[
FlakeOccurrence(name='step2', failure='testX'),
FlakeOccurrence(name='step3', failure='step3'),
])
f = Flake(name='testX',
count_day=10,
occurrences=[occ1.put(), occ2.put()],
is_step=True,
issue_id=123456)
return f, [occ1, occ2]
def _create_flake(self):
tf = datetime.datetime(2016, 8, 6, 10, 20, 30)
ts = tf - datetime.timedelta(hours=1)
tf2 = tf - datetime.timedelta(days=5)
ts2 = tf2 - datetime.timedelta(hours=1)
p = PatchsetBuilderRuns(issue=123456, patchset=1, master='tryserver.test',
builder='test-builder').put()
br_f0 = BuildRun(parent=p, buildnumber=0, result=2, time_started=ts2,
time_finished=tf2).put()
br_f1 = BuildRun(parent=p, buildnumber=1, result=2, time_started=ts,
time_finished=tf).put()
br_s1 = BuildRun(parent=p, buildnumber=2, result=0, time_started=ts,
time_finished=tf).put()
br_f2 = BuildRun(parent=p, buildnumber=3, result=4, time_started=ts,
time_finished=tf).put()
br_s2 = BuildRun(parent=p, buildnumber=4, result=0, time_started=ts,
time_finished=tf).put()
occ_key1 = FlakyRun(failure_run=br_f0, success_run=br_s2,
flakes=[
FlakeOccurrence(name='foo (x)', failure='foo.bar'),
FlakeOccurrence(name='foo (x)', failure='other')],
failure_run_time_started=ts2,
failure_run_time_finished=tf2).put()
occ_key2 = FlakyRun(failure_run=br_f1, success_run=br_s1,
flakes=[
FlakeOccurrence(name='bar (y)', failure='foo.bar')],
failure_run_time_started=ts,
failure_run_time_finished=tf).put()
occ_key3 = FlakyRun(failure_run=br_f2, success_run=br_s2,
flakes=[
FlakeOccurrence(
name='foo (x)', failure='foo.bar', issue_id=100),
FlakeOccurrence(
name='bar (y)', failure='foo.bar', issue_id=200)],
failure_run_time_started=ts,
failure_run_time_finished=tf).put()
return Flake(name='foo.bar', count_day=10, is_step=False,
occurrences=[occ_key1, occ_key2, occ_key3])
def post(self):
if (not self.request.get('failure_run_key') or
not self.request.get('success_run_key')):
self.response.set_status(400, 'Invalid request parameters')
return
failure_run = ndb.Key(urlsafe=self.request.get('failure_run_key')).get()
success_run = ndb.Key(urlsafe=self.request.get('success_run_key')).get()
flaky_run = FlakyRun(
failure_run=failure_run.key,
failure_run_time_started=failure_run.time_started,
failure_run_time_finished=failure_run.time_finished,
success_run=success_run.key)
failure_time = failure_run.time_finished
patchset_builder_runs = failure_run.key.parent().get()
master = BuildRun.removeMasterPrefix(patchset_builder_runs.master)
url = ('https://chrome-build-extract.appspot.com/p/' + master +
'/builders/' + patchset_builder_runs.builder +'/builds/' +
str(failure_run.buildnumber) + '?json=1')
urlfetch.set_default_fetch_deadline(60)
logging.info('get_flaky_run_reason ' + url)
response = urlfetch.fetch(url)
if response.status_code >= 400 and response.status_code <= 599:
logging.error('The request to %s has returned %d: %s', url,
response.status_code, response.content)
self.response.set_status(500, 'Failed to fetch build.')
return
json_result = json.loads(response.content)
steps = json_result['steps']
failed_steps = []
passed_steps = []
for step in steps:
result = step['results'][0]
if build_result.isResultSuccess(result):
passed_steps.append(step)
continue
if not build_result.isResultFailure(result):
continue
step_name = step['name']
step_text = ' '.join(step['text'])
if step_name in IGNORED_STEPS:
continue
# Custom (non-trivial) rules for ignoring flakes in certain steps:
# - [swarming] ...: summary step would also be red (do not double count)
# - Patch failure: ingore non-infra failures as they are typically due to
# changes in the code on HEAD
# - bot_update PATCH FAILED: Duplicates failure in 'Patch failure' step.
# - ... (retry summary): this is an artificial step to fail the build due
# to another step that has failed earlier (do not double count).
if (step_name.startswith('[swarming]') or
(step_name == 'Patch failure' and result != build_result.EXCEPTION) or
(step_name == 'bot_update' and 'PATCH FAILED' in step_text)):
continue
failed_steps.append(step)
steps_to_ignore = []
for step in failed_steps:
step_name = step['name']
if '(with patch)' in step_name:
# Ignore any steps from the same test suite, which is determined by the
# normalized step name. Additionally, if the step fails without patch,
# ignore the original step as well because tree is busted.
normalized_step_name = normalize_test_type(step_name, True)
for other_step in failed_steps:
if other_step == step:
continue
normalized_other_step_name = normalize_test_type(
other_step['name'], True)
if normalized_other_step_name == normalized_step_name:
steps_to_ignore.append(other_step['name'])
if '(without patch)' in other_step['name']:
steps_to_ignore.append(step['name'])
flakes_to_update = []
for step in failed_steps:
step_name = step['name']
if step_name in steps_to_ignore:
continue
flakes, is_step = self.get_flakes(
master, patchset_builder_runs.builder, failure_run.buildnumber, step)
for flake in flakes:
flake_occurrence = FlakeOccurrence(name=step_name, failure=flake)
flaky_run.flakes.append(flake_occurrence)
flakes_to_update.append((flake, is_step))
# Do not create FlakyRuns if all failed steps have been ignored.
if not flaky_run.flakes:
return
flaky_run_key = flaky_run.put()
for flake, is_step in flakes_to_update:
self.add_failure_to_flake(flake, flaky_run_key, failure_time, is_step)
self.flaky_runs.increment_by(1)
def post(self):
if (not self.request.get('failure_run_key') or
not self.request.get('success_run_key')):
self.response.set_status(400, 'Invalid request parameters')
return
failure_run = ndb.Key(urlsafe=self.request.get('failure_run_key')).get()
success_run = ndb.Key(urlsafe=self.request.get('success_run_key')).get()
flaky_run = FlakyRun(
failure_run=failure_run.key,
failure_run_time_started=failure_run.time_started,
failure_run_time_finished=failure_run.time_finished,
success_run=success_run.key)
failure_time = failure_run.time_finished
patchset_builder_runs = failure_run.key.parent().get()
master = BuildRun.removeMasterPrefix(patchset_builder_runs.master)
url = ('https://luci-milo.appspot.com/'
'prpc/milo.Buildbot/GetBuildbotBuildJSON')
request = json.dumps({
'master': master,
'builder': patchset_builder_runs.builder,
'buildNum': failure_run.buildnumber,
})
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json',
}
urlfetch.set_default_fetch_deadline(60)
logging.info('get_flaky_run_reason: %s, %s', url, request)
response = urlfetch.fetch(
url, payload=request, method=urlfetch.POST, headers=headers,
validate_certificate=True)
if response.status_code != 200:
logging.error('The request to %s has returned %d: %s', url,
response.status_code, response.content)
self.response.set_status(500, 'Failed to fetch build.')
return
content = response.content
if content.startswith(_MILO_RESPONSE_PREFIX):
content = content[len(_MILO_RESPONSE_PREFIX):]
data = json.loads(content)['data']
json_result = json.loads(base64.b64decode(data))
steps = json_result['steps']
failed_steps = []
passed_steps = []
for step in steps:
result = step['results'][0]
if build_result.isResultSuccess(result):
passed_steps.append(step)
continue
if not build_result.isResultFailure(result):
continue
# For Luci builds, some steps don't have step text anymore. Such steps
# include 'Failure reason', 'analyze', etc.
step_text = ' '.join(step['text'] or [])
step_name = step['name']
if step_name in IGNORED_STEPS:
continue
# Custom (non-trivial) rules for ignoring flakes in certain steps:
# - [swarming] ...: summary step would also be red (do not double count)
# - Patch failure: ingore non-infra failures as they are typically due to
# changes in the code on HEAD
# - bot_update PATCH FAILED: Duplicates failure in 'Patch failure' step.
# - ... (retry summary): this is an artificial step to fail the build due
# to another step that has failed earlier (do not double count).
if (step_name.startswith('[swarming]') or
(step_name == 'Patch failure' and result != build_result.EXCEPTION) or
(step_name == 'bot_update' and 'PATCH FAILED' in step_text)):
continue
failed_steps.append(step)
steps_to_ignore = []
for step in failed_steps:
step_name = step['name']
if '(with patch)' in step_name:
# Ignore any steps from the same test suite, which is determined by the
# normalized step name. Additionally, if the step fails without patch,
# ignore the original step as well because tree is busted.
normalized_step_name = normalize_test_type(step_name, True)
for other_step in failed_steps:
if other_step == step:
continue
normalized_other_step_name = normalize_test_type(
other_step['name'], True)
if normalized_other_step_name == normalized_step_name:
steps_to_ignore.append(other_step['name'])
if '(without patch)' in other_step['name']:
steps_to_ignore.append(step['name'])
flakes_to_update = []
for step in failed_steps:
step_name = step['name']
if step_name in steps_to_ignore:
continue
flakes, is_step = self.get_flakes(
master, patchset_builder_runs.builder, failure_run.buildnumber, step)
if is_step and not is_infra_step_flake(step_name):
continue # Ignore flakes of non-infra steps.
for flake in flakes:
flake_occurrence = FlakeOccurrence(name=step_name, failure=flake)
flaky_run.flakes.append(flake_occurrence)
flakes_to_update.append((flake, is_step))
# Do not create FlakyRuns if all failed steps have been ignored.
if not flaky_run.flakes:
return
flaky_run_key = flaky_run.put()
for flake, is_step in flakes_to_update:
if self.is_duplicate_occurrence(flake, flaky_run):
logging.info('Not adding duplicate occurrence for the same CL')
continue
self.add_failure_to_flake(flake, flaky_run_key, failure_time, is_step)
self.flaky_runs.increment_by(1)