def get_flakes(cls, mastername, buildername, buildnumber, step):
# If test results were invalid, report whole step as flaky.
steptext = ' '.join(step['text'])
stepname = normalize_test_type(step['name'])
if 'TEST RESULTS WERE INVALID' in steptext:
return [stepname]
url = TEST_RESULTS_URL_TEMPLATE % {
'mastername': urllib2.quote(mastername),
'buildername': urllib2.quote(buildername),
'buildnumber': urllib2.quote(str(buildnumber)),
'stepname': urllib2.quote(stepname),
}
try:
result = urlfetch.fetch(url)
if result.status_code >= 200 and result.status_code < 400:
json_result = json.loads(result.content)
_, failed, _ = cls._flatten_tests(
json_result.get('tests', {}),
json_result.get('path_delimiter', '/'))
return failed
if result.status_code == 404:
# This is quite a common case (only some failing steps are actually
# running tests and reporting results to flakiness dashboard).
logging.info('Failed to retrieve JSON from %s', url)
else:
logging.exception('Failed to retrieve JSON from %s', url)
except Exception:
logging.exception('Failed to retrieve or parse JSON from %s', url)
return [stepname]
def get_flakes(cls, mastername, buildername, buildnumber, step):
"""Returns a list of flakes in a given step.
It can either be entire step or a list of specific tests.
Args:
mastername: Master name on which step has been run.
buildername: Builder name on which step has been run.
buildnume: Number of the build in which step has been run.
step: Step name.
Returns:
(flakes, is_step), where flakes is a list of flake names and is_step is
True when the whole step is a flake, in which case flakes is a list
containing a single entry - the name of the step.
"""
# If test results were invalid, report whole step as flaky.
# For Luci builds, some steps don't have step text anymore. Such steps
# include 'Failure reason', 'analyze', etc.
steptext = ' '.join(step['text'] or [])
stepname = normalize_test_type(step['name'])
if 'TEST RESULTS WERE INVALID' in steptext:
return [stepname], True
url = TEST_RESULTS_URL_TEMPLATE % {
'mastername': urllib2.quote(mastername),
'buildername': urllib2.quote(buildername),
'buildnumber': urllib2.quote(str(buildnumber)),
'stepname': urllib2.quote(stepname),
}
try:
result = urlfetch.fetch(url)
if result.status_code >= 200 and result.status_code < 400:
json_result = json.loads(result.content)
_, failed, _ = cls._flatten_tests(
json_result.get('tests', {}),
json_result.get('path_delimiter', '/'))
if len(failed) > MAX_INDIVIDUAL_FLAKES_PER_STEP:
return [stepname], True
return failed, False
if result.status_code == 404:
# This is quite a common case (only some failing steps are actually
# running tests and reporting results to flakiness dashboard).
logging.info('Failed to retrieve JSON from %s', url)
else:
logging.exception('Failed to retrieve JSON from %s', url)
except Exception:
logging.exception('Failed to retrieve or parse JSON from %s', url)
return [stepname], True
def get(self):
search = self.request.get('q')
flake = Flake.query().filter(Flake.name == search).get()
if flake:
self.redirect('/all_flake_occurrences?key=%s' %
flake.key.urlsafe())
return
# Users might search using full step name. Try normalizing it before
# searching. Note that this won't find flakes in a step where
# chromium-try-flakes was able to determine which test has failed. Instead,
# users should search using the test name.
normalized_step_name = normalize_test_type(search)
flake = Flake.query().filter(Flake.name == normalized_step_name).get()
if flake:
self.redirect('/all_flake_occurrences?key=%s' %
flake.key.urlsafe())
return
self.response.write('No flake entry found for ' + search)
def test_ignores_with_patch_when_asked_to(self):
self.assertEqual(
'base_unittests',
util.normalize_test_type(
'base_unittests on Windows XP (with patch)',
ignore_with_patch=True))
def test_removes_instrumentation_test_prefix(self):
self.assertEqual(
'content_shell_test_apk (with patch)',
util.normalize_test_type(
'Instrumentation test content_shell_test_apk (with patch)'))
def test_without_patch_is_discarded(self):
self.assertEqual(
'base_unittests',
util.normalize_test_type(
'base_unittests on ATI GPU on Windows (without patch) on Windows'
))
def test_on_platform_with_parens_with_patch_even_more_noise(self):
self.assertEqual(
'base_unittests (with patch)',
util.normalize_test_type(
'base_unittests (ATI GPU) on Windows (with patch) on Windows'))
def test_on_platform_with_patch(self):
self.assertEqual(
'base_unittests (with patch)',
util.normalize_test_type(
'base_unittests on Windows XP (with patch)'))
def test_on_platform(self):
self.assertEqual(
'base_unittests',
util.normalize_test_type('base_unittests on Windows XP'))
def test_already_clean_name(self):
self.assertEqual('base_unittests',
util.normalize_test_type('base_unittests'))
def post(self):
if (not self.request.get('failure_run_key') or
not self.request.get('success_run_key')):
self.response.set_status(400, 'Invalid request parameters')
return
failure_run = ndb.Key(urlsafe=self.request.get('failure_run_key')).get()
success_run = ndb.Key(urlsafe=self.request.get('success_run_key')).get()
flaky_run = FlakyRun(
failure_run=failure_run.key,
failure_run_time_started=failure_run.time_started,
failure_run_time_finished=failure_run.time_finished,
success_run=success_run.key)
failure_time = failure_run.time_finished
patchset_builder_runs = failure_run.key.parent().get()
master = BuildRun.removeMasterPrefix(patchset_builder_runs.master)
url = ('https://chrome-build-extract.appspot.com/p/' + master +
'/builders/' + patchset_builder_runs.builder +'/builds/' +
str(failure_run.buildnumber) + '?json=1')
urlfetch.set_default_fetch_deadline(60)
logging.info('get_flaky_run_reason ' + url)
response = urlfetch.fetch(url)
if response.status_code >= 400 and response.status_code <= 599:
logging.error('The request to %s has returned %d: %s', url,
response.status_code, response.content)
self.response.set_status(500, 'Failed to fetch build.')
return
json_result = json.loads(response.content)
steps = json_result['steps']
failed_steps = []
passed_steps = []
for step in steps:
result = step['results'][0]
if build_result.isResultSuccess(result):
passed_steps.append(step)
continue
if not build_result.isResultFailure(result):
continue
step_name = step['name']
step_text = ' '.join(step['text'])
if step_name in IGNORED_STEPS:
continue
# Custom (non-trivial) rules for ignoring flakes in certain steps:
# - [swarming] ...: summary step would also be red (do not double count)
# - Patch failure: ingore non-infra failures as they are typically due to
# changes in the code on HEAD
# - bot_update PATCH FAILED: Duplicates failure in 'Patch failure' step.
# - ... (retry summary): this is an artificial step to fail the build due
# to another step that has failed earlier (do not double count).
if (step_name.startswith('[swarming]') or
(step_name == 'Patch failure' and result != build_result.EXCEPTION) or
(step_name == 'bot_update' and 'PATCH FAILED' in step_text)):
continue
failed_steps.append(step)
steps_to_ignore = []
for step in failed_steps:
step_name = step['name']
if '(with patch)' in step_name:
# Ignore any steps from the same test suite, which is determined by the
# normalized step name. Additionally, if the step fails without patch,
# ignore the original step as well because tree is busted.
normalized_step_name = normalize_test_type(step_name, True)
for other_step in failed_steps:
if other_step == step:
continue
normalized_other_step_name = normalize_test_type(
other_step['name'], True)
if normalized_other_step_name == normalized_step_name:
steps_to_ignore.append(other_step['name'])
if '(without patch)' in other_step['name']:
steps_to_ignore.append(step['name'])
flakes_to_update = []
for step in failed_steps:
step_name = step['name']
if step_name in steps_to_ignore:
continue
flakes, is_step = self.get_flakes(
master, patchset_builder_runs.builder, failure_run.buildnumber, step)
for flake in flakes:
flake_occurrence = FlakeOccurrence(name=step_name, failure=flake)
flaky_run.flakes.append(flake_occurrence)
flakes_to_update.append((flake, is_step))
# Do not create FlakyRuns if all failed steps have been ignored.
if not flaky_run.flakes:
return
flaky_run_key = flaky_run.put()
for flake, is_step in flakes_to_update:
self.add_failure_to_flake(flake, flaky_run_key, failure_time, is_step)
self.flaky_runs.increment_by(1)
def post(self):
if (not self.request.get('failure_run_key') or
not self.request.get('success_run_key')):
self.response.set_status(400, 'Invalid request parameters')
return
failure_run = ndb.Key(urlsafe=self.request.get('failure_run_key')).get()
success_run = ndb.Key(urlsafe=self.request.get('success_run_key')).get()
flaky_run = FlakyRun(
failure_run=failure_run.key,
failure_run_time_started=failure_run.time_started,
failure_run_time_finished=failure_run.time_finished,
success_run=success_run.key)
failure_time = failure_run.time_finished
patchset_builder_runs = failure_run.key.parent().get()
master = BuildRun.removeMasterPrefix(patchset_builder_runs.master)
url = ('https://luci-milo.appspot.com/'
'prpc/milo.Buildbot/GetBuildbotBuildJSON')
request = json.dumps({
'master': master,
'builder': patchset_builder_runs.builder,
'buildNum': failure_run.buildnumber,
})
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json',
}
urlfetch.set_default_fetch_deadline(60)
logging.info('get_flaky_run_reason: %s, %s', url, request)
response = urlfetch.fetch(
url, payload=request, method=urlfetch.POST, headers=headers,
validate_certificate=True)
if response.status_code != 200:
logging.error('The request to %s has returned %d: %s', url,
response.status_code, response.content)
self.response.set_status(500, 'Failed to fetch build.')
return
content = response.content
if content.startswith(_MILO_RESPONSE_PREFIX):
content = content[len(_MILO_RESPONSE_PREFIX):]
data = json.loads(content)['data']
json_result = json.loads(base64.b64decode(data))
steps = json_result['steps']
failed_steps = []
passed_steps = []
for step in steps:
result = step['results'][0]
if build_result.isResultSuccess(result):
passed_steps.append(step)
continue
if not build_result.isResultFailure(result):
continue
# For Luci builds, some steps don't have step text anymore. Such steps
# include 'Failure reason', 'analyze', etc.
step_text = ' '.join(step['text'] or [])
step_name = step['name']
if step_name in IGNORED_STEPS:
continue
# Custom (non-trivial) rules for ignoring flakes in certain steps:
# - [swarming] ...: summary step would also be red (do not double count)
# - Patch failure: ingore non-infra failures as they are typically due to
# changes in the code on HEAD
# - bot_update PATCH FAILED: Duplicates failure in 'Patch failure' step.
# - ... (retry summary): this is an artificial step to fail the build due
# to another step that has failed earlier (do not double count).
if (step_name.startswith('[swarming]') or
(step_name == 'Patch failure' and result != build_result.EXCEPTION) or
(step_name == 'bot_update' and 'PATCH FAILED' in step_text)):
continue
failed_steps.append(step)
steps_to_ignore = []
for step in failed_steps:
step_name = step['name']
if '(with patch)' in step_name:
# Ignore any steps from the same test suite, which is determined by the
# normalized step name. Additionally, if the step fails without patch,
# ignore the original step as well because tree is busted.
normalized_step_name = normalize_test_type(step_name, True)
for other_step in failed_steps:
if other_step == step:
continue
normalized_other_step_name = normalize_test_type(
other_step['name'], True)
if normalized_other_step_name == normalized_step_name:
steps_to_ignore.append(other_step['name'])
if '(without patch)' in other_step['name']:
steps_to_ignore.append(step['name'])
flakes_to_update = []
for step in failed_steps:
step_name = step['name']
if step_name in steps_to_ignore:
continue
flakes, is_step = self.get_flakes(
master, patchset_builder_runs.builder, failure_run.buildnumber, step)
if is_step and not is_infra_step_flake(step_name):
continue # Ignore flakes of non-infra steps.
for flake in flakes:
flake_occurrence = FlakeOccurrence(name=step_name, failure=flake)
flaky_run.flakes.append(flake_occurrence)
flakes_to_update.append((flake, is_step))
# Do not create FlakyRuns if all failed steps have been ignored.
if not flaky_run.flakes:
return
flaky_run_key = flaky_run.put()
for flake, is_step in flakes_to_update:
if self.is_duplicate_occurrence(flake, flaky_run):
logging.info('Not adding duplicate occurrence for the same CL')
continue
self.add_failure_to_flake(flake, flaky_run_key, failure_time, is_step)
self.flaky_runs.increment_by(1)
def show_all_flakes(flake, show_all):
from_index = 0 if show_all else -MAX_OCCURRENCES_DEFAULT
occurrences = filterNone(ndb.get_multi(flake.occurrences[from_index:]))
failure_runs_keys = []
patchsets_keys = []
flakes = []
step_names = set()
for o in occurrences:
failure_runs_keys.append(o.failure_run)
patchsets_keys.append(o.failure_run.parent())
matching_flakes = [f for f in o.flakes if f.failure == flake.name]
flakes.append(matching_flakes)
step_names.update(normalize_test_type(f.name) for f in matching_flakes)
failure_runs = filterNone(ndb.get_multi(failure_runs_keys))
patchsets = filterNone(ndb.get_multi(patchsets_keys))
class FailureRunExtended:
def __init__(self, url, milo_url, patchset_url, builder, formatted_time,
issue_ids, time_finished):
self.url = url
self.milo_url = milo_url
self.patchset_url = patchset_url
self.builder = builder
self.formatted_time = formatted_time
self.issue_ids = issue_ids
self.time_finished = time_finished
failure_runs_extended = []
for index, fr in enumerate(failure_runs):
failure_runs_extended.append(FailureRunExtended(
fr.getURL(),
fr.getMiloURL(),
patchsets[index].getURL(),
patchsets[index].builder,
fr.time_finished.strftime('%Y-%m-%d %H:%M:%S UTC'),
set([f.issue_id for f in flakes[index] if f.issue_id > 0]),
fr.time_finished,
))
# Do simple sorting to make reading easier.
failure_runs_extended = sorted(
failure_runs_extended, key=RunsSortFunction, reverse=True)
# Group flaky runs into periods separated by at least 3 days.
grouped_runs = []
if failure_runs_extended:
current_group = [failure_runs_extended[0]]
for f in failure_runs_extended[1:]:
if current_group[-1].time_finished - f.time_finished < MAX_GROUP_DISTANCE:
current_group.append(f)
else:
grouped_runs.append(current_group)
current_group = [f]
grouped_runs.append(current_group)
show_all_link = (len(flake.occurrences) > MAX_OCCURRENCES_DEFAULT and
not show_all)
data = {
'flake': flake,
'grouped_runs': grouped_runs,
'show_all_link': show_all_link,
'time_now': datetime.datetime.utcnow(),
}
if not flake.is_step:
data['flakiness_dashboard_urls'] = [
{
'url': FLAKINESS_DASHBOARD_URL % {
'normalized_step_name': step_name,
'test_name': flake.name
},
'step_name': step_name,
} for step_name in step_names
]
return data
