def post(self): if (not self.request.get('failure_run_key') or not self.request.get('success_run_key')): self.response.set_status(400, 'Invalid request parameters') return failure_run = ndb.Key(urlsafe=self.request.get('failure_run_key')).get() success_run = ndb.Key(urlsafe=self.request.get('success_run_key')).get() flaky_run = FlakyRun( failure_run=failure_run.key, failure_run_time_started=failure_run.time_started, failure_run_time_finished=failure_run.time_finished, success_run=success_run.key) failure_time = failure_run.time_finished patchset_builder_runs = failure_run.key.parent().get() master = BuildRun.removeMasterPrefix(patchset_builder_runs.master) url = ('https://chrome-build-extract.appspot.com/p/' + master + '/builders/' + patchset_builder_runs.builder +'/builds/' + str(failure_run.buildnumber) + '?json=1') urlfetch.set_default_fetch_deadline(60) logging.info('get_flaky_run_reason ' + url) response = urlfetch.fetch(url) if response.status_code >= 400 and response.status_code <= 599: logging.error('The request to %s has returned %d: %s', url, response.status_code, response.content) self.response.set_status(500, 'Failed to fetch build.') return json_result = json.loads(response.content) steps = json_result['steps'] failed_steps = [] passed_steps = [] for step in steps: result = step['results'][0] if build_result.isResultSuccess(result): passed_steps.append(step) continue if not build_result.isResultFailure(result): continue step_name = step['name'] step_text = ' '.join(step['text']) if step_name in IGNORED_STEPS: continue # Custom (non-trivial) rules for ignoring flakes in certain steps: # - [swarming] ...: summary step would also be red (do not double count) # - Patch failure: ingore non-infra failures as they are typically due to # changes in the code on HEAD # - bot_update PATCH FAILED: Duplicates failure in 'Patch failure' step. # - ... (retry summary): this is an artificial step to fail the build due # to another step that has failed earlier (do not double count). if (step_name.startswith('[swarming]') or (step_name == 'Patch failure' and result != build_result.EXCEPTION) or (step_name == 'bot_update' and 'PATCH FAILED' in step_text)): continue failed_steps.append(step) steps_to_ignore = [] for step in failed_steps: step_name = step['name'] if '(with patch)' in step_name: # Ignore any steps from the same test suite, which is determined by the # normalized step name. Additionally, if the step fails without patch, # ignore the original step as well because tree is busted. normalized_step_name = normalize_test_type(step_name, True) for other_step in failed_steps: if other_step == step: continue normalized_other_step_name = normalize_test_type( other_step['name'], True) if normalized_other_step_name == normalized_step_name: steps_to_ignore.append(other_step['name']) if '(without patch)' in other_step['name']: steps_to_ignore.append(step['name']) flakes_to_update = [] for step in failed_steps: step_name = step['name'] if step_name in steps_to_ignore: continue flakes, is_step = self.get_flakes( master, patchset_builder_runs.builder, failure_run.buildnumber, step) for flake in flakes: flake_occurrence = FlakeOccurrence(name=step_name, failure=flake) flaky_run.flakes.append(flake_occurrence) flakes_to_update.append((flake, is_step)) # Do not create FlakyRuns if all failed steps have been ignored. if not flaky_run.flakes: return flaky_run_key = flaky_run.put() for flake, is_step in flakes_to_update: self.add_failure_to_flake(flake, flaky_run_key, failure_time, is_step) self.flaky_runs.increment_by(1)
def post(self): if (not self.request.get('failure_run_key') or not self.request.get('success_run_key')): self.response.set_status(400, 'Invalid request parameters') return failure_run = ndb.Key(urlsafe=self.request.get('failure_run_key')).get() success_run = ndb.Key(urlsafe=self.request.get('success_run_key')).get() flaky_run = FlakyRun( failure_run=failure_run.key, failure_run_time_started=failure_run.time_started, failure_run_time_finished=failure_run.time_finished, success_run=success_run.key) failure_time = failure_run.time_finished patchset_builder_runs = failure_run.key.parent().get() master = BuildRun.removeMasterPrefix(patchset_builder_runs.master) url = ('https://luci-milo.appspot.com/' 'prpc/milo.Buildbot/GetBuildbotBuildJSON') request = json.dumps({ 'master': master, 'builder': patchset_builder_runs.builder, 'buildNum': failure_run.buildnumber, }) headers = { 'Content-Type': 'application/json', 'Accept': 'application/json', } urlfetch.set_default_fetch_deadline(60) logging.info('get_flaky_run_reason: %s, %s', url, request) response = urlfetch.fetch( url, payload=request, method=urlfetch.POST, headers=headers, validate_certificate=True) if response.status_code != 200: logging.error('The request to %s has returned %d: %s', url, response.status_code, response.content) self.response.set_status(500, 'Failed to fetch build.') return content = response.content if content.startswith(_MILO_RESPONSE_PREFIX): content = content[len(_MILO_RESPONSE_PREFIX):] data = json.loads(content)['data'] json_result = json.loads(base64.b64decode(data)) steps = json_result['steps'] failed_steps = [] passed_steps = [] for step in steps: result = step['results'][0] if build_result.isResultSuccess(result): passed_steps.append(step) continue if not build_result.isResultFailure(result): continue # For Luci builds, some steps don't have step text anymore. Such steps # include 'Failure reason', 'analyze', etc. step_text = ' '.join(step['text'] or []) step_name = step['name'] if step_name in IGNORED_STEPS: continue # Custom (non-trivial) rules for ignoring flakes in certain steps: # - [swarming] ...: summary step would also be red (do not double count) # - Patch failure: ingore non-infra failures as they are typically due to # changes in the code on HEAD # - bot_update PATCH FAILED: Duplicates failure in 'Patch failure' step. # - ... (retry summary): this is an artificial step to fail the build due # to another step that has failed earlier (do not double count). if (step_name.startswith('[swarming]') or (step_name == 'Patch failure' and result != build_result.EXCEPTION) or (step_name == 'bot_update' and 'PATCH FAILED' in step_text)): continue failed_steps.append(step) steps_to_ignore = [] for step in failed_steps: step_name = step['name'] if '(with patch)' in step_name: # Ignore any steps from the same test suite, which is determined by the # normalized step name. Additionally, if the step fails without patch, # ignore the original step as well because tree is busted. normalized_step_name = normalize_test_type(step_name, True) for other_step in failed_steps: if other_step == step: continue normalized_other_step_name = normalize_test_type( other_step['name'], True) if normalized_other_step_name == normalized_step_name: steps_to_ignore.append(other_step['name']) if '(without patch)' in other_step['name']: steps_to_ignore.append(step['name']) flakes_to_update = [] for step in failed_steps: step_name = step['name'] if step_name in steps_to_ignore: continue flakes, is_step = self.get_flakes( master, patchset_builder_runs.builder, failure_run.buildnumber, step) if is_step and not is_infra_step_flake(step_name): continue # Ignore flakes of non-infra steps. for flake in flakes: flake_occurrence = FlakeOccurrence(name=step_name, failure=flake) flaky_run.flakes.append(flake_occurrence) flakes_to_update.append((flake, is_step)) # Do not create FlakyRuns if all failed steps have been ignored. if not flaky_run.flakes: return flaky_run_key = flaky_run.put() for flake, is_step in flakes_to_update: if self.is_duplicate_occurrence(flake, flaky_run): logging.info('Not adding duplicate occurrence for the same CL') continue self.add_failure_to_flake(flake, flaky_run_key, failure_time, is_step) self.flaky_runs.increment_by(1)