class ArtifactLogAdder(object): def __init__(self, task_name): log.info('Initializing ArtifactLogAdder.') self.config = Config(task_name) self.utils = Utils(self.config) self.task = task_name def run(self): bugswarmapi = DatabaseAPI(token=DATABASE_PIPELINE_TOKEN) if os.path.isfile('../cache-dependency/output/{}.csv'.format( self.task)) is False: log.error('cache-dependency output CSV does not exist for task {}'. format(self.task)) sys.exit() cached_image_tags = set() with open('../cache-dependency/output/{}.csv'.format(self.task)) as f: for row in f: # This assumes format '<image tag>, <succeed/error>, <size>, <size increase>' row_list = row.split(', ') if row_list[1] == 'succeed': cached_image_tags.add(row_list[0]) for image_tag in cached_image_tags: response = bugswarmapi.find_artifact(image_tag) if not response.ok: log.error('Artifact not found: {}'.format(image_tag)) continue artifact = response.json() job_id = { 'failed': artifact['failed_job']['job_id'], 'passed': artifact['passed_job']['job_id'], } job_orig_log = { 'failed': os.getcwd() + '/' + self.utils.get_orig_log_path(job_id['failed']), 'passed': os.getcwd() + '/' + self.utils.get_orig_log_path(job_id['passed']), } for f_or_p in ['failed', 'passed']: response = bugswarmapi.set_build_log(str(job_id[f_or_p]), job_orig_log[f_or_p]) if response.ok: log.info( '{} build log with ID {} successfully set for artifact: {}' .format(f_or_p, job_id[f_or_p], image_tag)) else: log.error( 'Error {} attempting to set {} build log with ID {} set for artifact: {}' .format(str(response), f_or_p, job_id[f_or_p], image_tag))
def _copy_original_log(utils: Utils, jobpair: JobPair): for j in jobpair.jobs: original_log_path = utils.get_orig_log_path(j.job_id) if not download_log(j.job_id, original_log_path): raise ReproduceError( 'Error while copying the original log for {}.'.format( j.job_id)) utils.copy_orig_log_into_jobpair_dir(j)
class ReproducedResultsAnalyzer(object): def __init__(self, input_file, runs, task_name): log.info('Initializing ReproducedResultsAnalyzer.') self.input_file = input_file self.runs = runs self.config = Config(task_name) self.utils = Utils(self.config) self.analyzer = analyzer.Analyzer() # Initializing pair_center should not be in _pre_analyze because we want the pairs to maintain state between # analyzing each run. self.pair_center = PairCenter(input_file, self.utils) # The below attributes are initialized in _pre_analyze. self.start_time = None self.reproduced_logs = None self.reproduced_logs_analyzed = None self.error_count = None def run(self): for i in range(1, self.runs + 1): self._pre_analyze() self._analyze(i) self._post_analyze(i) self._show_reproducibility() self._write_output_json() log.info('Done!') def _pre_analyze(self): """ Reset state before analyzing the next run. """ self.start_time = time.time() self.reproduced_logs = {} self.reproduced_logs_analyzed = 0 self.error_count = 0 # Reset the match type flag before each run for r in self.pair_center.repos: for bp in self.pair_center.repos[r].buildpairs: bp.set_match_type.value = False def _analyze(self, run): """ Analyze a single run of reproduced results. For each job in a jobpair, check if the reproduced log exists in the task folder. If it does, then download the original Travis log. Finally, analyze and compare the two logs. """ for r in self.pair_center.repos: for bp in self.pair_center.repos[r].buildpairs: for jp in bp.jobpairs: for j in jp.jobs: try: analyzed_reproduced_log = analyze_and_compare(self, j, run) if analyzed_reproduced_log: self.reproduced_logs_analyzed += 1 except Exception as e: log.error('Encountered an error while analyzing and comparing {}: {}'.format(j.job_name, e)) self.error_count += 1 self.pair_center.update_buildpair_done_status() self.pair_center.assign_pair_match_types() self.pair_center.assign_pair_match_history(run) self.pair_center.assign_pair_patch_history(run) def _post_analyze(self, run): """ This function is called after analyzing each run. Print statistics like how many pairs matched and time elapsed and then visualize the match history after this run. """ log.info('Done analyzing run {}.'.format(run)) self._visualize_match_history() log.info('{} reproduced logs analyzed and {} errors in run {}.' .format(self.reproduced_logs_analyzed, self.error_count, run)) # Print a blank line to separate each run. log.info() mmm = self.utils.construct_mmm_count(self.pair_center) aaa = self.utils.construct_aaa_count(self.pair_center) log.debug('Match types in run {}: m1-m2-m3: {} a1-a2-a3: {}.'.format(run, mmm, aaa)) def _write_output_json(self): log.info('Writing output JSON annotated with match history.') pairs = read_json(self.input_file) # Write default attributes. for p in pairs: for jp in p['jobpairs']: jp['match_history'] = {} jp['failed_job']['match_history'] = {} jp['passed_job']['match_history'] = {} jp['failed_job']['orig_result'] = '' jp['passed_job']['orig_result'] = '' jp['failed_job']['mismatch_attrs'] = [] jp['passed_job']['mismatch_attrs'] = [] jp['failed_job']['pip_patch'] = False jp['passed_job']['pip_patch'] = False for p in pairs: repo = p['repo'] if repo not in self.pair_center.repos: continue # Try to find this build pair in pair center. for bp in self.pair_center.repos[repo].buildpairs: if p['failed_build']['build_id'] == bp.builds[0].build_id: # Found build pair in pair center. # Optional: Write buildpair match type. # This is not used since we switched to jobpair packaging. p['match'] = bp.match.value trigger_sha = p['failed_build']['head_sha'] # Similarly, for each job pair in build pair, try to find it in the pair center. for jp in p['jobpairs']: # For a build that has some jobs filtered and some jobs not filtered, # the job cannot be found in paircenter. if jp['is_filtered']: continue found_in_paircenter = False for jobpair in bp.jobpairs: if str(jobpair.jobs[0].job_id) == str(jp['failed_job']['job_id']): found_in_paircenter = True # Write jobpair match history, analyzed results, and mismatched attributes. jp['match_history'] = jobpair.match_history jp['failed_job']['match_history'] = jobpair.failed_job_match_history jp['passed_job']['match_history'] = jobpair.passed_job_match_history jp['failed_job']['orig_result'] = jobpair.jobs[0].orig_result jp['passed_job']['orig_result'] = jobpair.jobs[1].orig_result jp['failed_job']['mismatch_attrs'] = jobpair.jobs[0].mismatch_attrs jp['passed_job']['mismatch_attrs'] = jobpair.jobs[1].mismatch_attrs jp['failed_job']['pip_patch'] = jobpair.jobs[0].pip_patch jp['passed_job']['pip_patch'] = jobpair.jobs[1].pip_patch if not found_in_paircenter: # If not found in pair center, this jobpair was filtered out. # In this case, we still analyze the original log to get as many attributes as possible. for i in range(2): job_name = 'failed_job' if i == 0 else 'passed_job' job_id = jp[job_name]['job_id'] original_log_path = self.utils.get_orig_log_path(job_id) if not download_log(job_id, original_log_path): continue original_result = self.analyzer.analyze_single_log(original_log_path, job_id, trigger_sha, repo) if 'not_in_supported_language' in original_result: continue jp[job_name]['orig_result'] = original_result raise RuntimeError('Unexpected state: Jobpair not found in pair center. Exiting.') os.makedirs(self.config.result_json_dir, exist_ok=True) filename = self.config.task + '.json' filepath = os.path.join(self.config.result_json_dir, filename) write_json(filepath, pairs) def _get_all_jobpairs_and_all_runs(self) -> Tuple[List[JobPair], List[str]]: all_jobpairs = [] for r in self.pair_center.repos: for bp in self.pair_center.repos[r].buildpairs: for jp in bp.jobpairs: all_jobpairs.append(jp) all_runs = [] for jp in all_jobpairs: for run in jp.match_history: all_runs.append(run) all_runs = list(set(all_runs)) all_runs.sort() return all_jobpairs, all_runs def _visualize_match_history(self): log.info('Visualizing match history:') log.info('N means no reproduced log exists. (An error occured in reproducer while reproducing the job.)') all_jobpairs, all_runs = self._get_all_jobpairs_and_all_runs() for jp in all_jobpairs: log.info(jp.full_name) match_histories = [ (jp.match_history, 'Job pair'), (jp.failed_job_match_history, 'Failed job'), (jp.passed_job_match_history, 'Passed job'), ] for match_history, history_name in match_histories: # Task name is run number 1-5 mh = [str(match_history.get(run, 'N')) for run in all_runs] if mh: full_history_name = '{} match history'.format(history_name) log.info('{:>24}:'.format(full_history_name), ' -> '.join(mh)) else: log.info('No match history. (This jobpair is not reproduced.)') def _show_reproducibility(self): log.info('Visualizing reproducibility:') all_jobpairs, all_runs = self._get_all_jobpairs_and_all_runs() if not all_jobpairs: log.info('Nothing to visualize since no jobs were run.') else: full_name_max_length = max([len(jp.full_name) for jp in all_jobpairs]) for jp in all_jobpairs: mh = [] for run in all_runs: run_result = jp.match_history.get(run) # run_result could be 'N', 0, or 1 if run_result != 1: mh.append(0) else: mh.append(run_result) # No reproducing runs were successful if all(v == 0 for v in mh): reproducibility = 'Unreproducible' # match history is all 1s, all runs reproducible elif all(mh): reproducibility = 'Reproducible' else: reproducibility = 'Flaky' log.info('{full_name: >{width}} job pair reproducibility: {result}' .format(width=full_name_max_length, full_name=jp.full_name, result=reproducibility)) # Print a blank separator line. log.info()