def extend_with_bug_commits(case_study: CaseStudy, cmap: CommitMap, report_type: tp.Type['BaseReport'], merge_stage: int, ignore_blocked: bool) -> None: """ Extend a case study with revisions that either introduced or fixed a bug as determined by the given SZZ tool. Args: case_study: to extend cmap: commit map to map revisions to unique IDs ignore_blocked: ignore_blocked revisions merge_stage: stage the revisions will be added to report_type: report to use for bug detection """ project_cls: tp.Type[Project] = get_project_cls_by_name( case_study.project_name) def load_bugs_from_szz_report( load_fun: tp.Callable[[Path], SZZReport] ) -> tp.Optional[tp.FrozenSet[RawBug]]: reports = get_processed_revisions_files(case_study.project_name, report_type) if not reports: LOG.warning(f"I could not find any {report_type} reports. " "Falling back to bug provider.") return None report = load_fun(reports[0]) return report.get_all_raw_bugs() bugs: tp.Optional[tp.FrozenSet[RawBug]] = None if report_type == SZZUnleashedReport: bugs = load_bugs_from_szz_report(load_szzunleashed_report) elif report_type == PyDrillerSZZReport: bugs = load_bugs_from_szz_report(load_pydriller_szz_report) else: LOG.warning( f"Report type {report_type} is not supported by this extender " f"strategy. Falling back to bug provider.") if bugs is None: bug_provider = BugProvider.get_provider_for_project( get_project_cls_by_name(case_study.project_name)) bugs = bug_provider.find_raw_bugs() revisions: tp.Set[FullCommitHash] = set() for bug in bugs: revisions.add(bug.fixing_commit) revisions.update(bug.introducing_commits) rev_list = list(revisions) if ignore_blocked: rev_list = filter_blocked_revisions(rev_list, project_cls) case_study.include_revisions([(rev, cmap.time_id(rev)) for rev in rev_list], merge_stage)
def _get_all_issue_events(project_name: str) -> tp.List[IssueEvent]: """ Loads and returns all issue events for a given project. Args: project_name: The name of the project to look in. Returns: A list of IssueEvent objects or None. """ github_repo_name = get_github_repo_name_for_project( get_project_cls_by_name(project_name)) if github_repo_name: def load_issue_events(github: Github) -> 'PaginatedList[IssueEvent]': if github_repo_name: return github.get_repo(github_repo_name).get_issues_events() raise AssertionError(f"{project_name} is not a github project") cache_file_name = github_repo_name.replace("/", "_") + "_issues_events" issue_events = get_cached_github_object_list(cache_file_name, load_issue_events) if issue_events: return issue_events return [] raise AssertionError(f"{project_name} is not a github project")
def extend_with_release_revs(case_study: CaseStudy, cmap: CommitMap, release_type: ReleaseType, ignore_blocked: bool, merge_stage: int) -> None: """ Extend a case study with revisions marked as a release. This extender relies on the project to determine appropriate revisions. Args: case_study: to extend cmap: commit map to map revisions to unique IDs ignore_blocked: ignore_blocked revisions merge_stage: stage the revisions will be added to release_type: release type to add """ project_cls: tp.Type[Project] = get_project_cls_by_name( case_study.project_name) release_provider = ReleaseProvider.get_provider_for_project(project_cls) release_revisions: tp.List[FullCommitHash] = [ revision for revision, release in release_provider.get_release_revisions(release_type) ] if ignore_blocked: release_revisions = filter_blocked_revisions(release_revisions, project_cls) case_study.include_revisions([(rev, cmap.time_id(rev)) for rev in release_revisions], merge_stage)
def plot(self, view_mode: bool) -> None: """Plots bug plot for the whole project.""" project_name = self.plot_kwargs['project'] project_repo = get_local_project_git(project_name) bug_provider = BugProvider.get_provider_for_project( get_project_cls_by_name(project_name)) pydriller_bugs = bug_provider.find_pygit_bugs() reports = get_processed_revisions_files(project_name, SZZUnleashedReport) szzunleashed_bugs = frozenset([ as_pygit_bug(raw_bug, project_repo) for raw_bug in SZZUnleashedReport(reports[0]).get_all_raw_bugs() ]) if self.__szz_tool == 'pydriller': self.__figure = _plot_chord_diagram_for_raw_bugs( project_name, project_repo, pydriller_bugs, self.__szz_tool) elif self.__szz_tool == 'szz_unleashed': self.__figure = _plot_chord_diagram_for_raw_bugs( project_name, project_repo, szzunleashed_bugs, self.__szz_tool) elif self.__szz_tool == 'szz_diff': self.__figure = _bug_data_diff_plot(project_name, project_repo, pydriller_bugs, szzunleashed_bugs) else: raise PlotDataEmpty
def tabulate(self, table_format: TableFormat, wrap_table: bool) -> str: project_name: str = self.table_kwargs['case_study'].project_name bug_provider = BugProvider.get_provider_for_project( get_project_cls_by_name(project_name) ) variables = [ "fixing hash", "fixing message", "fixing author", "issue_number" ] pybugs = bug_provider.find_pygit_bugs() data_rows = [[ pybug.fixing_commit.hex, pybug.fixing_commit.message, pybug.fixing_commit.author.name, pybug.issue_id ] for pybug in pybugs] bug_df = pd.DataFrame(columns=variables, data=np.array(data_rows)) kwargs: tp.Dict[str, tp.Any] = {"bold_rows": True} if table_format.is_latex(): kwargs["multicolumn_format"] = "c" kwargs["longtable"] = True return dataframe_to_table( bug_df, table_format, wrap_table, wrap_landscape=True, **kwargs )
def revisions_of_project(self) -> None: """Generate the Revision list for the selected project if select specific is enabled.""" self.strategie_forms.setCurrentIndex( GenerationStrategie.SELECT_REVISION.value) if self.selected_project != self.revision_list_project: self.revision_details.setText("Loading Revisions") self.revision_details.repaint() get_local_project_git(self.selected_project).remotes[0].fetch() git_path = get_local_project_git_path(self.selected_project) initial_commit = get_initial_commit(git_path).hash commits = get_all_revisions_between(initial_commit, 'HEAD', FullCommitHash, git_path) commit_lookup_helper = create_commit_lookup_helper( self.selected_project) project = get_project_cls_by_name(self.selected_project) repo_name = get_primary_project_source(self.selected_project).local commits = map(lambda commit: CommitRepoPair(commit, repo_name), commits) cmap = get_commit_map(self.selected_project) commit_model = CommitTableModel( list(map(commit_lookup_helper, commits)), cmap, project) self.proxy_model.setSourceModel(commit_model) self.revision_list_project = self.selected_project self.revision_details.clear() self.revision_details.update()
def project_cls(self) -> tp.Type[bb.Project]: """ Look up the BenchBuild project for this case study. Returns: project class """ return get_project_cls_by_name(self.project_name)
def _gen_overview_data(tag_blocked: bool, **kwargs: tp.Any) -> tp.Dict[str, tp.List[int]]: case_study: CaseStudy = kwargs["case_study"] project_name = case_study.project_name commit_map: CommitMap = get_commit_map(project_name) project = get_project_cls_by_name(project_name) if 'report_type' in kwargs: result_file_type: tp.Type[BaseReport] = kwargs['report_type'] else: result_file_type = EmptyReport positions: tp.Dict[str, tp.List[int]] = { "background": [], "blocked": [], "blocked_all": [], "compile_error": [], "failed": [], "missing": [], "success": [] } for c_hash, index in commit_map.mapping_items(): if not case_study.has_revision(ShortCommitHash(c_hash)): positions["background"].append(index) if hasattr(project, "is_blocked_revision" ) and project.is_blocked_revision(c_hash)[0]: positions["blocked_all"].append(index) revisions = FileStatusDatabase.get_data_for_project( project_name, ["revision", "time_id", "file_status"], commit_map, case_study, result_file_type=result_file_type, tag_blocked=tag_blocked) positions["success"] = ( revisions[revisions["file_status"] == FileStatusExtension.SUCCESS. get_status_extension()])["time_id"].tolist() positions["failed"] = ( revisions[revisions["file_status"] == FileStatusExtension.FAILED. get_status_extension()])["time_id"].tolist() positions["blocked"] = ( revisions[revisions["file_status"] == FileStatusExtension.BLOCKED. get_status_extension()])["time_id"].tolist() positions["blocked_all"].extend( (revisions[revisions["file_status"] == FileStatusExtension.BLOCKED. get_status_extension()])["time_id"].tolist()) positions["missing"] = ( revisions[revisions["file_status"] == FileStatusExtension.MISSING. get_status_extension()])["time_id"].tolist() positions["compile_error"] = ( revisions[revisions["file_status"] == FileStatusExtension. COMPILE_ERROR.get_status_extension()])["time_id"].tolist() return positions
def tabulate(self, table_format: TableFormat, wrap_table: bool) -> str: case_studies = get_loaded_paper_config().get_all_case_studies() cs_data: tp.List[pd.DataFrame] = [] for case_study in case_studies: project_name = case_study.project_name commit_map = get_commit_map(project_name) project_cls = get_project_cls_by_name(project_name) project_repo = get_local_project_git(project_name) project_path = project_repo.path[:-5] project_git = git["-C", project_path] revisions = sorted( case_study.revisions, key=commit_map.time_id, reverse=True ) revision = revisions[0] rev_range = revision.hash if revision else "HEAD" cs_dict = { project_name: { "Domain": str(project_cls.DOMAIN)[0].upper() + str(project_cls.DOMAIN)[1:], "LOC": calc_repo_loc(project_repo, rev_range), "Commits": int(project_git("rev-list", "--count", rev_range)), "Authors": len( project_git("shortlog", "-s", rev_range).splitlines() ) } } if revision: cs_dict[project_name]["Revision"] = revision.short_hash cs_data.append(pd.DataFrame.from_dict(cs_dict, orient="index")) df = pd.concat(cs_data).sort_index() kwargs: tp.Dict[str, tp.Any] = {"bold_rows": True} if table_format.is_latex(): kwargs["multicolumn_format"] = "c" kwargs["multirow"] = True return dataframe_to_table( df, table_format, wrap_table, wrap_landscape=True, **kwargs )
def show_project_data(self, index: QModelIndex) -> None: """Update the project data field.""" project_name = index.data() if self.selected_project != project_name: self.selected_project = project_name project = get_project_cls_by_name(project_name) project_info = f"{project_name.upper()} : " \ f"\nDomain: {project.DOMAIN}" \ f"\nSource: " \ f"{bb.source.primary(*project.SOURCE).remote}" self.project_details.setText(project_info) self.project_details.update() if self.strategie_forms.currentIndex( ) == GenerationStrategie.SELECT_REVISION.value: self.revisions_of_project()
def extend_with_distrib_sampling(case_study: CaseStudy, cmap: CommitMap, sampling_method: NormalSamplingMethod, merge_stage: int, num_rev: int, ignore_blocked: bool, only_code_commits: bool) -> None: """ Extend a case study by sampling 'num_rev' new revisions, according to distribution specified with kwargs['distribution']. Args: case_study: to extend cmap: commit map to map revisions to unique IDs sampling_method: distribution to use for sampling merge_stage: stage the revisions will be added to num_rev: number of revisions to add ignore_blocked: ignore_blocked revisions only_code_commits: exclude commits which don't change code """ is_blocked: tp.Callable[[ShortCommitHash, tp.Type[Project]], bool] = lambda rev, _: False if ignore_blocked: is_blocked = is_revision_blocked is_code_commit: tp.Callable[[ShortCommitHash], bool] = lambda rev: True if only_code_commits: churn_conf = ChurnConfig.create_c_style_languages_config() project_git_path = get_local_project_git_path(case_study.project_name) def is_c_cpp_code_commit(commit: ShortCommitHash) -> bool: return contains_source_code(commit, project_git_path, churn_conf) is_code_commit = is_c_cpp_code_commit # Needs to be sorted so the propability distribution over the length # of the list is the same as the distribution over the commits age history project_cls = get_project_cls_by_name(case_study.project_name) revision_list = [ (FullCommitHash(rev), idx) for rev, idx in sorted(list(cmap.mapping_items()), key=lambda x: x[1]) if not case_study.has_revision_in_stage(ShortCommitHash(rev), merge_stage) and not is_blocked(ShortCommitHash(rev), project_cls) and is_code_commit(ShortCommitHash(rev)) ] case_study.include_revisions( sampling_method.sample_n(revision_list, num_rev), merge_stage)
def extend_with_smooth_revs(case_study: CaseStudy, cmap: CommitMap, boundary_gradient: int, ignore_blocked: bool, plot: Plot, merge_stage: int) -> None: """ Extend a case study with extra revisions that could smooth plot curves. This can remove steep gradients that result from missing certain revisions when sampling. Args: case_study: to extend cmap: commit map to map revisions to unique IDs ignore_blocked: ignore_blocked revisions merge_stage: stage the revisions will be added to plot: Plot to calculate new revisions from. boundary_gradient: Maximal expected gradient in percent between two revisions """ # convert input to float % gradient = boundary_gradient / float(100) print("Using boundary gradient: ", gradient) new_revisions = plot.calc_missing_revisions(gradient) if ignore_blocked: new_revisions = set( filter_blocked_revisions( list(new_revisions), get_project_cls_by_name(case_study.project_name))) # Remove revision that are already present in another stage. new_revisions = { rev for rev in new_revisions if not case_study.has_revision(rev) } if new_revisions: print("Found new revisions: ", new_revisions) case_study.include_revisions([(rev, cmap.time_id(rev)) for rev in new_revisions], merge_stage) else: print("No new revisions found that where not already " "present in the case study.")
def get_tagged_revision( revision: ShortCommitHash, project_name: str, result_file_type: tp.Type[BaseReport] ) -> FileStatusExtension: """ Calculates the file status for a revision. If two files exists the newest is considered for detecting the status. Args: revision: the revision to get the status for project_name: target project result_file_type: the type of the result file Returns: the status for the revision """ project_cls = get_project_cls_by_name(project_name) result_files = __get_result_files_dict(project_name, result_file_type) if revision not in result_files.keys(): return FileStatusExtension.MISSING return __get_tag_for_revision( revision, result_files[revision], project_cls, result_file_type )
def extend_with_revs_per_year(case_study: CaseStudy, cmap: CommitMap, merge_stage: int, ignore_blocked: bool, git_path: str, revs_per_year: int, revs_year_sep: bool) -> None: """ Extend a case_study with ``revs_per_year`` revisions per year. Args: case_study: to extend cmap: commit map to map revisions to unique IDs merge_stage: stage to add the new revisions to ignore_blocked: ignore blocked revisions' git_path: git path to the project revs_per_year: revisions to add per year revs_year_sep: put revisions in separate stages for each year """ def parse_int_string(string: tp.Optional[str]) -> tp.Optional[int]: if string is None: return None try: return int(string) except ValueError: return None def get_or_create_stage_for_year(year: int) -> int: stages = case_study.stages num_stages = len(stages) for stage_index in range(num_stages): stage_year = parse_int_string(stages[stage_index].name) if stage_year is None: continue if stage_year == year: return stage_index if stage_year > year: continue if stage_year < year: case_study.insert_empty_stage(stage_index) case_study.name_stage(stage_index, str(year)) return stage_index case_study.insert_empty_stage(num_stages) case_study.name_stage(num_stages, str(year)) return num_stages repo = pygit2.Repository(pygit2.discover_repository(git_path)) last_commit = repo[repo.head.target] commits: tp.DefaultDict[int, tp.List[FullCommitHash]] = defaultdict( list) # maps year -> list of commits for commit in repo.walk(last_commit.id, pygit2.GIT_SORT_TIME): commit_date = datetime.utcfromtimestamp(commit.commit_time) commits[commit_date.year].append( FullCommitHash.from_pygit_commit(commit)) new_rev_items = [] # new revisions that get added to to case_study for year, commits_in_year in commits.items(): samples = min(len(commits_in_year), revs_per_year) sample_commit_indices = sorted( random.sample(range(len(commits_in_year)), samples)) for commit_index in sample_commit_indices: commit_hash = commits_in_year[commit_index] if ignore_blocked and is_revision_blocked( commit_hash.to_short_commit_hash(), get_project_cls_by_name(case_study.project_name)): continue time_id = cmap.time_id(commit_hash) new_rev_items.append((commit_hash, time_id)) if revs_year_sep: merge_stage = get_or_create_stage_for_year(year) case_study.include_revisions(new_rev_items, merge_stage, True) new_rev_items.clear()
def get_revisions_status_for_case_study( case_study: CaseStudy, result_file_type: tp.Type[BaseReport], stage_num: int = -1, tag_blocked: bool = True, experiment_type: tp.Optional[tp.Type[VersionExperiment]] = None ) -> tp.List[tp.Tuple[ShortCommitHash, FileStatusExtension]]: """ Computes the file status for all revisions in this case study. Args: case_study: to work on result_file_type: report type of the result files stage_num: only consider a specific stage of the case study tag_blocked: if true, also blocked commits are tagged Returns: a list of (revision, status) tuples """ try: project_cls = get_project_cls_by_name(case_study.project_name) except LookupError: # Return an empty list should a project name not exist. return [] if experiment_type: tagged_revisions = get_tagged_experiment_specific_revisions( project_cls, result_file_type, tag_blocked, experiment_type) else: tagged_revisions = get_tagged_revisions(project_cls, result_file_type, tag_blocked) def filtered_tagged_revs( rev_provider: tp.Iterable[FullCommitHash] ) -> tp.List[tp.Tuple[ShortCommitHash, FileStatusExtension]]: filtered_revisions = [] for rev in rev_provider: short_rev = rev.to_short_commit_hash() found = False for tagged_rev in tagged_revisions: if short_rev == tagged_rev[0]: filtered_revisions.append(tagged_rev) found = True break if not found: if tag_blocked and is_revision_blocked(short_rev, project_cls): filtered_revisions.append( (short_rev, FileStatusExtension.BLOCKED)) else: filtered_revisions.append( (short_rev, FileStatusExtension.MISSING)) return filtered_revisions if stage_num == -1: return filtered_tagged_revs(case_study.revisions) if stage_num < case_study.num_stages: stage = case_study.stages[stage_num] return filtered_tagged_revs(stage.revisions) return []
def test_project_lookup_by_name(self) -> None: """Check if we can load project classes from their name.""" grav_prj_cls = get_project_cls_by_name("gravity") self.assertEqual(grav_prj_cls, Gravity)