def get_descriptions(prs: PullRequests) -> Dict[str, List[Description]]: descriptions = {} # type: Dict[str, List[Description]] repos = {} # type: Dict[str, Repository] for pr in prs: # See https://github.com/PyGithub/PyGithub/issues/2202, # obj._rawData doesn't spend additional API requests # We'll save some requests # pylint: disable=protected-access repo_name = pr._rawData["base"]["repo"]["full_name"] # type: ignore # pylint: enable=protected-access if repo_name not in repos: repos[repo_name] = pr.base.repo in_changelog = False merge_commit = pr.merge_commit_sha try: runner.run(f"git rev-parse '{merge_commit}'") except CalledProcessError: # It's possible that commit not in the repo, just continue logging.info("PR %s does not belong to the repo", pr.number) continue in_changelog = merge_commit in SHA_IN_CHANGELOG if in_changelog: desc = generate_description(pr, repos[repo_name]) if desc is not None: if desc.category not in descriptions: descriptions[desc.category] = [] descriptions[desc.category].append(desc) for descs in descriptions.values(): descs.sort() return descriptions
def update_contributors( relative_contributors_path: str = GENERATED_CONTRIBUTORS, force: bool = False, raise_error: bool = False, ): # Check if we have shallow checkout by comparing number of lines # '--is-shallow-repository' is in git since 2.15, 2017-10-30 if git_runner.run("git rev-parse --is-shallow-repository") == "true" and not force: logging.warning("The repository is shallow, refusing to update contributors") if raise_error: raise RuntimeError("update_contributors executed on a shallow repository") return # format: " 1016 Alexey Arno" shortlog = git_runner.run("git shortlog HEAD --summary") contributors = sorted( [c.split(maxsplit=1)[-1].replace('"', r"\"") for c in shortlog.split("\n")], ) contributors = [f' "{c}",' for c in contributors] executer = p.relpath(p.realpath(__file__), git_runner.cwd) content = CONTRIBUTORS_TEMPLATE.format( executer=executer, contributors="\n".join(contributors) ) contributors_path = get_abs_path(relative_contributors_path) with open(contributors_path, "w", encoding="utf-8") as cfd: cfd.write(content)
def main(): log_levels = [logging.WARN, logging.INFO, logging.DEBUG] args = parse_args() logging.basicConfig( format= "%(asctime)s %(levelname)-8s [%(filename)s:%(lineno)d]:\n%(message)s", level=log_levels[min(args.verbose, 2)], ) if args.debug_helpers: logging.getLogger("github_helper").setLevel(logging.DEBUG) logging.getLogger("git_helper").setLevel(logging.DEBUG) # Create a cache directory if not p.isdir(CACHE_PATH): os.mkdir(CACHE_PATH, 0o700) # Get the full repo if is_shallow(): logging.info("Unshallow repository") runner.run("git fetch --unshallow", stderr=DEVNULL) logging.info("Fetching all tags") runner.run("git fetch --tags", stderr=DEVNULL) check_refs(args.from_ref, args.to_ref, args.with_testing_tags) set_sha_in_changelog() logging.info("Using %s..%s as changelog interval", FROM_REF, TO_REF) # use merge-base commit as a starting point, if used ref in another branch base_commit = runner.run( f"git merge-base '{FROM_REF}^{{}}' '{TO_REF}^{{}}'") # Get starting and ending dates for gathering PRs # Add one day after and before to mitigate TZ possible issues # `tag^{}` format gives commit ref when we have annotated tags # format %cs gives a committer date, works better for cherry-picked commits from_date = runner.run(f"git log -1 --format=format:%cs '{base_commit}'") to_date = runner.run(f"git log -1 --format=format:%cs '{TO_REF}^{{}}'") merged = ( date.fromisoformat(from_date) - timedelta(1), date.fromisoformat(to_date) + timedelta(1), ) # Get all PRs for the given time frame global gh gh = GitHub(args.gh_user_or_token, args.gh_password, per_page=100, pool_size=args.jobs) gh.cache_path = CACHE_PATH query = f"type:pr repo:{args.repo} is:merged" prs = gh.get_pulls_from_search(query=query, merged=merged, sort="created") descriptions = get_descriptions(prs) write_changelog(args.output, descriptions)
def main(): log_levels = [logging.CRITICAL, logging.WARN, logging.INFO, logging.DEBUG] args = parse_args() logging.basicConfig( format= "%(asctime)s %(levelname)-8s [%(filename)s:%(lineno)d]:\n%(message)s", level=log_levels[min(args.verbose, 3)], ) # Create a cache directory if not p.isdir(CACHE_PATH): os.mkdir(CACHE_PATH, 0o700) # Get the full repo if is_shallow(): logging.info("Unshallow repository") runner.run("git fetch --unshallow", stderr=DEVNULL) logging.info("Fetching all tags") runner.run("git fetch --tags", stderr=DEVNULL) check_refs(args.from_ref, args.to_ref, args.with_testing_tags) set_sha_in_changelog() logging.info("Using %s..%s as changelog interval", FROM_REF, TO_REF) # Get starting and ending dates for gathering PRs # Add one day after and before to mitigate TZ possible issues # `tag^{}` format gives commit ref when we have annotated tags # format %cs gives a committer date, works better for cherry-picked commits from_date = runner.run(f"git log -1 --format=format:%cs '{FROM_REF}^{{}}'") from_date = (date.fromisoformat(from_date) - timedelta(1)).isoformat() to_date = runner.run(f"git log -1 --format=format:%cs '{TO_REF}^{{}}'") to_date = (date.fromisoformat(to_date) + timedelta(1)).isoformat() # Get all PRs for the given time frame global GitHub GitHub = Github(args.gh_user_or_token, args.gh_password, per_page=100, pool_size=args.jobs) query = f"type:pr repo:{args.repo} is:merged merged:{from_date}..{to_date}" repo = GitHub.get_repo(args.repo) api_prs = GitHub.search_issues(query=query, sort="created") logging.info("Found %s PRs for the query: '%s'", api_prs.totalCount, query) issues = [] # type: List[Issue] while True: try: for issue in api_prs: issues.append(issue) break except RateLimitExceededException: sleep_on_rate_limit() descriptions = get_descriptions(repo, issues, args.jobs) write_changelog(args.output, descriptions)
def run(self): while not self.queue.empty(): try: issue = self.queue.get() # type: Issue except Empty: break # possible race condition, just continue api_pr = get_pull_cached(self.repo, issue.number, issue.updated_at) in_changelog = False merge_commit = api_pr.merge_commit_sha try: runner.run(f"git rev-parse '{merge_commit}'") except CalledProcessError: # It's possible that commit not in the repo, just continue logging.info("PR %s does not belong to the repo", api_pr.number) continue in_changelog = merge_commit in SHA_IN_CHANGELOG if in_changelog: desc = generate_description(api_pr, self.repo) if desc is not None: self.response.append(desc) self.queue.task_done()
def receive_prs_for_backport(self): # The commit is the oldest open release branch's merge-base since_commit = git_runner( f"git merge-base {self.remote}/{self.release_branches[0]} " f"{self.remote}/{self.default_branch}") since_date = date.fromisoformat( git_runner.run(f"git log -1 --format=format:%cs {since_commit}")) # To not have a possible TZ issues tomorrow = date.today() + timedelta(days=1) logging.info("Receive PRs suppose to be backported") self.prs_for_backport = self.gh.get_pulls_from_search( query=f"{self._query} -label:pr-backported", label=",".join(self.labels_to_backport + [Labels.LABEL_MUST_BACKPORT]), merged=[since_date, tomorrow], ) logging.info( "PRs to be backported:\n %s", "\n ".join([pr.html_url for pr in self.prs_for_backport]), )
def check_refs(from_ref: Optional[str], to_ref: str): global FROM_REF, TO_REF TO_REF = to_ref # Check TO_REF runner.run(f"git rev-parse {TO_REF}") # Check from_ref if from_ref is None: # Get all tags pointing to TO_REF tags = runner.run(f"git tag --points-at '{TO_REF}^{{}}'") logging.info("All tags pointing to %s:\n%s", TO_REF, tags) exclude = " ".join([f"--exclude='{tag}'" for tag in tags.split("\n")]) FROM_REF = runner.run( f"git describe --abbrev=0 --tags {exclude} '{TO_REF}'") else: runner.run(f"git rev-parse {FROM_REF}") FROM_REF = from_ref
def main(): log_levels = [logging.CRITICAL, logging.WARN, logging.INFO, logging.DEBUG] args = parse_args() logging.basicConfig( format= "%(asctime)s %(levelname)-8s [%(filename)s:%(lineno)d]:\n%(message)s", level=log_levels[min(args.verbose, 3)], ) # Get the full repo if is_shallow(): logging.info("Unshallow repository") runner.run("git fetch --unshallow", stderr=DEVNULL) logging.info("Fetching all tags") runner.run("git fetch --tags", stderr=DEVNULL) check_refs(args.from_ref, args.to_ref) set_sha_in_changelog() logging.info("Using %s..%s as changelog interval", FROM_REF, TO_REF) # Get starting and ending dates for gathering PRs # Add one day after and before to mitigate TZ possible issues # `tag^{}` format gives commit ref when we have annotated tags from_date = runner.run(f"git log -1 --format=format:%as '{FROM_REF}^{{}}'") from_date = (date.fromisoformat(from_date) - timedelta(1)).isoformat() to_date = runner.run(f"git log -1 --format=format:%as '{TO_REF}^{{}}'") to_date = (date.fromisoformat(to_date) + timedelta(1)).isoformat() # Get all PRs for the given time frame gh = Github(args.gh_user_or_token, args.gh_password, per_page=100, pool_size=args.jobs) query = f"type:pr repo:{args.repo} is:merged merged:{from_date}..{to_date}" repo = gh.get_repo(args.repo) api_prs = gh.search_issues(query=query, sort="created") logging.info("Found %s PRs for the query: '%s'", api_prs.totalCount, query) pr_numbers = [pr.number for pr in api_prs] descriptions = get_descriptions(repo, pr_numbers, args.jobs) write_changelog(args.output, descriptions)
def set_sha_in_changelog(): global SHA_IN_CHANGELOG SHA_IN_CHANGELOG = runner.run( f"git log --format=format:%H {FROM_REF}..{TO_REF}").split("\n")