def get_descriptions(prs: PullRequests) -> Dict[str, List[Description]]:
descriptions = {} # type: Dict[str, List[Description]]
repos = {} # type: Dict[str, Repository]
for pr in prs:
# See https://github.com/PyGithub/PyGithub/issues/2202,
# obj._rawData doesn't spend additional API requests
# We'll save some requests
# pylint: disable=protected-access
repo_name = pr._rawData["base"]["repo"]["full_name"] # type: ignore
# pylint: enable=protected-access
if repo_name not in repos:
repos[repo_name] = pr.base.repo
in_changelog = False
merge_commit = pr.merge_commit_sha
try:
runner.run(f"git rev-parse '{merge_commit}'")
except CalledProcessError:
# It's possible that commit not in the repo, just continue
logging.info("PR %s does not belong to the repo", pr.number)
continue
in_changelog = merge_commit in SHA_IN_CHANGELOG
if in_changelog:
desc = generate_description(pr, repos[repo_name])
if desc is not None:
if desc.category not in descriptions:
descriptions[desc.category] = []
descriptions[desc.category].append(desc)
for descs in descriptions.values():
descs.sort()
return descriptions
def update_contributors(
relative_contributors_path: str = GENERATED_CONTRIBUTORS,
force: bool = False,
raise_error: bool = False,
):
# Check if we have shallow checkout by comparing number of lines
# '--is-shallow-repository' is in git since 2.15, 2017-10-30
if git_runner.run("git rev-parse --is-shallow-repository") == "true" and not force:
logging.warning("The repository is shallow, refusing to update contributors")
if raise_error:
raise RuntimeError("update_contributors executed on a shallow repository")
return
# format: " 1016 Alexey Arno"
shortlog = git_runner.run("git shortlog HEAD --summary")
contributors = sorted(
[c.split(maxsplit=1)[-1].replace('"', r"\"") for c in shortlog.split("\n")],
)
contributors = [f' "{c}",' for c in contributors]
executer = p.relpath(p.realpath(__file__), git_runner.cwd)
content = CONTRIBUTORS_TEMPLATE.format(
executer=executer, contributors="\n".join(contributors)
)
contributors_path = get_abs_path(relative_contributors_path)
with open(contributors_path, "w", encoding="utf-8") as cfd:
cfd.write(content)
def main():
log_levels = [logging.WARN, logging.INFO, logging.DEBUG]
args = parse_args()
logging.basicConfig(
format=
"%(asctime)s %(levelname)-8s [%(filename)s:%(lineno)d]:\n%(message)s",
level=log_levels[min(args.verbose, 2)],
)
if args.debug_helpers:
logging.getLogger("github_helper").setLevel(logging.DEBUG)
logging.getLogger("git_helper").setLevel(logging.DEBUG)
# Create a cache directory
if not p.isdir(CACHE_PATH):
os.mkdir(CACHE_PATH, 0o700)
# Get the full repo
if is_shallow():
logging.info("Unshallow repository")
runner.run("git fetch --unshallow", stderr=DEVNULL)
logging.info("Fetching all tags")
runner.run("git fetch --tags", stderr=DEVNULL)
check_refs(args.from_ref, args.to_ref, args.with_testing_tags)
set_sha_in_changelog()
logging.info("Using %s..%s as changelog interval", FROM_REF, TO_REF)
# use merge-base commit as a starting point, if used ref in another branch
base_commit = runner.run(
f"git merge-base '{FROM_REF}^{{}}' '{TO_REF}^{{}}'")
# Get starting and ending dates for gathering PRs
# Add one day after and before to mitigate TZ possible issues
# `tag^{}` format gives commit ref when we have annotated tags
# format %cs gives a committer date, works better for cherry-picked commits
from_date = runner.run(f"git log -1 --format=format:%cs '{base_commit}'")
to_date = runner.run(f"git log -1 --format=format:%cs '{TO_REF}^{{}}'")
merged = (
date.fromisoformat(from_date) - timedelta(1),
date.fromisoformat(to_date) + timedelta(1),
)
# Get all PRs for the given time frame
global gh
gh = GitHub(args.gh_user_or_token,
args.gh_password,
per_page=100,
pool_size=args.jobs)
gh.cache_path = CACHE_PATH
query = f"type:pr repo:{args.repo} is:merged"
prs = gh.get_pulls_from_search(query=query, merged=merged, sort="created")
descriptions = get_descriptions(prs)
write_changelog(args.output, descriptions)
def main():
log_levels = [logging.CRITICAL, logging.WARN, logging.INFO, logging.DEBUG]
args = parse_args()
logging.basicConfig(
format=
"%(asctime)s %(levelname)-8s [%(filename)s:%(lineno)d]:\n%(message)s",
level=log_levels[min(args.verbose, 3)],
)
# Create a cache directory
if not p.isdir(CACHE_PATH):
os.mkdir(CACHE_PATH, 0o700)
# Get the full repo
if is_shallow():
logging.info("Unshallow repository")
runner.run("git fetch --unshallow", stderr=DEVNULL)
logging.info("Fetching all tags")
runner.run("git fetch --tags", stderr=DEVNULL)
check_refs(args.from_ref, args.to_ref, args.with_testing_tags)
set_sha_in_changelog()
logging.info("Using %s..%s as changelog interval", FROM_REF, TO_REF)
# Get starting and ending dates for gathering PRs
# Add one day after and before to mitigate TZ possible issues
# `tag^{}` format gives commit ref when we have annotated tags
# format %cs gives a committer date, works better for cherry-picked commits
from_date = runner.run(f"git log -1 --format=format:%cs '{FROM_REF}^{{}}'")
from_date = (date.fromisoformat(from_date) - timedelta(1)).isoformat()
to_date = runner.run(f"git log -1 --format=format:%cs '{TO_REF}^{{}}'")
to_date = (date.fromisoformat(to_date) + timedelta(1)).isoformat()
# Get all PRs for the given time frame
global GitHub
GitHub = Github(args.gh_user_or_token,
args.gh_password,
per_page=100,
pool_size=args.jobs)
query = f"type:pr repo:{args.repo} is:merged merged:{from_date}..{to_date}"
repo = GitHub.get_repo(args.repo)
api_prs = GitHub.search_issues(query=query, sort="created")
logging.info("Found %s PRs for the query: '%s'", api_prs.totalCount, query)
issues = [] # type: List[Issue]
while True:
try:
for issue in api_prs:
issues.append(issue)
break
except RateLimitExceededException:
sleep_on_rate_limit()
descriptions = get_descriptions(repo, issues, args.jobs)
write_changelog(args.output, descriptions)
def run(self):
while not self.queue.empty():
try:
issue = self.queue.get() # type: Issue
except Empty:
break # possible race condition, just continue
api_pr = get_pull_cached(self.repo, issue.number, issue.updated_at)
in_changelog = False
merge_commit = api_pr.merge_commit_sha
try:
runner.run(f"git rev-parse '{merge_commit}'")
except CalledProcessError:
# It's possible that commit not in the repo, just continue
logging.info("PR %s does not belong to the repo",
api_pr.number)
continue
in_changelog = merge_commit in SHA_IN_CHANGELOG
if in_changelog:
desc = generate_description(api_pr, self.repo)
if desc is not None:
self.response.append(desc)
self.queue.task_done()
def receive_prs_for_backport(self):
# The commit is the oldest open release branch's merge-base
since_commit = git_runner(
f"git merge-base {self.remote}/{self.release_branches[0]} "
f"{self.remote}/{self.default_branch}")
since_date = date.fromisoformat(
git_runner.run(f"git log -1 --format=format:%cs {since_commit}"))
# To not have a possible TZ issues
tomorrow = date.today() + timedelta(days=1)
logging.info("Receive PRs suppose to be backported")
self.prs_for_backport = self.gh.get_pulls_from_search(
query=f"{self._query} -label:pr-backported",
label=",".join(self.labels_to_backport +
[Labels.LABEL_MUST_BACKPORT]),
merged=[since_date, tomorrow],
)
logging.info(
"PRs to be backported:\n %s",
"\n ".join([pr.html_url for pr in self.prs_for_backport]),
)
def check_refs(from_ref: Optional[str], to_ref: str):
global FROM_REF, TO_REF
TO_REF = to_ref
# Check TO_REF
runner.run(f"git rev-parse {TO_REF}")
# Check from_ref
if from_ref is None:
# Get all tags pointing to TO_REF
tags = runner.run(f"git tag --points-at '{TO_REF}^{{}}'")
logging.info("All tags pointing to %s:\n%s", TO_REF, tags)
exclude = " ".join([f"--exclude='{tag}'" for tag in tags.split("\n")])
FROM_REF = runner.run(
f"git describe --abbrev=0 --tags {exclude} '{TO_REF}'")
else:
runner.run(f"git rev-parse {FROM_REF}")
FROM_REF = from_ref
def main():
log_levels = [logging.CRITICAL, logging.WARN, logging.INFO, logging.DEBUG]
args = parse_args()
logging.basicConfig(
format=
"%(asctime)s %(levelname)-8s [%(filename)s:%(lineno)d]:\n%(message)s",
level=log_levels[min(args.verbose, 3)],
)
# Get the full repo
if is_shallow():
logging.info("Unshallow repository")
runner.run("git fetch --unshallow", stderr=DEVNULL)
logging.info("Fetching all tags")
runner.run("git fetch --tags", stderr=DEVNULL)
check_refs(args.from_ref, args.to_ref)
set_sha_in_changelog()
logging.info("Using %s..%s as changelog interval", FROM_REF, TO_REF)
# Get starting and ending dates for gathering PRs
# Add one day after and before to mitigate TZ possible issues
# `tag^{}` format gives commit ref when we have annotated tags
from_date = runner.run(f"git log -1 --format=format:%as '{FROM_REF}^{{}}'")
from_date = (date.fromisoformat(from_date) - timedelta(1)).isoformat()
to_date = runner.run(f"git log -1 --format=format:%as '{TO_REF}^{{}}'")
to_date = (date.fromisoformat(to_date) + timedelta(1)).isoformat()
# Get all PRs for the given time frame
gh = Github(args.gh_user_or_token,
args.gh_password,
per_page=100,
pool_size=args.jobs)
query = f"type:pr repo:{args.repo} is:merged merged:{from_date}..{to_date}"
repo = gh.get_repo(args.repo)
api_prs = gh.search_issues(query=query, sort="created")
logging.info("Found %s PRs for the query: '%s'", api_prs.totalCount, query)
pr_numbers = [pr.number for pr in api_prs]
descriptions = get_descriptions(repo, pr_numbers, args.jobs)
write_changelog(args.output, descriptions)
def set_sha_in_changelog():
global SHA_IN_CHANGELOG
SHA_IN_CHANGELOG = runner.run(
f"git log --format=format:%H {FROM_REF}..{TO_REF}").split("\n")
