def init(self, data='{}', commit=True):
logger.debug(('Initializing new repository for package with '
'ID {}').format(self.pk))
with GitRepo.init() as repo:
# Create, add, and commit the data file to the repo
logger.debug('Adding and committing data to the repo at {}'
.format(repo.path))
datafilename = pathjoin(repo.path, 'data.json')
with open(datafilename, 'w') as datafile:
datafile.write(data)
repo.add(datafilename)
repo.commit(message='Initial Commit')
# Store the repo
repo_url = self._make_repo_url()
logger.debug('Copying the bare repo at {}/.git to {}'
.format(repo.path, repo_url))
copytree(pathjoin(repo.path, '.git'), repo_url)
# Save the repo location
self.repo = repo_url
if commit:
self.save()
def get_data(self):
logger.debug('Cloning repository for package with ID {}'.format(self.pk))
with GitRepo.clone(self.repo) as repo:
datafilename = pathjoin(repo.path, 'data.json')
with open(datafilename) as datafile:
data = json.load(datafile)
return data
def sign(self, signing_actor, signing_datetime=None, ):
if signing_datetime is None:
signing_datetime = now()
with GitRepo.clone(self.repo) as repo:
tagname = '{}.{}'.format(hash(signing_actor),
hash(signing_datetime))
repo.tag(tagname,
"{} took action {} at {}".format(signing_actor, 'sign',
signing_datetime))
repo.push(tags=True)
def merge_into(self, repo: GitRepo, dry_run: bool = False) -> None:
# Raises exception if matching rule is not found
find_matching_merge_rule(self, repo)
if repo.current_branch() != self.default_branch():
repo.checkout(self.default_branch())
if not self.is_ghstack_pr():
msg = self.get_title() + "\n\n" + self.get_body()
msg += f"\nPull Request resolved: {self.get_pr_url()}\n"
pr_branch_name = f"__pull-request-{self.pr_num}__init__"
repo.fetch(f"pull/{self.pr_num}/head", pr_branch_name)
repo._run_git("merge", "--squash", pr_branch_name)
repo._run_git("commit", f"--author=\"{self.get_author()}\"", "-m",
msg)
else:
self.merge_ghstack_into(repo)
repo.push(self.default_branch(), dry_run)
def try_revert(repo: GitRepo,
pr: GitHubPR,
*,
dry_run: bool = False,
comment_id: Optional[int] = None,
reason: Optional[str] = None) -> None:
def post_comment(msg: str) -> None:
gh_post_pr_comment(pr.org, pr.project, pr.pr_num, msg, dry_run=dry_run)
if not pr.is_closed():
return post_comment(f"Can't revert open PR #{pr.pr_num}")
comment = pr.get_last_comment(
) if comment_id is None else pr.get_comment_by_id(comment_id)
if not RE_REVERT_CMD.match(
comment.body_text) and not RE_REVERT_CMD_CLI.match(
comment.body_text):
raise RuntimeError(
f"Comment {comment.body_text} does not seem to be a valid revert command"
)
if comment.editor_login is not None:
return post_comment("Don't want to revert based on edited command")
author_association = comment.author_association
author_login = comment.author_login
# For some reason, one can not be a member of private repo, only CONTRIBUTOR
expected_association = "CONTRIBUTOR" if pr.is_base_repo_private(
) else "MEMBER"
if author_association != expected_association and author_association != "OWNER":
return post_comment(
f"Will not revert as @{author_login} is not a {expected_association}, but {author_association}"
)
skip_internal_checks = can_skip_internal_checks(pr, comment_id)
# Raises exception if matching rule is not found, but ignores all status checks
find_matching_merge_rule(pr,
repo,
force=True,
skip_internal_checks=skip_internal_checks)
commit_sha = pr.get_merge_commit()
if commit_sha is None:
commits = repo.commits_resolving_gh_pr(pr.pr_num)
if len(commits) == 0:
raise RuntimeError("Can't find any commits resolving PR")
commit_sha = commits[0]
msg = repo.commit_message(commit_sha)
rc = RE_DIFF_REV.search(msg)
if rc is not None and not can_skip_internal_checks:
raise RuntimeError(
f"Can't revert PR that was landed via phabricator as {rc.group(1)}"
)
revert_msg = f"\nReverted {pr.get_pr_url()} on behalf of {prefix_with_github_url(author_login)}"
revert_msg += f" due to {reason}\n" if reason is not None else "\n"
repo.checkout(pr.default_branch())
repo.revert(commit_sha)
msg = repo.commit_message("HEAD")
msg = re.sub(RE_PULL_REQUEST_RESOLVED, "", msg)
msg += revert_msg
repo.amend_commit_message(msg)
repo.push(pr.default_branch(), dry_run)
post_comment(
f"@{pr.get_pr_creator_login()} your PR has been successfully reverted."
)
if not dry_run:
gh_add_labels(pr.org, pr.project, pr.pr_num, ["reverted"])
gh_post_commit_comment(pr.org, pr.project, commit_sha, revert_msg)
def merge_into(self,
repo: GitRepo,
*,
force: bool = False,
dry_run: bool = False,
comment_id: Optional[int] = None) -> None:
# Raises exception if matching rule is not found
find_matching_merge_rule(self,
repo,
force=force,
skip_internal_checks=can_skip_internal_checks(
self, comment_id))
if repo.current_branch() != self.default_branch():
repo.checkout(self.default_branch())
if not self.is_ghstack_pr():
# Adding the url here makes it clickable within the Github UI
approved_by_urls = ', '.join(
prefix_with_github_url(login)
for login in self.get_approved_by())
msg = self.get_title() + f" (#{self.pr_num})\n\n" + self.get_body()
msg += f"\nPull Request resolved: {self.get_pr_url()}\n"
msg += f"Approved by: {approved_by_urls}\n"
pr_branch_name = f"__pull-request-{self.pr_num}__init__"
repo.fetch(f"pull/{self.pr_num}/head", pr_branch_name)
repo._run_git("merge", "--squash", pr_branch_name)
repo._run_git("commit", f"--author=\"{self.get_author()}\"", "-m",
msg)
else:
self.merge_ghstack_into(repo, force, comment_id=comment_id)
repo.push(self.default_branch(), dry_run)
if not dry_run:
gh_add_labels(self.org, self.project, self.pr_num, ["merged"])
def merge(pr_num: int,
repo: GitRepo,
dry_run: bool = False,
force: bool = False,
comment_id: Optional[int] = None,
mandatory_only: bool = False,
on_green: bool = False,
timeout_minutes: int = 400,
stale_pr_days: int = 3) -> None:
repo = GitRepo(get_git_repo_dir(), get_git_remote_name())
org, project = repo.gh_owner_and_name()
pr = GitHubPR(org, project, pr_num)
initial_commit_sha = pr.last_commit()['oid']
check_for_sev(org, project, force)
if force or can_skip_internal_checks(pr, comment_id):
# do not wait for any pending signals if PR is closed as part of co-development process
return pr.merge_into(repo,
dry_run=dry_run,
force=force,
comment_id=comment_id)
if (datetime.utcnow() - pr.last_pushed_at()).days > stale_pr_days:
raise RuntimeError(
"This PR is too stale; the last push date was more than 3 days ago. Please rebase and try again."
)
start_time = time.time()
last_exception = ''
elapsed_time = 0.0
while elapsed_time < timeout_minutes * 60:
check_for_sev(org, project, force)
current_time = time.time()
elapsed_time = current_time - start_time
print(
f"Attempting merge of https://github.com/{org}/{project}/pull/{pr_num} ({elapsed_time / 60} minutes elapsed)"
)
pr = GitHubPR(org, project, pr_num)
if initial_commit_sha != pr.last_commit()['oid']:
raise RuntimeError(
"New commits were pushed while merging. Please rerun the merge command."
)
try:
find_matching_merge_rule(pr, repo)
pending = pr_get_pending_checks(pr)
failing = pr_get_failed_checks(pr)
if (not mandatory_only and on_green) and len(failing) > 0:
raise RuntimeError(
f"{len(failing)} additional jobs have failed, first few of them are: "
+ ' ,'.join(f"[{x[0]}]({x[1]})" for x in failing[:5]))
if (not mandatory_only and on_green) and len(pending) > 0:
raise MandatoryChecksMissingError(
f"Still waiting for {len(pending)} additional jobs to finish, "
+
f"first few of them are: {' ,'.join(x[0] for x in pending[:5])}"
)
return pr.merge_into(repo,
dry_run=dry_run,
force=force,
comment_id=comment_id)
except MandatoryChecksMissingError as ex:
last_exception = str(ex)
print(
f"Merge of https://github.com/{org}/{project}/pull/{pr_num} failed due to: {ex}. Retrying in 5 min"
)
time.sleep(5 * 60)
# Finally report timeout back
msg = f"Merged timed out after {timeout_minutes} minutes. Please contact the pytorch_dev_infra team."
msg += f"The last exception was: {last_exception}"
if not dry_run:
gh_add_labels(org, project, pr_num, ["land-failed"])
raise RuntimeError(msg)
def test_lint_fails(self, mocked_gql: Any) -> None:
"Tests that PR fails mandatory lint check"
pr = GitHubPR("pytorch", "pytorch", 74649)
repo = GitRepo(get_git_repo_dir(), get_git_remote_name())
self.assertRaises(RuntimeError,
lambda: find_matching_merge_rule(pr, repo))
def test_match_rules(self, mocked_gql: Any) -> None:
"Tests that PR passes merge rules"
pr = GitHubPR("pytorch", "pytorch", 71759)
repo = GitRepo(get_git_repo_dir(), get_git_remote_name())
self.assertTrue(find_matching_merge_rule(pr, repo) is not None)
def try_revert(repo: GitRepo, pr: GitHubPR, dry_run: bool = False) -> None:
def post_comment(msg: str) -> None:
gh_post_comment(pr.org, pr.project, pr.pr_num, msg, dry_run=dry_run)
if not pr.is_closed():
return post_comment(f"Can't revert open PR #{pr.pr_num}")
if not RE_REVERT_CMD.match(pr.get_comment_body()):
raise RuntimeError(
f"Comment {pr.get_comment_body()} does not seem to be a valid revert command"
)
if pr.get_comment_editor_login() is not None:
return post_comment("Don't want to revert based on edited command")
author_association = pr.get_comment_author_association()
author_login = pr.get_comment_author_login()
# For some reason, one can not be a member of private repo, only CONTRIBUTOR
expected_association = "CONTRIBUTOR" if pr.is_base_repo_private(
) else "MEMBER"
if author_association != expected_association and author_association != "OWNER":
return post_comment(
f"Will not revert as @{author_login} is not a {expected_association}, but {author_association}"
)
# Raises exception if matching rule is not found
find_matching_merge_rule(pr, repo)
commit_sha = pr.get_merge_commit()
if commit_sha is None:
commits = repo.commits_resolving_gh_pr(pr.pr_num)
if len(commits) == 0:
raise RuntimeError("Can't find any commits resolving PR")
commit_sha = commits[0]
msg = repo.commit_message(commit_sha)
rc = RE_DIFF_REV.search(msg)
if rc is not None:
raise RuntimeError(
f"Can't revert PR that was landed via phabricator as {rc.group(1)}"
)
repo.checkout(pr.default_branch())
repo.revert(commit_sha)
msg = repo.commit_message("HEAD")
msg = re.sub(RE_PULL_REQUEST_RESOLVED, "", msg)
msg += f"\nReverted {pr.get_pr_url()} on behalf of @{author_login}\n"
repo.amend_commit_message(msg)
repo.push(pr.default_branch(), dry_run)
if not dry_run:
gh_add_labels(pr.org, pr.project, pr.pr_num, ["reverted"])
def rebase_ghstack_onto(pr: GitHubPR,
repo: GitRepo,
onto_branch: str,
dry_run: bool = False) -> None:
if subprocess.run([sys.executable, "-m", "ghstack", "--help"],
capture_output=True).returncode != 0:
subprocess.run([sys.executable, "-m", "pip", "install", "ghstack"])
orig_ref = f"{re.sub(r'/head$', '/orig', pr.head_ref())}"
onto_branch = f"refs/remotes/origin/{onto_branch}"
repo.fetch(orig_ref, orig_ref)
repo._run_git("rebase", onto_branch, orig_ref)
os.environ["OAUTH_TOKEN"] = os.environ["GITHUB_TOKEN"]
with open('.ghstackrc', 'w+') as f:
f.write('[ghstack]\n' + "github_url=github.com\n" +
"github_username=pytorchmergebot\n" + "remote_name=origin")
if dry_run:
print("Don't know how to dry-run ghstack")
else:
ghstack_result = subprocess.run(["ghstack"], capture_output=True)
push_result = ghstack_result.stdout.decode("utf-8")
print(push_result)
if ghstack_result.returncode != 0:
raise Exception(f"\n```{push_result}```")
# The contents of a successful push result should look like:
# Summary of changes (ghstack 0.6.0)
# - Updated https://github.com/clee2000/random-testing/pull/2
# - Updated https://github.com/clee2000/random-testing/pull/1
# Facebook employees can import your changes by running
# (on a Facebook machine):
# ghimport -s https://github.com/clee2000/random-testing/pull/2
# If you want to work on this diff stack on another machine:
# ghstack checkout https://github.com/clee2000/random-testing/pull/2
org, project = repo.gh_owner_and_name()
for line in push_result.splitlines():
if "Updated" in line:
pr_num = int(line.split("/")[-1])
if pr_num != pr.pr_num:
gh_post_comment(
pr.org,
pr.project,
pr_num,
f"Rebased `{orig_ref}` onto `{onto_branch}` because #{pr.pr_num} was rebased, "
"please pull locally before adding more changes (for example, via `ghstack "
+
f"checkout https://github.com/{org}/{project}/pull/{pr_num}`)",
dry_run=dry_run)
else:
gh_post_comment(
pr.org,
pr.project,
pr_num,
f"Successfully rebased `{orig_ref}` onto `{onto_branch}`, please pull locally "
+
"before adding more changes (for example, via `ghstack "
+
f"checkout https://github.com/{org}/{project}/pull/{pr.pr_num}`)",
dry_run=dry_run)
if f"Skipped https://github.com/{org}/{project}/pull/{pr.pr_num}" in push_result:
gh_post_comment(
pr.org,
pr.project,
pr.pr_num,
f"Tried to rebase and push PR #{pr.pr_num}, but it was already up to date",
dry_run=dry_run)
