def init(self, data='{}', commit=True): logger.debug(('Initializing new repository for package with ' 'ID {}').format(self.pk)) with GitRepo.init() as repo: # Create, add, and commit the data file to the repo logger.debug('Adding and committing data to the repo at {}' .format(repo.path)) datafilename = pathjoin(repo.path, 'data.json') with open(datafilename, 'w') as datafile: datafile.write(data) repo.add(datafilename) repo.commit(message='Initial Commit') # Store the repo repo_url = self._make_repo_url() logger.debug('Copying the bare repo at {}/.git to {}' .format(repo.path, repo_url)) copytree(pathjoin(repo.path, '.git'), repo_url) # Save the repo location self.repo = repo_url if commit: self.save()
def get_data(self): logger.debug('Cloning repository for package with ID {}'.format(self.pk)) with GitRepo.clone(self.repo) as repo: datafilename = pathjoin(repo.path, 'data.json') with open(datafilename) as datafile: data = json.load(datafile) return data
def sign(self, signing_actor, signing_datetime=None, ): if signing_datetime is None: signing_datetime = now() with GitRepo.clone(self.repo) as repo: tagname = '{}.{}'.format(hash(signing_actor), hash(signing_datetime)) repo.tag(tagname, "{} took action {} at {}".format(signing_actor, 'sign', signing_datetime)) repo.push(tags=True)
def merge_into(self, repo: GitRepo, dry_run: bool = False) -> None: # Raises exception if matching rule is not found find_matching_merge_rule(self, repo) if repo.current_branch() != self.default_branch(): repo.checkout(self.default_branch()) if not self.is_ghstack_pr(): msg = self.get_title() + "\n\n" + self.get_body() msg += f"\nPull Request resolved: {self.get_pr_url()}\n" pr_branch_name = f"__pull-request-{self.pr_num}__init__" repo.fetch(f"pull/{self.pr_num}/head", pr_branch_name) repo._run_git("merge", "--squash", pr_branch_name) repo._run_git("commit", f"--author=\"{self.get_author()}\"", "-m", msg) else: self.merge_ghstack_into(repo) repo.push(self.default_branch(), dry_run)
def try_revert(repo: GitRepo, pr: GitHubPR, *, dry_run: bool = False, comment_id: Optional[int] = None, reason: Optional[str] = None) -> None: def post_comment(msg: str) -> None: gh_post_pr_comment(pr.org, pr.project, pr.pr_num, msg, dry_run=dry_run) if not pr.is_closed(): return post_comment(f"Can't revert open PR #{pr.pr_num}") comment = pr.get_last_comment( ) if comment_id is None else pr.get_comment_by_id(comment_id) if not RE_REVERT_CMD.match( comment.body_text) and not RE_REVERT_CMD_CLI.match( comment.body_text): raise RuntimeError( f"Comment {comment.body_text} does not seem to be a valid revert command" ) if comment.editor_login is not None: return post_comment("Don't want to revert based on edited command") author_association = comment.author_association author_login = comment.author_login # For some reason, one can not be a member of private repo, only CONTRIBUTOR expected_association = "CONTRIBUTOR" if pr.is_base_repo_private( ) else "MEMBER" if author_association != expected_association and author_association != "OWNER": return post_comment( f"Will not revert as @{author_login} is not a {expected_association}, but {author_association}" ) skip_internal_checks = can_skip_internal_checks(pr, comment_id) # Raises exception if matching rule is not found, but ignores all status checks find_matching_merge_rule(pr, repo, force=True, skip_internal_checks=skip_internal_checks) commit_sha = pr.get_merge_commit() if commit_sha is None: commits = repo.commits_resolving_gh_pr(pr.pr_num) if len(commits) == 0: raise RuntimeError("Can't find any commits resolving PR") commit_sha = commits[0] msg = repo.commit_message(commit_sha) rc = RE_DIFF_REV.search(msg) if rc is not None and not can_skip_internal_checks: raise RuntimeError( f"Can't revert PR that was landed via phabricator as {rc.group(1)}" ) revert_msg = f"\nReverted {pr.get_pr_url()} on behalf of {prefix_with_github_url(author_login)}" revert_msg += f" due to {reason}\n" if reason is not None else "\n" repo.checkout(pr.default_branch()) repo.revert(commit_sha) msg = repo.commit_message("HEAD") msg = re.sub(RE_PULL_REQUEST_RESOLVED, "", msg) msg += revert_msg repo.amend_commit_message(msg) repo.push(pr.default_branch(), dry_run) post_comment( f"@{pr.get_pr_creator_login()} your PR has been successfully reverted." ) if not dry_run: gh_add_labels(pr.org, pr.project, pr.pr_num, ["reverted"]) gh_post_commit_comment(pr.org, pr.project, commit_sha, revert_msg)
def merge_into(self, repo: GitRepo, *, force: bool = False, dry_run: bool = False, comment_id: Optional[int] = None) -> None: # Raises exception if matching rule is not found find_matching_merge_rule(self, repo, force=force, skip_internal_checks=can_skip_internal_checks( self, comment_id)) if repo.current_branch() != self.default_branch(): repo.checkout(self.default_branch()) if not self.is_ghstack_pr(): # Adding the url here makes it clickable within the Github UI approved_by_urls = ', '.join( prefix_with_github_url(login) for login in self.get_approved_by()) msg = self.get_title() + f" (#{self.pr_num})\n\n" + self.get_body() msg += f"\nPull Request resolved: {self.get_pr_url()}\n" msg += f"Approved by: {approved_by_urls}\n" pr_branch_name = f"__pull-request-{self.pr_num}__init__" repo.fetch(f"pull/{self.pr_num}/head", pr_branch_name) repo._run_git("merge", "--squash", pr_branch_name) repo._run_git("commit", f"--author=\"{self.get_author()}\"", "-m", msg) else: self.merge_ghstack_into(repo, force, comment_id=comment_id) repo.push(self.default_branch(), dry_run) if not dry_run: gh_add_labels(self.org, self.project, self.pr_num, ["merged"])
def merge(pr_num: int, repo: GitRepo, dry_run: bool = False, force: bool = False, comment_id: Optional[int] = None, mandatory_only: bool = False, on_green: bool = False, timeout_minutes: int = 400, stale_pr_days: int = 3) -> None: repo = GitRepo(get_git_repo_dir(), get_git_remote_name()) org, project = repo.gh_owner_and_name() pr = GitHubPR(org, project, pr_num) initial_commit_sha = pr.last_commit()['oid'] check_for_sev(org, project, force) if force or can_skip_internal_checks(pr, comment_id): # do not wait for any pending signals if PR is closed as part of co-development process return pr.merge_into(repo, dry_run=dry_run, force=force, comment_id=comment_id) if (datetime.utcnow() - pr.last_pushed_at()).days > stale_pr_days: raise RuntimeError( "This PR is too stale; the last push date was more than 3 days ago. Please rebase and try again." ) start_time = time.time() last_exception = '' elapsed_time = 0.0 while elapsed_time < timeout_minutes * 60: check_for_sev(org, project, force) current_time = time.time() elapsed_time = current_time - start_time print( f"Attempting merge of https://github.com/{org}/{project}/pull/{pr_num} ({elapsed_time / 60} minutes elapsed)" ) pr = GitHubPR(org, project, pr_num) if initial_commit_sha != pr.last_commit()['oid']: raise RuntimeError( "New commits were pushed while merging. Please rerun the merge command." ) try: find_matching_merge_rule(pr, repo) pending = pr_get_pending_checks(pr) failing = pr_get_failed_checks(pr) if (not mandatory_only and on_green) and len(failing) > 0: raise RuntimeError( f"{len(failing)} additional jobs have failed, first few of them are: " + ' ,'.join(f"[{x[0]}]({x[1]})" for x in failing[:5])) if (not mandatory_only and on_green) and len(pending) > 0: raise MandatoryChecksMissingError( f"Still waiting for {len(pending)} additional jobs to finish, " + f"first few of them are: {' ,'.join(x[0] for x in pending[:5])}" ) return pr.merge_into(repo, dry_run=dry_run, force=force, comment_id=comment_id) except MandatoryChecksMissingError as ex: last_exception = str(ex) print( f"Merge of https://github.com/{org}/{project}/pull/{pr_num} failed due to: {ex}. Retrying in 5 min" ) time.sleep(5 * 60) # Finally report timeout back msg = f"Merged timed out after {timeout_minutes} minutes. Please contact the pytorch_dev_infra team." msg += f"The last exception was: {last_exception}" if not dry_run: gh_add_labels(org, project, pr_num, ["land-failed"]) raise RuntimeError(msg)
def test_lint_fails(self, mocked_gql: Any) -> None: "Tests that PR fails mandatory lint check" pr = GitHubPR("pytorch", "pytorch", 74649) repo = GitRepo(get_git_repo_dir(), get_git_remote_name()) self.assertRaises(RuntimeError, lambda: find_matching_merge_rule(pr, repo))
def test_match_rules(self, mocked_gql: Any) -> None: "Tests that PR passes merge rules" pr = GitHubPR("pytorch", "pytorch", 71759) repo = GitRepo(get_git_repo_dir(), get_git_remote_name()) self.assertTrue(find_matching_merge_rule(pr, repo) is not None)
def try_revert(repo: GitRepo, pr: GitHubPR, dry_run: bool = False) -> None: def post_comment(msg: str) -> None: gh_post_comment(pr.org, pr.project, pr.pr_num, msg, dry_run=dry_run) if not pr.is_closed(): return post_comment(f"Can't revert open PR #{pr.pr_num}") if not RE_REVERT_CMD.match(pr.get_comment_body()): raise RuntimeError( f"Comment {pr.get_comment_body()} does not seem to be a valid revert command" ) if pr.get_comment_editor_login() is not None: return post_comment("Don't want to revert based on edited command") author_association = pr.get_comment_author_association() author_login = pr.get_comment_author_login() # For some reason, one can not be a member of private repo, only CONTRIBUTOR expected_association = "CONTRIBUTOR" if pr.is_base_repo_private( ) else "MEMBER" if author_association != expected_association and author_association != "OWNER": return post_comment( f"Will not revert as @{author_login} is not a {expected_association}, but {author_association}" ) # Raises exception if matching rule is not found find_matching_merge_rule(pr, repo) commit_sha = pr.get_merge_commit() if commit_sha is None: commits = repo.commits_resolving_gh_pr(pr.pr_num) if len(commits) == 0: raise RuntimeError("Can't find any commits resolving PR") commit_sha = commits[0] msg = repo.commit_message(commit_sha) rc = RE_DIFF_REV.search(msg) if rc is not None: raise RuntimeError( f"Can't revert PR that was landed via phabricator as {rc.group(1)}" ) repo.checkout(pr.default_branch()) repo.revert(commit_sha) msg = repo.commit_message("HEAD") msg = re.sub(RE_PULL_REQUEST_RESOLVED, "", msg) msg += f"\nReverted {pr.get_pr_url()} on behalf of @{author_login}\n" repo.amend_commit_message(msg) repo.push(pr.default_branch(), dry_run) if not dry_run: gh_add_labels(pr.org, pr.project, pr.pr_num, ["reverted"])
def rebase_ghstack_onto(pr: GitHubPR, repo: GitRepo, onto_branch: str, dry_run: bool = False) -> None: if subprocess.run([sys.executable, "-m", "ghstack", "--help"], capture_output=True).returncode != 0: subprocess.run([sys.executable, "-m", "pip", "install", "ghstack"]) orig_ref = f"{re.sub(r'/head$', '/orig', pr.head_ref())}" onto_branch = f"refs/remotes/origin/{onto_branch}" repo.fetch(orig_ref, orig_ref) repo._run_git("rebase", onto_branch, orig_ref) os.environ["OAUTH_TOKEN"] = os.environ["GITHUB_TOKEN"] with open('.ghstackrc', 'w+') as f: f.write('[ghstack]\n' + "github_url=github.com\n" + "github_username=pytorchmergebot\n" + "remote_name=origin") if dry_run: print("Don't know how to dry-run ghstack") else: ghstack_result = subprocess.run(["ghstack"], capture_output=True) push_result = ghstack_result.stdout.decode("utf-8") print(push_result) if ghstack_result.returncode != 0: raise Exception(f"\n```{push_result}```") # The contents of a successful push result should look like: # Summary of changes (ghstack 0.6.0) # - Updated https://github.com/clee2000/random-testing/pull/2 # - Updated https://github.com/clee2000/random-testing/pull/1 # Facebook employees can import your changes by running # (on a Facebook machine): # ghimport -s https://github.com/clee2000/random-testing/pull/2 # If you want to work on this diff stack on another machine: # ghstack checkout https://github.com/clee2000/random-testing/pull/2 org, project = repo.gh_owner_and_name() for line in push_result.splitlines(): if "Updated" in line: pr_num = int(line.split("/")[-1]) if pr_num != pr.pr_num: gh_post_comment( pr.org, pr.project, pr_num, f"Rebased `{orig_ref}` onto `{onto_branch}` because #{pr.pr_num} was rebased, " "please pull locally before adding more changes (for example, via `ghstack " + f"checkout https://github.com/{org}/{project}/pull/{pr_num}`)", dry_run=dry_run) else: gh_post_comment( pr.org, pr.project, pr_num, f"Successfully rebased `{orig_ref}` onto `{onto_branch}`, please pull locally " + "before adding more changes (for example, via `ghstack " + f"checkout https://github.com/{org}/{project}/pull/{pr.pr_num}`)", dry_run=dry_run) if f"Skipped https://github.com/{org}/{project}/pull/{pr.pr_num}" in push_result: gh_post_comment( pr.org, pr.project, pr.pr_num, f"Tried to rebase and push PR #{pr.pr_num}, but it was already up to date", dry_run=dry_run)