def __init__(self, token, owner, name, team): self._gh = RemoteRepo(token, owner=owner, name=name, team=team, max_page_size=30, min_page_size=7) self._token = token self.default_branch_name = self._gh.default_branch self.ssh_url = self._gh.ssh_url
def __init__(self, token, owner, name, team, pr_number, target_branch): self._gh = RemoteRepo(token, owner=owner, name=name, team=team) self._pr = self._gh.get_pull_request(pr_number) self.ssh_url = self._gh.ssh_url # TODO: check if pull-request is merged. self.merge_commit_oid = self._pr['mergeCommit']['oid'] self.target_branch = target_branch self.backport_branch = 'backport/{branch}/{pr}'.format(branch=target_branch, pr=pr_number) self.cherrypick_branch = 'cherrypick/{branch}/{oid}'.format(branch=target_branch, oid=self.merge_commit_oid)
class Backport: def __init__(self, token, owner, name, team): self._gh = RemoteRepo(token, owner=owner, name=name, team=team, max_page_size=30, min_page_size=7) self._token = token self.default_branch_name = self._gh.default_branch self.ssh_url = self._gh.ssh_url def getPullRequests(self, from_commit): return self._gh.get_pull_requests(from_commit) def getBranchesWithRelease(self): branches = set() for pull_request in self._gh.find_pull_requests("release"): branches.add(pull_request['headRefName']) return branches def execute(self, repo, upstream, until_commit, run_cherrypick): repo = LocalRepo(repo, upstream, self.default_branch_name) all_branches = repo.get_release_branches( ) # [(branch_name, base_commit)] release_branches = self.getBranchesWithRelease() branches = [] # iterate over all branches to preserve their precedence. for branch in all_branches: if branch[0] in release_branches: branches.append(branch) if not branches: logging.info('No release branches found!') return for branch in branches: logging.info('Found release branch: %s', branch[0]) if not until_commit: until_commit = branches[0][1] pull_requests = self.getPullRequests(until_commit) backport_map = {} RE_MUST_BACKPORT = re.compile(r'^v(\d+\.\d+)-must-backport$') RE_NO_BACKPORT = re.compile(r'^v(\d+\.\d+)-no-backport$') RE_BACKPORTED = re.compile(r'^v(\d+\.\d+)-backported$') # pull-requests are sorted by ancestry from the most recent. for pr in pull_requests: while repo.comparator(branches[-1][1]) >= repo.comparator( pr['mergeCommit']['oid']): logging.info( "PR #{} is already inside {}. Dropping this branch for further PRs" .format(pr['number'], branches[-1][0])) branches.pop() logging.info("Processing PR #{}".format(pr['number'])) assert len(branches) branch_set = set([branch[0] for branch in branches]) # First pass. Find all must-backports for label in pr['labels']['nodes']: if label['name'] == 'pr-bugfix' or label[ 'name'] == 'pr-must-backport': backport_map[pr['number']] = branch_set.copy() continue matched = RE_MUST_BACKPORT.match(label['name']) if matched: if pr['number'] not in backport_map: backport_map[pr['number']] = set() backport_map[pr['number']].add(matched.group(1)) # Second pass. Find all no-backports for label in pr['labels']['nodes']: if label['name'] == 'pr-no-backport' and pr[ 'number'] in backport_map: del backport_map[pr['number']] break matched_no_backport = RE_NO_BACKPORT.match(label['name']) matched_backported = RE_BACKPORTED.match(label['name']) if matched_no_backport and pr[ 'number'] in backport_map and matched_no_backport.group( 1) in backport_map[pr['number']]: backport_map[pr['number']].remove( matched_no_backport.group(1)) logging.info('\tskipping %s because of forced no-backport', matched_no_backport.group(1)) elif matched_backported and pr[ 'number'] in backport_map and matched_backported.group( 1) in backport_map[pr['number']]: backport_map[pr['number']].remove( matched_backported.group(1)) logging.info( '\tskipping %s because it\'s already backported manually', matched_backported.group(1)) for pr, branches in list(backport_map.items()): logging.info('PR #%s needs to be backported to:', pr) for branch in branches: logging.info('\t%s, and the status is: %s', branch, run_cherrypick(self._token, pr, branch)) # print API costs logging.info('\nGitHub API total costs per query:') for name, value in list(self._gh.api_costs.items()): logging.info('%s : %s', name, value)
class CherryPick: class Status(Enum): DISCARDED = 'discarded' NOT_INITIATED = 'not started' FIRST_MERGEABLE = 'waiting for 1st stage' FIRST_CONFLICTS = 'conflicts on 1st stage' SECOND_MERGEABLE = 'waiting for 2nd stage' SECOND_CONFLICTS = 'conflicts on 2nd stage' MERGED = 'backported' def _run(self, args): logging.info(subprocess.check_output(args)) def __init__(self, token, owner, name, team, pr_number, target_branch): self._gh = RemoteRepo(token, owner=owner, name=name, team=team) self._pr = self._gh.get_pull_request(pr_number) self.ssh_url = self._gh.ssh_url # TODO: check if pull-request is merged. self.merge_commit_oid = self._pr['mergeCommit']['oid'] self.target_branch = target_branch self.backport_branch = 'backport/{branch}/{pr}'.format( branch=target_branch, pr=pr_number) self.cherrypick_branch = 'cherrypick/{branch}/{oid}'.format( branch=target_branch, oid=self.merge_commit_oid) def getCherryPickPullRequest(self): return self._gh.find_pull_request(base=self.backport_branch, head=self.cherrypick_branch) def createCherryPickPullRequest(self, repo_path): DESCRIPTION = ( 'This pull-request is a first step of an automated backporting.\n' 'It contains changes like after calling a local command `git cherry-pick`.\n' 'If you intend to continue backporting this changes, then resolve all conflicts if any.\n' 'Otherwise, if you do not want to backport them, then just close this pull-request.\n' '\n' 'The check results does not matter at this step - you can safely ignore them.\n' 'Also this pull-request will be merged automatically as it reaches the mergeable state, but you always can merge it manually.\n' ) # FIXME: replace with something better than os.system() git_prefix = [ 'git', '-C', repo_path, '-c', '[email protected]', '-c', 'user.name=robot-clickhouse' ] base_commit_oid = self._pr['mergeCommit']['parents']['nodes'][0]['oid'] # Create separate branch for backporting, and make it look like real cherry-pick. self._run(git_prefix + ['checkout', '-f', self.target_branch]) self._run(git_prefix + ['checkout', '-B', self.backport_branch]) self._run(git_prefix + ['merge', '-s', 'ours', '--no-edit', base_commit_oid]) # Create secondary branch to allow pull request with cherry-picked commit. self._run( git_prefix + ['branch', '-f', self.cherrypick_branch, self.merge_commit_oid]) self._run(git_prefix + [ 'push', '-f', 'origin', '{branch}:{branch}'.format( branch=self.backport_branch) ]) self._run(git_prefix + [ 'push', '-f', 'origin', '{branch}:{branch}'.format( branch=self.cherrypick_branch) ]) # Create pull-request like a local cherry-pick pr = self._gh.create_pull_request( source=self.cherrypick_branch, target=self.backport_branch, title='Cherry pick #{number} to {target}: {title}'.format( number=self._pr['number'], target=self.target_branch, title=self._pr['title'].replace('"', '\\"')), description='Original pull-request #{}\n\n{}'.format( self._pr['number'], DESCRIPTION)) # FIXME: use `team` to leave a single eligible assignee. self._gh.add_assignee(pr, self._pr['author']) self._gh.add_assignee(pr, self._pr['mergedBy']) self._gh.set_label(pr, "do not test") self._gh.set_label(pr, "pr-cherrypick") return pr def mergeCherryPickPullRequest(self, cherrypick_pr): return self._gh.merge_pull_request(cherrypick_pr['id']) def getBackportPullRequest(self): return self._gh.find_pull_request(base=self.target_branch, head=self.backport_branch) def createBackportPullRequest(self, cherrypick_pr, repo_path): DESCRIPTION = ( 'This pull-request is a last step of an automated backporting.\n' 'Treat it as a standard pull-request: look at the checks and resolve conflicts.\n' 'Merge it only if you intend to backport changes to the target branch, otherwise just close it.\n' ) git_prefix = [ 'git', '-C', repo_path, '-c', '[email protected]', '-c', 'user.name=robot-clickhouse' ] pr_title = 'Backport #{number} to {target}: {title}'.format( number=self._pr['number'], target=self.target_branch, title=self._pr['title'].replace('"', '\\"')) self._run(git_prefix + ['checkout', '-f', self.backport_branch]) self._run(git_prefix + ['pull', '--ff-only', 'origin', self.backport_branch]) self._run(git_prefix + [ 'reset', '--soft', self._run(git_prefix + ['merge-base', self.target_branch, self.backport_branch]) ]) self._run(git_prefix + ['commit', '-a', '-m', pr_title]) self._run(git_prefix + [ 'push', '-f', 'origin', '{branch}:{branch}'.format( branch=self.backport_branch) ]) pr = self._gh.create_pull_request( source=self.backport_branch, target=self.target_branch, title=pr_title, description= 'Original pull-request #{}\nCherry-pick pull-request #{}\n\n{}'. format(self._pr['number'], cherrypick_pr['number'], DESCRIPTION)) # FIXME: use `team` to leave a single eligible assignee. self._gh.add_assignee(pr, self._pr['author']) self._gh.add_assignee(pr, self._pr['mergedBy']) self._gh.set_label(pr, "pr-backport") return pr def execute(self, repo_path, dry_run=False): pr1 = self.getCherryPickPullRequest() if not pr1: if not dry_run: pr1 = self.createCherryPickPullRequest(repo_path) logging.debug('Created PR with cherry-pick of %s to %s: %s', self._pr['number'], self.target_branch, pr1['url']) else: return CherryPick.Status.NOT_INITIATED else: logging.debug('Found PR with cherry-pick of %s to %s: %s', self._pr['number'], self.target_branch, pr1['url']) if not pr1['merged'] and pr1[ 'mergeable'] == 'MERGEABLE' and not pr1['closed']: if not dry_run: pr1 = self.mergeCherryPickPullRequest(pr1) logging.debug('Merged PR with cherry-pick of %s to %s: %s', self._pr['number'], self.target_branch, pr1['url']) if not pr1['merged']: logging.debug('Waiting for PR with cherry-pick of %s to %s: %s', self._pr['number'], self.target_branch, pr1['url']) if pr1['closed']: return CherryPick.Status.DISCARDED elif pr1['mergeable'] == 'CONFLICTING': return CherryPick.Status.FIRST_CONFLICTS else: return CherryPick.Status.FIRST_MERGEABLE pr2 = self.getBackportPullRequest() if not pr2: if not dry_run: pr2 = self.createBackportPullRequest(pr1, repo_path) logging.debug('Created PR with backport of %s to %s: %s', self._pr['number'], self.target_branch, pr2['url']) else: return CherryPick.Status.FIRST_MERGEABLE else: logging.debug('Found PR with backport of %s to %s: %s', self._pr['number'], self.target_branch, pr2['url']) if pr2['merged']: return CherryPick.Status.MERGED elif pr2['closed']: return CherryPick.Status.DISCARDED elif pr2['mergeable'] == 'CONFLICTING': return CherryPick.Status.SECOND_CONFLICTS else: return CherryPick.Status.SECOND_MERGEABLE
class Backport: def __init__(self, token, owner, name, team): self._gh = RemoteRepo(token, owner=owner, name=name, team=team, max_page_size=30, min_page_size=7) self._token = token self.default_branch_name = self._gh.default_branch self.ssh_url = self._gh.ssh_url def getPullRequests(self, from_commit): return self._gh.get_pull_requests(from_commit) def execute(self, repo, til, number, run_cherrypick): repo = LocalRepo(repo, 'origin', self.default_branch_name) branches = repo.get_release_branches()[ -number:] # [(branch_name, base_commit)] if not branches: logging.info('No release branches found!') return for branch in branches: logging.info('Found release branch: %s', branch[0]) if not til: til = branches[0][1] prs = self.getPullRequests(til) backport_map = {} RE_MUST_BACKPORT = re.compile(r'^v(\d+\.\d+)-must-backport$') RE_NO_BACKPORT = re.compile(r'^v(\d+\.\d+)-no-backport$') RE_BACKPORTED = re.compile(r'^v(\d+\.\d+)-backported$') # pull-requests are sorted by ancestry from the least recent. for pr in prs: while repo.comparator(branches[-1][1]) >= repo.comparator( pr['mergeCommit']['oid']): logging.info( "PR #{} is already inside {}. Dropping this branch for further PRs" .format(pr['number'], branches[-1][0])) branches.pop() logging.info("Processing PR #{}".format(pr['number'])) assert len(branches) branch_set = set([branch[0] for branch in branches]) # First pass. Find all must-backports for label in pr['labels']['nodes']: if label['name'].startswith( 'pr-') and label['color'] == 'ff0000': backport_map[pr['number']] = branch_set.copy() continue m = RE_MUST_BACKPORT.match(label['name']) if m: if pr['number'] not in backport_map: backport_map[pr['number']] = set() backport_map[pr['number']].add(m.group(1)) # Second pass. Find all no-backports for label in pr['labels']['nodes']: if label['name'] == 'pr-no-backport' and pr[ 'number'] in backport_map: del backport_map[pr['number']] break m1 = RE_NO_BACKPORT.match(label['name']) m2 = RE_BACKPORTED.match(label['name']) if m1 and pr['number'] in backport_map and m1.group( 1) in backport_map[pr['number']]: backport_map[pr['number']].remove(m1.group(1)) logging.info('\tskipping %s because of forced no-backport', m1.group(1)) elif m2 and pr['number'] in backport_map and m2.group( 1) in backport_map[pr['number']]: backport_map[pr['number']].remove(m2.group(1)) logging.info( '\tskipping %s because it\'s already backported manually', m2.group(1)) for pr, branches in backport_map.items(): logging.info('PR #%s needs to be backported to:', pr) for branch in branches: logging.info('\t%s, and the status is: %s', branch, run_cherrypick(self._token, pr, branch)) # print API costs logging.info('\nGitHub API total costs per query:') for name, value in self._gh.api_costs.items(): logging.info('%s : %s', name, value)
class CherryPick: class Status(Enum): DISCARDED = "discarded" NOT_INITIATED = "not started" FIRST_MERGEABLE = "waiting for 1st stage" FIRST_CONFLICTS = "conflicts on 1st stage" SECOND_MERGEABLE = "waiting for 2nd stage" SECOND_CONFLICTS = "conflicts on 2nd stage" MERGED = "backported" def _run(self, args): out = subprocess.check_output(args).rstrip() logging.debug(out) return out def __init__(self, token, owner, name, team, pr_number, target_branch): self._gh = RemoteRepo(token, owner=owner, name=name, team=team) self._pr = self._gh.get_pull_request(pr_number) self.ssh_url = self._gh.ssh_url # TODO: check if pull-request is merged. self.merge_commit_oid = self._pr["mergeCommit"]["oid"] self.target_branch = target_branch self.backport_branch = "backport/{branch}/{pr}".format( branch=target_branch, pr=pr_number) self.cherrypick_branch = "cherrypick/{branch}/{oid}".format( branch=target_branch, oid=self.merge_commit_oid) def getCherryPickPullRequest(self): return self._gh.find_pull_request(base=self.backport_branch, head=self.cherrypick_branch) def createCherryPickPullRequest(self, repo_path): DESCRIPTION = ( "This pull-request is a first step of an automated backporting.\n" "It contains changes like after calling a local command `git cherry-pick`.\n" "If you intend to continue backporting this changes, then resolve all conflicts if any.\n" "Otherwise, if you do not want to backport them, then just close this pull-request.\n" "\n" "The check results does not matter at this step - you can safely ignore them.\n" "Also this pull-request will be merged automatically as it reaches the mergeable state, but you always can merge it manually.\n" ) # FIXME: replace with something better than os.system() git_prefix = [ "git", "-C", repo_path, "-c", "[email protected]", "-c", "user.name=robot-clickhouse", ] base_commit_oid = self._pr["mergeCommit"]["parents"]["nodes"][0]["oid"] # Create separate branch for backporting, and make it look like real cherry-pick. self._run(git_prefix + ["checkout", "-f", self.target_branch]) self._run(git_prefix + ["checkout", "-B", self.backport_branch]) self._run(git_prefix + ["merge", "-s", "ours", "--no-edit", base_commit_oid]) # Create secondary branch to allow pull request with cherry-picked commit. self._run( git_prefix + ["branch", "-f", self.cherrypick_branch, self.merge_commit_oid]) self._run(git_prefix + [ "push", "-f", "origin", "{branch}:{branch}".format(branch=self.backport_branch), ]) self._run(git_prefix + [ "push", "-f", "origin", "{branch}:{branch}".format(branch=self.cherrypick_branch), ]) # Create pull-request like a local cherry-pick pr = self._gh.create_pull_request( source=self.cherrypick_branch, target=self.backport_branch, title="Cherry pick #{number} to {target}: {title}".format( number=self._pr["number"], target=self.target_branch, title=self._pr["title"].replace('"', '\\"'), ), description="Original pull-request #{}\n\n{}".format( self._pr["number"], DESCRIPTION), ) # FIXME: use `team` to leave a single eligible assignee. self._gh.add_assignee(pr, self._pr["author"]) self._gh.add_assignee(pr, self._pr["mergedBy"]) self._gh.set_label(pr, "do not test") self._gh.set_label(pr, "pr-cherrypick") return pr def mergeCherryPickPullRequest(self, cherrypick_pr): return self._gh.merge_pull_request(cherrypick_pr["id"]) def getBackportPullRequest(self): return self._gh.find_pull_request(base=self.target_branch, head=self.backport_branch) def createBackportPullRequest(self, cherrypick_pr, repo_path): DESCRIPTION = ( "This pull-request is a last step of an automated backporting.\n" "Treat it as a standard pull-request: look at the checks and resolve conflicts.\n" "Merge it only if you intend to backport changes to the target branch, otherwise just close it.\n" ) git_prefix = [ "git", "-C", repo_path, "-c", "[email protected]", "-c", "user.name=robot-clickhouse", ] pr_title = "Backport #{number} to {target}: {title}".format( number=self._pr["number"], target=self.target_branch, title=self._pr["title"].replace('"', '\\"'), ) self._run(git_prefix + ["checkout", "-f", self.backport_branch]) self._run(git_prefix + ["pull", "--ff-only", "origin", self.backport_branch]) self._run(git_prefix + [ "reset", "--soft", self._run(git_prefix + [ "merge-base", "origin/" + self.target_branch, self.backport_branch, ]), ]) self._run(git_prefix + ["commit", "-a", "--allow-empty", "-m", pr_title]) self._run(git_prefix + [ "push", "-f", "origin", "{branch}:{branch}".format(branch=self.backport_branch), ]) pr = self._gh.create_pull_request( source=self.backport_branch, target=self.target_branch, title=pr_title, description= "Original pull-request #{}\nCherry-pick pull-request #{}\n\n{}". format(self._pr["number"], cherrypick_pr["number"], DESCRIPTION), ) # FIXME: use `team` to leave a single eligible assignee. self._gh.add_assignee(pr, self._pr["author"]) self._gh.add_assignee(pr, self._pr["mergedBy"]) self._gh.set_label(pr, "pr-backport") return pr def execute(self, repo_path, dry_run=False): pr1 = self.getCherryPickPullRequest() if not pr1: if not dry_run: pr1 = self.createCherryPickPullRequest(repo_path) logging.debug( "Created PR with cherry-pick of %s to %s: %s", self._pr["number"], self.target_branch, pr1["url"], ) else: return CherryPick.Status.NOT_INITIATED else: logging.debug( "Found PR with cherry-pick of %s to %s: %s", self._pr["number"], self.target_branch, pr1["url"], ) if not pr1["merged"] and pr1[ "mergeable"] == "MERGEABLE" and not pr1["closed"]: if not dry_run: pr1 = self.mergeCherryPickPullRequest(pr1) logging.debug( "Merged PR with cherry-pick of %s to %s: %s", self._pr["number"], self.target_branch, pr1["url"], ) if not pr1["merged"]: logging.debug( "Waiting for PR with cherry-pick of %s to %s: %s", self._pr["number"], self.target_branch, pr1["url"], ) if pr1["closed"]: return CherryPick.Status.DISCARDED elif pr1["mergeable"] == "CONFLICTING": return CherryPick.Status.FIRST_CONFLICTS else: return CherryPick.Status.FIRST_MERGEABLE pr2 = self.getBackportPullRequest() if not pr2: if not dry_run: pr2 = self.createBackportPullRequest(pr1, repo_path) logging.debug( "Created PR with backport of %s to %s: %s", self._pr["number"], self.target_branch, pr2["url"], ) else: return CherryPick.Status.FIRST_MERGEABLE else: logging.debug( "Found PR with backport of %s to %s: %s", self._pr["number"], self.target_branch, pr2["url"], ) if pr2["merged"]: return CherryPick.Status.MERGED elif pr2["closed"]: return CherryPick.Status.DISCARDED elif pr2["mergeable"] == "CONFLICTING": return CherryPick.Status.SECOND_CONFLICTS else: return CherryPick.Status.SECOND_MERGEABLE