def __init__(self, token, owner, name, team):
self._gh = RemoteRepo(token,
owner=owner,
name=name,
team=team,
max_page_size=30,
min_page_size=7)
self._token = token
self.default_branch_name = self._gh.default_branch
self.ssh_url = self._gh.ssh_url
def __init__(self, token, owner, name, team, pr_number, target_branch):
self._gh = RemoteRepo(token, owner=owner, name=name, team=team)
self._pr = self._gh.get_pull_request(pr_number)
self.ssh_url = self._gh.ssh_url
# TODO: check if pull-request is merged.
self.merge_commit_oid = self._pr['mergeCommit']['oid']
self.target_branch = target_branch
self.backport_branch = 'backport/{branch}/{pr}'.format(branch=target_branch, pr=pr_number)
self.cherrypick_branch = 'cherrypick/{branch}/{oid}'.format(branch=target_branch, oid=self.merge_commit_oid)
class Backport:
def __init__(self, token, owner, name, team):
self._gh = RemoteRepo(token,
owner=owner,
name=name,
team=team,
max_page_size=30,
min_page_size=7)
self._token = token
self.default_branch_name = self._gh.default_branch
self.ssh_url = self._gh.ssh_url
def getPullRequests(self, from_commit):
return self._gh.get_pull_requests(from_commit)
def getBranchesWithRelease(self):
branches = set()
for pull_request in self._gh.find_pull_requests("release"):
branches.add(pull_request['headRefName'])
return branches
def execute(self, repo, upstream, until_commit, run_cherrypick):
repo = LocalRepo(repo, upstream, self.default_branch_name)
all_branches = repo.get_release_branches(
) # [(branch_name, base_commit)]
release_branches = self.getBranchesWithRelease()
branches = []
# iterate over all branches to preserve their precedence.
for branch in all_branches:
if branch[0] in release_branches:
branches.append(branch)
if not branches:
logging.info('No release branches found!')
return
for branch in branches:
logging.info('Found release branch: %s', branch[0])
if not until_commit:
until_commit = branches[0][1]
pull_requests = self.getPullRequests(until_commit)
backport_map = {}
RE_MUST_BACKPORT = re.compile(r'^v(\d+\.\d+)-must-backport$')
RE_NO_BACKPORT = re.compile(r'^v(\d+\.\d+)-no-backport$')
RE_BACKPORTED = re.compile(r'^v(\d+\.\d+)-backported$')
# pull-requests are sorted by ancestry from the most recent.
for pr in pull_requests:
while repo.comparator(branches[-1][1]) >= repo.comparator(
pr['mergeCommit']['oid']):
logging.info(
"PR #{} is already inside {}. Dropping this branch for further PRs"
.format(pr['number'], branches[-1][0]))
branches.pop()
logging.info("Processing PR #{}".format(pr['number']))
assert len(branches)
branch_set = set([branch[0] for branch in branches])
# First pass. Find all must-backports
for label in pr['labels']['nodes']:
if label['name'] == 'pr-bugfix' or label[
'name'] == 'pr-must-backport':
backport_map[pr['number']] = branch_set.copy()
continue
matched = RE_MUST_BACKPORT.match(label['name'])
if matched:
if pr['number'] not in backport_map:
backport_map[pr['number']] = set()
backport_map[pr['number']].add(matched.group(1))
# Second pass. Find all no-backports
for label in pr['labels']['nodes']:
if label['name'] == 'pr-no-backport' and pr[
'number'] in backport_map:
del backport_map[pr['number']]
break
matched_no_backport = RE_NO_BACKPORT.match(label['name'])
matched_backported = RE_BACKPORTED.match(label['name'])
if matched_no_backport and pr[
'number'] in backport_map and matched_no_backport.group(
1) in backport_map[pr['number']]:
backport_map[pr['number']].remove(
matched_no_backport.group(1))
logging.info('\tskipping %s because of forced no-backport',
matched_no_backport.group(1))
elif matched_backported and pr[
'number'] in backport_map and matched_backported.group(
1) in backport_map[pr['number']]:
backport_map[pr['number']].remove(
matched_backported.group(1))
logging.info(
'\tskipping %s because it\'s already backported manually',
matched_backported.group(1))
for pr, branches in list(backport_map.items()):
logging.info('PR #%s needs to be backported to:', pr)
for branch in branches:
logging.info('\t%s, and the status is: %s', branch,
run_cherrypick(self._token, pr, branch))
# print API costs
logging.info('\nGitHub API total costs per query:')
for name, value in list(self._gh.api_costs.items()):
logging.info('%s : %s', name, value)
class CherryPick:
class Status(Enum):
DISCARDED = 'discarded'
NOT_INITIATED = 'not started'
FIRST_MERGEABLE = 'waiting for 1st stage'
FIRST_CONFLICTS = 'conflicts on 1st stage'
SECOND_MERGEABLE = 'waiting for 2nd stage'
SECOND_CONFLICTS = 'conflicts on 2nd stage'
MERGED = 'backported'
def _run(self, args):
logging.info(subprocess.check_output(args))
def __init__(self, token, owner, name, team, pr_number, target_branch):
self._gh = RemoteRepo(token, owner=owner, name=name, team=team)
self._pr = self._gh.get_pull_request(pr_number)
self.ssh_url = self._gh.ssh_url
# TODO: check if pull-request is merged.
self.merge_commit_oid = self._pr['mergeCommit']['oid']
self.target_branch = target_branch
self.backport_branch = 'backport/{branch}/{pr}'.format(
branch=target_branch, pr=pr_number)
self.cherrypick_branch = 'cherrypick/{branch}/{oid}'.format(
branch=target_branch, oid=self.merge_commit_oid)
def getCherryPickPullRequest(self):
return self._gh.find_pull_request(base=self.backport_branch,
head=self.cherrypick_branch)
def createCherryPickPullRequest(self, repo_path):
DESCRIPTION = (
'This pull-request is a first step of an automated backporting.\n'
'It contains changes like after calling a local command `git cherry-pick`.\n'
'If you intend to continue backporting this changes, then resolve all conflicts if any.\n'
'Otherwise, if you do not want to backport them, then just close this pull-request.\n'
'\n'
'The check results does not matter at this step - you can safely ignore them.\n'
'Also this pull-request will be merged automatically as it reaches the mergeable state, but you always can merge it manually.\n'
)
# FIXME: replace with something better than os.system()
git_prefix = [
'git', '-C', repo_path, '-c',
'[email protected]', '-c',
'user.name=robot-clickhouse'
]
base_commit_oid = self._pr['mergeCommit']['parents']['nodes'][0]['oid']
# Create separate branch for backporting, and make it look like real cherry-pick.
self._run(git_prefix + ['checkout', '-f', self.target_branch])
self._run(git_prefix + ['checkout', '-B', self.backport_branch])
self._run(git_prefix +
['merge', '-s', 'ours', '--no-edit', base_commit_oid])
# Create secondary branch to allow pull request with cherry-picked commit.
self._run(
git_prefix +
['branch', '-f', self.cherrypick_branch, self.merge_commit_oid])
self._run(git_prefix + [
'push', '-f', 'origin', '{branch}:{branch}'.format(
branch=self.backport_branch)
])
self._run(git_prefix + [
'push', '-f', 'origin', '{branch}:{branch}'.format(
branch=self.cherrypick_branch)
])
# Create pull-request like a local cherry-pick
pr = self._gh.create_pull_request(
source=self.cherrypick_branch,
target=self.backport_branch,
title='Cherry pick #{number} to {target}: {title}'.format(
number=self._pr['number'],
target=self.target_branch,
title=self._pr['title'].replace('"', '\\"')),
description='Original pull-request #{}\n\n{}'.format(
self._pr['number'], DESCRIPTION))
# FIXME: use `team` to leave a single eligible assignee.
self._gh.add_assignee(pr, self._pr['author'])
self._gh.add_assignee(pr, self._pr['mergedBy'])
self._gh.set_label(pr, "do not test")
self._gh.set_label(pr, "pr-cherrypick")
return pr
def mergeCherryPickPullRequest(self, cherrypick_pr):
return self._gh.merge_pull_request(cherrypick_pr['id'])
def getBackportPullRequest(self):
return self._gh.find_pull_request(base=self.target_branch,
head=self.backport_branch)
def createBackportPullRequest(self, cherrypick_pr, repo_path):
DESCRIPTION = (
'This pull-request is a last step of an automated backporting.\n'
'Treat it as a standard pull-request: look at the checks and resolve conflicts.\n'
'Merge it only if you intend to backport changes to the target branch, otherwise just close it.\n'
)
git_prefix = [
'git', '-C', repo_path, '-c',
'[email protected]', '-c',
'user.name=robot-clickhouse'
]
pr_title = 'Backport #{number} to {target}: {title}'.format(
number=self._pr['number'],
target=self.target_branch,
title=self._pr['title'].replace('"', '\\"'))
self._run(git_prefix + ['checkout', '-f', self.backport_branch])
self._run(git_prefix +
['pull', '--ff-only', 'origin', self.backport_branch])
self._run(git_prefix + [
'reset', '--soft',
self._run(git_prefix +
['merge-base', self.target_branch, self.backport_branch])
])
self._run(git_prefix + ['commit', '-a', '-m', pr_title])
self._run(git_prefix + [
'push', '-f', 'origin', '{branch}:{branch}'.format(
branch=self.backport_branch)
])
pr = self._gh.create_pull_request(
source=self.backport_branch,
target=self.target_branch,
title=pr_title,
description=
'Original pull-request #{}\nCherry-pick pull-request #{}\n\n{}'.
format(self._pr['number'], cherrypick_pr['number'], DESCRIPTION))
# FIXME: use `team` to leave a single eligible assignee.
self._gh.add_assignee(pr, self._pr['author'])
self._gh.add_assignee(pr, self._pr['mergedBy'])
self._gh.set_label(pr, "pr-backport")
return pr
def execute(self, repo_path, dry_run=False):
pr1 = self.getCherryPickPullRequest()
if not pr1:
if not dry_run:
pr1 = self.createCherryPickPullRequest(repo_path)
logging.debug('Created PR with cherry-pick of %s to %s: %s',
self._pr['number'], self.target_branch,
pr1['url'])
else:
return CherryPick.Status.NOT_INITIATED
else:
logging.debug('Found PR with cherry-pick of %s to %s: %s',
self._pr['number'], self.target_branch, pr1['url'])
if not pr1['merged'] and pr1[
'mergeable'] == 'MERGEABLE' and not pr1['closed']:
if not dry_run:
pr1 = self.mergeCherryPickPullRequest(pr1)
logging.debug('Merged PR with cherry-pick of %s to %s: %s',
self._pr['number'], self.target_branch,
pr1['url'])
if not pr1['merged']:
logging.debug('Waiting for PR with cherry-pick of %s to %s: %s',
self._pr['number'], self.target_branch, pr1['url'])
if pr1['closed']:
return CherryPick.Status.DISCARDED
elif pr1['mergeable'] == 'CONFLICTING':
return CherryPick.Status.FIRST_CONFLICTS
else:
return CherryPick.Status.FIRST_MERGEABLE
pr2 = self.getBackportPullRequest()
if not pr2:
if not dry_run:
pr2 = self.createBackportPullRequest(pr1, repo_path)
logging.debug('Created PR with backport of %s to %s: %s',
self._pr['number'], self.target_branch,
pr2['url'])
else:
return CherryPick.Status.FIRST_MERGEABLE
else:
logging.debug('Found PR with backport of %s to %s: %s',
self._pr['number'], self.target_branch, pr2['url'])
if pr2['merged']:
return CherryPick.Status.MERGED
elif pr2['closed']:
return CherryPick.Status.DISCARDED
elif pr2['mergeable'] == 'CONFLICTING':
return CherryPick.Status.SECOND_CONFLICTS
else:
return CherryPick.Status.SECOND_MERGEABLE
class Backport:
def __init__(self, token, owner, name, team):
self._gh = RemoteRepo(token,
owner=owner,
name=name,
team=team,
max_page_size=30,
min_page_size=7)
self._token = token
self.default_branch_name = self._gh.default_branch
self.ssh_url = self._gh.ssh_url
def getPullRequests(self, from_commit):
return self._gh.get_pull_requests(from_commit)
def execute(self, repo, til, number, run_cherrypick):
repo = LocalRepo(repo, 'origin', self.default_branch_name)
branches = repo.get_release_branches()[
-number:] # [(branch_name, base_commit)]
if not branches:
logging.info('No release branches found!')
return
for branch in branches:
logging.info('Found release branch: %s', branch[0])
if not til:
til = branches[0][1]
prs = self.getPullRequests(til)
backport_map = {}
RE_MUST_BACKPORT = re.compile(r'^v(\d+\.\d+)-must-backport$')
RE_NO_BACKPORT = re.compile(r'^v(\d+\.\d+)-no-backport$')
RE_BACKPORTED = re.compile(r'^v(\d+\.\d+)-backported$')
# pull-requests are sorted by ancestry from the least recent.
for pr in prs:
while repo.comparator(branches[-1][1]) >= repo.comparator(
pr['mergeCommit']['oid']):
logging.info(
"PR #{} is already inside {}. Dropping this branch for further PRs"
.format(pr['number'], branches[-1][0]))
branches.pop()
logging.info("Processing PR #{}".format(pr['number']))
assert len(branches)
branch_set = set([branch[0] for branch in branches])
# First pass. Find all must-backports
for label in pr['labels']['nodes']:
if label['name'].startswith(
'pr-') and label['color'] == 'ff0000':
backport_map[pr['number']] = branch_set.copy()
continue
m = RE_MUST_BACKPORT.match(label['name'])
if m:
if pr['number'] not in backport_map:
backport_map[pr['number']] = set()
backport_map[pr['number']].add(m.group(1))
# Second pass. Find all no-backports
for label in pr['labels']['nodes']:
if label['name'] == 'pr-no-backport' and pr[
'number'] in backport_map:
del backport_map[pr['number']]
break
m1 = RE_NO_BACKPORT.match(label['name'])
m2 = RE_BACKPORTED.match(label['name'])
if m1 and pr['number'] in backport_map and m1.group(
1) in backport_map[pr['number']]:
backport_map[pr['number']].remove(m1.group(1))
logging.info('\tskipping %s because of forced no-backport',
m1.group(1))
elif m2 and pr['number'] in backport_map and m2.group(
1) in backport_map[pr['number']]:
backport_map[pr['number']].remove(m2.group(1))
logging.info(
'\tskipping %s because it\'s already backported manually',
m2.group(1))
for pr, branches in backport_map.items():
logging.info('PR #%s needs to be backported to:', pr)
for branch in branches:
logging.info('\t%s, and the status is: %s', branch,
run_cherrypick(self._token, pr, branch))
# print API costs
logging.info('\nGitHub API total costs per query:')
for name, value in self._gh.api_costs.items():
logging.info('%s : %s', name, value)
class CherryPick:
class Status(Enum):
DISCARDED = "discarded"
NOT_INITIATED = "not started"
FIRST_MERGEABLE = "waiting for 1st stage"
FIRST_CONFLICTS = "conflicts on 1st stage"
SECOND_MERGEABLE = "waiting for 2nd stage"
SECOND_CONFLICTS = "conflicts on 2nd stage"
MERGED = "backported"
def _run(self, args):
out = subprocess.check_output(args).rstrip()
logging.debug(out)
return out
def __init__(self, token, owner, name, team, pr_number, target_branch):
self._gh = RemoteRepo(token, owner=owner, name=name, team=team)
self._pr = self._gh.get_pull_request(pr_number)
self.ssh_url = self._gh.ssh_url
# TODO: check if pull-request is merged.
self.merge_commit_oid = self._pr["mergeCommit"]["oid"]
self.target_branch = target_branch
self.backport_branch = "backport/{branch}/{pr}".format(
branch=target_branch, pr=pr_number)
self.cherrypick_branch = "cherrypick/{branch}/{oid}".format(
branch=target_branch, oid=self.merge_commit_oid)
def getCherryPickPullRequest(self):
return self._gh.find_pull_request(base=self.backport_branch,
head=self.cherrypick_branch)
def createCherryPickPullRequest(self, repo_path):
DESCRIPTION = (
"This pull-request is a first step of an automated backporting.\n"
"It contains changes like after calling a local command `git cherry-pick`.\n"
"If you intend to continue backporting this changes, then resolve all conflicts if any.\n"
"Otherwise, if you do not want to backport them, then just close this pull-request.\n"
"\n"
"The check results does not matter at this step - you can safely ignore them.\n"
"Also this pull-request will be merged automatically as it reaches the mergeable state, but you always can merge it manually.\n"
)
# FIXME: replace with something better than os.system()
git_prefix = [
"git",
"-C",
repo_path,
"-c",
"[email protected]",
"-c",
"user.name=robot-clickhouse",
]
base_commit_oid = self._pr["mergeCommit"]["parents"]["nodes"][0]["oid"]
# Create separate branch for backporting, and make it look like real cherry-pick.
self._run(git_prefix + ["checkout", "-f", self.target_branch])
self._run(git_prefix + ["checkout", "-B", self.backport_branch])
self._run(git_prefix +
["merge", "-s", "ours", "--no-edit", base_commit_oid])
# Create secondary branch to allow pull request with cherry-picked commit.
self._run(
git_prefix +
["branch", "-f", self.cherrypick_branch, self.merge_commit_oid])
self._run(git_prefix + [
"push",
"-f",
"origin",
"{branch}:{branch}".format(branch=self.backport_branch),
])
self._run(git_prefix + [
"push",
"-f",
"origin",
"{branch}:{branch}".format(branch=self.cherrypick_branch),
])
# Create pull-request like a local cherry-pick
pr = self._gh.create_pull_request(
source=self.cherrypick_branch,
target=self.backport_branch,
title="Cherry pick #{number} to {target}: {title}".format(
number=self._pr["number"],
target=self.target_branch,
title=self._pr["title"].replace('"', '\\"'),
),
description="Original pull-request #{}\n\n{}".format(
self._pr["number"], DESCRIPTION),
)
# FIXME: use `team` to leave a single eligible assignee.
self._gh.add_assignee(pr, self._pr["author"])
self._gh.add_assignee(pr, self._pr["mergedBy"])
self._gh.set_label(pr, "do not test")
self._gh.set_label(pr, "pr-cherrypick")
return pr
def mergeCherryPickPullRequest(self, cherrypick_pr):
return self._gh.merge_pull_request(cherrypick_pr["id"])
def getBackportPullRequest(self):
return self._gh.find_pull_request(base=self.target_branch,
head=self.backport_branch)
def createBackportPullRequest(self, cherrypick_pr, repo_path):
DESCRIPTION = (
"This pull-request is a last step of an automated backporting.\n"
"Treat it as a standard pull-request: look at the checks and resolve conflicts.\n"
"Merge it only if you intend to backport changes to the target branch, otherwise just close it.\n"
)
git_prefix = [
"git",
"-C",
repo_path,
"-c",
"[email protected]",
"-c",
"user.name=robot-clickhouse",
]
pr_title = "Backport #{number} to {target}: {title}".format(
number=self._pr["number"],
target=self.target_branch,
title=self._pr["title"].replace('"', '\\"'),
)
self._run(git_prefix + ["checkout", "-f", self.backport_branch])
self._run(git_prefix +
["pull", "--ff-only", "origin", self.backport_branch])
self._run(git_prefix + [
"reset",
"--soft",
self._run(git_prefix + [
"merge-base",
"origin/" + self.target_branch,
self.backport_branch,
]),
])
self._run(git_prefix +
["commit", "-a", "--allow-empty", "-m", pr_title])
self._run(git_prefix + [
"push",
"-f",
"origin",
"{branch}:{branch}".format(branch=self.backport_branch),
])
pr = self._gh.create_pull_request(
source=self.backport_branch,
target=self.target_branch,
title=pr_title,
description=
"Original pull-request #{}\nCherry-pick pull-request #{}\n\n{}".
format(self._pr["number"], cherrypick_pr["number"], DESCRIPTION),
)
# FIXME: use `team` to leave a single eligible assignee.
self._gh.add_assignee(pr, self._pr["author"])
self._gh.add_assignee(pr, self._pr["mergedBy"])
self._gh.set_label(pr, "pr-backport")
return pr
def execute(self, repo_path, dry_run=False):
pr1 = self.getCherryPickPullRequest()
if not pr1:
if not dry_run:
pr1 = self.createCherryPickPullRequest(repo_path)
logging.debug(
"Created PR with cherry-pick of %s to %s: %s",
self._pr["number"],
self.target_branch,
pr1["url"],
)
else:
return CherryPick.Status.NOT_INITIATED
else:
logging.debug(
"Found PR with cherry-pick of %s to %s: %s",
self._pr["number"],
self.target_branch,
pr1["url"],
)
if not pr1["merged"] and pr1[
"mergeable"] == "MERGEABLE" and not pr1["closed"]:
if not dry_run:
pr1 = self.mergeCherryPickPullRequest(pr1)
logging.debug(
"Merged PR with cherry-pick of %s to %s: %s",
self._pr["number"],
self.target_branch,
pr1["url"],
)
if not pr1["merged"]:
logging.debug(
"Waiting for PR with cherry-pick of %s to %s: %s",
self._pr["number"],
self.target_branch,
pr1["url"],
)
if pr1["closed"]:
return CherryPick.Status.DISCARDED
elif pr1["mergeable"] == "CONFLICTING":
return CherryPick.Status.FIRST_CONFLICTS
else:
return CherryPick.Status.FIRST_MERGEABLE
pr2 = self.getBackportPullRequest()
if not pr2:
if not dry_run:
pr2 = self.createBackportPullRequest(pr1, repo_path)
logging.debug(
"Created PR with backport of %s to %s: %s",
self._pr["number"],
self.target_branch,
pr2["url"],
)
else:
return CherryPick.Status.FIRST_MERGEABLE
else:
logging.debug(
"Found PR with backport of %s to %s: %s",
self._pr["number"],
self.target_branch,
pr2["url"],
)
if pr2["merged"]:
return CherryPick.Status.MERGED
elif pr2["closed"]:
return CherryPick.Status.DISCARDED
elif pr2["mergeable"] == "CONFLICTING":
return CherryPick.Status.SECOND_CONFLICTS
else:
return CherryPick.Status.SECOND_MERGEABLE
