def __call__(self, target):
check_root()
to_fetch = None
if issue_tracker_tool.ISSUE_BE_LIKE.match(target) is None:
# doesn't look like an issue
to_fetch = [target]
resolve_errors(git_tool.recurse_submodules(lambda: git_tool.fetch(to_fetch)), **FETCH_HANDLING)
kind = None
if all(x[1] for x in git_tool.recurse_submodules(lambda: git_tool.is_remote_branch(target))):
kind = "branch"
elif all(x[1] for x in git_tool.recurse_submodules(lambda: git_tool.is_tag(target))):
kind = "tag"
elif issue_tracker_tool.validate_issue(target) is not None:
_target = target
target = branchname_from_issue(target, test=True)
kind = "branch"
if target is None:
print ("Can't find any matching branches. You probably didn't run 'new-feature'")
do_new_feature = raw_input("Do you want to run it now [y/N]?")
if do_new_feature in YES:
new_feature(_target,labels=None)
target = branchname_from_issue(_target)
else:
print RED("Please run new-feature first")
sys.exit(1)
if kind is not None:
resolve_errors(git_tool.recurse_submodules(lambda: git_tool.checkout_remote(target, kind)),
title="Checking out %s %s..." % (kind, target))
else:
print "Failed to resolve target '%s'" % target
def __call__(self, build_no, project="Android,Context,Discovery"):
check_root()
tag = "cibuild_%s" % build_no
resolve_errors(git_tool.recurse_submodules(lambda: git_tool.fetch()), **FETCH_HANDLING)
# make sure we get the correct week even if we freeze on Sunday
now = datetime.datetime.now()
day = datetime.timedelta(days=1)
cal = (now+day).isocalendar()
year = cal[0]
pulse = cal[1]
branchname = "rc-%d-%02d" % (year, pulse)
resolve_errors(git_tool.recurse_submodules(lambda: git_tool.new_branch(branchname, tag)),
title="Freezing into %s" % branchname)
resolve_errors(git_tool.recurse_submodules(lambda: git_tool.forcepush(branchname)),
title="Pushing %s" % branchname)
def protect():
repo = git_tool.get_repo()
success = cr_tool.protect_branch(repo, branchname)
return None, None if success else "Failed"
def mark_issues_to_rc():
"""
mark issues as IN_RC:
we search Jira for all issues with IN_MASTER status and then exclude issues not found in the git log
of the branchname
"""
jql = 'project in ({0}) and status="{1}"'.format(project, issue_tracker_tool.JiraStatus.LANDED_IN_MASTER)
issues = issue_tracker_tool.search(jql)
keys_in_master = [issue.key for issue in issues]
exclude = issues_not_in_branch(branchname)
print 'excluding issues not in {0}'.format(branchname), exclude
for key in set(keys_in_master) - set(exclude):
print "{0}: marking in rc".format(key)
issue_tracker_tool.mark_in_rc(key, branchname)
resolve_errors(git_tool.recurse_submodules(protect), title="Protecting %s" % branchname)
mark_issues_to_rc()
print BOLD("Initiating a build")
rc_build_no, success = builder_tool.build_launcher(branch=branchname,
profiles=["Debug"],
block=True,
gitlab_token=cr_tool.get_gitlab_token(),
split_apks="yes")
if success:
print BOLD("Updating Rollout")
rollout_tool.create_release(rc_build_no, 0) # freezing a release is always with distribution=0
print BOLD("Sending event to Timebox")
timebox_tool.send_event('freeze', rc_build_no, '0', pulse, builder_tool.get_build_path(str(rc_build_no)))
def __call__(self):
check_root()
uninited_modules = git_tool.get_uninited_submodules()
if uninited_modules:
display_uninited_modules_instructions(uninited_modules)
return
current_branch = get_current_branch()
resolve_errors(git_tool.recurse_submodules(lambda: git_tool.fetch([current_branch])), **FETCH_HANDLING)
resolve_errors(git_tool.recurse_submodules(lambda: git_tool.rebase_if_needed(current_branch)), **REBASE_HANDLING)
resolve_errors(git_tool.fix_refs(), title="Fixing submodule references")
resolve_errors(git_tool.recurse_submodules(git_tool.push), title="Pushing...")
def __call__(self, what):
resolve_errors(git_tool.recurse_submodules(lambda: git_tool.fetch()), **FETCH_HANDLING)
if self.is_issue(what):
branchname = branchname_from_issue(what, test=True, squashed=True)
else:
branchname = what
def approve():
if git_tool.is_merged_with("origin/%s" % branchname, ref="origin/master"):
return None, None
added = self.APPROVAL_FUNC(git_tool.get_repo(), branchname)
return added, None
resolve_errors(git_tool.recurse_submodules(approve), title="Marking your approvals...")
def branchname_from_issue(issue, test=False, squashed=False, exit_on_fail=True):
possible_branchnames = git_tool.recurse_submodules(lambda: get_branches_for_issue(issue, squashed), False, False)
possible_branchname_set = set()
for ret in possible_branchnames:
submodule, branchnames = ret
branchnames = filter(lambda x: x.endswith('__sq') == squashed, branchnames)
if len(branchnames) == 1:
possible_branchname_set.add(branchnames[0])
elif len(branchnames) > 1:
print "Found multiple possible branches in submodule %s: %r" % (submodule, branchnames)
sys.exit(1)
if len(possible_branchname_set) == 1:
return possible_branchname_set.pop()
if len(possible_branchname_set) == 0:
if test:
return None
print "Failed to find branches matching issue %s" % issue
sys.exit(1) if exit_on_fail else None
if len(possible_branchname_set) > 1:
if test:
return None
print "Found multiple possible branches for issue: %r" % possible_branchname_set
sys.exit(1) if exit_on_fail else None
return None
def process(self, assigner, assignee, build_number, issue, title, apk_path, squashed_branch):
hipchat_assigner = user_db.get_user_for_service(assigner, 'hipchat')
hipchat_assignee = user_db.get_user_for_service(assignee, 'hipchat')
jenkins_job_url = builder_tool.get_build_path(build_number)
jira_issue_url = issue_tracker_tool.get_issue_url(issue.key)
issue_tracker_tool.comment_issue(issue.key, "Build {0}, [Download Link|{1}]".format(build_number, apk_path))
im_tool.send_message(hipchat_assigner, "Build {0} ({1}) is successful, listing MRs... [{2}, {3}]".format(build_number, issue, jenkins_job_url, jira_issue_url))
im_tool.send_message(hipchat_assignee, "Build {0} ({1}) is successful, MRs are coming your way... [{2}, {3}]".format(build_number, issue, jenkins_job_url, jira_issue_url))
def open_mr():
git_tool.fetch(['master'])
if git_tool.is_branch_diverged('origin/master'):
mr = cr_tool.create_mr(git_tool.get_repo(), squashed_branch, assignee, "WIP: "+title)
cr_tool.approve_build(git_tool.get_repo(), squashed_branch, build_number)
im_tool.send_message(hipchat_assigner, "MR {0} has been assigned to @{1}".format(mr, hipchat_assignee))
im_tool.send_message(hipchat_assignee, "Hey, @{0} sent you a MR to review: {1}".format(hipchat_assigner, mr))
return mr, None
return None, None
resolve_errors(git_tool.recurse_submodules(open_mr), title="Opening MRs...")
update_jira(partial(issue_tracker_tool.send_to_cr, issue.key, user_db.get_user_for_service(assignee, 'jira')),
'Requested code review')
def __call__(self, checks=True, push=True):
check_root()
if checks:
check_config()
uninited_modules = git_tool.get_uninited_submodules()
if uninited_modules:
display_uninited_modules_instructions(uninited_modules)
return
resolve_errors(git_tool.recurse_submodules(lambda: git_tool.fetch(['master'])), **FETCH_HANDLING)
resolve_errors(git_tool.recurse_submodules(lambda: git_tool.rebase_if_needed('master')), **REBASE_HANDLING)
resolve_errors(git_tool.fix_refs(), title="Fixing submodule references")
if push:
resolve_errors(git_tool.recurse_submodules(git_tool.forcepush), title="Pushing...")
def get_current_branch():
current_branches = git_tool.recurse_submodules(git_tool.get_current_branch, False, False, concurrent=True)
grouped_branches = itertools.groupby(sorted(current_branches, key=itemgetter(1)), itemgetter(1))
grouped_branches = {k: map(itemgetter(0), v) for k, v in grouped_branches}
if len(grouped_branches) != 1:
print "Invalid state: on more than one branch:"
for branch_name, submodules in grouped_branches.iteritems():
print '{}:'.format(branch_name)
for submodule in submodules:
print '\t{}'.format(submodule)
print
sys.exit(1)
return grouped_branches.keys()[0]
def __call__(self, namespace):
check_root()
branch = get_current_branch()
authors = Counter()
if is_squashed_branch(branch):
print "You are trying to submit a squashed branch doll, go back to the original branch, it'll surely work!"
return
squashed_branch = append_squashed_branch_suffix(branch)
issue = issue_from_branchname(branch)
issue = issue_tracker_tool.validate_issue(issue)
title = issue.fields.summary
title = "%s: %s" % (issue.key, title)
uninited_modules = git_tool.get_uninited_submodules()
if uninited_modules:
display_uninited_modules_instructions(uninited_modules)
return
resolve_errors(git_tool.recurse_submodules(lambda: git_tool.fetch(['master'])), **FETCH_HANDLING)
resolve_errors(git_tool.recurse_submodules(lambda: git_tool.rebase_if_needed('master')), **REBASE_HANDLING)
def get_original_author():
if git_tool.is_branch_diverged('origin/master'):
authors[git_tool.get_author()] += 1
list(git_tool.recurse_submodules(get_original_author))
author = authors.most_common(1)[0][0] if authors else None
def create_squashed_branch():
git_tool.gitcmd(['branch', '-f', squashed_branch, branch])
return git_tool.gitcmd(['checkout', squashed_branch])
def squash():
ret = err = None
if git_tool.is_branch_diverged('origin/master'):
ret, err = git_tool.squash(title, author)
return ret, err
def push_and_back():
git_tool.gitcmd(['checkout', branch])
return git_tool.forcepush(squashed_branch)
resolve_errors(git_tool.recurse_submodules(git_tool.forcepush), title="Pushing...")
resolve_errors(git_tool.recurse_submodules(create_squashed_branch), title="Creating squashed branch...")
resolve_errors(git_tool.recurse_submodules(squash), title="Squashing...")
resolve_errors(git_tool.fix_refs(), title="Fixing refs...")
resolve_errors(git_tool.recurse_submodules(push_and_back), title="Pushing squashed branch...")
self.process(namespace, squashed_branch)
def __call__(self, message):
check_root()
current_branches = get_current_branch()
current_issue = issue_from_branchname(current_branches)
message = "%s: %s" % (current_issue, message)
def commit_if_necessary():
status = git_tool.status()
if status is None:
return None, "Status is none??"
if len(status.staged) > 0:
return git_tool.gitcmd(["commit", "-nm", message])
return None, None
resolve_errors(git_tool.recurse_submodules(commit_if_necessary))
def __call__(self, issue):
print BOLD("Aborting issue {}...".format(issue))
branchname = branchname_from_issue(issue, squashed=True, exit_on_fail=False)
def close():
if git_tool.is_merged_with("origin/%s" % branchname, ref="origin/master"):
return None, None
closed = cr_tool.close_mr(git_tool.get_repo(), branchname)
return closed, None
# close open MRs, if there are any
if branchname:
resolve_errors(git_tool.recurse_submodules(close), title="Closing open MRs...")
else:
print BOLD("No open MRs to close")
# move Jira ticket to "Aborted"
print BOLD("Aborting Jira ticket...")
issue_tracker_tool.abort(issue)
print BOLD("DONE")
def __call__(self, issue, branchname):
check_root()
work_on(branchname)
def do_the_cherry():
if os.path.exists('.git/CHERRY_PICK_HEAD'):
ret, err = git_tool.gitcmd(['cherry-pick', '--allow-empty-message', '--continue'])
if err is not None:
return ret, err
ret, err = git_tool.gitcmd(
['log', '--grep=%s' % issue, '--reverse', '--date-order', '--cherry', '--oneline',
'%s..origin/master' % branchname])
if err is not None:
return ret, err
commits = [x[1:].split() for x in ret.split('\n') if len(x) > 1]
commits = [x[0] for x in commits if len(x) > 0]
if len(commits) > 0:
print BOLD("applying " + ", ".join(UNDERLINE(x) for x in commits))
commits = ['cherry-pick'] + commits
else:
return None, None
return git_tool.gitcmd(commits)
return resolve_errors(git_tool.recurse_submodules(do_the_cherry), title="Cherry Picking...")
def __call__(self, issue, dry_run):
check_root()
branch = get_current_branch()
if branch != 'master':
work_on('master')
# list of reverted commits and their 'revert commits'
excluded_commits = set()
def should_revert(commit):
message = commit['message']
if "This reverts commit" in message:
revert_commit = commit['id']
reverted_commit = message.split("This reverts commit", 1)[1].strip(' .')
excluded_commits.add(revert_commit)
excluded_commits.add(reverted_commit)
print '{repo}: {reverted_commit} is already reverted by {revert_commit}'\
.format(repo=git_tool.get_repo(), reverted_commit=reverted_commit, revert_commit=revert_commit)
return False
return not commit['id'] in excluded_commits
def do_the_revert():
GIT_COMMIT_FIELDS = ['id', 'author_name', 'author_email', 'date', 'message']
GIT_LOG_FORMAT = ['%H', '%an', '%ae', '%ad', '%B']
GIT_LOG_FORMAT = '%x1f'.join(GIT_LOG_FORMAT) + '%x1e'
REVERT_TEMPLATE = """Revert "{message}"\n\nThis reverts commit {commit}."""
# ":" is added to issue id to prevent mistakes (--grep AN-xxx:)
(log, err) = git_tool.gitcmd(['log', '--grep', issue+':', '--format={}'.format(GIT_LOG_FORMAT)])
if not log:
return log, err
# turn log into a list of dictionaries for for easier use
log = log.strip('\n\x1e').split("\x1e")
log = [row.strip().split("\x1f") for row in log]
log = [dict(zip(GIT_COMMIT_FIELDS, row)) for row in log]
# filter only those commits which have not been reverted already
revert_candidates = filter(should_revert, log)
ret = None if not revert_candidates else '\n'.join(map(str, revert_candidates))
if not dry_run:
for commit in revert_candidates:
# try reverting
revert_ret, revert_err = git_tool.gitcmd(['revert','--no-edit',commit['id']])
submodules = None
if revert_err is not None:
# try resolving submodule reference issues automatically
gitstatus = git_tool.status()
if submodules is None:
submodules = git_tool.get_submodules()
if submodules is None:
return ret, err
non_subs = set(gitstatus.conflict) - set(submodules)
if len(non_subs) > 0:
print "Conflict in non submodules %r" % non_subs
return revert_ret, revert_err
for sub in gitstatus.conflict:
print "Adding %s" % sub
add_ret, add_err = git_tool.gitcmd(['add', sub])
if add_err is not None:
return revert_ret, revert_err
commit_ret, commit_err = git_tool.gitcmd(['commit', '-m', REVERT_TEMPLATE.format(message=commit['message'], commit=commit['id'])])
return commit_ret, commit_err
if revert_err:
ret = revert_ret
err = revert_err
return ret, err
if not dry_run:
input = raw_input(RED("Are you sure you want backout {issue}? "
"(It is advised that you use '-n' to initiate a dry run first) [y/N]".format(issue=issue)))
if input not in ['Y', 'y']:
sys.exit(0)
ret = resolve_errors(git_tool.recurse_submodules(do_the_revert), title="Reverting...")
#there was an error during resolve_errors, don't reopen the ticket just yet
if not ret:
print "OMG! There was an error during revert. Fix it and try again..."
exit(1)
if not dry_run:
print 'Stopping Progress on Jira issue {issue}'.format(issue=issue)
if issue_tracker_tool.stop_progress(issue):
print 'Successfully stopped progress on {issue}'.format(issue=issue)
else:
print 'Could not stop progress on {issue}'.format(issue=issue)
return ret
def __call__(self, what, ok_no_ff):
check_root()
if get_current_branch() != 'master':
print "You should be on the master branch before landing a feature, sweetie!"
return
fetched = False
if issue_tracker_tool.ISSUE_BE_LIKE.match(what) is not None:
issue = what
branchname = branchname_from_issue(issue, test=True, squashed=True)
if branchname is None:
resolve_errors(git_tool.recurse_submodules(lambda: git_tool.fetch()), **FETCH_HANDLING)
fetched = True
branchname = branchname_from_issue(what, squashed=True)
else:
branchname = what
issue = issue_from_branchname(what, squashed=True)
if not is_squashed_branch(branchname):
print YELLOW("Warning: landning non-squashed branch!")
if issue is None:
print YELLOW("Warning: unknown issue!")
def is_ready_to_land():
if not fetched:
git_tool.fetch([branchname, 'master'])
if git_tool.is_merged_with("origin/%s" % branchname, ref="origin/master"):
return None, None
err = []
if not git_tool.is_merged_with("origin/master", ref="origin/%s" % branchname):
err.append("Not rebased!")
approvals = list(cr_tool.get_signed_comments(git_tool.get_repo(), branchname))
body = lambda x: x['body']
built = filter(compose(cr_tool.ApproveBuild.filter_message, body), approvals)
cr = filter(compose(cr_tool.ApproveCR.filter_message, body), approvals)
qa = filter(compose(cr_tool.ApproveQA.filter_message, body), approvals)
ui = filter(compose(cr_tool.ApproveUITests.filter_message, body), approvals)
probe = filter(compose(cr_tool.ApproveProbeTests.filter_message, body), approvals)
ret = []
if len(built) == 0:
err.append("Wasn't built!")
else:
ret.extend('Built by %s at %s' % (x['author']['name'], x['created_at']) for x in built)
if len(cr) == 0:
err.append("Wasn't reviewed!")
else:
ret.extend('Reviewd by %s at %s' % (x['author']['name'], x['created_at']) for x in cr)
if len(qa) == 0:
err.append("Didn't pass QA!")
else:
ret.extend("Passed QA's %s at %s" % (x['author']['name'], x['created_at']) for x in qa)
if len(ui) == 0:
err.append("Didn't pass UI tests!")
else:
ret.extend("Passed UI Tests at %s" % x['created_at'] for x in ui)
if len(probe) == 0:
err.append("Didn't pass Probe benchmark test")
else:
ret.extend("Passed Probe benchmark test at %s" % x['created_at'] for x in probe)
return "\n".join(ret), (None if len(err) == 0 else "\n".join(err))
resolve_errors(git_tool.recurse_submodules(is_ready_to_land), **APPROVAL_HANDLING)
def do_push():
return git_tool.push('master')
def do_merge():
ret, err = git_tool.gitcmd(['merge', '--ff' if ok_no_ff else '--ff-only', 'origin/%s' % branchname])
if err is not None:
return ret, err
return ret, None
merge_success = resolve_errors(git_tool.recurse_submodules(do_merge), **MERGE_HANDLING)
if merge_success:
resolve_errors(git_tool.fix_refs(), title="Fixing submodule references")
push_success = resolve_errors(git_tool.recurse_submodules(do_push), title="Pushing...")
# Make sure all MR for this issue in gitlab are closed
def check_open_mr():
repo = git_tool.get_repo()
project = cr_tool.get_project(repo)
mr = cr_tool.get_open_mr(project, branchname)
if mr is not None:
print RED("ALERT: merge request is still open in GitLab for {}!".format(repo))
return None, "ALERT: merge request is still open in GitLab"
return None, None
gitlab_check_success = resolve_errors(git_tool.recurse_submodules(check_open_mr), title="Checking GitLab status...")
if not gitlab_check_success:
print RED("ERROR: there are still open MR in GitLab after land for {}, something is terribly wrong!!".format(issue))
if push_success and issue is not None:
update_jira(partial(issue_tracker_tool.land, issue), 'landed in master')
return push_success
return False
def __call__(self):
check_root()
branch = get_current_branch()
resolve_errors(git_tool.recurse_submodules(git_tool.forcepush), title="Pushing...")
print BOLD("Building")
builder_tool.build_launcher(branch, ["Debug"], False)
