def needed_on_branches(bug):
"""Based on blocking flags and attachment flags, determine which
branches the bug needs uplifting to"""
needed_on = []
fixed_on = fixed_on_branches(bug)
branches = c.read_value('repository.enabled_branches')
blocking_flag = bug.get(c.read_value('rules.blocking_flag'))
blocking_rules = c.read_value('rules.blocking_rules')
patch_rules = c.read_value('rules.patch_rules')
def _a(x):
for y in x:
if not y in needed_on and not y in fixed_on and y in branches:
needed_on.append(y)
if blocking_flag in blocking_rules.keys():
_a(blocking_rules[blocking_flag])
for flag in patch_rules.keys():
for a in bug.get('attachments', []):
for f in a.get('flags', []):
if f['name'] == flag and f['status'] == '+':
_a(patch_rules[flag])
return needed_on
def build_uplift_requirements(repo_dir):
if os.path.exists(requirements_file) and util.ask_yn("Found existing requirements. Should they be used?"):
bug_info = util.read_json(requirements_file)
else:
bug_info = {}
enabled_branches = c.read_value('repository.enabled_branches')
all_queries = c.read_value('queries')
queries = []
for branch in enabled_branches:
queries.extend(all_queries[branch])
bugs = [x for x in find_bugs(queries) if not is_skipable(x)]
print "Fetching bug data"
for bug_id in bugs:
if is_skipable(bug_id):
continue
bug = bzapi.fetch_complete_bug(bug_id)
print "+",
needed_on = branch_logic.needed_on_branches(bug)
if len(needed_on) == 0:
continue
b = bug_info[bug_id] = {}
b['needed_on'] = needed_on
b['already_fixed_on'] = branch_logic.fixed_on_branches(bug)
b['summary'] = bug['summary']
print "\nFinished fetching bug data"
util.write_json(requirements_file, bug_info)
return bug_info
def flags_to_set(for_branches):
"""Take a list of branches and return a dictionary that contains
pairings of flag name and flag value. For the purposes of this
program, we always use 'fixed'."""
fb = []
branches = c.read_value('repository.enabled_branches')
status_flags = c.read_value('rules.status_flags')
for b in [x for x in for_branches if x in branches]:
if b in status_flags.keys():
fb.append(status_flags[b])
return dict([(x, 'fixed') for x in fb])
def do_query(url, method='get', retry=False, **kwargs):
"""Light wrapper around the BzAPI which takes an API url,
fetches the data then returns the data as a Python
dictionary. Only API errors are caught, and those are
raised as FailedBZAPICall exceptions"""
for i in range(0, c.read_value('bugzilla.api.attempts')):
try:
return _raw_query(method, url, i+1, **kwargs)
except Exception, e:
print "Query attempt %i failed: %s" % (i, e)
time.sleep(c.read_value('bugzilla.api.delay'))
def display_uplift_report(report, max_summary=90):
"""Generate a PrettyTable that shows the bug id, branch status
and first up to 100 chars of the summary"""
branches = c.read_value('repository.enabled_branches')
headers = ['Bug'] + ['%s commit' % x for x in ['master'] + branches] + ['Summary']
t = pt.PrettyTable(headers, sortby="Bug")
t.align['Bug'] = "l"
t.align['Summary'] = "l"
for bug_id in [x for x in report.keys() if not uplift.is_skipable(x)]:
bug = report[bug_id]
row = [bug_id]
master_commits = bug['commits']
row.append("\n".join([x[:7] for x in master_commits]) if len(master_commits) > 0 else "skipped")
for branch in branches:
branch_commits = []
for mcommit in master_commits:
if bug.has_key('uplift_status'):
if branch in bug['uplift_status'][mcommit]['success'].keys():
branch_commit = bug['uplift_status'][mcommit]['success'][branch]
if branch_commit == mcommit:
branch_commits.append("+++")
else:
branch_commits.append(branch_commit)
elif branch in bug['uplift_status'][mcommit]['failure']:
branch_commits.append("failed")
else:
branch_commits.append("---")
if len(branch_commits) == 0:
row.append("---")
else:
row.append("\n".join([x[:7] for x in branch_commits]))
t.add_row(row + [trim_words(bug['summary'])])
return t
def compute_url(query, endpoint):
"""This is where we assemble the query. We add in the BZ credentials
here so that they don't end up in other parts of the program"""
full_query = copy.deepcopy(query)
full_query.update(credentials)
api_host = c.read_value('bugzilla.api.host')
return "%s%s%s?%s" % (api_host, "" if api_host.endswith("/") else "/", endpoint, urllib.urlencode(full_query))
def create_gaia(repo_dir, gaia_url):
# These two lines are stupid. They break subtlely when
# repo_dir isn't an absolute path.
repo_dir_p = os.path.split(repo_dir.rstrip(os.sep))[0]
cache_dir = _cache_dir(os.path.abspath(repo_dir))
# Initialize or update the cached copy of gaia
if not os.path.isdir(cache_dir):
print "Initial clone of Gaia cache directory"
git_op(["clone", "--mirror", gaia_url, cache_dir],
workdir=os.path.split(cache_dir.rstrip(os.sep))[0])
else:
print "Fetching updates to Gaia cache directory"
git_op(["fetch", "--all"], workdir=cache_dir)
# Because we do all of the repository creation locally (i.e. cheaply), we don't
# really want to risk having bad commits left around, so we delete the repo
print "Deleting Gaia scratch directory"
delete_gaia(repo_dir)
# Let's create the working copy of gaia. We want to clone it from the
# cache, fix the remotes and create the remote references in the local
# copy by fetching from the actual remote. We fetch the actual remote's
# references because we want to create a copy of gaia that doesn't need
# to use the cached copy when pushing changes
print "Cloning Gaia scratch from cache"
git_op(["clone", "file://%s" % cache_dir, repo_dir], workdir=repo_dir_p)
git_op(["remote", "rename", "origin", "cache"], workdir=repo_dir)
git_op(["remote", "add", "origin", gaia_url], workdir=repo_dir)
print "Fetching remote references"
git_op(["fetch", "origin"], workdir=repo_dir)
branches = c.read_value('repository.enabled_branches')
for branch in branches + ['master']:
recreate_branch(repo_dir, branch, remote="origin")
def fixed_on_branches(bug):
"""Take a bug dictionary and use the bugzilla flags to determine
which branches the bug is fixed or verifed on. This does not
look at the contents of the repository because that's impossible
to do correctly. If you git revert a commit, the original commit
is still 'on' the branch, but substantively absent"""
status_flags = c.read_value('rules.status_flags')
branches = c.read_value('repository.enabled_branches')
# I should figure out what this variable actually is and rename it!
_branches = dict([(status_flags[k],k) for k in status_flags.keys()])
b = []
for x in ('fixed', 'verified'):
for flag in _branches.keys():
if bug.get(flag) == x and not _branches[flag] in b:
if _branches[flag] in branches:
b.append(_branches[flag])
return b
def push(repo_dir):
branches = c.read_value('repository.enabled_branches')
preview_push_info = git.push(repo_dir, remote="origin", branches=branches, dry_run=True)
print "If you push, you'd be pushing: "
_display_push_info(preview_push_info)
if not util.ask_yn('Do you wish to push?'):
return None
for i in range(5):
try:
rv = git.push(repo_dir, remote="origin", branches=branches, dry_run=False)
util.write_json(push_info_file, rv)
print "Push attempt %d worked" % int(i+1)
return rv
except:
print "Push attempt %d failed" % int(i+1)
raise git.PushFailure("remote %s branches %s" % (remote, util.e_join(branches)))
def display_uplift_requirements(requirements, max_summary=90):
"""Generate a PrettyTable that shows the bug id, branch status
and first up to 100 chars of the summary"""
branches = c.read_value('repository.enabled_branches')
headers = ['Bug'] + ['%s status' % x for x in branches] + ['Summary']
t = pt.PrettyTable(headers, sortby="Bug")
t.align['Bug'] = "l"
t.align['Summary'] = "l"
for bug_id in [x for x in requirements.keys() if not uplift.is_skipable(x)]:
bug = requirements[bug_id]
row = [bug_id]
needed_on = bug['needed_on']
fixed_on = bug['already_fixed_on']
for branch in branches:
if branch in fixed_on:
row.append("fixed")
elif branch in needed_on:
row.append("needed")
else:
row.append("---")
t.add_row(row + [trim_words(bug['summary'])])
return t
def main():
args = sys.argv
gaia_url = c.read_value('repository.url')
gaia_path = os.path.join(os.getcwd(), gaia_url.split('/')[-1])
if len(args) < 2:
print "You must specify a command"
exit(1)
cmd = args[1]
cmd_args = args[2:]
if cmd == 'show':
bugs = uplift.build_uplift_requirements(gaia_path)
print "\n\nRequirements for Bug uplift:"
print reporting.display_uplift_requirements(bugs)
elif cmd == 'uplift':
requirements = uplift.build_uplift_requirements(gaia_path)
full_requirements = find_commits.for_all_bugs(gaia_path, requirements)
print "\n\nUplift requirements:"
print reporting.display_uplift_requirements(full_requirements)
uplift_report = uplift.uplift(gaia_path, gaia_url, full_requirements)
print reporting.display_uplift_report(uplift_report)
try:
push_info = uplift.push(gaia_path)
if push_info:
try:
reporting.comment(gaia_path, uplift_report)
except Exception as e:
print "ERROR: Commenting on the bugs failed"
print " Fix the error and try again with:"
print " uplift comments %s" % uplift.uplift_dated_file
print "DEBUG INFO FOLLOWS:"
print e
traceback.print_exc()
else:
print "To replay the comments for this push, run:"
print " uplift comments %s" % uplift.uplift_dated_file
except git.PushFailure:
print "ERROR: Pushing failed. Try doing another uplift, and tell it to 'reuse' commits"
exit(1)
elif cmd == 'update':
git.create_gaia(gaia_path, gaia_url)
elif cmd == 'merge':
merge_hd.merge(gaia_path, gaia_url, cmd_args[0], cmd_args[1])
elif cmd == 'comments':
if len(cmd_args) == 0:
uplift_report_file = uplift.uplift_report_file
elif len(cmd_args) == 1 and os.path.exists(cmd_args[0]):
uplift_report_file = os.path.abspath(cmd_args[0])
else:
print "ERROR: missing or too many json files"
exit(1)
with open(uplift_report_file, 'rb') as f:
uplift_report = json.load(f)
reporting.comment(gaia_path, uplift_report)
elif cmd == 'merge-comments':
merge_hd.comment(gaia_path, cmd_args[0], cmd_args[1])
elif cmd == "sort-commits":
if len(cmd_args) < 3:
print "ERROR: You must have a branch and at least one commit to sort"
exit(1)
branch = cmd_args[1]
commits = cmd_args[2:]
print "->".join(git.sort_commits(gaia_path, commits, branch))
else:
print "ERROR: You did not specify a command!"
exit(1)
