def addremote(self, repo, fetch=True):
os.chdir(self.directory)
cmd = shell('git remote | grep ^%s$' % repo.name)
if cmd.returncode != 0:
shell('git remote add %s %s' % (repo.name, repo.url))
if fetch:
cmd = shell('git fetch %s' % (repo.name))
if cmd.returncode != 0:
raise RemoteFetchError
self.remotes[repo.name] = repo
def format_patch(self, recombination):
shell(
'git fetch replica +refs/changes/*:refs/remotes/replica/changes/*')
cmd = shell('git checkout remotes/replica/changes/%s/%s/%s' %
(recombination.number[-2:], recombination.number,
recombination.patchset_number))
cmd = shell('git show --pretty=format:"" HEAD --patch-with-stat')
diff = '\n'.join(cmd.output)
if not diff:
raise Error
#diff = 'diff --git a/test-requirements.txt b/test-requirements.txt\nindex 509587b..829b6d6 100644\n--- a/test-requirements.txt\n+++ b/test-requirements.txt\n@@ -1,3 +1,4 @@\n+# ifjoweijf\n # The order of packages is significant, because pip processes them in the order\n # of appearance. Changing the order has an impact on the overall integration\n # process, which may cause wedges in the gate later.\n'
cmd = shell('git format-patch %s^..%s --stdout' %
(recombination.main_source.revision,
recombination.main_source.revision))
patch = '\n'.join(cmd.output)
rs = re.search("Subject: \[PATCH\] ", patch)
mpatch = patch[:rs.end()]
cmd = shell('git --version | sed -e "s/git version //"')
gitver = cmd.output[0]
ampatch = mpatch + recombination.backport_change.commit_message + "\n---\n" + diff + "\n--\n%s\n" % gitver
log.debugvar('ampatch')
cmd = shell('git checkout -B %s remotes/replica/%s' %
(recombination.backport_change.branch,
recombination.backport_change.branch))
cmd = shell('git am --abort')
cmd = shell('git am', stdin=ampatch)
def upload_change(self, branch, topic, reviewers=None, successremove=True):
command = 'git push %s HEAD:refs/drafts/%s/%s' % (self.name, branch,
topic)
if reviewers:
command = "%s%%" % command
for reviewer in reviewers:
command = command + "r=%s," % reviewer
command.rstrip(',')
# FIXME: check upload results in another way
shell('git checkout %s' % branch)
#cmd = shell('git review -D -r %s -t "%s" %s' % (self.name, topic, branch))
#for line in cmd.output:
# if 'Nothing to do' in line:
# log.debug("trying alternative upload method")
# shell("git push %s HEAD:refs/drafts/%s/%s" % (self.name, branch, topic))
# break
shell(command)
cmd = shell(
'ssh %s gerrit query --current-patch-set --format json "topic:%s AND status:open"'
% (self.host, topic))
shell('git checkout parking')
log.debug(pprint.pformat(cmd.output))
if not cmd.output[:-1] and successremove:
shell('git push replica :%s' % branch)
return None
gerrit_infos = json.loads(cmd.output[:-1][0])
infos = self.normalize_infos(gerrit_infos)
return infos
def submit_change(self, number, patchset):
shell('ssh %s gerrit review --publish --project %s %s,%s' %
(self.host, self.project_name, number, patchset))
shell('ssh %s gerrit review --submit --project %s %s,%s' %
(self.host, self.project_name, number, patchset))
cmd = shell(
'ssh %s gerrit query --format json "change:%s AND status:merged"' %
(self.host, number))
if cmd.output[:-1]:
return True
return False
def __init__(self, directory):
self.directory = directory
self.remotes = dict()
try:
os.mkdir(self.directory)
except OSError:
pass
os.chdir(self.directory)
try:
os.stat(".git")
except OSError:
shell('git init')
def get_changes_data(self,
search_values,
search_field='commit',
results_key='revision',
branch=None):
if type(search_values) is str or type(search_values) is unicode:
search_values = [search_values]
if search_field != 'commit':
log.error('Tracked repo search does not support search by %s' %
search_field)
return None
changes_data = dict()
os.chdir(self.directory)
for revision in search_values:
infos = {}
cmd = shell('git show -s --pretty=format:"%%H %%P" %s' %
(revision))
infos['id'], infos['parent'] = cmd.output[0].split(' ')[0:2]
infos['revision'] = infos['id']
if not branch:
log.error("for git repositories you must specify a branch")
sys.exit(1)
else:
infos['branch'] = branch
infos['project-name'] = self.project_name
changes_data[infos[results_key]] = infos
return changes_data
def add_gerrit_remote(self,
name,
location,
project_name,
fetch=True,
fetch_changes=True):
repo = Gerrit(name, location, project_name)
self.addremote(repo, fetch=fetch)
repo.local_track = TrackedRepo(name, self.directory, project_name)
if fetch_changes:
shell('git fetch %s +refs/changes/*:refs/remotes/%s/changes/*' %
(name, name))
try:
os.stat(".git/hooks/commit-msg")
except OSError:
shell('scp -p %s:hooks/commit-msg .git/hooks/' % location)
def suggest_conflict_solution(self, recombination):
patches_branch = recombination.patches_source.branch
pick_revision = recombination.main_source.revision
suggested_solution = None
log.info("Trying to find a possible cause")
cmd = shell('git show -s --pretty=format:"%%an <%%ae>" %s' %
pick_revision)
author = cmd.output[0]
cmd = shell('git show -s --pretty=format:"%%at" %s' % pick_revision)
date = cmd.output[0]
cmd = shell(
'git log --pretty=raw --author="%s" | grep -B 3 "%s" | grep commit\ | sed -e "s/commit //g"'
% (author, date))
if cmd.output:
suggested_solution = "Commit %s from upstream was already cherry-picked as %s in %s patches branch" % (
pick_revision, cmd.output[0], patches_branch)
return suggested_solution
def get_commits(self,
revision_start,
revision_end,
first_parent=True,
reverse=True,
no_merges=False):
os.chdir(self.directory)
options = ''
commit_list = list()
log.debug("Interval: %s..%s" % (revision_start, revision_end))
os.chdir(self.directory)
shell('git checkout parking')
if reverse:
options = '%s --reverse' % options
if first_parent:
options = '%s --first-parent' % options
if no_merges:
options = '%s --no-merges' % options
cmd = shell('git rev-list %s --pretty="%%H" %s..%s | grep -v ^commit' %
(options, revision_start, revision_end))
for commit_hash in cmd.output:
commit = dict()
commit['hash'] = commit_hash
cmd = shell('git show -s --pretty="%%P" %s' % commit_hash)
commit['parents'] = cmd.output[0].split(' ')
cmd = shell('git show -s --pretty="%%B" %s' % commit_hash)
commit['body'] = cmd.output
if len(commit['parents']) > 1:
commit['subcommits'] = self.get_commits(commit['parents'][0],
commit['parents'][1],
first_parent=False,
reverse=False)
commit_list.append(commit)
return commit_list
def commit_recomb(self, recombination):
pick_revision = recombination.main_source.revision
merge_revision = recombination.patches_source.revision
main_source_name = recombination.main_source_name
patches_source_name = recombination.patches_source_name
main_branch = recombination.main_source.branch
fd, commit_message_filename = tempfile.mkstemp(prefix="recomb-",
suffix=".yaml",
text=True)
os.close(fd)
commit_data = recombination.get_commit_message_data()
with open(commit_message_filename, 'w') as commit_message_file:
# We have to be sure this is the first line in yaml document
commit_message_file.write(
"Recombination: %s:%s-%s:%s~%s\n\n" %
(main_source_name, pick_revision[:6], patches_source_name,
merge_revision[:6], main_branch))
yaml.dump(commit_data,
commit_message_file,
default_flow_style=False,
indent=4,
canonical=False,
default_style=False)
cmd = shell("git commit -F %s" % (commit_message_filename))
# If two changes with the exact content are merged upstream
# the above command will succeed but nothing will be committed.
# and recombination upload will fail due to no change.
# this assures that we will always commit something to upload
for line in cmd.output:
if 'nothing to commit' in line or 'nothing added' in line:
shell("git commit --allow-empty -F %s" %
(commit_message_filename))
#logsummary.warning('Contents in commit %s have been merged twice in upstream' % pick_revision)
break
os.unlink(commit_message_filename)
def query_changes_json(self, query, comments=False):
changes_infos = list()
cmd = shell(
'ssh %s gerrit query --comments --current-patch-set --format json %s'
% (self.host, query))
log.debug(pprint.pformat(cmd.output))
for change_json in cmd.output:
if change_json != '':
change = json.loads(change_json)
if "type" not in change or (change['type'] != 'stats'
and change['type'] != 'error'):
changes_infos.append(change)
log.debug("end query json")
return changes_infos
def comment_change(self,
number,
patchset,
comment_message,
verified=None,
code_review=None):
review_input = dict()
review_input['labels'] = dict()
review_input['message'] = comment_message
if code_review:
review_input['labels']['Code-Review'] = code_review
if verified:
review_input['labels']['Verified'] = verified
json_input = json.dumps(review_input, ensure_ascii=False)
cmd = shell("echo '%s' | ssh %s gerrit review --json %s,%s" %
(json_input, self.host, number, patchset))
def fetch_recombinations(self, test_basedir, status, recomb_id=None):
untested_recombs = self.recomb_remote.get_untested_recombs_infos(
recomb_id=recomb_id)
dirlist = dict()
os.chdir(self.directory)
change_dir = os.getcwd()
shell('git checkout parking')
for recomb in untested_recombs:
recomb_dir = "%s/%s/code" % (self.project_name, recomb['number'])
recomb_branch = 'remotes/%s/changes/%s/%s/%s' % (
self.recomb_remote.name, recomb['number'][-2:],
recomb['number'], recomb['currentPatchSet']['number'])
shell('git checkout %s' % recomb_branch)
shutil.rmtree(test_basedir + "/" + recomb_dir, ignore_errors=True)
shutil.copytree(change_dir,
test_basedir + "/" + recomb_dir,
ignore=shutil.ignore_patterns('.git*'))
shell('git checkout parking')
dirlist[recomb['number']] = recomb_dir
return dirlist
def publish_change(self, number, patchset):
shell('ssh %s gerrit review --publish --project %s %s,%s' %
(self.host, self.project_name, number, patchset))
def abandon_change(self, number, patchset):
shell('ssh %s gerrit review --abandon --project %s %s,%s' %
(self.host, self.project_name, number, patchset))
def reject_change(self, number, patchset):
shell('ssh %s gerrit review --code-review -2 --verified -1 %s,%s' %
(self.host, number, patchset))
def cherrypick_recombine(self, recombination, permanent_patches=None):
#shell('git fetch replica')
#shell('git fetch original')
pick_revision = recombination.main_source.revision
merge_revision = recombination.patches_source.revision
cmd = shell('git branch --list %s' % recombination.branch)
if cmd.output:
cmd = shell('git branch -D %s' % recombination.branch)
cmd = shell('git branch -r --list replica/%s' % recombination.branch)
if cmd.output:
cmd = shell('git push replica :%s' % recombination.branch)
cmd = shell('git checkout -b %s %s' %
(recombination.branch, merge_revision))
log.info("Creating remote disposable branch on replica")
cmd = shell('git push replica HEAD:%s' % recombination.branch)
cmd = shell('git cherry-pick --no-commit %s' % (pick_revision))
# if merge fails, push empty change, and comment with git status.
# TO FIND existing commit in patches (conflict resolution suggestions)
# for commit in $(git rev-list --reverse --max-count 1000 --no-merges remotes/original/master); do AUTHOR=$(git show -s --pretty=format:"%an <%ae>" $commit); DATE=$(git show -s --pretty=format:"%at" $commit); CORRES=$(git log --pretty=raw --author="$AUTHOR" | grep -B 3 "$DATE" | grep commit\ | sed -e "s/commit //g"); if [ ! -z $CORRES ] ; then echo $commit in original/master is present in patches as $CORRES; fi; done
if cmd.returncode != 0:
#if cmd.returncode == 0:
failure_cause = None
log.error("Recombination Failed")
cmd = shell('git status --porcelain')
status = ''
suggested_solution = ''
try:
if recombination.backport_change.exist_different:
pass
except AttributeError:
pass
if failure_cause == "conflict":
conflicts = cmd.output
recombination.backport_change.commit_message = self.add_conflicts_string(
conflicts, recombination.backport_change.commit_message)
status = '\n '.join([''] + conflicts)
# TODO: add diff3 conflict blocks to output to status
for filestatus in conflicts:
filename = filestatus[2:] # re.sub('^[A-Z]*\ ', '')
with open(filename) as conflict_file:
filecontent = conflict_file.read()
for lineno, line in enum(filecontent.split('\n')):
rs = re.search('^<<<<<<', line)
if rs is not None:
block_start = line
rs = re.search('^>>>>>>', line)
if rs is not None:
block_end = line
block = '\n'.join(
filecontent.split('\n')[block_start:block_end])
diffs[filename] = block
suggested_solution = self.suggest_conflict_solution(
recombination)
cmd = shell('git cherry-pick --abort')
recombination.status = "FAILED"
self.commit_recomb(recombination)
raise RecombinationFailed(status, suggested_solution)
else:
recombination.status = "SUCCESSFUL"
self.commit_recomb(recombination)
def get_revision(self, ref):
os.chdir(self.directory)
# works with both tags and branches
cmd = shell('git rev-list -n 1 %s' % ref)
revision = cmd.output[0].rstrip('\n')
return revision
def approve_change(self, number, patchset):
shell('ssh %s gerrit review --code-review 2 --verified 1 %s,%s' %
(self.host, number, patchset))
def delete_remote_branches(self, remote_name, branches):
os.chdir(self.directory)
for branch in branches:
shell('git push %s :%s' % (remote_name, branch))
def merge_recombine(self, recombination):
shell('git fetch replica')
shell('git fetch original')
removed_commits = list()
pick_revision = recombination.main_source.revision
merge_revision = recombination.patches_source.revision
patches_branch = recombination.patches_source.branch
target_replacement_branch = recombination.target_replacement_branch
# Branch prep
# local patches branch
shell('git checkout -B recomb_attempt-%s-base %s' %
(patches_branch, merge_revision))
# local recomb branch
cmd = shell('git rev-parse %s~1' % pick_revision)
starting_revision = cmd.output[0]
shell('git checkout -B %s %s' %
(recombination.branch, starting_revision))
log.info("Creating remote disposable branch on replica")
cmd = shell('git push replica HEAD:%s' % recombination.branch)
if cmd.returncode != 0:
raise PushError
patches_removal_queue = list()
merge = shell("git merge --stat --squash --no-commit %s %s" %
(pick_revision, merge_revision))
if merge.returncode != 0:
attempt_number = 0
patches_base = dict()
log.error("first attempt at merge failed")
cmd = shell('git status --porcelain')
prev_conflict_status = cmd.output
cmd = shell('git merge-base %s %s' %
(pick_revision, merge_revision))
ancestor = cmd.output[0]
cmd = shell(
'git rev-list --reverse --first-parent %s..remotes/replica/%s'
% (ancestor, patches_branch))
patches_removal_queue = cmd.output
for commit in cmd.output:
cmd = shell('git show --pretty=format:"" %s' % commit,
show_stdout=False)
diff = '\n'.join(cmd.output)
hash_object = hashlib.sha1(diff)
patches_base[hash_object.hexdigest()] = commit
if patches_removal_queue:
log.warning("attempting automatic resolution")
else:
log.error("automatic resolution impossible")
else:
log.info("Merge successful")
retry_branch = None
while merge.returncode != 0 and patches_removal_queue:
attempt_number += 1
shell('git reset --hard %s' % recombination.branch)
shell('git checkout recomb_attempt-%s-base' % patches_branch)
retry_branch = 'recomb_attempt-%s-retry_%s' % (patches_branch,
attempt_number)
shell('git checkout -b %s' % (retry_branch))
# Rebasing changes all the commits hashes after "commit"
next_patch_toremove = patches_removal_queue.pop(0)
shell('git rebase -p --onto %s^ %s' %
(next_patch_toremove, next_patch_toremove))
cmd = shell('git rev-parse %s' % retry_branch)
retry_merge_revision = cmd.output[0]
shell('git checkout %s' % recombination.branch)
merge = shell("git merge --stat --squash --no-commit %s %s" %
(pick_revision, retry_merge_revision))
if merge.returncode != 0:
log.warning("automatic resolution attempt %d failed" %
attempt_number)
cmd = shell('git status --porcelain')
conflict_status = cmd.output
diff = difflib.Differ()
same_status = list(
diff.compare(prev_conflict_status, conflict_status))
if not same_status:
removed_commits.append(next_patch_toremove)
# removing this patch did not solve everything, but did
# something nonetheless, keep it removed and try to remove
# something else too
# change the base, recalculate every patch commit hash since
# rebase changed everything
shell('git branch -D recomb_attempt-%s-base' %
patches_branch)
shell(
'git checkout -B recomb_attempt-%s-base %s' %
patches_branch, retry_merge_revision)
cmd = shell(
'git rev-list --reverse --first-parent %s..%s' %
(ancestor, retry_branch))
patches_removal_queue = cmd.output
prev_conflict_status = conflict_status
shell('git branch -D %s' % retry_branch)
else:
logsummary.warning(
"automatic resolution attempt %d succeeded" %
attempt_number)
merge_revision = retry_merge_revision
removed_commits.append(next_patch_toremove)
logsummary.info("removed commits")
logsummary.info(removed_commits)
logsummary.info(
"removed commits (commit hash relative to starting patches branch)"
)
logsummary.info('removed_commits_deashed')
if merge.returncode != 0:
logsummary.error("automatic resolution failed")
shell('git push replica :%s' % recombination.branch)
else:
logsummary.info("Recombination successful")
# create new patches-branch
# TODO: understand if this can be a automatic task or we just notify someone
if retry_branch:
shell('git push replica :%s' % patches_branch)
shell('git push replica %s:refs/heads/%s' %
(retry_branch, patches_branch))
shell('git branch -D %s' % retry_branch)
recombination.removed_patches_commits = removed_commits
recombination.status = "SUCCESSFUL"
self.commit_recomb(recombination)
# Create target branch replacement for this recombination
shell('git checkout -B %s %s' %
(target_replacement_branch, starting_revision))
cmd = shell("git merge --log --no-edit %s %s" %
(pick_revision, merge_revision))
if cmd.returncode == 0:
shell('git push replica HEAD:%s' % target_replacement_branch)
shell('git checkout parking')
#shell('git branch -D %s' % recombination_branch)
shell('git branch -D recomb_attempt-%s-base' % patches_branch)
shell('git branch -D %s' % target_replacement_branch)
def remove_commits(self, branch, removed_commits, remote=''):
shell('git branch --track %s%s %s' (remote, branch, branch))
shell('git checkout %s' % branch)
for commit in removed_commits:
cmd = shell('git show -s %s' % commit)
if cmd.output:
shell('git rebase -p --onto %s^ %s' % (commit, commit))
log.info('removed commit %s from branch %s' % (commit, branch))
else:
break
if remote:
shell('git push -f %s HEAD:%s' % (remote, branch))
log.info('Pushed modified branch on remote')
shell('git checkout parking')
def delete_branch(self, branch):
os.chdir(self.directory)
shell('git checkout parking')
shell('git branch -D %s' % branch)
def track_branch(self, branch, remote_branch):
os.chdir(self.directory)
shell('git checkout parking')
shell('git branch --track %s %s' % (branch, remote_branch))
def list_branches(self, remote_name, pattern=''):
os.chdir(self.directory)
cmd = shell(
'git for-each-ref --format="%%(refname)" refs/remotes/%s/%s | sed -e "s/refs\/remotes\/%s\///"'
% (remote_name, pattern, remote_name))
return cmd.output
def __init__(self, project_name, directory):
super(Underlayer, self).__init__(directory)
self.project_name = project_name
shell('git config diff.renames copy')
shell('git config diff.renamelimit 10000')
shell('git config merge.conflictstyle diff3')
# TODO: remove all local branches
# git for-each-ref --format="%(refname)" refs/heads | sed -e "s/refs\/heads//"
# for branch in local_branches:
# shell('git branch -D %s' % branch)
self.mirror_remote = None
cmd = shell('git checkout parking')
if cmd.returncode != 0:
shell('git checkout --orphan parking')
shell('git commit --allow-empty -a -m "parking"')
self.branch_maps = dict()
self.branch_maps['original->replica'] = dict()
self.branch_maps['patches->replica'] = dict()
self.branch_maps['target->replica'] = dict()
self.branch_maps['original->target'] = dict()
self.branch_maps['patches->target'] = dict()
self.branch_maps['replica->target'] = dict()
self.branch_maps['original->patches'] = dict()
self.branch_maps['replica->patches'] = dict()
self.branch_maps['target->patches'] = dict()
def sync_replica(self, replica_branch, revision):
os.chdir(self.directory)
shell('git fetch replica')
shell('git branch --track replica-%s remotes/replica/%s' %
(replica_branch, replica_branch))
shell('git checkout replica-%s' % replica_branch)
cmd = shell('git merge --ff-only %s' % revision)
if cmd.returncode != 0:
log.debug(cmd.output)
log.critical("Error merging. Exiting")
raise MergeError
cmd = shell('git push replica HEAD:%s' % replica_branch)
if cmd.returncode != 0:
log.debug(cmd.output)
log.critical("Error pushing the merge. Exiting")
raise PushError
shell('git checkout parking')
shell('git branch -D replica-%s' % replica_branch)
def update_target_branch(self, target_replacement_branch, target_branch):
shell('git fetch replica')
shell('git checkout remotes/replica/%s' % (target_replacement_branch))
shell('git push -f replica HEAD:%s' % (target_branch))
shell('git checkout parking')
shell('git push replica :%s ' % target_replacement_branch)
def get_recombinations_from_original(self, original_branch, original_ids,
diversity_refname,
replication_strategy, replica_lock):
patches_branch = self.branch_maps['original->patches'][original_branch]
diversity_revision = self.get_revision(diversity_refname)
diversity_change = self.patches_remote.local_track.get_change(
diversity_revision, branch=patches_branch)
recombinations = OrderedDict()
original_changes = self.original_remote.get_changes(
list(original_ids), branch=original_branch)
recomb_data = self.recomb_remote.get_changes_data(list(original_ids),
search_field='topic',
results_key='topic')
log.debugvar('original_changes')
for change_id in original_ids:
if replication_strategy == "lock-and-backports":
recomb_class = EvolutionDiversityRecombination
elif replication_strategy == "change-by-change":
recomb_class = OriginalDiversityRecombination
new_recomb = True
if change_id in recomb_data:
try:
recombination = recomb_class(self, self.recomb_remote)
recombination.load_change_data(
recomb_data[change_id],
original_remote=self.original_remote,
patches_remote=self.patches_remote,
diversity_change=diversity_change)
new_recomb = False
except RecombinationCanceledError:
del (recombination)
if new_recomb:
recombination = recomb_class(self, self.recomb_remote)
# Set real commit as revision
original_changes[change_id].revision = original_ids[change_id]
if replication_strategy == "lock-and-backports":
lock_revision = self.get_revision(replica_lock)
cmd = shell(
'git show -s --pretty=format:"%%an <%%ae>" %s' %
original_ids[change_id])
author = cmd.output[0]
cmd = shell('git show -s --pretty=format:"%%at" %s' %
original_ids[change_id])
date = cmd.output[0]
cmd = shell(
'git log --pretty=raw --author="%s" %s..%s | grep -B 3 "%s" | grep commit\ | sed -e "s/commit //g"'
% (author, lock_revision, diversity_revision, date))
if cmd.output:
backport_change = self.patches_remote.get_change(
cmd.output[0], search_field='commit')
# TODO: evaluate body diff.
# if body_diff:
# log.warning ('backport is present but patch differs')
# backport_change.exist_different = True
else:
backport_change = Change(remote=self.patches_remote)
# backport_change.branch = self.underlayer.branch_maps['patches']['original'][self.evolution_change.branch]
recombination.initialize(
self.recomb_remote,
evolution_change=original_changes[change_id],
diversity_change=diversity_change,
backport_change=backport_change)
elif replication_strategy == "change-by-change":
recombination.initialize(
self.recomb_remote,
original_change=original_changes[change_id],
diversity_change=diversity_change)
recombinations[change_id] = recombination
log.debugvar('change_id')
recomb = recombinations[change_id].__dict__
log.debugvar('recomb')
return recombinations
def revision_exists(self, remote, revision, branch):
cmd = shell("git ")
return True
