def filter(self, commit_iter):
self.log.info(
"""
Filtering out all commits that have a Change-Id that matches one
found in the given search ref: %s
""", self.search_ref)
for commit in commit_iter:
change_id = self._get_change_id(commit)
# if there is no change_id to compare against, return the commit
if not change_id:
self.log.debug(
"""
Including change missing 'Change-Id'
Commit: %s %s
Message: %s
""", commit.short, commit.message.splitlines()[0],
commit.message)
yield commit
continue
# retrieve all matching commits because we need to check
# each match for whether the changeId is actually in
# the footer or just included as a reference.
matching_commits = Commit.iter_items(self.repo,
self._get_rev_range(),
regexp_ignore_case=True,
grep="^%s$" % change_id)
duplicate_change_id = None
for possible in matching_commits:
duplicate_change_id = self._get_change_id(possible)
if duplicate_change_id == change_id:
break
if duplicate_change_id and duplicate_change_id == change_id:
self.log.debug(
"""
Skipping duplicate Change-Id in search ref
%s
Commit: %s %s
""", change_id, commit.short,
commit.message.splitlines()[0])
continue
# no match in the search ref, so include commit
self.log.debug(
"""
Including unmatched change
%s
Commit: %s %s
""", change_id, commit.short,
commit.message.splitlines()[0])
yield commit
def _check_tree_state(self):
expected = getattr(self, 'expect_found', None)
# even if empty want to confirm that find no changes applied,
# otherwise confirm we find the expected number of changes.
if expected is not None:
if len(list(Commit.new(self.repo,
self.target_branch).parents)) > 1:
changes = list(
Commit.iter_items(
self.repo,
'%s..%s^2' %
(self.upstream_branch, self.target_branch),
topo_order=True))
else:
# allow checking that nothing was rebased
changes = []
self.assertThat(
len(changes), Equals(len(expected)),
"should only have seen %s changes, got: %s" %
(len(expected), ", ".join([
"%s:%s" % (commit.hexsha, commit.message.splitlines()[0])
for commit in changes
])))
# expected should be listed in order from oldest to newest, so
# reverse changes to match as it would be newest to oldest.
changes.reverse()
for commit, node in zip(changes, expected):
if node == "MERGE":
continue
subject = commit.message.splitlines()[0]
node_subject = self.gittree.graph[node].message.splitlines()[0]
self.assertThat(
subject, Equals(node_subject),
"subject '%s' of commit '%s' does not match "
"subject '%s' of node '%s'" %
(subject, commit.hexsha, node_subject, node))
def _check_tree_state(self):
expected = getattr(self, 'expect_found', None)
# even if empty want to confirm that find no changes applied,
# otherwise confirm we find the expected number of changes.
if expected is not None:
if len(list(Commit.new(self.repo,
self.target_branch).parents)) > 1:
changes = list(Commit.iter_items(
self.repo,
'%s..%s^2' % (self.upstream_branch, self.target_branch),
topo_order=True))
else:
# allow checking that nothing was rebased
changes = []
self.assertThat(
len(changes), Equals(len(expected)),
"should only have seen %s changes, got: %s" %
(len(expected),
", ".join(["%s:%s" % (commit.hexsha,
commit.message.splitlines()[0])
for commit in changes])))
# expected should be listed in order from oldest to newest, so
# reverse changes to match as it would be newest to oldest.
changes.reverse()
for commit, node in zip(changes, expected):
if node == "MERGE":
continue
subject = commit.message.splitlines()[0]
node_subject = self.gittree.graph[node].message.splitlines()[0]
self.assertThat(subject, Equals(node_subject),
"subject '%s' of commit '%s' does not match "
"subject '%s' of node '%s'" % (
subject, commit.hexsha, node_subject,
node))
def already_synced(self, strategy):
"""Check if already synced
Check if we are already up to date or if there are changes to
be applied.
"""
# if last commit in the strategy was a merge, then the additional
# branches that were merged in previously can be extracted based on
# the commits merged.
if len(strategy) > 0:
prev_import_merge = strategy[-1]
else:
# no changes carried?
prev_import_merge = None
additional_commits = None
if prev_import_merge and len(prev_import_merge.parents) > 1:
additional_commits = {
commit for commit in prev_import_merge.parents
if commit.hexsha != strategy.previous_upstream.hexsha}
if (additional_commits and
len(self.extra_branches) != len(additional_commits)):
self.log.warning("""
**************** WARNING ****************
Previous import merged additional branches but none have
been specified on the command line for this import.\n""")
# detect if nothing to do
if (strategy.previous_upstream.hexsha ==
self.git.rev_parse(self.upstream)):
self.log.notice("%s already at latest upstream commit: '%s'",
self.branch, strategy.previous_upstream)
if additional_commits is None:
self.log.notice("Nothing to be imported")
return True
else:
new_additional_commits = {Commit.new(self.repo, branch)
for branch in self.extra_branches}
if new_additional_commits == additional_commits:
self.log.notice(
"""
No updated additional branch given, nothing to be done
""")
return True
return False
def find(self):
"""
Searches the git history of the target branch for a commit message
containing the pattern given in the constructor. This is used as a base
commit from which to return a list of commits since this point.
"""
commits = Commit.iter_items(self.repo, self.branch, grep=self.pattern,
max_count=1, extended_regexp=True)
self.commit = next(commits, None)
if not self.commit:
raise RuntimeError("Failed to locate a pattern match")
self.log.debug("Commit matching search pattern is: '%s'",
self.commit.hexsha)
return self.commit.hexsha
def test_interactive(self):
upstream_branch = self.branches['upstream'][0]
target_branch = self.branches['head'][0]
cmdline = self.parser.get_default('script_cmdline') + self.parser_args
# ensure interactive mode cannot hang tests
def kill(proc_pid):
process = psutil.Process(proc_pid)
for proc in process.children(recursive=True):
try:
proc.kill()
except OSError:
continue
try:
process.kill()
except OSError:
pass
def get_output(proc):
self.output = proc.communicate()[0]
proc = subprocess.Popen(cmdline,
stdin=open(os.devnull, "r"),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, close_fds=True,
cwd=self.testrepo.path)
proc_thread = threading.Thread(target=get_output, args=[proc])
proc_thread.start()
proc_thread.join(getattr(self, 'timeout', 5))
if proc_thread.is_alive():
kill(proc.pid)
proc_thread.join()
self.addDetail('subprocess-output',
text_content(self.output.decode('utf-8')))
raise Exception('Process #%d killed after timeout' % proc.pid)
self.addDetail('subprocess-output',
text_content(self.output.decode('utf-8')))
self.assertThat(proc.returncode, Equals(0))
expected = getattr(self, 'expect_rebased', [])
if expected:
changes = list(Commit.iter_items(
self.repo, '%s..%s^2' % (upstream_branch, target_branch)))
self.assertThat(
len(changes), Equals(len(expected)),
"should only have seen %s changes, got: %s" %
(len(expected),
", ".join(["%s:%s" % (commit.hexsha,
commit.message.splitlines()[0])
for commit in changes])))
# expected should be listed in order from oldest to newest, so
# reverse changes to match as it would be newest to oldest.
changes.reverse()
for commit, node in zip(changes, expected):
subject = commit.message.splitlines()[0]
node_subject = self.gittree.graph[node].message.splitlines()[0]
self.assertThat(subject, Equals(node_subject),
"subject '%s' of commit '%s' does not match "
"subject '%s' of node '%s'" % (
subject, commit.hexsha, node_subject,
node))
import_branch = [head for head in self.repo.heads
if str(head).startswith("import") and
not str(head).endswith("-base")]
self.assertThat(self.git.rev_parse(import_branch),
Not(Equals(self.git.rev_parse(target_branch))),
"Import branch and target should have identical "
"contents, but not be the same")
# allow disabling of checking the merge commit contents
# as some tests won't result in an import
if getattr(self, 'check_merge', True):
commit_message = self.git.log(target_branch, n=1)
self.assertThat(commit_message,
Contains("of '%s' into '%s'" % (upstream_branch,
target_branch)))
# make sure the final state of merge is correct
self.assertThat(
self.repo.git.rev_parse("%s^{tree}" % target_branch),
Equals(self.repo.git.rev_parse(
"%s^2^{tree}" % target_branch)),
"--finish option failed to merge correctly")
# allow additional test specific verification methods below
extra_test_func = getattr(self, '_verify_%s' % self.name, None)
if extra_test_func:
extra_test_func()
def filter(self, commit_iter):
self.log.info(
"""
Filtering out all commits marked with a Superseded-by Change-Id
which is present in '%s'
""", self.search_ref)
supersede_re = re.compile('^%s\s*(.+)\s*$' %
lib.SUPERSEDE_HEADER,
re.IGNORECASE | re.MULTILINE)
for commit in commit_iter:
commit_note = commit.note(note_ref=lib.IMPORT_NOTE_REF)
# include non-annotated commits
if not commit_note:
yield commit
continue
# include annotated commits which don't have a SUPERSEDE_HEADER
superseding_change_ids = supersede_re.findall(commit_note)
if not superseding_change_ids:
yield commit
continue
# search for all the change-ids in matches (egrep regex)
commits_grep_re = '^Change-Id:\\s*\(%s\)\\s*$' % \
'\|'.join(superseding_change_ids)
# retrieve all matching commits because we need to check
# each match for whether the changeId is actually in
# the footer or just included as a reference.
matching_commits = Commit.iter_items(self.repo,
self._get_rev_range(),
regexp_ignore_case=True,
grep=commits_grep_re)
for possible in matching_commits:
change_id = self._get_change_id(possible)
if change_id:
superseding_change_ids.remove(change_id)
# include commits which have some superseding change-ids not
# present in upstream
if superseding_change_ids:
self.log.debug(
"""
Including commit '%s %s'
because the following superseding change-ids have not been
found:
%s
""", commit.short, commit.message.splitlines()[0],
'\n '.join(superseding_change_ids))
yield commit
continue
self.log.debug(
"""
Filtering out commit '%s %s'
because it has been marked as superseded by the following
note:
%s
""", commit.short, commit.message.splitlines()[0],
commit_note)
def list(self, upstream=None):
"""
Returns a list of Commit objects, between the '<commitish>' revision
given in the constructor, and the commit object returned by the find()
method. If given an upstream branch, uses --cherry-pick/--left-only to
exclude commits that are identical to those already on the upstream
branch.
"""
if not self.commit:
self.find()
revision_spec = "{0}..{1}".format(self.commit.hexsha, self.branch)
# search for previous import commit first, if found, wish to include
# the discarded parents as part of the set of changes to ignore in the
# final list.
self.log.debug(
"""
Searching for previous merges that exclude one side of the history
since the last import.
git rev-list --ancestry-path --merges %s
""", revision_spec)
merge_list = list(Commit.iter_items(self.repo, revision_spec,
topo_order=True,
ancestry_path=True, merges=True))
# to handle special case where a merge of a commit introduces
# nothing to the tree, need to spot when such commits are based
# off of the import set, so as not to accidentally exclude
strip_commits = set()
ancestry_commits = [
commit.hexsha
for commit in Commit.iter_items(
self.repo, revision_spec, topo_order=True,
ancestry_path=True, merges=False)
]
self.log.debug("Ancestry commits: %s", ancestry_commits)
extra_args = []
previous_import = None
for mergecommit, parent in ((mc, p)
for mc in merge_list
for p in mc.parents):
# inspect each
(previous_import_candidate,
ignores,
prune_list) = self._check_merge_is_previous(
mergecommit, parent, merge_list[-1], ancestry_commits)
if ignores:
self.log.debug(
"""
Adding following to ignore list:
%s
""", "\n ".join(ignores))
extra_args.extend(ignores)
if prune_list:
self.log.debug(
"""
Adding following commits to be pruned afterwards:
%s
""", "\n ".join(prune_list))
strip_commits.update(prune_list)
if previous_import_candidate:
previous_import = previous_import_candidate
self.log.info(
"""
Found possible previous import merge:
%s
""", previous_import)
if previous_import:
self.log.info(
"""
Found possible previous import merge:
%s
""", previous_import)
# walk the tree and find all commits that lie in the path between the
# commit found by find() and head of the branch in two steps, to
# ensure a deterministic order between what is from the previous
# upstream to the last import, and from that import to what is on
# the tip of the head to avoid inversion where older commits
# started before the previous import merge and approved afterwards
# are not sorted by 'rev-list' predictably.
commit_list = []
if upstream is None:
if previous_import:
search_list = [
(previous_import, self.branch, None),
(self.commit, previous_import, None),
]
else:
search_list = [(self.commit, self.branch, None)]
rev_spec = "{0}..{1}"
git_args = {}
else:
if previous_import:
search_list = [
(self.branch, upstream, "^%s" % previous_import),
(previous_import, upstream, "^%s~1" % previous_import)
]
else:
search_list = [(self.branch, upstream, None)]
rev_spec = "{0}...{1}"
git_args = {'cherry_pick': True,
'left_only': True,
'full_history': True,
}
extra_args.append("^%s" % self.commit)
for start, end, exclude in search_list:
extra = list(extra_args)
if exclude:
extra.append(exclude)
extra.extend(["--", "."])
revision_spec = rev_spec.format(start, end)
self.log.info(
"""
Walking the changes between found commit and target, excluding
those behind the previous import or merged as an additional
branch during the previous import
git rev-list --topo-order %s %s %s
""", ' '.join(self.git.transform_kwargs(**git_args)),
revision_spec, " ".join(extra))
commit_list.append(
Commit._iter_from_process_or_stream(
self.repo,
self.git.rev_list(revision_spec,
*extra,
as_process=True,
topo_order=True,
**git_args)))
# chain the filters as generators so that we don't need to allocate new
# lists for each step in the filter chain.
commit_list = itertools.chain(*commit_list)
# strip commits that cannot be excluded through revision specifications
# or without removing additional history
commit_list = [c for c in commit_list if c.hexsha not in strip_commits]
for f in self.filters:
commit_list = f.filter(commit_list)
commits = list(commit_list)
self.log.debug(
"""
commits found:
%s
""", ("\n" + " " * 4).join([c.hexsha for c in commits]))
return commits
def test_interactive(self):
upstream_branch = self.branches['upstream'][0]
target_branch = self.branches['head'][0]
cmdline = self.parser.get_default('script_cmdline') + self.parser_args
# ensure interactive mode cannot hang tests
def kill(proc_pid):
process = psutil.Process(proc_pid)
for proc in process.children(recursive=True):
try:
proc.kill()
except OSError:
continue
try:
process.kill()
except OSError:
pass
def get_output(proc):
self.output = proc.communicate()[0]
proc = subprocess.Popen(cmdline,
stdin=open(os.devnull, "r"),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=True,
cwd=self.testrepo.path)
proc_thread = threading.Thread(target=get_output, args=[proc])
proc_thread.start()
proc_thread.join(getattr(self, 'timeout', 5))
if proc_thread.is_alive():
kill(proc.pid)
proc_thread.join()
self.addDetail('subprocess-output',
text_content(self.output.decode('utf-8')))
raise Exception('Process #%d killed after timeout' % proc.pid)
self.addDetail('subprocess-output',
text_content(self.output.decode('utf-8')))
self.assertThat(proc.returncode, Equals(0))
expected = getattr(self, 'expect_rebased', [])
if expected:
changes = list(
Commit.iter_items(
self.repo, '%s..%s^2' % (upstream_branch, target_branch)))
self.assertThat(
len(changes), Equals(len(expected)),
"should only have seen %s changes, got: %s" %
(len(expected), ", ".join([
"%s:%s" % (commit.hexsha, commit.message.splitlines()[0])
for commit in changes
])))
# expected should be listed in order from oldest to newest, so
# reverse changes to match as it would be newest to oldest.
changes.reverse()
for commit, node in zip(changes, expected):
subject = commit.message.splitlines()[0]
node_subject = self.gittree.graph[node].message.splitlines()[0]
self.assertThat(
subject, Equals(node_subject),
"subject '%s' of commit '%s' does not match "
"subject '%s' of node '%s'" %
(subject, commit.hexsha, node_subject, node))
import_branch = [
head for head in self.repo.heads if str(head).startswith("import")
and not str(head).endswith("-base")
]
self.assertThat(
self.git.rev_parse(import_branch),
Not(Equals(self.git.rev_parse(target_branch))),
"Import branch and target should have identical "
"contents, but not be the same")
# allow disabling of checking the merge commit contents
# as some tests won't result in an import
if getattr(self, 'check_merge', True):
commit_message = self.git.log(target_branch, n=1)
self.assertThat(
commit_message,
Contains("of '%s' into '%s'" %
(upstream_branch, target_branch)))
# make sure the final state of merge is correct
self.assertThat(
self.repo.git.rev_parse("%s^{tree}" % target_branch),
Equals(self.repo.git.rev_parse("%s^2^{tree}" % target_branch)),
"--finish option failed to merge correctly")
# allow additional test specific verification methods below
extra_test_func = getattr(self, '_verify_%s' % self.name, None)
if extra_test_func:
extra_test_func()