def filter(self, commit_iter): self.log.info( """ Filtering out all commits that have a Change-Id that matches one found in the given search ref: %s """, self.search_ref) for commit in commit_iter: change_id = self._get_change_id(commit) # if there is no change_id to compare against, return the commit if not change_id: self.log.debug( """ Including change missing 'Change-Id' Commit: %s %s Message: %s """, commit.short, commit.message.splitlines()[0], commit.message) yield commit continue # retrieve all matching commits because we need to check # each match for whether the changeId is actually in # the footer or just included as a reference. matching_commits = Commit.iter_items(self.repo, self._get_rev_range(), regexp_ignore_case=True, grep="^%s$" % change_id) duplicate_change_id = None for possible in matching_commits: duplicate_change_id = self._get_change_id(possible) if duplicate_change_id == change_id: break if duplicate_change_id and duplicate_change_id == change_id: self.log.debug( """ Skipping duplicate Change-Id in search ref %s Commit: %s %s """, change_id, commit.short, commit.message.splitlines()[0]) continue # no match in the search ref, so include commit self.log.debug( """ Including unmatched change %s Commit: %s %s """, change_id, commit.short, commit.message.splitlines()[0]) yield commit
def _check_tree_state(self): expected = getattr(self, 'expect_found', None) # even if empty want to confirm that find no changes applied, # otherwise confirm we find the expected number of changes. if expected is not None: if len(list(Commit.new(self.repo, self.target_branch).parents)) > 1: changes = list( Commit.iter_items( self.repo, '%s..%s^2' % (self.upstream_branch, self.target_branch), topo_order=True)) else: # allow checking that nothing was rebased changes = [] self.assertThat( len(changes), Equals(len(expected)), "should only have seen %s changes, got: %s" % (len(expected), ", ".join([ "%s:%s" % (commit.hexsha, commit.message.splitlines()[0]) for commit in changes ]))) # expected should be listed in order from oldest to newest, so # reverse changes to match as it would be newest to oldest. changes.reverse() for commit, node in zip(changes, expected): if node == "MERGE": continue subject = commit.message.splitlines()[0] node_subject = self.gittree.graph[node].message.splitlines()[0] self.assertThat( subject, Equals(node_subject), "subject '%s' of commit '%s' does not match " "subject '%s' of node '%s'" % (subject, commit.hexsha, node_subject, node))
def _check_tree_state(self): expected = getattr(self, 'expect_found', None) # even if empty want to confirm that find no changes applied, # otherwise confirm we find the expected number of changes. if expected is not None: if len(list(Commit.new(self.repo, self.target_branch).parents)) > 1: changes = list(Commit.iter_items( self.repo, '%s..%s^2' % (self.upstream_branch, self.target_branch), topo_order=True)) else: # allow checking that nothing was rebased changes = [] self.assertThat( len(changes), Equals(len(expected)), "should only have seen %s changes, got: %s" % (len(expected), ", ".join(["%s:%s" % (commit.hexsha, commit.message.splitlines()[0]) for commit in changes]))) # expected should be listed in order from oldest to newest, so # reverse changes to match as it would be newest to oldest. changes.reverse() for commit, node in zip(changes, expected): if node == "MERGE": continue subject = commit.message.splitlines()[0] node_subject = self.gittree.graph[node].message.splitlines()[0] self.assertThat(subject, Equals(node_subject), "subject '%s' of commit '%s' does not match " "subject '%s' of node '%s'" % ( subject, commit.hexsha, node_subject, node))
def already_synced(self, strategy): """Check if already synced Check if we are already up to date or if there are changes to be applied. """ # if last commit in the strategy was a merge, then the additional # branches that were merged in previously can be extracted based on # the commits merged. if len(strategy) > 0: prev_import_merge = strategy[-1] else: # no changes carried? prev_import_merge = None additional_commits = None if prev_import_merge and len(prev_import_merge.parents) > 1: additional_commits = { commit for commit in prev_import_merge.parents if commit.hexsha != strategy.previous_upstream.hexsha} if (additional_commits and len(self.extra_branches) != len(additional_commits)): self.log.warning(""" **************** WARNING **************** Previous import merged additional branches but none have been specified on the command line for this import.\n""") # detect if nothing to do if (strategy.previous_upstream.hexsha == self.git.rev_parse(self.upstream)): self.log.notice("%s already at latest upstream commit: '%s'", self.branch, strategy.previous_upstream) if additional_commits is None: self.log.notice("Nothing to be imported") return True else: new_additional_commits = {Commit.new(self.repo, branch) for branch in self.extra_branches} if new_additional_commits == additional_commits: self.log.notice( """ No updated additional branch given, nothing to be done """) return True return False
def find(self): """ Searches the git history of the target branch for a commit message containing the pattern given in the constructor. This is used as a base commit from which to return a list of commits since this point. """ commits = Commit.iter_items(self.repo, self.branch, grep=self.pattern, max_count=1, extended_regexp=True) self.commit = next(commits, None) if not self.commit: raise RuntimeError("Failed to locate a pattern match") self.log.debug("Commit matching search pattern is: '%s'", self.commit.hexsha) return self.commit.hexsha
def test_interactive(self): upstream_branch = self.branches['upstream'][0] target_branch = self.branches['head'][0] cmdline = self.parser.get_default('script_cmdline') + self.parser_args # ensure interactive mode cannot hang tests def kill(proc_pid): process = psutil.Process(proc_pid) for proc in process.children(recursive=True): try: proc.kill() except OSError: continue try: process.kill() except OSError: pass def get_output(proc): self.output = proc.communicate()[0] proc = subprocess.Popen(cmdline, stdin=open(os.devnull, "r"), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True, cwd=self.testrepo.path) proc_thread = threading.Thread(target=get_output, args=[proc]) proc_thread.start() proc_thread.join(getattr(self, 'timeout', 5)) if proc_thread.is_alive(): kill(proc.pid) proc_thread.join() self.addDetail('subprocess-output', text_content(self.output.decode('utf-8'))) raise Exception('Process #%d killed after timeout' % proc.pid) self.addDetail('subprocess-output', text_content(self.output.decode('utf-8'))) self.assertThat(proc.returncode, Equals(0)) expected = getattr(self, 'expect_rebased', []) if expected: changes = list(Commit.iter_items( self.repo, '%s..%s^2' % (upstream_branch, target_branch))) self.assertThat( len(changes), Equals(len(expected)), "should only have seen %s changes, got: %s" % (len(expected), ", ".join(["%s:%s" % (commit.hexsha, commit.message.splitlines()[0]) for commit in changes]))) # expected should be listed in order from oldest to newest, so # reverse changes to match as it would be newest to oldest. changes.reverse() for commit, node in zip(changes, expected): subject = commit.message.splitlines()[0] node_subject = self.gittree.graph[node].message.splitlines()[0] self.assertThat(subject, Equals(node_subject), "subject '%s' of commit '%s' does not match " "subject '%s' of node '%s'" % ( subject, commit.hexsha, node_subject, node)) import_branch = [head for head in self.repo.heads if str(head).startswith("import") and not str(head).endswith("-base")] self.assertThat(self.git.rev_parse(import_branch), Not(Equals(self.git.rev_parse(target_branch))), "Import branch and target should have identical " "contents, but not be the same") # allow disabling of checking the merge commit contents # as some tests won't result in an import if getattr(self, 'check_merge', True): commit_message = self.git.log(target_branch, n=1) self.assertThat(commit_message, Contains("of '%s' into '%s'" % (upstream_branch, target_branch))) # make sure the final state of merge is correct self.assertThat( self.repo.git.rev_parse("%s^{tree}" % target_branch), Equals(self.repo.git.rev_parse( "%s^2^{tree}" % target_branch)), "--finish option failed to merge correctly") # allow additional test specific verification methods below extra_test_func = getattr(self, '_verify_%s' % self.name, None) if extra_test_func: extra_test_func()
def filter(self, commit_iter): self.log.info( """ Filtering out all commits marked with a Superseded-by Change-Id which is present in '%s' """, self.search_ref) supersede_re = re.compile('^%s\s*(.+)\s*$' % lib.SUPERSEDE_HEADER, re.IGNORECASE | re.MULTILINE) for commit in commit_iter: commit_note = commit.note(note_ref=lib.IMPORT_NOTE_REF) # include non-annotated commits if not commit_note: yield commit continue # include annotated commits which don't have a SUPERSEDE_HEADER superseding_change_ids = supersede_re.findall(commit_note) if not superseding_change_ids: yield commit continue # search for all the change-ids in matches (egrep regex) commits_grep_re = '^Change-Id:\\s*\(%s\)\\s*$' % \ '\|'.join(superseding_change_ids) # retrieve all matching commits because we need to check # each match for whether the changeId is actually in # the footer or just included as a reference. matching_commits = Commit.iter_items(self.repo, self._get_rev_range(), regexp_ignore_case=True, grep=commits_grep_re) for possible in matching_commits: change_id = self._get_change_id(possible) if change_id: superseding_change_ids.remove(change_id) # include commits which have some superseding change-ids not # present in upstream if superseding_change_ids: self.log.debug( """ Including commit '%s %s' because the following superseding change-ids have not been found: %s """, commit.short, commit.message.splitlines()[0], '\n '.join(superseding_change_ids)) yield commit continue self.log.debug( """ Filtering out commit '%s %s' because it has been marked as superseded by the following note: %s """, commit.short, commit.message.splitlines()[0], commit_note)
def list(self, upstream=None): """ Returns a list of Commit objects, between the '<commitish>' revision given in the constructor, and the commit object returned by the find() method. If given an upstream branch, uses --cherry-pick/--left-only to exclude commits that are identical to those already on the upstream branch. """ if not self.commit: self.find() revision_spec = "{0}..{1}".format(self.commit.hexsha, self.branch) # search for previous import commit first, if found, wish to include # the discarded parents as part of the set of changes to ignore in the # final list. self.log.debug( """ Searching for previous merges that exclude one side of the history since the last import. git rev-list --ancestry-path --merges %s """, revision_spec) merge_list = list(Commit.iter_items(self.repo, revision_spec, topo_order=True, ancestry_path=True, merges=True)) # to handle special case where a merge of a commit introduces # nothing to the tree, need to spot when such commits are based # off of the import set, so as not to accidentally exclude strip_commits = set() ancestry_commits = [ commit.hexsha for commit in Commit.iter_items( self.repo, revision_spec, topo_order=True, ancestry_path=True, merges=False) ] self.log.debug("Ancestry commits: %s", ancestry_commits) extra_args = [] previous_import = None for mergecommit, parent in ((mc, p) for mc in merge_list for p in mc.parents): # inspect each (previous_import_candidate, ignores, prune_list) = self._check_merge_is_previous( mergecommit, parent, merge_list[-1], ancestry_commits) if ignores: self.log.debug( """ Adding following to ignore list: %s """, "\n ".join(ignores)) extra_args.extend(ignores) if prune_list: self.log.debug( """ Adding following commits to be pruned afterwards: %s """, "\n ".join(prune_list)) strip_commits.update(prune_list) if previous_import_candidate: previous_import = previous_import_candidate self.log.info( """ Found possible previous import merge: %s """, previous_import) if previous_import: self.log.info( """ Found possible previous import merge: %s """, previous_import) # walk the tree and find all commits that lie in the path between the # commit found by find() and head of the branch in two steps, to # ensure a deterministic order between what is from the previous # upstream to the last import, and from that import to what is on # the tip of the head to avoid inversion where older commits # started before the previous import merge and approved afterwards # are not sorted by 'rev-list' predictably. commit_list = [] if upstream is None: if previous_import: search_list = [ (previous_import, self.branch, None), (self.commit, previous_import, None), ] else: search_list = [(self.commit, self.branch, None)] rev_spec = "{0}..{1}" git_args = {} else: if previous_import: search_list = [ (self.branch, upstream, "^%s" % previous_import), (previous_import, upstream, "^%s~1" % previous_import) ] else: search_list = [(self.branch, upstream, None)] rev_spec = "{0}...{1}" git_args = {'cherry_pick': True, 'left_only': True, 'full_history': True, } extra_args.append("^%s" % self.commit) for start, end, exclude in search_list: extra = list(extra_args) if exclude: extra.append(exclude) extra.extend(["--", "."]) revision_spec = rev_spec.format(start, end) self.log.info( """ Walking the changes between found commit and target, excluding those behind the previous import or merged as an additional branch during the previous import git rev-list --topo-order %s %s %s """, ' '.join(self.git.transform_kwargs(**git_args)), revision_spec, " ".join(extra)) commit_list.append( Commit._iter_from_process_or_stream( self.repo, self.git.rev_list(revision_spec, *extra, as_process=True, topo_order=True, **git_args))) # chain the filters as generators so that we don't need to allocate new # lists for each step in the filter chain. commit_list = itertools.chain(*commit_list) # strip commits that cannot be excluded through revision specifications # or without removing additional history commit_list = [c for c in commit_list if c.hexsha not in strip_commits] for f in self.filters: commit_list = f.filter(commit_list) commits = list(commit_list) self.log.debug( """ commits found: %s """, ("\n" + " " * 4).join([c.hexsha for c in commits])) return commits
def test_interactive(self): upstream_branch = self.branches['upstream'][0] target_branch = self.branches['head'][0] cmdline = self.parser.get_default('script_cmdline') + self.parser_args # ensure interactive mode cannot hang tests def kill(proc_pid): process = psutil.Process(proc_pid) for proc in process.children(recursive=True): try: proc.kill() except OSError: continue try: process.kill() except OSError: pass def get_output(proc): self.output = proc.communicate()[0] proc = subprocess.Popen(cmdline, stdin=open(os.devnull, "r"), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True, cwd=self.testrepo.path) proc_thread = threading.Thread(target=get_output, args=[proc]) proc_thread.start() proc_thread.join(getattr(self, 'timeout', 5)) if proc_thread.is_alive(): kill(proc.pid) proc_thread.join() self.addDetail('subprocess-output', text_content(self.output.decode('utf-8'))) raise Exception('Process #%d killed after timeout' % proc.pid) self.addDetail('subprocess-output', text_content(self.output.decode('utf-8'))) self.assertThat(proc.returncode, Equals(0)) expected = getattr(self, 'expect_rebased', []) if expected: changes = list( Commit.iter_items( self.repo, '%s..%s^2' % (upstream_branch, target_branch))) self.assertThat( len(changes), Equals(len(expected)), "should only have seen %s changes, got: %s" % (len(expected), ", ".join([ "%s:%s" % (commit.hexsha, commit.message.splitlines()[0]) for commit in changes ]))) # expected should be listed in order from oldest to newest, so # reverse changes to match as it would be newest to oldest. changes.reverse() for commit, node in zip(changes, expected): subject = commit.message.splitlines()[0] node_subject = self.gittree.graph[node].message.splitlines()[0] self.assertThat( subject, Equals(node_subject), "subject '%s' of commit '%s' does not match " "subject '%s' of node '%s'" % (subject, commit.hexsha, node_subject, node)) import_branch = [ head for head in self.repo.heads if str(head).startswith("import") and not str(head).endswith("-base") ] self.assertThat( self.git.rev_parse(import_branch), Not(Equals(self.git.rev_parse(target_branch))), "Import branch and target should have identical " "contents, but not be the same") # allow disabling of checking the merge commit contents # as some tests won't result in an import if getattr(self, 'check_merge', True): commit_message = self.git.log(target_branch, n=1) self.assertThat( commit_message, Contains("of '%s' into '%s'" % (upstream_branch, target_branch))) # make sure the final state of merge is correct self.assertThat( self.repo.git.rev_parse("%s^{tree}" % target_branch), Equals(self.repo.git.rev_parse("%s^2^{tree}" % target_branch)), "--finish option failed to merge correctly") # allow additional test specific verification methods below extra_test_func = getattr(self, '_verify_%s' % self.name, None) if extra_test_func: extra_test_func()