def _get_version_tags_on_branch(reporoot, branch):
"""Return tags from the branch, in date order.
Need to get the list of tags in the right order, because the topo
search breaks the date ordering. Use git to ask for the tags in
order, rather than trying to sort them, because many repositories
have "non-standard" tags or have renumbered projects (from
date-based to SemVer), for which sorting would require complex
logic.
"""
tags = []
tag_cmd = [
'git',
'log',
'--simplify-by-decoration',
'--pretty="%d"',
]
if branch:
tag_cmd.append(branch)
LOG.debug('running %s' % ' '.join(tag_cmd))
tag_results = utils.check_output(tag_cmd, cwd=reporoot)
LOG.debug(tag_results)
for line in tag_results.splitlines():
LOG.debug('line %r' % line)
for match in TAG_RE.findall(line):
tags.append(match)
return tags
def _get_version_tags_on_branch(reporoot, branch):
"""Return tags from the branch, in date order.
Need to get the list of tags in the right order, because the topo
search breaks the date ordering. Use git to ask for the tags in
order, rather than trying to sort them, because many repositories
have "non-standard" tags or have renumbered projects (from
date-based to SemVer), for which sorting would require complex
logic.
"""
tags = []
tag_cmd = [
'git', 'log',
'--simplify-by-decoration',
'--pretty="%d"',
]
if branch:
tag_cmd.append(branch)
LOG.debug('running %s' % ' '.join(tag_cmd))
tag_results = utils.check_output(tag_cmd, cwd=reporoot)
LOG.debug(tag_results)
for line in tag_results.splitlines():
LOG.debug('line %r' % line)
for match in TAG_RE.findall(line):
tags.append(match)
return tags
def setUp(self):
super(GPGKeyFixture, self).setUp()
tempdir = self.useFixture(fixtures.TempDir())
gnupg_version_re = re.compile("^gpg\s.*\s([\d+])\.([\d+])\.([\d+])")
gnupg_version = utils.check_output(["gpg", "--version"], cwd=tempdir.path)
for line in gnupg_version.split("\n"):
gnupg_version = gnupg_version_re.match(line)
if gnupg_version:
gnupg_version = (int(gnupg_version.group(1)), int(gnupg_version.group(2)), int(gnupg_version.group(3)))
break
else:
if gnupg_version is None:
gnupg_version = (0, 0, 0)
config_file = tempdir.path + "/key-config"
f = open(config_file, "wt")
try:
if gnupg_version[0] == 2 and gnupg_version[1] >= 1:
f.write(
"""
%no-protection
%transient-key
"""
)
f.write(
"""
%no-ask-passphrase
Key-Type: RSA
Name-Real: Example Key
Name-Comment: N/A
Name-Email: [email protected]
Expire-Date: 2d
Preferences: (setpref)
%commit
"""
)
finally:
f.close()
# Note that --quick-random (--debug-quick-random in GnuPG 2.x)
# does not have a corresponding preferences file setting and
# must be passed explicitly on the command line instead
if gnupg_version[0] == 1:
gnupg_random = "--quick-random"
elif gnupg_version[0] >= 2:
gnupg_random = "--debug-quick-random"
else:
gnupg_random = ""
cmd = ["gpg", "--gen-key", "--batch"]
if gnupg_random:
cmd.append(gnupg_random)
cmd.append(config_file)
subprocess.check_call(
cmd,
cwd=tempdir.path,
# Direct stderr to its own pipe, from which we don't read,
# to quiet the commands.
stderr=subprocess.PIPE,
)
def get_file_at_commit(reporoot, filename, sha):
"Return the contents of the file if it exists at the commit, or None."
try:
return utils.check_output(
['git', 'show', '%s:%s' % (sha, filename)],
cwd=reporoot,
)
except subprocess.CalledProcessError:
return None
def get_file_at_commit(reporoot, filename, sha):
"Return the contents of the file if it exists at the commit, or None."
try:
return utils.check_output(
['git', 'show', '%s:%s' % (sha, filename)],
cwd=reporoot,
)
except subprocess.CalledProcessError:
return None
def test_remote_branches(self):
self.repo.git("checkout", "2.0.0")
self.repo.git("checkout", "-b", "stable/2")
self.repo.git("checkout", "master")
scanner1 = scanner.Scanner(self.c)
head1 = scanner1._get_ref("stable/2")
self.assertIsNotNone(head1)
print("head1", head1)
# Create a second repository by cloning the first.
print(utils.check_output(["git", "clone", self.reporoot, "reporoot2"], cwd=self.temp_dir))
reporoot2 = os.path.join(self.temp_dir, "reporoot2")
print(utils.check_output(["git", "remote", "update"], cwd=reporoot2))
print(utils.check_output(["git", "remote", "-v"], cwd=reporoot2))
print(utils.check_output(["find", ".git/refs"], cwd=reporoot2))
print(utils.check_output(["git", "branch", "-a"], cwd=reporoot2))
c2 = config.Config(reporoot2)
scanner2 = scanner.Scanner(c2)
head2 = scanner2._get_ref("origin/stable/2")
self.assertIsNotNone(head2)
self.assertEqual(head1, head2)
def setUp(self):
super(GPGKeyFixture, self).setUp()
tempdir = self.useFixture(fixtures.TempDir())
gnupg_version_re = re.compile('^gpg\s.*\s([\d+])\.([\d+])\.([\d+])')
gnupg_version = utils.check_output(['gpg', '--version'],
cwd=tempdir.path)
for line in gnupg_version[0].split('\n'):
gnupg_version = gnupg_version_re.match(line)
if gnupg_version:
gnupg_version = (int(gnupg_version.group(1)),
int(gnupg_version.group(2)),
int(gnupg_version.group(3)))
break
else:
if gnupg_version is None:
gnupg_version = (0, 0, 0)
config_file = tempdir.path + '/key-config'
f = open(config_file, 'wt')
try:
if gnupg_version[0] == 2 and gnupg_version[1] >= 1:
f.write("""
%no-protection
%transient-key
""")
f.write("""
%no-ask-passphrase
Key-Type: RSA
Name-Real: Example Key
Name-Comment: N/A
Name-Email: [email protected]
Expire-Date: 2d
Preferences: (setpref)
%commit
""")
finally:
f.close()
# Note that --quick-random (--debug-quick-random in GnuPG 2.x)
# does not have a corresponding preferences file setting and
# must be passed explicitly on the command line instead
# if gnupg_version[0] == 1:
# gnupg_random = '--quick-random'
# elif gnupg_version[0] >= 2:
# gnupg_random = '--debug-quick-random'
# else:
# gnupg_random = ''
subprocess.check_call(
['gpg', '--gen-key', '--batch',
# gnupg_random,
config_file],
cwd=tempdir.path)
def setUp(self):
super(GPGKeyFixture, self).setUp()
tempdir = self.useFixture(fixtures.TempDir())
gnupg_version_re = re.compile('^gpg\s.*\s([\d+])\.([\d+])\.([\d+])')
gnupg_version = utils.check_output(['gpg', '--version'],
cwd=tempdir.path)
for line in gnupg_version.split('\n'):
gnupg_version = gnupg_version_re.match(line)
if gnupg_version:
gnupg_version = (int(gnupg_version.group(1)),
int(gnupg_version.group(2)),
int(gnupg_version.group(3)))
break
else:
if gnupg_version is None:
gnupg_version = (0, 0, 0)
config_file = tempdir.path + '/key-config'
f = open(config_file, 'wt')
try:
if gnupg_version[0] == 2 and gnupg_version[1] >= 1:
f.write("""
%no-protection
%transient-key
""")
f.write("""
%no-ask-passphrase
Key-Type: RSA
Name-Real: Example Key
Name-Comment: N/A
Name-Email: [email protected]
Expire-Date: 2d
Preferences: (setpref)
%commit
""")
finally:
f.close()
# Note that --quick-random (--debug-quick-random in GnuPG 2.x)
# does not have a corresponding preferences file setting and
# must be passed explicitly on the command line instead
if gnupg_version[0] == 1:
gnupg_random = '--quick-random'
elif gnupg_version[0] >= 2:
gnupg_random = '--debug-quick-random'
else:
gnupg_random = ''
cmd = ['gpg', '--gen-key', '--batch']
if gnupg_random:
cmd.append(gnupg_random)
cmd.append(config_file)
subprocess.check_call(cmd, cwd=tempdir.path)
def _get_current_version(reporoot, branch=None):
"""Return the current version of the repository.
If the repo appears to contain a python project, use setup.py to
get the version so pbr (if used) can do its thing. Otherwise, use
git describe.
"""
cmd = ['git', 'describe', '--tags']
if branch is not None:
cmd.append(branch)
try:
result = utils.check_output(cmd, cwd=reporoot).strip()
if '-' in result:
# Descriptions that come after a commit look like
# 2.0.0-1-abcde, and we want to remove the SHA value from
# the end since we only care about the version number
# itself, but we need to recognize that the change is
# unreleased so keep the -1 part.
result, dash, ignore = result.rpartition('-')
except subprocess.CalledProcessError:
# This probably means there are no tags.
result = '0.0.0'
return result
def _get_current_version(reporoot, branch=None):
"""Return the current version of the repository.
If the repo appears to contain a python project, use setup.py to
get the version so pbr (if used) can do its thing. Otherwise, use
git describe.
"""
cmd = ["git", "describe", "--tags"]
if branch is not None:
cmd.append(branch)
try:
result = utils.check_output(cmd, cwd=reporoot).strip()
if "-" in result:
# Descriptions that come after a commit look like
# 2.0.0-1-abcde, and we want to remove the SHA value from
# the end since we only care about the version number
# itself, but we need to recognize that the change is
# unreleased so keep the -1 part.
result, dash, ignore = result.rpartition("-")
except subprocess.CalledProcessError:
# This probably means there are no tags.
result = "0.0.0"
return result
def get_notes_by_version(reporoot, notesdir, branch=None):
"""Return an OrderedDict mapping versions to lists of notes files.
The versions are presented in reverse chronological order.
Notes files are associated with the earliest version for which
they were available, regardless of whether they changed later.
"""
versions = []
earliest_seen = collections.OrderedDict()
# Determine the current version, which might be an unreleased or dev
# version.
current_version = _get_current_version(reporoot, branch)
# print('current_version = %s' % current_version)
# Remember the most current filename for each id, to allow for
# renames.
last_name_by_id = {}
# FIXME(dhellmann): This might need to be more line-oriented for
# longer histories.
log_cmd = ['git', 'log', '--pretty=%x00%H %d', '--name-only']
if branch is not None:
log_cmd.append(branch)
log_cmd.extend(['--', notesdir])
history_results = utils.check_output(log_cmd, cwd=reporoot)
history = history_results.split('\x00')
current_version = current_version
for h in history:
h = h.strip()
if not h:
continue
# print(h)
hlines = h.splitlines()
# The first line of the block will include the SHA and may
# include tags, the other lines are filenames.
sha = hlines[0].split(' ')[0]
tags = _TAG_PAT.findall(hlines[0])
filenames = hlines[2:]
# If there are no tags in this block, assume the most recently
# seen version.
if not tags:
tags = [current_version]
else:
current_version = tags[0]
# Remember each version we have seen.
if current_version not in versions:
versions.append(current_version)
# Remember the files seen, using their UUID suffix as a unique id.
for f in filenames:
# Updated as older tags are found, handling edits to release
# notes.
uniqueid = _get_unique_id(f)
earliest_seen[uniqueid] = tags[0]
if uniqueid in last_name_by_id:
# We already have a filename for this id from a
# new commit, so use that one in case the name has
# changed.
continue
if _file_exists_at_commit(reporoot, f, sha):
# Remember this filename as the most recent version of
# the unique id we have seen, in case the name
# changed from an older commit.
last_name_by_id[uniqueid] = (f, sha)
# Invert earliest_seen to make a list of notes files for each
# version.
files_and_tags = collections.OrderedDict()
for v in versions:
files_and_tags[v] = []
# Produce a list of the actual files present in the repository. If
# a note is removed, this step should let us ignore it.
for uniqueid, version in earliest_seen.items():
base, sha = last_name_by_id[uniqueid]
files_and_tags[version].append((base, sha))
for version, filenames in files_and_tags.items():
files_and_tags[version] = list(reversed(filenames))
return files_and_tags
def _run_git(self, *args):
return utils.check_output(
['git'] + list(args),
cwd=self.reporoot,
)
def get_notes_by_version(reporoot, notesdir, branch=None):
"""Return an OrderedDict mapping versions to lists of notes files.
The versions are presented in reverse chronological order.
Notes files are associated with the earliest version for which
they were available, regardless of whether they changed later.
"""
LOG.debug("scanning %s/%s (branch=%s)" % (reporoot, notesdir, branch))
# Determine all of the tags known on the branch, in their date
# order. We scan the commit history in topological order to ensure
# we have the commits in the right version, so we might encounter
# the tags in a different order during that phase.
versions_by_date = _get_version_tags_on_branch(reporoot, branch)
LOG.debug("versions by date %r" % (versions_by_date,))
versions = []
earliest_seen = collections.OrderedDict()
# Determine the current version, which might be an unreleased or
# dev version if there are unreleased commits at the head of the
# branch in question. Since the version may not already be known,
# make sure it is in the list of versions by date. And since it is
# the most recent version, go ahead and insert it at the front of
# the list.
current_version = _get_current_version(reporoot, branch)
LOG.debug("current repository version: %s" % current_version)
if current_version not in versions_by_date:
LOG.debug("adding %s to versions by date" % current_version)
versions_by_date.insert(0, current_version)
# Remember the most current filename for each id, to allow for
# renames.
last_name_by_id = {}
# FIXME(dhellmann): This might need to be more line-oriented for
# longer histories.
log_cmd = [
"git",
"log",
"--topo-order", # force traversal order rather than date order
"--pretty=%x00%H %d", # output contents in parsable format
"--name-only", # only include the names of the files in the patch
]
if branch is not None:
log_cmd.append(branch)
LOG.debug("running %s" % " ".join(log_cmd))
history_results = utils.check_output(log_cmd, cwd=reporoot)
history = history_results.split("\x00")
current_version = current_version
for h in history:
h = h.strip()
if not h:
continue
# print(h)
hlines = h.splitlines()
# The first line of the block will include the SHA and may
# include tags, the other lines are filenames.
sha = hlines[0].split(" ")[0]
tags = _TAG_PAT.findall(hlines[0])
# Filter the files based on the notes directory we were
# given. We cannot do this in the git log command directly
# because it means we end up skipping some of the tags if the
# commits being tagged don't include any release note
# files. Even if this list ends up empty, we continue doing
# the other processing so that we record all of the known
# versions.
filenames = [f for f in hlines[2:] if fnmatch.fnmatch(f, notesdir + "/*.yaml")]
# If there are no tags in this block, assume the most recently
# seen version.
if not tags:
tags = [current_version]
else:
current_version = tags[0]
LOG.debug("%s has tags, updating current version to %s" % (sha, current_version))
# Remember each version we have seen.
if current_version not in versions:
LOG.debug("%s is a new version" % current_version)
versions.append(current_version)
LOG.debug("%s contains files %s" % (sha, filenames))
# Remember the files seen, using their UUID suffix as a unique id.
for f in filenames:
# Updated as older tags are found, handling edits to release
# notes.
LOG.debug("setting earliest reference to %s to %s" % (f, tags[0]))
uniqueid = _get_unique_id(f)
earliest_seen[uniqueid] = tags[0]
if uniqueid in last_name_by_id:
# We already have a filename for this id from a
# new commit, so use that one in case the name has
# changed.
LOG.debug("%s was seen before" % f)
continue
if _file_exists_at_commit(reporoot, f, sha):
# Remember this filename as the most recent version of
# the unique id we have seen, in case the name
# changed from an older commit.
last_name_by_id[uniqueid] = (f, sha)
LOG.debug("remembering %s as filename for %s" % (f, uniqueid))
# Invert earliest_seen to make a list of notes files for each
# version.
files_and_tags = collections.OrderedDict()
for v in versions:
files_and_tags[v] = []
# Produce a list of the actual files present in the repository. If
# a note is removed, this step should let us ignore it.
for uniqueid, version in earliest_seen.items():
try:
base, sha = last_name_by_id[uniqueid]
files_and_tags[version].append((base, sha))
except KeyError:
# Unable to find the file again, skip it to avoid breaking
# the build.
msg = ("[reno] unable to find file associated " "with unique id %r, skipping") % uniqueid
LOG.debug(msg)
print(msg, file=sys.stderr)
# Only return the parts of files_and_tags that actually have
# filenames associated with the versions.
trimmed = collections.OrderedDict()
for ov in versions_by_date:
if not files_and_tags.get(ov):
continue
# Sort the notes associated with the version so they are in a
# deterministic order, to avoid having the same data result in
# different output depending on random factors. Earlier
# versions of the scanner assumed the notes were recorded in
# chronological order based on the commit date, but with the
# change to use topological sorting that is no longer
# necessarily true. We want the notes to always show up in the
# same order, but it doesn't really matter what order that is,
# so just sort based on the unique id.
trimmed[ov] = sorted(files_and_tags[ov])
return trimmed
def get_notes_by_version(reporoot, notesdir, branch=None,
collapse_pre_releases=True,
earliest_version=None):
"""Return an OrderedDict mapping versions to lists of notes files.
The versions are presented in reverse chronological order.
Notes files are associated with the earliest version for which
they were available, regardless of whether they changed later.
:param reporoot: Path to the root of the git repository.
:type reporoot: str
:param notesdir: The directory under *reporoot* with the release notes.
:type notesdir: str
:param branch: The name of the branch to scan. Defaults to current.
:type branch: str
:param collapse_pre_releases: When true, merge pre-release versions
into the final release, if it is present.
:type collapse_pre_releases: bool
"""
LOG.debug('scanning %s/%s (branch=%s)' % (reporoot, notesdir, branch))
# Determine all of the tags known on the branch, in their date
# order. We scan the commit history in topological order to ensure
# we have the commits in the right version, so we might encounter
# the tags in a different order during that phase.
versions_by_date = _get_version_tags_on_branch(reporoot, branch)
LOG.debug('versions by date %r' % (versions_by_date,))
versions = []
earliest_seen = collections.OrderedDict()
# Determine the current version, which might be an unreleased or
# dev version if there are unreleased commits at the head of the
# branch in question. Since the version may not already be known,
# make sure it is in the list of versions by date. And since it is
# the most recent version, go ahead and insert it at the front of
# the list.
current_version = _get_current_version(reporoot, branch)
LOG.debug('current repository version: %s' % current_version)
if current_version not in versions_by_date:
LOG.debug('adding %s to versions by date' % current_version)
versions_by_date.insert(0, current_version)
# Remember the most current filename for each id, to allow for
# renames.
last_name_by_id = {}
# Remember uniqueids that have had files deleted.
uniqueids_deleted = collections.defaultdict(set)
# FIXME(dhellmann): This might need to be more line-oriented for
# longer histories.
log_cmd = [
'git', 'log',
'--topo-order', # force traversal order rather than date order
'--pretty=%x00%H %d', # output contents in parsable format
'--name-only' # only include the names of the files in the patch
]
if branch is not None:
log_cmd.append(branch)
LOG.debug('running %s' % ' '.join(log_cmd))
history_results = utils.check_output(log_cmd, cwd=reporoot)
history = history_results.split('\x00')
current_version = current_version
for h in history:
h = h.strip()
if not h:
continue
# print(h)
hlines = h.splitlines()
# The first line of the block will include the SHA and may
# include tags, the other lines are filenames.
sha = hlines[0].split(' ')[0]
tags = TAG_RE.findall(hlines[0])
# Filter the files based on the notes directory we were
# given. We cannot do this in the git log command directly
# because it means we end up skipping some of the tags if the
# commits being tagged don't include any release note
# files. Even if this list ends up empty, we continue doing
# the other processing so that we record all of the known
# versions.
filenames = [
f
for f in hlines[2:]
if fnmatch.fnmatch(f, notesdir + '/*.yaml')
]
# If there are no tags in this block, assume the most recently
# seen version.
if not tags:
tags = [current_version]
else:
current_version = tags[0]
LOG.debug('%s has tags %s (%r), updating current version to %s' %
(sha, tags, hlines[0], current_version))
# Remember each version we have seen.
if current_version not in versions:
LOG.debug('%s is a new version' % current_version)
versions.append(current_version)
LOG.debug('%s contains files %s' % (sha, filenames))
# Remember the files seen, using their UUID suffix as a unique id.
for f in filenames:
# Updated as older tags are found, handling edits to release
# notes.
uniqueid = _get_unique_id(f)
LOG.debug('%s: found file %s',
uniqueid, f)
LOG.debug('%s: setting earliest reference to %s' %
(uniqueid, tags[0]))
earliest_seen[uniqueid] = tags[0]
if uniqueid in last_name_by_id:
# We already have a filename for this id from a
# new commit, so use that one in case the name has
# changed.
LOG.debug('%s: was seen before in %s',
uniqueid, last_name_by_id[uniqueid])
continue
elif _file_exists_at_commit(reporoot, f, sha):
LOG.debug('%s: looking for %s in deleted files %s',
uniqueid, f, uniqueids_deleted[uniqueid])
if f in uniqueids_deleted[uniqueid]:
# The file exists in the commit, but was deleted
# later in the history.
LOG.debug('%s: skipping deleted file %s',
uniqueid, f)
else:
# Remember this filename as the most recent version of
# the unique id we have seen, in case the name
# changed from an older commit.
last_name_by_id[uniqueid] = (f, sha)
LOG.debug('%s: remembering %s as filename',
uniqueid, f)
else:
# Track files that have been deleted. The rename logic
# above checks for repeated references to files that
# are deleted later, and the inversion logic below
# checks for any remaining values and skips those
# entries.
LOG.debug('%s: saw a file that no longer exists',
uniqueid)
uniqueids_deleted[uniqueid].add(f)
LOG.debug('%s: deleted files %s',
uniqueid, uniqueids_deleted[uniqueid])
# Invert earliest_seen to make a list of notes files for each
# version.
files_and_tags = collections.OrderedDict()
for v in versions:
files_and_tags[v] = []
# Produce a list of the actual files present in the repository. If
# a note is removed, this step should let us ignore it.
for uniqueid, version in earliest_seen.items():
try:
base, sha = last_name_by_id[uniqueid]
if base in uniqueids_deleted.get(uniqueid, set()):
LOG.debug('skipping deleted note %s' % uniqueid)
continue
files_and_tags[version].append((base, sha))
except KeyError:
# Unable to find the file again, skip it to avoid breaking
# the build.
msg = ('[reno] unable to find file associated '
'with unique id %r, skipping') % uniqueid
LOG.debug(msg)
print(msg, file=sys.stderr)
# Combine pre-releases into the final release, if we are told to
# and the final release exists.
if collapse_pre_releases:
collapsing = files_and_tags
files_and_tags = collections.OrderedDict()
for ov in versions_by_date:
if ov not in collapsing:
# We don't need to collapse this one because there are
# no notes attached to it.
continue
pre_release_match = PRE_RELEASE_RE.search(ov)
LOG.debug('checking %r', ov)
if pre_release_match:
# Remove the trailing pre-release part of the version
# from the string.
pre_rel_str = pre_release_match.groups()[0]
canonical_ver = ov[:-len(pre_rel_str)].rstrip('.')
if canonical_ver not in versions_by_date:
# This canonical version was never tagged, so we
# do not want to collapse the pre-releases. Reset
# to the original version.
canonical_ver = ov
else:
LOG.debug('combining into %r', canonical_ver)
else:
canonical_ver = ov
if canonical_ver not in files_and_tags:
files_and_tags[canonical_ver] = []
files_and_tags[canonical_ver].extend(collapsing[ov])
# Only return the parts of files_and_tags that actually have
# filenames associated with the versions.
trimmed = collections.OrderedDict()
for ov in versions_by_date:
if not files_and_tags.get(ov):
continue
# Sort the notes associated with the version so they are in a
# deterministic order, to avoid having the same data result in
# different output depending on random factors. Earlier
# versions of the scanner assumed the notes were recorded in
# chronological order based on the commit date, but with the
# change to use topological sorting that is no longer
# necessarily true. We want the notes to always show up in the
# same order, but it doesn't really matter what order that is,
# so just sort based on the unique id.
trimmed[ov] = sorted(files_and_tags[ov])
# If we have been told to stop at a version, we can do that
# now.
if earliest_version and ov == earliest_version:
break
LOG.debug('[reno] found %d versions and %d files',
len(trimmed.keys()), sum(len(ov) for ov in trimmed.values()))
return trimmed
def get_notes_by_version(reporoot,
notesdir,
branch=None,
collapse_pre_releases=True,
earliest_version=None):
"""Return an OrderedDict mapping versions to lists of notes files.
The versions are presented in reverse chronological order.
Notes files are associated with the earliest version for which
they were available, regardless of whether they changed later.
:param reporoot: Path to the root of the git repository.
:type reporoot: str
:param notesdir: The directory under *reporoot* with the release notes.
:type notesdir: str
:param branch: The name of the branch to scan. Defaults to current.
:type branch: str
:param collapse_pre_releases: When true, merge pre-release versions
into the final release, if it is present.
:type collapse_pre_releases: bool
"""
LOG.debug('scanning %s/%s (branch=%s)' % (reporoot, notesdir, branch))
# Determine all of the tags known on the branch, in their date
# order. We scan the commit history in topological order to ensure
# we have the commits in the right version, so we might encounter
# the tags in a different order during that phase.
versions_by_date = _get_version_tags_on_branch(reporoot, branch)
LOG.debug('versions by date %r' % (versions_by_date, ))
versions = []
earliest_seen = collections.OrderedDict()
# Determine the current version, which might be an unreleased or
# dev version if there are unreleased commits at the head of the
# branch in question. Since the version may not already be known,
# make sure it is in the list of versions by date. And since it is
# the most recent version, go ahead and insert it at the front of
# the list.
current_version = _get_current_version(reporoot, branch)
LOG.debug('current repository version: %s' % current_version)
if current_version not in versions_by_date:
LOG.debug('adding %s to versions by date' % current_version)
versions_by_date.insert(0, current_version)
# Remember the most current filename for each id, to allow for
# renames.
last_name_by_id = {}
# Remember uniqueids that have had files deleted.
uniqueids_deleted = collections.defaultdict(set)
# FIXME(dhellmann): This might need to be more line-oriented for
# longer histories.
log_cmd = [
'git',
'log',
'--topo-order', # force traversal order rather than date order
'--pretty=%x00%H %d', # output contents in parsable format
'--name-only' # only include the names of the files in the patch
]
if branch is not None:
log_cmd.append(branch)
LOG.debug('running %s' % ' '.join(log_cmd))
history_results = utils.check_output(log_cmd, cwd=reporoot)
history = history_results.split('\x00')
current_version = current_version
for h in history:
h = h.strip()
if not h:
continue
# print(h)
hlines = h.splitlines()
# The first line of the block will include the SHA and may
# include tags, the other lines are filenames.
sha = hlines[0].split(' ')[0]
tags = TAG_RE.findall(hlines[0])
# Filter the files based on the notes directory we were
# given. We cannot do this in the git log command directly
# because it means we end up skipping some of the tags if the
# commits being tagged don't include any release note
# files. Even if this list ends up empty, we continue doing
# the other processing so that we record all of the known
# versions.
filenames = [
f for f in hlines[2:] if fnmatch.fnmatch(f, notesdir + '/*.yaml')
]
# If there are no tags in this block, assume the most recently
# seen version.
if not tags:
tags = [current_version]
else:
current_version = tags[0]
LOG.debug('%s has tags %s (%r), updating current version to %s' %
(sha, tags, hlines[0], current_version))
# Remember each version we have seen.
if current_version not in versions:
LOG.debug('%s is a new version' % current_version)
versions.append(current_version)
LOG.debug('%s contains files %s' % (sha, filenames))
# Remember the files seen, using their UUID suffix as a unique id.
for f in filenames:
# Updated as older tags are found, handling edits to release
# notes.
uniqueid = _get_unique_id(f)
LOG.debug('%s: found file %s', uniqueid, f)
LOG.debug('%s: setting earliest reference to %s' %
(uniqueid, tags[0]))
earliest_seen[uniqueid] = tags[0]
if uniqueid in last_name_by_id:
# We already have a filename for this id from a
# new commit, so use that one in case the name has
# changed.
LOG.debug('%s: was seen before in %s', uniqueid,
last_name_by_id[uniqueid])
continue
elif _file_exists_at_commit(reporoot, f, sha):
LOG.debug('%s: looking for %s in deleted files %s', uniqueid,
f, uniqueids_deleted[uniqueid])
if f in uniqueids_deleted[uniqueid]:
# The file exists in the commit, but was deleted
# later in the history.
LOG.debug('%s: skipping deleted file %s', uniqueid, f)
else:
# Remember this filename as the most recent version of
# the unique id we have seen, in case the name
# changed from an older commit.
last_name_by_id[uniqueid] = (f, sha)
LOG.debug('%s: remembering %s as filename', uniqueid, f)
else:
# Track files that have been deleted. The rename logic
# above checks for repeated references to files that
# are deleted later, and the inversion logic below
# checks for any remaining values and skips those
# entries.
LOG.debug('%s: saw a file that no longer exists', uniqueid)
uniqueids_deleted[uniqueid].add(f)
LOG.debug('%s: deleted files %s', uniqueid,
uniqueids_deleted[uniqueid])
# Invert earliest_seen to make a list of notes files for each
# version.
files_and_tags = collections.OrderedDict()
for v in versions:
files_and_tags[v] = []
# Produce a list of the actual files present in the repository. If
# a note is removed, this step should let us ignore it.
for uniqueid, version in earliest_seen.items():
try:
base, sha = last_name_by_id[uniqueid]
if base in uniqueids_deleted.get(uniqueid, set()):
LOG.debug('skipping deleted note %s' % uniqueid)
continue
files_and_tags[version].append((base, sha))
except KeyError:
# Unable to find the file again, skip it to avoid breaking
# the build.
msg = ('[reno] unable to find file associated '
'with unique id %r, skipping') % uniqueid
LOG.debug(msg)
print(msg, file=sys.stderr)
# Combine pre-releases into the final release, if we are told to
# and the final release exists.
if collapse_pre_releases:
collapsing = files_and_tags
files_and_tags = collections.OrderedDict()
for ov in versions_by_date:
if ov not in collapsing:
# We don't need to collapse this one because there are
# no notes attached to it.
continue
pre_release_match = PRE_RELEASE_RE.search(ov)
LOG.debug('checking %r', ov)
if pre_release_match:
# Remove the trailing pre-release part of the version
# from the string.
pre_rel_str = pre_release_match.groups()[0]
canonical_ver = ov[:-len(pre_rel_str)].rstrip('.')
if canonical_ver not in versions_by_date:
# This canonical version was never tagged, so we
# do not want to collapse the pre-releases. Reset
# to the original version.
canonical_ver = ov
else:
LOG.debug('combining into %r', canonical_ver)
else:
canonical_ver = ov
if canonical_ver not in files_and_tags:
files_and_tags[canonical_ver] = []
files_and_tags[canonical_ver].extend(collapsing[ov])
# Only return the parts of files_and_tags that actually have
# filenames associated with the versions.
trimmed = collections.OrderedDict()
for ov in versions_by_date:
if not files_and_tags.get(ov):
continue
# Sort the notes associated with the version so they are in a
# deterministic order, to avoid having the same data result in
# different output depending on random factors. Earlier
# versions of the scanner assumed the notes were recorded in
# chronological order based on the commit date, but with the
# change to use topological sorting that is no longer
# necessarily true. We want the notes to always show up in the
# same order, but it doesn't really matter what order that is,
# so just sort based on the unique id.
trimmed[ov] = sorted(files_and_tags[ov])
# If we have been told to stop at a version, we can do that
# now.
if earliest_version and ov == earliest_version:
break
LOG.debug('[reno] found %d versions and %d files', len(trimmed.keys()),
sum(len(ov) for ov in trimmed.values()))
return trimmed
def git(self, *args):
self.logger.debug("$ git %s", " ".join(args))
output = utils.check_output(["git"] + list(args), cwd=self.reporoot)
self.logger.debug(output)
return output
def _run_git(self, *args):
return utils.check_output(
['git'] + list(args),
cwd=self.reporoot,
)
