def get_sdist_name(workdir, repo):
"Check out the code."
dest = os.path.join(workdir, repo)
setup_path = os.path.join(dest, 'setup.py')
if not os.path.exists(setup_path):
LOG.debug('did not find %s, maybe %s is not a python project',
setup_path, repo)
return None
use_tox = repo.endswith('/pbr')
if use_tox and not os.path.exists(os.path.join(dest, '.tox', 'venv')):
# Use tox to set up a virtualenv so we can install the
# dependencies for the package. This only seems to be
# necessary for pbr, but...
processutils.check_output(
['tox', '-e', 'venv', '--notest'],
cwd=dest,
)
if use_tox:
python = '.tox/venv/bin/python'
else:
python = 'python'
# Run it once and discard the result to ensure any setup_requires
# dependencies are installed.
cmd = [python, 'setup.py', '--name']
processutils.check_output(cmd, cwd=dest)
# Run it again to get a clean version of the name.
LOG.debug('Running: %s in %s' % (' '.join(cmd), dest))
out = processutils.check_output(cmd, cwd=dest).decode('utf-8')
LOG.debug('Results: %s' % (out, ))
name = out.splitlines()[-1].strip()
return name
def check_branch_sha(workdir, repo, series, sha):
"""Check if the SHA is in the targeted branch.
The SHA must appear on a stable/$series branch (if it exists) or
master (if stable/$series does not exist). It is up to the
reviewer to verify that releases from master are in a sensible
location relative to other existing branches.
We do not compare $series against the existing branches ordering
because that would prevent us from retroactively creating a stable
branch for a project after a later stable branch is created (i.e.,
if stable/N exists we could not create stable/N-1).
"""
remote_match = 'remotes/origin/stable/%s' % series
try:
containing_branches = _filter_branches(
processutils.check_output(
['git', 'branch', '-a', '--contains', sha],
cwd=os.path.join(workdir, repo),
).decode('utf-8')
)
# If the patch is on the named branch, everything is fine.
if remote_match in containing_branches:
LOG.debug('found %s branch', remote_match)
return True
LOG.debug('did not find %s in branches containing %s: %s',
remote_match, sha, containing_branches)
# If the expected branch does not exist yet, this may be a
# late release attempt to create that branch or just a project
# that hasn't branched, yet, and is releasing from master for
# that series. Allow the release, as long as it is on the
# master branch.
all_branches = _filter_branches(
processutils.check_output(
['git', 'branch', '-a'],
cwd=os.path.join(workdir, repo),
).decode('utf-8')
)
if remote_match not in all_branches:
if 'master' in containing_branches:
LOG.debug('did not find %s but SHA is on master',
remote_match)
return True
if 'origin/master' in containing_branches:
LOG.debug('did not find %s but SHA is on origin/master',
remote_match)
return True
# At this point we know the release is not from the required
# branch and it is not from master, which means it is the
# wrong branch and should not be allowed.
LOG.debug('did not find SHA on %s or master or origin/master',
remote_match)
return False
except processutils.CalledProcessError as e:
LOG.error('failed checking SHA on branch: %s [%s]' % (e, e.output.strip()))
return False
def commit_exists(workdir, repo, ref):
"""Return boolean specifying whether the reference exists in the repository.
The commit must have been merged into the repository, but this
check does not enforce any branch membership.
"""
try:
processutils.check_output(
['git', 'show', ref],
cwd=os.path.join(workdir, repo),
).decode('utf-8')
except processutils.CalledProcessError as err:
LOG.error('Could not find {}: {}'.format(ref, err))
return False
return True
def get_branches(workdir, repo):
try:
output = processutils.check_output(
['git', 'branch', '-a'],
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
).decode('utf-8').strip()
# Example output:
# * (no branch)
# master
# stable/mitaka
# stable/newton
# stable/ocata
# remotes/origin/HEAD -> origin/master
# remotes/origin/master
# remotes/origin/stable/mitaka
# remotes/origin/stable/newton
# remotes/origin/stable/ocata
results = []
for line in output.splitlines():
branch = line.strip().lstrip('*').strip()
if branch.startswith('('):
continue
if '->' in branch:
continue
results.append(branch)
return results
except processutils.CalledProcessError as e:
LOG.error('failed to retrieve list of branches: %s [%s]',
e, e.output.strip())
return []
def setUp(self):
super(GPGKeyFixture, self).setUp()
# Force a temporary home directory with a short path so the
# gpg commands do not complain about an excessively long
# value.
self.useFixture(fixtures.TempHomeDir('/tmp'))
tempdir = self.useFixture(fixtures.TempDir('/tmp'))
gnupg_version_re = re.compile(r'^gpg\s.*\s([\d+])\.([\d+])\.([\d+])')
gnupg_version = processutils.check_output(
['gpg', '--version'], cwd=tempdir.path).decode('utf-8')
for line in gnupg_version.split('\n'):
gnupg_version = gnupg_version_re.match(line)
if gnupg_version:
gnupg_version = (int(gnupg_version.group(1)),
int(gnupg_version.group(2)),
int(gnupg_version.group(3)))
break
else:
if gnupg_version is None:
gnupg_version = (0, 0, 0)
config_file = tempdir.path + '/key-config'
LOG.debug('creating gpg config file in %s', config_file)
with open(config_file, 'wt') as f:
if gnupg_version[0] == 2 and gnupg_version[1] >= 1:
f.write(
textwrap.dedent("""
%no-protection
%transient-key
"""))
f.write(
textwrap.dedent("""
%no-ask-passphrase
Key-Type: RSA
Name-Real: Example Key
Name-Comment: N/A
Name-Email: [email protected]
Expire-Date: 2d
%commit
"""))
# Note that --quick-random (--debug-quick-random in GnuPG 2.x)
# does not have a corresponding preferences file setting and
# must be passed explicitly on the command line instead
if gnupg_version[0] == 1:
gnupg_random = '--quick-random'
elif gnupg_version[0] >= 2:
gnupg_random = '--debug-quick-random'
else:
gnupg_random = ''
cmd = ['gpg', '--gen-key', '--batch']
if gnupg_random:
cmd.append(gnupg_random)
cmd.append('key-config')
LOG.debug('generating gpg key')
processutils.check_call(cmd, cwd=tempdir.path)
def find_modified_deliverable_files():
"Return a list of files modified by the most recent commit."
results = processutils.check_output(
['git', 'diff', '--name-only', '--pretty=format:', 'HEAD^']
).decode('utf-8')
filenames = [
l.strip()
for l in results.splitlines()
if (l.startswith('deliverables/'))
]
return filenames
def check_ancestry(workdir, repo, old_version, sha):
"Check if the SHA is in the ancestry of the previous version."
try:
ancestors = processutils.check_output(
['git', 'log', '--oneline', '--ancestry-path',
'%s..%s' % (old_version, sha)],
cwd=os.path.join(workdir, repo),
).decode('utf-8').strip()
return bool(ancestors)
except processutils.CalledProcessError as e:
LOG.error('failed checking ancestry: %s [%s]' % (e, e.output.strip()))
return False
def get_head(workdir, repo):
cmd = ['git', 'log', '-n', '1', '--pretty=tformat:%h']
try:
return processutils.check_output(
cmd,
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
).decode('utf-8').strip()
except processutils.CalledProcessError as e:
LOG.warning('failed to retrieve HEAD: %s [%s]',
e, e.output.strip())
return None
def add_tag(workdir, repo, tag, sha):
cmd = ['git', 'tag', '-m', 'temporary tag', tag, sha]
try:
LOG.info(' '.join(cmd))
return processutils.check_output(
cmd,
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
).decode('utf-8').strip()
except processutils.CalledProcessError as e:
LOG.warning('failed to add tag: %s [%s]', e, e.output.strip())
return None
def get_branch_base(workdir, repo, branch):
"Return SHA at base of branch."
# http://stackoverflow.com/questions/1527234/finding-a-branch-point-with-git
# git rev-list $(git rev-list --first-parent ^origin/stable/newton master | tail -n1)^^!
#
# Determine the first parent.
cmd = [
'git',
'rev-list',
'--first-parent',
'^origin/{}'.format(branch),
'master',
]
try:
parents = processutils.check_output(
cmd,
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
).decode('utf-8').strip()
except processutils.CalledProcessError as e:
LOG.warning('failed to retrieve branch base: %s [%s]',
e, e.output.strip())
return None
parent = parents.splitlines()[-1]
# Now get the ^^! commit
cmd = [
'git',
'rev-list',
'{}^^!'.format(parent),
]
try:
return processutils.check_output(
cmd,
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
).decode('utf-8').strip()
except processutils.CalledProcessError as e:
LOG.warning('failed to retrieve branch base: %s [%s]',
e, e.output.strip())
return None
def build_sdist(workdir, repo):
"""Build the sdist."""
dest = os.path.join(workdir, repo)
build_path = os.path.join(dest, 'dist')
if os.path.exists(build_path):
# sdist already built, skip rebuilding it
return
setup_path = os.path.join(dest, 'setup.py')
if not os.path.exists(setup_path):
LOG.debug('did not find %s, maybe %s is not a python project',
setup_path, repo)
return
use_tox = repo.endswith('/pbr')
if use_tox and not os.path.exists(os.path.join(dest, '.tox', 'venv')):
# Use tox to set up a virtualenv so we can install the
# dependencies for the package. This only seems to be
# necessary for pbr, but...
processutils.check_output(
['tox', '-e', 'venv', '--notest'],
cwd=dest,
)
if use_tox:
python = '.tox/venv/bin/python3'
else:
python = 'python3'
# Set some flags to turn off pbr functionality that we don't need.
flags = {
'SKIP_GENERATE_RENO': '1',
'SKIP_GENERATE_AUTHORS': '1',
'SKIP_WRITE_GIT_CHANGELOG': '1',
}
cmd = [python, 'setup.py', 'sdist', 'bdist_wheel']
processutils.check_call(
cmd,
cwd=dest,
env=flags)
def stable_branch_exists(workdir, repo, series):
"Does the stable/series branch exist?"
remote_match = 'remotes/origin/stable/%s' % series
try:
containing_branches = _filter_branches(
processutils.check_output(
['git', 'branch', '-a'],
cwd=os.path.join(workdir, repo),
).decode('utf-8'))
return (remote_match in containing_branches)
except processutils.CalledProcessError as e:
LOG.error('failed checking for branch: %s [%s]', e, e.output.strip())
return False
def get_latest_tag(workdir, repo, sha=None):
cmd = ['git', 'describe', '--abbrev=0', '--always']
if sha is not None:
cmd.append(sha)
try:
return processutils.check_output(
cmd,
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
).decode('utf-8').strip()
except processutils.CalledProcessError as e:
LOG.warning('failed to retrieve latest tag: %s [%s]', e,
e.output.strip())
return None
def sha_for_tag(workdir, repo, version):
"""Return the SHA for a given tag"""
# git log 2.3.11 -n 1 --pretty=format:%H
try:
actual_sha = processutils.check_output(
['git', 'log', str(version), '-n', '1', '--pretty=format:%H'],
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
).decode('utf-8')
actual_sha = actual_sha.strip()
except processutils.CalledProcessError as e:
LOG.info('ERROR getting SHA for tag %r: %s [%s]',
version, e, e.output.strip())
actual_sha = ''
return actual_sha
def branches_containing(workdir, repo, ref):
try:
output = processutils.check_output(
['git', 'branch', '-r', '--contains', ref],
cwd=os.path.join(workdir, repo),
stderr=subprocess.STDOUT,
).decode('utf-8').strip()
# Example output:
# origin/stable/ocata
results = []
for line in output.splitlines():
results.append(line.strip())
return results
except processutils.CalledProcessError as e:
LOG.error('failed to retrieve list of branches containing %s: %s [%s]',
ref, e, e.output.strip())
return []
def changes_since(workdir, repo, ref):
"""Get all changes between the last ref and the current point.
:param workdir: The git repo working directory.
:param repo: The name of the repo.
:param ref: The starting ref.
:returns: Merged commits between the two points.
"""
try:
changes = processutils.check_output(
['git', 'log', '--decorate', '--no-merges', '--pretty=oneline',
"%s..HEAD" % ref],
cwd=os.path.join(workdir, repo),
).decode('utf-8').strip()
except processutils.CalledProcessError as err:
LOG.error('Could not find {}: {}'.format(ref, err))
changes = ''
return changes
def ensure_basic_git_config(workdir, repo, settings):
"""Given a repo directory and a settings dict, set local config values
if those settings are not already defined.
"""
dest = os.path.join(workdir, repo)
for key, value in settings.items():
LOG.info('looking for git config {}'.format(key))
try:
existing = processutils.check_output(
['git', 'config', '--get', key],
cwd=dest,
).decode('utf-8').strip()
LOG.info('using existing setting of {}: {!r}'.format(key, existing))
except processutils.CalledProcessError:
LOG.info('updating setting of {} to {!r}'.format(key, value))
processutils.check_call(
['git', 'config', key, value],
cwd=dest,
)
def branch_exists(workdir, repo, prefix, identifier):
"""Does the prefix/identifier branch exist.
Checks if a named branch already exists.
:param workdir: The working directory for the local clone.
:param repo: The name of the repo.
:param prefix: The branch prefix (e.g. "stable" or "bugfix").
:param idenifier: The branch identifier (series name or version).
"""
remote_match = 'remotes/origin/{}/{}'.format(prefix, identifier)
try:
containing_branches = _filter_branches(
processutils.check_output(
['git', 'branch', '-a'],
cwd=os.path.join(workdir, repo),
).decode('utf-8')
)
LOG.debug('looking for %s', remote_match)
LOG.debug('found branches: %s', containing_branches)
return (remote_match in containing_branches)
except processutils.CalledProcessError as e:
LOG.error('failed checking for branch: %s [%s]', e, e.output.strip())
return False
def git(self, *args):
output = processutils.check_output(
['git'] + list(args),
cwd=self.path,
)
return output
