def checkout_ref(workdir, repo, ref, messages=None):
"""Checkout a specific ref in the repo."""
LOG.debug('Resetting the repository %s to HEAD', repo)
# Reset the repo to HEAD just in case any ot the previous steps
# updated a checked in file. If this fails we continue to try the
# requested ref as that shoudl give us a more complete error log.
try:
processutils.check_call(
['git', 'reset', '--hard'],
cwd=os.path.join(workdir, repo))
except processutils.CalledProcessError as err:
if messages:
messages.warning(
'Could not reset repository {} to HEAD: {}'.format(
repo, err))
LOG.debug('Checking out repository %s to %s', repo, ref)
try:
processutils.check_call(
['git', 'checkout', ref],
cwd=os.path.join(workdir, repo))
except processutils.CalledProcessError as err:
if messages:
messages.error(
'Could not checkout repository {} at {}: {}'.format(
repo, ref, err))
return False
return True
def get_requirements_at_ref(workdir, repo, ref):
"""Check out the repo at the ref and load the list of requirements."""
body = ''
try:
dest = gitutils.clone_repo(workdir, repo, ref=ref)
processutils.check_call(['python3', 'setup.py', 'sdist'], cwd=dest)
sdist_name = pythonutils.get_sdist_name(workdir, repo)
requirements_filename = os.path.join(
dest,
sdist_name + '.egg-info',
'requires.txt',
)
if os.path.exists(requirements_filename):
with open(requirements_filename, 'r') as f:
body = f.read()
else:
# The package has no dependencies.
pass
except Exception:
# We've had a few cases where a previous version had an issue and could
# no longer be installed. In this case, just move along.
LOG.warning('Unable to create sdist, unable to get requirements.')
LOG.warning('!!! Perform manual comparison for requirements changes'
'!!!')
return parse_requirements(body)
def setUp(self):
super(GPGKeyFixture, self).setUp()
# Force a temporary home directory with a short path so the
# gpg commands do not complain about an excessively long
# value.
self.useFixture(fixtures.TempHomeDir('/tmp'))
tempdir = self.useFixture(fixtures.TempDir('/tmp'))
gnupg_version_re = re.compile(r'^gpg\s.*\s([\d+])\.([\d+])\.([\d+])')
gnupg_version = processutils.check_output(
['gpg', '--version'], cwd=tempdir.path).decode('utf-8')
for line in gnupg_version.split('\n'):
gnupg_version = gnupg_version_re.match(line)
if gnupg_version:
gnupg_version = (int(gnupg_version.group(1)),
int(gnupg_version.group(2)),
int(gnupg_version.group(3)))
break
else:
if gnupg_version is None:
gnupg_version = (0, 0, 0)
config_file = tempdir.path + '/key-config'
LOG.debug('creating gpg config file in %s', config_file)
with open(config_file, 'wt') as f:
if gnupg_version[0] == 2 and gnupg_version[1] >= 1:
f.write(
textwrap.dedent("""
%no-protection
%transient-key
"""))
f.write(
textwrap.dedent("""
%no-ask-passphrase
Key-Type: RSA
Name-Real: Example Key
Name-Comment: N/A
Name-Email: [email protected]
Expire-Date: 2d
%commit
"""))
# Note that --quick-random (--debug-quick-random in GnuPG 2.x)
# does not have a corresponding preferences file setting and
# must be passed explicitly on the command line instead
if gnupg_version[0] == 1:
gnupg_random = '--quick-random'
elif gnupg_version[0] >= 2:
gnupg_random = '--debug-quick-random'
else:
gnupg_random = ''
cmd = ['gpg', '--gen-key', '--batch']
if gnupg_random:
cmd.append(gnupg_random)
cmd.append('key-config')
LOG.debug('generating gpg key')
processutils.check_call(cmd, cwd=tempdir.path)
def check_readme_format(workdir, repo):
"Verify that the README format looks OK."
dest = os.path.join(workdir, repo)
setup_path = os.path.join(dest, 'setup.py')
if not os.path.exists(setup_path):
LOG.debug('did not find %s, maybe %s is not a python project',
setup_path, repo)
return None
# NOTE(dhellmann): This relies on validate being run via tox so
# that python3 is present and the docutils package is installed.
processutils.check_call(
['python3', 'setup.py', 'check', '--restructuredtext', '--strict'],
cwd=dest,
)
def clone_repo(workdir, repo, ref=None, branch=None):
"Check out the code."
LOG.debug('Checking out repository {} to {}'.format(
repo, branch or ref or 'master'))
cmd = [
'./tools/clone_repo.sh',
'--workspace', workdir,
]
if ref:
cmd.extend(['--ref', ref])
if branch:
cmd.extend(['--branch', branch])
cmd.append(repo)
processutils.check_call(cmd)
dest = os.path.join(workdir, repo)
return dest
def get_requirements_at_ref(workdir, repo, ref):
"Check out the repo at the ref and load the list of requirements."
dest = gitutils.clone_repo(workdir, repo, ref=ref)
processutils.check_call(['python', 'setup.py', 'sdist'], cwd=dest)
sdist_name = pythonutils.get_sdist_name(workdir, repo)
requirements_filename = os.path.join(
dest,
sdist_name + '.egg-info',
'requires.txt',
)
if os.path.exists(requirements_filename):
with open(requirements_filename, 'r') as f:
body = f.read()
else:
# The package has no dependencies.
body = ''
return parse_requirements(body)
def ensure_basic_git_config(workdir, repo, settings):
"""Given a repo directory and a settings dict, set local config values
if those settings are not already defined.
"""
dest = os.path.join(workdir, repo)
for key, value in settings.items():
LOG.info('looking for git config {}'.format(key))
try:
existing = processutils.check_output(
['git', 'config', '--get', key],
cwd=dest,
).decode('utf-8').strip()
LOG.info('using existing setting of {}: {!r}'.format(key, existing))
except processutils.CalledProcessError:
LOG.info('updating setting of {} to {!r}'.format(key, value))
processutils.check_call(
['git', 'config', key, value],
cwd=dest,
)
def check_readme_format(workdir, repo):
"Verify that the README format looks OK."
dest = os.path.join(workdir, repo)
setup_path = os.path.join(dest, 'setup.py')
if not os.path.exists(setup_path):
LOG.debug('did not find %s, maybe %s is not a python project',
setup_path, repo)
return None
# Check if the sdist build has been done
build_path = os.path.join(dest, 'dist')
if not os.path.exists(build_path):
build_sdist(workdir, repo)
# NOTE(dhellmann): This relies on validate being run via tox so
# that python3 is present and the twine package is installed.
processutils.check_call(
['twine', 'check', os.path.join(build_path, '*')],
cwd=dest,
)
def build_sdist(workdir, repo):
"""Build the sdist."""
dest = os.path.join(workdir, repo)
build_path = os.path.join(dest, 'dist')
if os.path.exists(build_path):
# sdist already built, skip rebuilding it
return
setup_path = os.path.join(dest, 'setup.py')
if not os.path.exists(setup_path):
LOG.debug('did not find %s, maybe %s is not a python project',
setup_path, repo)
return
use_tox = repo.endswith('/pbr')
if use_tox and not os.path.exists(os.path.join(dest, '.tox', 'venv')):
# Use tox to set up a virtualenv so we can install the
# dependencies for the package. This only seems to be
# necessary for pbr, but...
processutils.check_output(
['tox', '-e', 'venv', '--notest'],
cwd=dest,
)
if use_tox:
python = '.tox/venv/bin/python3'
else:
python = 'python3'
# Set some flags to turn off pbr functionality that we don't need.
flags = {
'SKIP_GENERATE_RENO': '1',
'SKIP_GENERATE_AUTHORS': '1',
'SKIP_WRITE_GIT_CHANGELOG': '1',
}
cmd = [python, 'setup.py', 'sdist', 'bdist_wheel']
processutils.check_call(
cmd,
cwd=dest,
env=flags)
