def get_sdist_name(workdir, repo): "Check out the code." dest = os.path.join(workdir, repo) setup_path = os.path.join(dest, 'setup.py') if not os.path.exists(setup_path): LOG.debug('did not find %s, maybe %s is not a python project', setup_path, repo) return None use_tox = repo.endswith('/pbr') if use_tox and not os.path.exists(os.path.join(dest, '.tox', 'venv')): # Use tox to set up a virtualenv so we can install the # dependencies for the package. This only seems to be # necessary for pbr, but... processutils.check_output( ['tox', '-e', 'venv', '--notest'], cwd=dest, ) if use_tox: python = '.tox/venv/bin/python' else: python = 'python' # Run it once and discard the result to ensure any setup_requires # dependencies are installed. cmd = [python, 'setup.py', '--name'] processutils.check_output(cmd, cwd=dest) # Run it again to get a clean version of the name. LOG.debug('Running: %s in %s' % (' '.join(cmd), dest)) out = processutils.check_output(cmd, cwd=dest).decode('utf-8') LOG.debug('Results: %s' % (out, )) name = out.splitlines()[-1].strip() return name
def check_branch_sha(workdir, repo, series, sha): """Check if the SHA is in the targeted branch. The SHA must appear on a stable/$series branch (if it exists) or master (if stable/$series does not exist). It is up to the reviewer to verify that releases from master are in a sensible location relative to other existing branches. We do not compare $series against the existing branches ordering because that would prevent us from retroactively creating a stable branch for a project after a later stable branch is created (i.e., if stable/N exists we could not create stable/N-1). """ remote_match = 'remotes/origin/stable/%s' % series try: containing_branches = _filter_branches( processutils.check_output( ['git', 'branch', '-a', '--contains', sha], cwd=os.path.join(workdir, repo), ).decode('utf-8') ) # If the patch is on the named branch, everything is fine. if remote_match in containing_branches: LOG.debug('found %s branch', remote_match) return True LOG.debug('did not find %s in branches containing %s: %s', remote_match, sha, containing_branches) # If the expected branch does not exist yet, this may be a # late release attempt to create that branch or just a project # that hasn't branched, yet, and is releasing from master for # that series. Allow the release, as long as it is on the # master branch. all_branches = _filter_branches( processutils.check_output( ['git', 'branch', '-a'], cwd=os.path.join(workdir, repo), ).decode('utf-8') ) if remote_match not in all_branches: if 'master' in containing_branches: LOG.debug('did not find %s but SHA is on master', remote_match) return True if 'origin/master' in containing_branches: LOG.debug('did not find %s but SHA is on origin/master', remote_match) return True # At this point we know the release is not from the required # branch and it is not from master, which means it is the # wrong branch and should not be allowed. LOG.debug('did not find SHA on %s or master or origin/master', remote_match) return False except processutils.CalledProcessError as e: LOG.error('failed checking SHA on branch: %s [%s]' % (e, e.output.strip())) return False
def commit_exists(workdir, repo, ref): """Return boolean specifying whether the reference exists in the repository. The commit must have been merged into the repository, but this check does not enforce any branch membership. """ try: processutils.check_output( ['git', 'show', ref], cwd=os.path.join(workdir, repo), ).decode('utf-8') except processutils.CalledProcessError as err: LOG.error('Could not find {}: {}'.format(ref, err)) return False return True
def get_branches(workdir, repo): try: output = processutils.check_output( ['git', 'branch', '-a'], cwd=os.path.join(workdir, repo), stderr=subprocess.STDOUT, ).decode('utf-8').strip() # Example output: # * (no branch) # master # stable/mitaka # stable/newton # stable/ocata # remotes/origin/HEAD -> origin/master # remotes/origin/master # remotes/origin/stable/mitaka # remotes/origin/stable/newton # remotes/origin/stable/ocata results = [] for line in output.splitlines(): branch = line.strip().lstrip('*').strip() if branch.startswith('('): continue if '->' in branch: continue results.append(branch) return results except processutils.CalledProcessError as e: LOG.error('failed to retrieve list of branches: %s [%s]', e, e.output.strip()) return []
def setUp(self): super(GPGKeyFixture, self).setUp() # Force a temporary home directory with a short path so the # gpg commands do not complain about an excessively long # value. self.useFixture(fixtures.TempHomeDir('/tmp')) tempdir = self.useFixture(fixtures.TempDir('/tmp')) gnupg_version_re = re.compile(r'^gpg\s.*\s([\d+])\.([\d+])\.([\d+])') gnupg_version = processutils.check_output( ['gpg', '--version'], cwd=tempdir.path).decode('utf-8') for line in gnupg_version.split('\n'): gnupg_version = gnupg_version_re.match(line) if gnupg_version: gnupg_version = (int(gnupg_version.group(1)), int(gnupg_version.group(2)), int(gnupg_version.group(3))) break else: if gnupg_version is None: gnupg_version = (0, 0, 0) config_file = tempdir.path + '/key-config' LOG.debug('creating gpg config file in %s', config_file) with open(config_file, 'wt') as f: if gnupg_version[0] == 2 and gnupg_version[1] >= 1: f.write( textwrap.dedent(""" %no-protection %transient-key """)) f.write( textwrap.dedent(""" %no-ask-passphrase Key-Type: RSA Name-Real: Example Key Name-Comment: N/A Name-Email: [email protected] Expire-Date: 2d %commit """)) # Note that --quick-random (--debug-quick-random in GnuPG 2.x) # does not have a corresponding preferences file setting and # must be passed explicitly on the command line instead if gnupg_version[0] == 1: gnupg_random = '--quick-random' elif gnupg_version[0] >= 2: gnupg_random = '--debug-quick-random' else: gnupg_random = '' cmd = ['gpg', '--gen-key', '--batch'] if gnupg_random: cmd.append(gnupg_random) cmd.append('key-config') LOG.debug('generating gpg key') processutils.check_call(cmd, cwd=tempdir.path)
def find_modified_deliverable_files(): "Return a list of files modified by the most recent commit." results = processutils.check_output( ['git', 'diff', '--name-only', '--pretty=format:', 'HEAD^'] ).decode('utf-8') filenames = [ l.strip() for l in results.splitlines() if (l.startswith('deliverables/')) ] return filenames
def check_ancestry(workdir, repo, old_version, sha): "Check if the SHA is in the ancestry of the previous version." try: ancestors = processutils.check_output( ['git', 'log', '--oneline', '--ancestry-path', '%s..%s' % (old_version, sha)], cwd=os.path.join(workdir, repo), ).decode('utf-8').strip() return bool(ancestors) except processutils.CalledProcessError as e: LOG.error('failed checking ancestry: %s [%s]' % (e, e.output.strip())) return False
def get_head(workdir, repo): cmd = ['git', 'log', '-n', '1', '--pretty=tformat:%h'] try: return processutils.check_output( cmd, cwd=os.path.join(workdir, repo), stderr=subprocess.STDOUT, ).decode('utf-8').strip() except processutils.CalledProcessError as e: LOG.warning('failed to retrieve HEAD: %s [%s]', e, e.output.strip()) return None
def add_tag(workdir, repo, tag, sha): cmd = ['git', 'tag', '-m', 'temporary tag', tag, sha] try: LOG.info(' '.join(cmd)) return processutils.check_output( cmd, cwd=os.path.join(workdir, repo), stderr=subprocess.STDOUT, ).decode('utf-8').strip() except processutils.CalledProcessError as e: LOG.warning('failed to add tag: %s [%s]', e, e.output.strip()) return None
def get_branch_base(workdir, repo, branch): "Return SHA at base of branch." # http://stackoverflow.com/questions/1527234/finding-a-branch-point-with-git # git rev-list $(git rev-list --first-parent ^origin/stable/newton master | tail -n1)^^! # # Determine the first parent. cmd = [ 'git', 'rev-list', '--first-parent', '^origin/{}'.format(branch), 'master', ] try: parents = processutils.check_output( cmd, cwd=os.path.join(workdir, repo), stderr=subprocess.STDOUT, ).decode('utf-8').strip() except processutils.CalledProcessError as e: LOG.warning('failed to retrieve branch base: %s [%s]', e, e.output.strip()) return None parent = parents.splitlines()[-1] # Now get the ^^! commit cmd = [ 'git', 'rev-list', '{}^^!'.format(parent), ] try: return processutils.check_output( cmd, cwd=os.path.join(workdir, repo), stderr=subprocess.STDOUT, ).decode('utf-8').strip() except processutils.CalledProcessError as e: LOG.warning('failed to retrieve branch base: %s [%s]', e, e.output.strip()) return None
def build_sdist(workdir, repo): """Build the sdist.""" dest = os.path.join(workdir, repo) build_path = os.path.join(dest, 'dist') if os.path.exists(build_path): # sdist already built, skip rebuilding it return setup_path = os.path.join(dest, 'setup.py') if not os.path.exists(setup_path): LOG.debug('did not find %s, maybe %s is not a python project', setup_path, repo) return use_tox = repo.endswith('/pbr') if use_tox and not os.path.exists(os.path.join(dest, '.tox', 'venv')): # Use tox to set up a virtualenv so we can install the # dependencies for the package. This only seems to be # necessary for pbr, but... processutils.check_output( ['tox', '-e', 'venv', '--notest'], cwd=dest, ) if use_tox: python = '.tox/venv/bin/python3' else: python = 'python3' # Set some flags to turn off pbr functionality that we don't need. flags = { 'SKIP_GENERATE_RENO': '1', 'SKIP_GENERATE_AUTHORS': '1', 'SKIP_WRITE_GIT_CHANGELOG': '1', } cmd = [python, 'setup.py', 'sdist', 'bdist_wheel'] processutils.check_call( cmd, cwd=dest, env=flags)
def stable_branch_exists(workdir, repo, series): "Does the stable/series branch exist?" remote_match = 'remotes/origin/stable/%s' % series try: containing_branches = _filter_branches( processutils.check_output( ['git', 'branch', '-a'], cwd=os.path.join(workdir, repo), ).decode('utf-8')) return (remote_match in containing_branches) except processutils.CalledProcessError as e: LOG.error('failed checking for branch: %s [%s]', e, e.output.strip()) return False
def get_latest_tag(workdir, repo, sha=None): cmd = ['git', 'describe', '--abbrev=0', '--always'] if sha is not None: cmd.append(sha) try: return processutils.check_output( cmd, cwd=os.path.join(workdir, repo), stderr=subprocess.STDOUT, ).decode('utf-8').strip() except processutils.CalledProcessError as e: LOG.warning('failed to retrieve latest tag: %s [%s]', e, e.output.strip()) return None
def sha_for_tag(workdir, repo, version): """Return the SHA for a given tag""" # git log 2.3.11 -n 1 --pretty=format:%H try: actual_sha = processutils.check_output( ['git', 'log', str(version), '-n', '1', '--pretty=format:%H'], cwd=os.path.join(workdir, repo), stderr=subprocess.STDOUT, ).decode('utf-8') actual_sha = actual_sha.strip() except processutils.CalledProcessError as e: LOG.info('ERROR getting SHA for tag %r: %s [%s]', version, e, e.output.strip()) actual_sha = '' return actual_sha
def branches_containing(workdir, repo, ref): try: output = processutils.check_output( ['git', 'branch', '-r', '--contains', ref], cwd=os.path.join(workdir, repo), stderr=subprocess.STDOUT, ).decode('utf-8').strip() # Example output: # origin/stable/ocata results = [] for line in output.splitlines(): results.append(line.strip()) return results except processutils.CalledProcessError as e: LOG.error('failed to retrieve list of branches containing %s: %s [%s]', ref, e, e.output.strip()) return []
def changes_since(workdir, repo, ref): """Get all changes between the last ref and the current point. :param workdir: The git repo working directory. :param repo: The name of the repo. :param ref: The starting ref. :returns: Merged commits between the two points. """ try: changes = processutils.check_output( ['git', 'log', '--decorate', '--no-merges', '--pretty=oneline', "%s..HEAD" % ref], cwd=os.path.join(workdir, repo), ).decode('utf-8').strip() except processutils.CalledProcessError as err: LOG.error('Could not find {}: {}'.format(ref, err)) changes = '' return changes
def ensure_basic_git_config(workdir, repo, settings): """Given a repo directory and a settings dict, set local config values if those settings are not already defined. """ dest = os.path.join(workdir, repo) for key, value in settings.items(): LOG.info('looking for git config {}'.format(key)) try: existing = processutils.check_output( ['git', 'config', '--get', key], cwd=dest, ).decode('utf-8').strip() LOG.info('using existing setting of {}: {!r}'.format(key, existing)) except processutils.CalledProcessError: LOG.info('updating setting of {} to {!r}'.format(key, value)) processutils.check_call( ['git', 'config', key, value], cwd=dest, )
def branch_exists(workdir, repo, prefix, identifier): """Does the prefix/identifier branch exist. Checks if a named branch already exists. :param workdir: The working directory for the local clone. :param repo: The name of the repo. :param prefix: The branch prefix (e.g. "stable" or "bugfix"). :param idenifier: The branch identifier (series name or version). """ remote_match = 'remotes/origin/{}/{}'.format(prefix, identifier) try: containing_branches = _filter_branches( processutils.check_output( ['git', 'branch', '-a'], cwd=os.path.join(workdir, repo), ).decode('utf-8') ) LOG.debug('looking for %s', remote_match) LOG.debug('found branches: %s', containing_branches) return (remote_match in containing_branches) except processutils.CalledProcessError as e: LOG.error('failed checking for branch: %s [%s]', e, e.output.strip()) return False
def git(self, *args): output = processutils.check_output( ['git'] + list(args), cwd=self.path, ) return output