def validate_new_releases(deliverable_info, filename,
team_data,
mk_warning, mk_error):
"""Apply validation rules that only apply to the current series.
"""
if not deliverable_info.get('releases'):
return
final_release = deliverable_info['releases'][-1]
deliverable_name = os.path.basename(filename)[:-5] # strip .yaml
expected_repos = set(
r.name
for r in governance.get_repositories(
team_data,
deliverable_name=deliverable_name,
)
)
link_mode = deliverable_info.get('artifact-link-mode', 'tarball')
if link_mode != 'none' and not expected_repos:
mk_error('unable to find deliverable %s in the governance list' %
deliverable_name)
actual_repos = set(
p['repo']
for p in final_release.get('projects', [])
)
for extra in actual_repos.difference(expected_repos):
mk_warning(
'release %s includes repository %s '
'that is not in the governance list' %
(final_release['version'], extra)
)
for missing in expected_repos.difference(actual_repos):
mk_warning(
'release %s is missing %s from the governance list' %
(final_release['version'], missing)
)
def test_by_deliverable(self):
repos = governance.get_repositories(
TEAM_DATA,
deliverable_name='release-tools',
)
self.assertEqual(
['openstack-infra/release-tools'],
[r.name for r in repos],
)
def test_by_deliverable(self):
repos = governance.get_repositories(
TEAM_DATA,
deliverable_name='release-tools',
)
self.assertEqual(
['openstack-infra/release-tools'],
[r.name for r in repos],
)
def test_code_only(self):
repos = governance.get_repositories(
TEAM_DATA,
code_only=True,
)
self.assertEqual(
sorted(['openstack/release-schedule-generator',
'openstack/release-test',
'openstack-infra/release-tools',
'openstack/releases',
'openstack/reno']),
sorted(r.name for r in repos),
)
def test_code_only(self):
repos = governance.get_repositories(
TEAM_DATA,
code_only=True,
)
self.assertEqual(
sorted(['openstack/release-schedule-generator',
'openstack/release-test',
'openstack-infra/release-tools',
'openstack/releases',
'openstack/reno']),
sorted(r.name for r in repos),
)
def test_by_team(self):
repos = governance.get_repositories(
TEAM_DATA,
team_name='Release Management',
)
self.assertEqual(
sorted(['openstack/release-schedule-generator',
'openstack/release-test',
'openstack-infra/release-tools',
'openstack/releases',
'openstack/reno',
'openstack-dev/specs-cookiecutter']),
sorted(r.name for r in repos),
)
def test_tags_positive(self):
repos = governance.get_repositories(
TEAM_DATA,
tags=['team:diverse-affiliation'],
)
self.assertEqual(
sorted(['openstack/release-schedule-generator',
'openstack/release-test',
'openstack-infra/release-tools',
'openstack/releases',
'openstack/reno',
'openstack-dev/specs-cookiecutter']),
sorted(r.name for r in repos),
)
def test_by_team(self):
repos = governance.get_repositories(
TEAM_DATA,
team_name='Release Management',
)
self.assertEqual(
sorted(['openstack/release-schedule-generator',
'openstack/release-test',
'openstack-infra/release-tools',
'openstack/releases',
'openstack/reno',
'openstack-dev/specs-cookiecutter']),
sorted(r.name for r in repos),
)
def test_tags_positive(self):
repos = governance.get_repositories(
TEAM_DATA,
tags=['team:diverse-affiliation'],
)
self.assertEqual(
sorted(['openstack/release-schedule-generator',
'openstack/release-test',
'openstack-infra/release-tools',
'openstack/releases',
'openstack/reno',
'openstack-dev/specs-cookiecutter']),
sorted(r.name for r in repos),
)
def validate_new_releases(deliverable_info, filename, team_data, mk_warning,
mk_error):
"""Apply validation rules that only apply to the current series.
"""
if not deliverable_info.get('releases'):
return
final_release = deliverable_info['releases'][-1]
deliverable_name = os.path.basename(filename)[:-5] # strip .yaml
expected_repos = set(r.name for r in governance.get_repositories(
team_data,
deliverable_name=deliverable_name,
))
link_mode = deliverable_info.get('artifact-link-mode', 'tarball')
if link_mode != 'none' and not expected_repos:
mk_error('unable to find deliverable %s in the governance list' %
deliverable_name)
actual_repos = set(p['repo'] for p in final_release.get('projects', []))
for extra in actual_repos.difference(expected_repos):
mk_warning('release %s includes repository %s '
'that is not in the governance list' %
(final_release['version'], extra))
for missing in expected_repos.difference(actual_repos):
mk_warning('release %s is missing %s from the governance list' %
(final_release['version'], missing))
def test_tag_negative(self):
repos = governance.get_repositories(
TEAM_DATA,
tags=['team:single-vendor'],
)
self.assertEqual([], list(repos))
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--verbose',
'-v',
action='store_true',
default=False,
)
parser.add_argument(
'--deliverables-dir',
default=openstack_releases.deliverable_dir,
help='location of deliverable files',
)
args = parser.parse_args()
config_filename = os.path.join(
appdirs.user_config_dir('openstack-release', 'openstack'),
'gerrit.ini',
)
config = configparser.ConfigParser()
config.read(config_filename, encoding='utf-8')
if not config.has_option('DEFAULT', 'username'):
parser.error('No username set in {}'.format(config_filename))
if not config.has_option('DEFAULT', 'password'):
parser.error('No password set in {}'.format(config_filename))
team_data = governance.get_team_data()
# Some deliverables were independent at one time but might not be
# any more, so compare the independent list with the current
# release series.
all_independent_deliverables = set(
name for team, series, name, deliv in deliverable.Deliverables(
root_dir=args.deliverables_dir,
collapse_history=True,
).get_deliverables(None, None))
current_deliverables = set(
name for team, series, name, deliv in deliverable.Deliverables(
root_dir=args.deliverables_dir,
collapse_history=True,
).get_deliverables(None, defaults.RELEASE))
independent_deliverables = all_independent_deliverables.difference(
current_deliverables)
gerrit = GerritClient(
config['DEFAULT']['username'],
config['DEFAULT']['password'],
)
for repo in governance.get_repositories(team_data, code_only=True):
if repo.deliverable.team.name in IGNORED_TEAMS:
if args.verbose:
print('{}: ignoring {} team'.format(
repo.name, repo.deliverable.team.name))
continue
if repo.deliverable.name in independent_deliverables:
if args.verbose:
print('{}: ignoring independent deliverable'.format(repo.name))
continue
acls = gerrit.get_access(repo.name)
local_tag_acls = acls.get('local', {}).get('refs/tags/*', {})
if local_tag_acls:
rules = local_tag_acls.get('permissions',
{}).get('pushSignedTag',
{}).get('rules', {})
if not rules and args.verbose:
print('{}: OK'.format(repo.name))
for group_id, permissions in rules.items():
group_details = gerrit.get_group(group_id)
group_name = group_details['name']
if group_name in ALLOWED:
if args.verbose:
print('{}: {} pushSignedTag OK'.format(
repo.name, group_name))
continue
if args.verbose:
print('{}: {} pushSignedTag WARNING'.format(
repo.name, group_name))
else:
print('{}: {} pushSignedTag'.format(repo.name, group_name))
def test_tag_negative(self):
repos = governance.get_repositories(
TEAM_DATA,
tags=['team:single-vendor'],
)
self.assertEqual([], list(repos))
def main():
if not sys.stdout.encoding:
# Wrap sys.stdout with a writer that knows how to handle
# encoding Unicode data.
import codecs
wrapped_stdout = codecs.getwriter('UTF-8')(sys.stdout)
sys.stdout = wrapped_stdout
parser = argparse.ArgumentParser()
parser.add_argument(
'--no-cleanup',
dest='cleanup',
default=True,
action='store_false',
help='do not remove temporary files',
)
parser.add_argument(
'--no-shortcut', '--force', '-f',
dest='shortcut',
default=True,
action='store_false',
help='if a tag has been applied, skip the repo',
)
parser.add_argument(
'input',
nargs='*',
help=('YAML files to validate, defaults to '
'files changed in the latest commit'),
)
args = parser.parse_args()
# Set up logging, including making some loggers quiet.
logging.basicConfig(
format='%(levelname)7s: %(message)s',
stream=sys.stdout,
level=logging.DEBUG,
)
logging.getLogger('urllib3.connectionpool').setLevel(logging.WARNING)
filenames = args.input or gitutils.find_modified_deliverable_files()
if not filenames:
print('no modified deliverable files, skipping report')
return 0
workdir = tempfile.mkdtemp(prefix='releases-')
print('creating temporary files in %s' % workdir)
def cleanup_workdir():
if args.cleanup:
shutil.rmtree(workdir, True)
else:
print('not cleaning up %s' % workdir)
atexit.register(cleanup_workdir)
team_data = governance.get_team_data()
official_repos = set(
r.name
for r in governance.get_repositories(team_data)
)
# Remove any inherited PAGER environment variable to avoid
# blocking the output waiting for input.
os.environ['PAGER'] = ''
for filename in filenames:
if not os.path.exists(filename):
print('\n%s was removed, skipping' % filename)
continue
print('\n' + ('=' * 80))
print('\nChecking %s\n' % filename)
deliv = deliverable.Deliverable.read_file(filename)
stable_branch = deliv.series not in ['independent', defaults.RELEASE]
# By default assume the project does not use milestones.
header('Release model')
print(deliv.model)
header('Team details')
if deliv.team:
team_name = deliv.team
team_dict = team_data.get(team_name)
if team_dict:
team = governance.Team(team_name, team_dict)
print('found team %s' % team_name)
print(' PTL : %(name)s (%(irc)s)' % team.ptl)
print(' Liaison: %s (%s)\n' % team.liaison)
team_deliv = team.deliverables.get(deliv.name)
if team_deliv:
print('found deliverable %s' % deliv.name)
for rn, repo in sorted(team_deliv.repositories.items()):
follows_stable_policy = 'stable:follows-policy' in repo.tags
print('\nrepo %s\ntags:' % repo.name)
for t in repo.tags:
print(' %s' % t)
print('')
if stable_branch and follows_stable_policy:
banner('Needs Stable Policy Review')
print()
else:
print(('no deliverable %r found for team %r, '
'cannot report on governance status') %
(deliv.name, team_name))
else:
print('no team %r found, cannot report on governance status' %
team_name)
else:
print('no team name given, cannot report on governance status')
# If there are no releases listed, this is probably a new
# deliverable file for initializing a new series. We don't
# need to list its changes.
if not deliv.is_released:
header('No releases')
print('no releases were found, assuming an initialization file')
continue
# assume the releases are in order and take the last one
new_release = deliv.releases[-1]
for project in new_release.projects:
tag_exists = gitutils.tag_exists(
project.repo.name,
new_release.version,
)
if tag_exists:
print('%s %s exists on git server already' %
(project.repo.name, new_release.version))
if args.shortcut:
print('skipping further processing')
continue
if project.repo.is_retired:
print('%s is retired' % (project.repo.name,))
if args.shortcut:
print('skipping further processing')
continue
# Start by checking out master, always. We need the repo
# checked out before we can tell if the stable branch
# really exists.
gitutils.clone_repo(
workdir,
project.repo.name,
branch='master',
)
# Set some git configuration values to allow us to perform
# local operations like tagging.
gitutils.ensure_basic_git_config(
workdir, project.repo.name,
{'user.email': '*****@*****.**',
'user.name': 'OpenStack Proposal Bot'},
)
# Determine which branch we should actually be looking
# at. Assume any series for which there is no stable
# branch will be on 'master'.
if gitutils.stable_branch_exists(workdir,
project.repo.name,
deliv.series):
branch = 'stable/' + deliv.series
else:
branch = 'master'
if branch != 'master':
# Check out the repo again to the right branch if we
# didn't get it the first time.
gitutils.clone_repo(
workdir,
project.repo.name,
branch=branch,
)
# look at the previous tag for the parent of the commit
# getting the new release
previous_tag = gitutils.get_latest_tag(
workdir,
project.repo.name,
'{}^'.format(project.hash)
)
try:
previous_release = deliv.get_release(previous_tag)
except ValueError:
previous_release = None
start_range = previous_tag
if previous_release:
previous_project = {
x.repo.name: x
for x in previous_release.projects
}.get(project.repo.name)
if previous_project is not None:
start_range = previous_tag
if start_range:
git_range = '%s..%s' % (start_range, project.hash)
else:
git_range = project.hash
# Show details about the commit being tagged.
header('Details for commit receiving new tag %s' %
new_release.version)
print('\ngit describe %s\n' % project.hash)
try:
subprocess.check_call(
['git', 'describe', project.hash],
cwd=os.path.join(workdir, project.repo.name),
)
except subprocess.CalledProcessError as e:
print('WARNING: Could not run git describe: %s' % e)
git_show(
workdir=workdir,
repo=project.repo.name,
title='Check existing tags',
ref=project.hash,
)
git_list_existing_branches(
workdir=workdir,
repo=project.repo.name,
)
branches = git_branch_contains(
workdir=workdir,
repo=project.repo.name,
title='Branches containing commit',
commit=project.hash,
)
header('Relationship to HEAD')
if deliv.is_independent:
if branches:
tag_branch = branches[0]
else:
tag_branch = branch
head_sha = gitutils.sha_for_tag(
workdir,
project.repo.name,
tag_branch,
)
print('HEAD of {} is {}'.format(tag_branch, head_sha))
else:
if (branch in branches) or (not branches):
tag_branch = branch
else:
tag_branch = branches[0]
head_sha = gitutils.sha_for_tag(
workdir,
project.repo.name,
tag_branch,
)
print('HEAD of {} is {}'.format(tag_branch, head_sha))
requested_sha = gitutils.sha_for_tag(
workdir,
project.repo.name,
project.hash,
)
# If the sha for HEAD and the requested release don't
# match, show any unreleased changes on the branch. We ask
# git to give us the real SHA for the requested release in
# case the deliverables file has the short version of the
# hash.
if head_sha == requested_sha:
print('\nRequest releases from HEAD on %s' % tag_branch)
else:
git_log(workdir, project.repo.name, 'Release will NOT include',
'%s..%s' % (requested_sha, head_sha),
extra_args=['--format=%h %ci %s'])
show_watched_queries(branch, project.repo.name)
# Show any requirements changes in the upcoming release.
# Include setup.cfg, in case the project uses "extras".
if start_range:
git_diff(workdir, project.repo.name, git_range, '*requirements*.txt',
'Requirements Changes %s' % git_range)
git_diff(workdir, project.repo.name, git_range, 'doc/requirements.txt',
'Doc Requirements Changes %s' % git_range)
git_diff(workdir, project.repo.name, git_range, 'setup.cfg',
'setup.cfg Changes %s' % git_range)
git_diff(workdir, project.repo.name, git_range, 'bindep.txt',
'bindep.txt Changes %s' % git_range)
# Before we try to determine if the previous release
# is an ancestor or produce the release notes we need
# the tag to exist in the local repository.
if not tag_exists:
header('Applying Temporary Tag')
print('\ngit tag {version} {hash}'.format(
version=new_release.version,
hash=project.hash,
))
subprocess.check_call(
['git', 'tag', new_release.version,
project.hash],
cwd=os.path.join(workdir, project.repo.name),
)
# Show any changes in the previous release but not in this
# release, in case someone picks an "early" SHA or a
# regular commit instead of the appropriate merge commit.
previous_tag_exists = False
if previous_release:
previous_tag_exists = gitutils.tag_exists(
project.repo.name,
previous_release.version,
)
if previous_tag_exists:
git_log(
workdir, project.repo.name,
'Patches in previous release but not in this one',
[previous_release.version,
'--not',
project.hash],
extra_args=['--topo-order', '--oneline', '--no-merges'],
)
# The tag will have been added as a local tag above if
# it does not already exist.
header('New release %s includes previous release %s' %
(new_release.version, previous_release.version))
print('\ngit tag --contains %s\n' %
previous_release.version)
containing_tags = subprocess.check_output(
['git', 'tag',
'--contains',
previous_release.version],
cwd=os.path.join(workdir, project.repo.name),
).decode('utf-8').split()
print('Containing tags:', containing_tags)
if new_release.version not in containing_tags:
print('WARNING: Missing %s' % new_release.version)
else:
print('Found new version %s' % new_release.version)
is_ancestor = gitutils.check_ancestry(
workdir,
project.repo.name,
previous_release.version,
project.hash,
)
if is_ancestor:
print('SHA found in descendants')
else:
print('SHA NOT FOUND in descendants')
# Show the changes since the last release, first as a
# graph view so we can check for bad merges, and then with
# more detail.
git_log(workdir, project.repo.name,
'Release %s will include' % new_release.version,
git_range,
extra_args=['--graph', '--oneline', '--decorate',
'--topo-order'])
git_log(workdir, project.repo.name,
'Details Contents',
git_range,
extra_args=['--no-merges', '--topo-order'])
# The tag will have been added as a local tag above if it does
# not already exist.
header('Release Notes')
try:
notes = release_notes.generate_release_notes(
repo=project.repo.name,
repo_path=os.path.join(workdir, project.repo.name),
start_revision=new_release.diff_start or start_range,
end_revision=new_release.version,
show_dates=True,
skip_requirement_merges=True,
is_stable=branch.startswith('stable/'),
series=deliv.series,
email='*****@*****.**',
email_from='*****@*****.**',
email_reply_to='*****@*****.**',
email_tags='',
include_pypi_link=False,
changes_only=False,
first_release=deliv.is_first_release,
repo_name=project.repo.name,
description='',
publishing_dir_name=project.repo.name,
)
except Exception as e:
logging.exception('Failed to produce release notes')
else:
print('\n')
print(notes)
if 'library' in deliv.type:
show_dependency_listings(
project.guess_sdist_name(),
official_repos,
)
return 0
