def validate_new_releases(deliverable_info, filename, team_data, mk_warning, mk_error): """Apply validation rules that only apply to the current series. """ if not deliverable_info.get('releases'): return final_release = deliverable_info['releases'][-1] deliverable_name = os.path.basename(filename)[:-5] # strip .yaml expected_repos = set( r.name for r in governance.get_repositories( team_data, deliverable_name=deliverable_name, ) ) link_mode = deliverable_info.get('artifact-link-mode', 'tarball') if link_mode != 'none' and not expected_repos: mk_error('unable to find deliverable %s in the governance list' % deliverable_name) actual_repos = set( p['repo'] for p in final_release.get('projects', []) ) for extra in actual_repos.difference(expected_repos): mk_warning( 'release %s includes repository %s ' 'that is not in the governance list' % (final_release['version'], extra) ) for missing in expected_repos.difference(actual_repos): mk_warning( 'release %s is missing %s from the governance list' % (final_release['version'], missing) )
def test_by_deliverable(self): repos = governance.get_repositories( TEAM_DATA, deliverable_name='release-tools', ) self.assertEqual( ['openstack-infra/release-tools'], [r.name for r in repos], )
def test_code_only(self): repos = governance.get_repositories( TEAM_DATA, code_only=True, ) self.assertEqual( sorted(['openstack/release-schedule-generator', 'openstack/release-test', 'openstack-infra/release-tools', 'openstack/releases', 'openstack/reno']), sorted(r.name for r in repos), )
def test_by_team(self): repos = governance.get_repositories( TEAM_DATA, team_name='Release Management', ) self.assertEqual( sorted(['openstack/release-schedule-generator', 'openstack/release-test', 'openstack-infra/release-tools', 'openstack/releases', 'openstack/reno', 'openstack-dev/specs-cookiecutter']), sorted(r.name for r in repos), )
def test_tags_positive(self): repos = governance.get_repositories( TEAM_DATA, tags=['team:diverse-affiliation'], ) self.assertEqual( sorted(['openstack/release-schedule-generator', 'openstack/release-test', 'openstack-infra/release-tools', 'openstack/releases', 'openstack/reno', 'openstack-dev/specs-cookiecutter']), sorted(r.name for r in repos), )
def validate_new_releases(deliverable_info, filename, team_data, mk_warning, mk_error): """Apply validation rules that only apply to the current series. """ if not deliverable_info.get('releases'): return final_release = deliverable_info['releases'][-1] deliverable_name = os.path.basename(filename)[:-5] # strip .yaml expected_repos = set(r.name for r in governance.get_repositories( team_data, deliverable_name=deliverable_name, )) link_mode = deliverable_info.get('artifact-link-mode', 'tarball') if link_mode != 'none' and not expected_repos: mk_error('unable to find deliverable %s in the governance list' % deliverable_name) actual_repos = set(p['repo'] for p in final_release.get('projects', [])) for extra in actual_repos.difference(expected_repos): mk_warning('release %s includes repository %s ' 'that is not in the governance list' % (final_release['version'], extra)) for missing in expected_repos.difference(actual_repos): mk_warning('release %s is missing %s from the governance list' % (final_release['version'], missing))
def test_tag_negative(self): repos = governance.get_repositories( TEAM_DATA, tags=['team:single-vendor'], ) self.assertEqual([], list(repos))
def main(): parser = argparse.ArgumentParser() parser.add_argument( '--verbose', '-v', action='store_true', default=False, ) parser.add_argument( '--deliverables-dir', default=openstack_releases.deliverable_dir, help='location of deliverable files', ) args = parser.parse_args() config_filename = os.path.join( appdirs.user_config_dir('openstack-release', 'openstack'), 'gerrit.ini', ) config = configparser.ConfigParser() config.read(config_filename, encoding='utf-8') if not config.has_option('DEFAULT', 'username'): parser.error('No username set in {}'.format(config_filename)) if not config.has_option('DEFAULT', 'password'): parser.error('No password set in {}'.format(config_filename)) team_data = governance.get_team_data() # Some deliverables were independent at one time but might not be # any more, so compare the independent list with the current # release series. all_independent_deliverables = set( name for team, series, name, deliv in deliverable.Deliverables( root_dir=args.deliverables_dir, collapse_history=True, ).get_deliverables(None, None)) current_deliverables = set( name for team, series, name, deliv in deliverable.Deliverables( root_dir=args.deliverables_dir, collapse_history=True, ).get_deliverables(None, defaults.RELEASE)) independent_deliverables = all_independent_deliverables.difference( current_deliverables) gerrit = GerritClient( config['DEFAULT']['username'], config['DEFAULT']['password'], ) for repo in governance.get_repositories(team_data, code_only=True): if repo.deliverable.team.name in IGNORED_TEAMS: if args.verbose: print('{}: ignoring {} team'.format( repo.name, repo.deliverable.team.name)) continue if repo.deliverable.name in independent_deliverables: if args.verbose: print('{}: ignoring independent deliverable'.format(repo.name)) continue acls = gerrit.get_access(repo.name) local_tag_acls = acls.get('local', {}).get('refs/tags/*', {}) if local_tag_acls: rules = local_tag_acls.get('permissions', {}).get('pushSignedTag', {}).get('rules', {}) if not rules and args.verbose: print('{}: OK'.format(repo.name)) for group_id, permissions in rules.items(): group_details = gerrit.get_group(group_id) group_name = group_details['name'] if group_name in ALLOWED: if args.verbose: print('{}: {} pushSignedTag OK'.format( repo.name, group_name)) continue if args.verbose: print('{}: {} pushSignedTag WARNING'.format( repo.name, group_name)) else: print('{}: {} pushSignedTag'.format(repo.name, group_name))
def main(): if not sys.stdout.encoding: # Wrap sys.stdout with a writer that knows how to handle # encoding Unicode data. import codecs wrapped_stdout = codecs.getwriter('UTF-8')(sys.stdout) sys.stdout = wrapped_stdout parser = argparse.ArgumentParser() parser.add_argument( '--no-cleanup', dest='cleanup', default=True, action='store_false', help='do not remove temporary files', ) parser.add_argument( '--no-shortcut', '--force', '-f', dest='shortcut', default=True, action='store_false', help='if a tag has been applied, skip the repo', ) parser.add_argument( 'input', nargs='*', help=('YAML files to validate, defaults to ' 'files changed in the latest commit'), ) args = parser.parse_args() # Set up logging, including making some loggers quiet. logging.basicConfig( format='%(levelname)7s: %(message)s', stream=sys.stdout, level=logging.DEBUG, ) logging.getLogger('urllib3.connectionpool').setLevel(logging.WARNING) filenames = args.input or gitutils.find_modified_deliverable_files() if not filenames: print('no modified deliverable files, skipping report') return 0 workdir = tempfile.mkdtemp(prefix='releases-') print('creating temporary files in %s' % workdir) def cleanup_workdir(): if args.cleanup: shutil.rmtree(workdir, True) else: print('not cleaning up %s' % workdir) atexit.register(cleanup_workdir) team_data = governance.get_team_data() official_repos = set( r.name for r in governance.get_repositories(team_data) ) # Remove any inherited PAGER environment variable to avoid # blocking the output waiting for input. os.environ['PAGER'] = '' for filename in filenames: if not os.path.exists(filename): print('\n%s was removed, skipping' % filename) continue print('\n' + ('=' * 80)) print('\nChecking %s\n' % filename) deliv = deliverable.Deliverable.read_file(filename) stable_branch = deliv.series not in ['independent', defaults.RELEASE] # By default assume the project does not use milestones. header('Release model') print(deliv.model) header('Team details') if deliv.team: team_name = deliv.team team_dict = team_data.get(team_name) if team_dict: team = governance.Team(team_name, team_dict) print('found team %s' % team_name) print(' PTL : %(name)s (%(irc)s)' % team.ptl) print(' Liaison: %s (%s)\n' % team.liaison) team_deliv = team.deliverables.get(deliv.name) if team_deliv: print('found deliverable %s' % deliv.name) for rn, repo in sorted(team_deliv.repositories.items()): follows_stable_policy = 'stable:follows-policy' in repo.tags print('\nrepo %s\ntags:' % repo.name) for t in repo.tags: print(' %s' % t) print('') if stable_branch and follows_stable_policy: banner('Needs Stable Policy Review') print() else: print(('no deliverable %r found for team %r, ' 'cannot report on governance status') % (deliv.name, team_name)) else: print('no team %r found, cannot report on governance status' % team_name) else: print('no team name given, cannot report on governance status') # If there are no releases listed, this is probably a new # deliverable file for initializing a new series. We don't # need to list its changes. if not deliv.is_released: header('No releases') print('no releases were found, assuming an initialization file') continue # assume the releases are in order and take the last one new_release = deliv.releases[-1] for project in new_release.projects: tag_exists = gitutils.tag_exists( project.repo.name, new_release.version, ) if tag_exists: print('%s %s exists on git server already' % (project.repo.name, new_release.version)) if args.shortcut: print('skipping further processing') continue if project.repo.is_retired: print('%s is retired' % (project.repo.name,)) if args.shortcut: print('skipping further processing') continue # Start by checking out master, always. We need the repo # checked out before we can tell if the stable branch # really exists. gitutils.clone_repo( workdir, project.repo.name, branch='master', ) # Set some git configuration values to allow us to perform # local operations like tagging. gitutils.ensure_basic_git_config( workdir, project.repo.name, {'user.email': '*****@*****.**', 'user.name': 'OpenStack Proposal Bot'}, ) # Determine which branch we should actually be looking # at. Assume any series for which there is no stable # branch will be on 'master'. if gitutils.stable_branch_exists(workdir, project.repo.name, deliv.series): branch = 'stable/' + deliv.series else: branch = 'master' if branch != 'master': # Check out the repo again to the right branch if we # didn't get it the first time. gitutils.clone_repo( workdir, project.repo.name, branch=branch, ) # look at the previous tag for the parent of the commit # getting the new release previous_tag = gitutils.get_latest_tag( workdir, project.repo.name, '{}^'.format(project.hash) ) try: previous_release = deliv.get_release(previous_tag) except ValueError: previous_release = None start_range = previous_tag if previous_release: previous_project = { x.repo.name: x for x in previous_release.projects }.get(project.repo.name) if previous_project is not None: start_range = previous_tag if start_range: git_range = '%s..%s' % (start_range, project.hash) else: git_range = project.hash # Show details about the commit being tagged. header('Details for commit receiving new tag %s' % new_release.version) print('\ngit describe %s\n' % project.hash) try: subprocess.check_call( ['git', 'describe', project.hash], cwd=os.path.join(workdir, project.repo.name), ) except subprocess.CalledProcessError as e: print('WARNING: Could not run git describe: %s' % e) git_show( workdir=workdir, repo=project.repo.name, title='Check existing tags', ref=project.hash, ) git_list_existing_branches( workdir=workdir, repo=project.repo.name, ) branches = git_branch_contains( workdir=workdir, repo=project.repo.name, title='Branches containing commit', commit=project.hash, ) header('Relationship to HEAD') if deliv.is_independent: if branches: tag_branch = branches[0] else: tag_branch = branch head_sha = gitutils.sha_for_tag( workdir, project.repo.name, tag_branch, ) print('HEAD of {} is {}'.format(tag_branch, head_sha)) else: if (branch in branches) or (not branches): tag_branch = branch else: tag_branch = branches[0] head_sha = gitutils.sha_for_tag( workdir, project.repo.name, tag_branch, ) print('HEAD of {} is {}'.format(tag_branch, head_sha)) requested_sha = gitutils.sha_for_tag( workdir, project.repo.name, project.hash, ) # If the sha for HEAD and the requested release don't # match, show any unreleased changes on the branch. We ask # git to give us the real SHA for the requested release in # case the deliverables file has the short version of the # hash. if head_sha == requested_sha: print('\nRequest releases from HEAD on %s' % tag_branch) else: git_log(workdir, project.repo.name, 'Release will NOT include', '%s..%s' % (requested_sha, head_sha), extra_args=['--format=%h %ci %s']) show_watched_queries(branch, project.repo.name) # Show any requirements changes in the upcoming release. # Include setup.cfg, in case the project uses "extras". if start_range: git_diff(workdir, project.repo.name, git_range, '*requirements*.txt', 'Requirements Changes %s' % git_range) git_diff(workdir, project.repo.name, git_range, 'doc/requirements.txt', 'Doc Requirements Changes %s' % git_range) git_diff(workdir, project.repo.name, git_range, 'setup.cfg', 'setup.cfg Changes %s' % git_range) git_diff(workdir, project.repo.name, git_range, 'bindep.txt', 'bindep.txt Changes %s' % git_range) # Before we try to determine if the previous release # is an ancestor or produce the release notes we need # the tag to exist in the local repository. if not tag_exists: header('Applying Temporary Tag') print('\ngit tag {version} {hash}'.format( version=new_release.version, hash=project.hash, )) subprocess.check_call( ['git', 'tag', new_release.version, project.hash], cwd=os.path.join(workdir, project.repo.name), ) # Show any changes in the previous release but not in this # release, in case someone picks an "early" SHA or a # regular commit instead of the appropriate merge commit. previous_tag_exists = False if previous_release: previous_tag_exists = gitutils.tag_exists( project.repo.name, previous_release.version, ) if previous_tag_exists: git_log( workdir, project.repo.name, 'Patches in previous release but not in this one', [previous_release.version, '--not', project.hash], extra_args=['--topo-order', '--oneline', '--no-merges'], ) # The tag will have been added as a local tag above if # it does not already exist. header('New release %s includes previous release %s' % (new_release.version, previous_release.version)) print('\ngit tag --contains %s\n' % previous_release.version) containing_tags = subprocess.check_output( ['git', 'tag', '--contains', previous_release.version], cwd=os.path.join(workdir, project.repo.name), ).decode('utf-8').split() print('Containing tags:', containing_tags) if new_release.version not in containing_tags: print('WARNING: Missing %s' % new_release.version) else: print('Found new version %s' % new_release.version) is_ancestor = gitutils.check_ancestry( workdir, project.repo.name, previous_release.version, project.hash, ) if is_ancestor: print('SHA found in descendants') else: print('SHA NOT FOUND in descendants') # Show the changes since the last release, first as a # graph view so we can check for bad merges, and then with # more detail. git_log(workdir, project.repo.name, 'Release %s will include' % new_release.version, git_range, extra_args=['--graph', '--oneline', '--decorate', '--topo-order']) git_log(workdir, project.repo.name, 'Details Contents', git_range, extra_args=['--no-merges', '--topo-order']) # The tag will have been added as a local tag above if it does # not already exist. header('Release Notes') try: notes = release_notes.generate_release_notes( repo=project.repo.name, repo_path=os.path.join(workdir, project.repo.name), start_revision=new_release.diff_start or start_range, end_revision=new_release.version, show_dates=True, skip_requirement_merges=True, is_stable=branch.startswith('stable/'), series=deliv.series, email='*****@*****.**', email_from='*****@*****.**', email_reply_to='*****@*****.**', email_tags='', include_pypi_link=False, changes_only=False, first_release=deliv.is_first_release, repo_name=project.repo.name, description='', publishing_dir_name=project.repo.name, ) except Exception as e: logging.exception('Failed to produce release notes') else: print('\n') print(notes) if 'library' in deliv.type: show_dependency_listings( project.guess_sdist_name(), official_repos, ) return 0