def _initialize_deliverable_data(app): global _deliverables series_status_data = series_status.SeriesStatus.from_directory( 'deliverables') deliverable.Deliverable.init_series_status_data(series_status_data) _deliverables = deliverable.Deliverables('deliverables')
def main(): parser = argparse.ArgumentParser() parser.add_argument( 'deliverable', help='the name of the deliverable, such as "nova" or "oslo.config"', ) parser.add_argument( '--series', default=defaults.RELEASE, help='the release series, such as "newton" or "ocata"', ) parser.add_argument( '--deliverables-dir', default=openstack_releases.deliverable_dir, help='location of deliverable files', ) args = parser.parse_args() # Deal with the inconsistency of the name for the independent # directory. series = args.series if series == 'independent': series = '_independent' all_deliv = deliverable.Deliverables( root_dir=args.deliverables_dir, collapse_history=False, ) for entry in all_deliv.get_deliverable_history(args.deliverable): deliv = deliverable.Deliverable(*entry) print(deliv.team) break
def main(): parser = argparse.ArgumentParser() parser.add_argument( 'deliverable', help='the name of the deliverable, such as "nova" or "oslo.config"', ) parser.add_argument( '--deliverables-dir', default=openstack_releases.deliverable_dir, help='location of deliverable files', ) parser.add_argument( '--file', default=False, action='store_true', help='deliverable arg is a file path rather than a std. deliverable') args = parser.parse_args() # If we've been told the 'deliverable' is infact a yaml file *or* the # deliverable contains a '/' just load that file directly if args.file or '/' in args.deliverable: deliv = deliverable.Deliverable.read_file(args.deliverable) else: all_deliv = deliverable.Deliverables( root_dir=args.deliverables_dir, collapse_history=False, ) deliv = next(all_deliv.get_deliverable_history(args.deliverable)) print(deliv.team)
def _initialize_deliverable_data(): global _deliverables global _series_status_data LOG.info('Loading deliverable data...') _series_status_data = series_status.SeriesStatus.default() deliverable.Deliverable.init_series_status_data(_series_status_data) _deliverables = deliverable.Deliverables('deliverables')
def _initialize_team_data(app): global _deliverables global _all_teams _deliverables = deliverable.Deliverables('deliverables') team_data = governance.get_team_data() for tn, td in team_data.items(): _all_teams[tn] = td for dn, dd in td['deliverables'].items(): _all_deliverable_types[dn] = _get_deliverable_type(dn, dd)
def main(): parser = argparse.ArgumentParser() parser.add_argument( 'old_series', help='the previous release series, such as "newton"', ) parser.add_argument( 'new_series', help='the new release series, such as "ocata"', ) parser.add_argument( '--deliverables-dir', default=openstack_releases.deliverable_dir, help='location of deliverable files', ) args = parser.parse_args() all_deliv = deliverable.Deliverables( root_dir=args.deliverables_dir, collapse_history=False, ) new_deliverables = set( deliv.name for deliv in all_deliv.get_deliverables(None, args.new_series) ) outdir = os.path.join(args.deliverables_dir, args.new_series) if not os.path.exists(outdir): print('creating output directory {}'.format(outdir)) os.mkdir(outdir) old_deliverables = all_deliv.get_deliverables(None, args.old_series) for deliv in old_deliverables: if deliv.name in new_deliverables: continue if not deliv.is_released and not deliv.branches: # There were no releases for the deliverable in the # previous series, stop carrying it over. print('{} skipped (no releases in {})'.format( deliv.name, args.old_series)) continue # Clean up some series-specific data that should not be copied # over. raw_data = deliv.data for key in IGNORE: if key in raw_data: del raw_data[key] outfilename = os.path.abspath( os.path.join(outdir, deliv.name + '.yaml') ) with open(outfilename, 'w', encoding='utf-8') as f: print('{} created'.format(outfilename)) f.write(yamlutils.dumps(raw_data))
def repositories_list(deliverables_dir, series): """Yields (team, repo) tuples for cycle-with-milestones deliverables""" deliverables = deliverable.Deliverables(deliverables_dir) for d in deliverables.get_deliverables(None, series): if d.model != 'cycle-with-milestones': continue if not d.repos: print('WARNING: no releases for {} in {}'.format(d.name, d.series)) for repo in sorted(d.repos): if repo not in EXCEPTIONS: yield (d.team, repo)
def main(): parser = argparse.ArgumentParser() parser.add_argument( '--deliverables-dir', default=openstack_releases.deliverable_dir, help='location of deliverable files', ) parser.add_argument( '--series', default=defaults.RELEASE, help='the release series, such as "newton" or "ocata"', ) args = parser.parse_args() all_deliv = deliverable.Deliverables( root_dir=args.deliverables_dir, collapse_history=False, ) interesting_deliverables = [ d for d in ( deliverable.Deliverable(t, s, dn, da) for t, s, dn, da in all_deliv.get_deliverables(None, args.series)) if d.model == MILESTONE ] team_data = governance.get_team_data() teams = {n.lower(): governance.Team(n, i) for n, i in team_data.items()} # Dump the dashboard data writer = csv.writer(sys.stdout) writer.writerow( ('Team', 'Deliverable Type', 'Deliverable Name', 'Pre-RC1', 'RC1', 'Branched at', 'Latest RC', 'Release Notes', 'Comments', 'PTL Nick', 'PTL Email', 'Liaison Nick', 'IRC Channel')) for deliv in sorted(interesting_deliverables, key=lambda x: (x.team, x.name)): team = teams[deliv.team.lower()] writer.writerow(( deliv.team.lower(), deliv.type, deliv.name, deliv.latest_release, '', # RC1 deliv.get_branch_location('stable/' + args.series), # branched at '', # latest RC deliv.release_notes, '', # Comments team.data['ptl']['irc'], team.data['ptl']['email'], team.liaison[1] or '', team.data.get('irc-channel')))
def main(): parser = argparse.ArgumentParser() parser.add_argument( 'old_series', help='the previous release series, such as "newton"', ) parser.add_argument( 'new_series', help='the new release series, such as "ocata"', ) parser.add_argument( '--deliverables-dir', default=openstack_releases.deliverable_dir, help='location of deliverable files', ) args = parser.parse_args() all_deliv = deliverable.Deliverables( root_dir=args.deliverables_dir, collapse_history=False, ) new_deliverables = set( name for team, series, name, data in all_deliv.get_deliverables(None, args.new_series) ) outdir = os.path.join(args.deliverables_dir, args.new_series) if not os.path.exists(outdir): print('creating output directory {}'.format(outdir)) os.mkdir(outdir) old_deliverables = all_deliv.get_deliverables(None, args.old_series) for team, series, name, data in old_deliverables: if name in new_deliverables: continue if not data.get('releases'): # There were no releases for the deliverable in the # previous series, stop carrying it over. print('{} skipped (no releases in {})'.format(name, args.old_series)) continue # Clean up some series-specific data that should not be copied # over. for key in ['releases', 'branches', 'release-notes']: if key in data: del data[key] outfilename = os.path.join(outdir, name + '.yaml') with open(outfilename, 'w') as f: print('{} created'.format(outfilename)) f.write('---\n') yaml.dump(data, f, default_flow_style=False)
def main(): parser = argparse.ArgumentParser() parser.add_argument( 'deliverable', help='the name of the deliverable, such as "nova" or "oslo.config"', ) parser.add_argument( '--deliverables-dir', default=openstack_releases.deliverable_dir, help='location of deliverable files', ) args = parser.parse_args() all_deliv = deliverable.Deliverables( root_dir=args.deliverables_dir, collapse_history=False, ) for deliv in all_deliv.get_deliverable_history(args.deliverable): print(deliv.team) break
def main(): parser = argparse.ArgumentParser() parser.add_argument( '--no-cleanup', dest='cleanup', default=True, action='store_false', help='do not remove temporary files', ) parser.add_argument( '--all', default=False, action='store_true', help='process all deliverables, including cycle-trailing ones', ) parser.add_argument( '--verbose', '-v', action='store_true', default=False, help='produce detailed output', ) parser.add_argument( '--canary', action='store_true', default=False, help='process only a canary final release on release-test', ) parser.add_argument( '--deliverables-dir', default=openstack_releases.deliverable_dir, help='location of deliverable files', ) parser.add_argument( 'prior_series', help='the name of the previous series', ) parser.add_argument( 'series', help='the name of the release series to work on' ) args = parser.parse_args() if args.verbose: def verbose(msg): print(msg) else: def verbose(msg): pass # Set up logging, including making some loggers quiet. logging.basicConfig( format='%(levelname)7s: %(message)s', stream=sys.stdout, level=logging.DEBUG if args.verbose else logging.INFO, ) logging.getLogger('urllib3.connectionpool').setLevel(logging.WARNING) deliverables_dir = args.deliverables_dir workdir = tempfile.mkdtemp(prefix='releases-') print('creating temporary files in %s' % workdir) def cleanup_workdir(): if args.cleanup: try: shutil.rmtree(workdir) except Exception: pass else: print('not cleaning up %s' % workdir) atexit.register(cleanup_workdir) verbose('Scanning {}/{}'.format(deliverables_dir, args.series)) all_deliv = deliverable.Deliverables( root_dir=args.deliverables_dir, collapse_history=False, ) for deliv in all_deliv.get_deliverables(None, args.series): if args.canary and deliv.name != "release-test": continue verbose('\n{} {}'.format(deliv.name, deliv.model)) if (deliv.model == 'cycle-trailing' or deliv.type == 'trailing'): verbose('# {} is a cycle-trailing project'.format( deliv.name)) if not args.all: continue if not deliv.releases: verbose('# no releases') continue latest_release = deliv.releases[-1] projects = latest_release.projects if not projects: verbose('# no projects in latest release') continue for pre_rel in ['a', 'b', 'rc']: if pre_rel in str(latest_release.version): break else: # we did not find any pre_rel verbose('# {} was not a release candidate'.format( latest_release.version)) continue # The new version is the same as the latest release version # without the pre-release component at the end. Make sure it # has 3 sets of digits. new_version = '.'.join( (latest_release.version.split('.')[:-1] + ['0'])[:3] ) branch = 'stable/{}'.format(args.prior_series) diff_start = get_prior_branch_point( workdir, projects[0].repo.name, branch, ) deliverable_data = deliv.data release_data = { 'version': new_version, 'projects': deliv.data['releases'][-1]['projects'], } if diff_start: release_data['diff-start'] = diff_start deliverable_data['releases'].append(release_data) print('new version for {}: {}'.format( deliv.name, new_version)) filename = os.path.join(deliverables_dir, deliv.filename) with open(filename, 'w', encoding='utf-8') as f: f.write(yamlutils.dumps(deliverable_data))
def main(): parser = argparse.ArgumentParser() output_mode = parser.add_mutually_exclusive_group() output_mode.add_argument( '-v', '--verbose', action='store_true', default=False, help='show more than the deliverable name', ) output_mode.add_argument( '-r', '--repos', action='store_true', default=False, help='show the repository names not deliverable names', ) parser.add_argument( '--team', help='the name of the project team, such as "Nova" or "Oslo"', ) parser.add_argument( '--deliverable', help='the name of the deliverable, such as "nova" or "oslo.config"', ) parser.add_argument( '--series', default=defaults.RELEASE, help='the release series, such as "newton" or "ocata"', ) parser.add_argument( '--csvfile', help='Save results (same as when --verbose) to CSV file', ) model = parser.add_mutually_exclusive_group() model.add_argument( '--model', help=('the release model, such as "cycle-with-milestones"' ' or "independent"'), ) model.add_argument( '--cycle-based', action='store_true', default=False, help='include all cycle-based code repositories', ) parser.add_argument( '--type', help='deliverable type, such as "library" or "service"', ) parser.add_argument( '--tag', default=[], action='append', help='look for one more more tags on the deliverable or team', ) parser.add_argument( '--deliverables-dir', default=openstack_releases.deliverable_dir, help='location of deliverable files', ) parser.add_argument( '--no-stable-branch', default=False, action='store_true', help='limit the list to deliverables without a stable branch', ) grp = parser.add_mutually_exclusive_group() grp.add_argument( '--unreleased', default=False, action='store_true', help='limit the list to deliverables not released in the cycle', ) grp.add_argument( '--missing-milestone', help=('deliverables that do not have the specified milestone as ' 'the most current release; for example 2 would look for .0b2 ' 'in the version number (implies --model cycle-with-milestones)'), ) grp.add_argument( '--missing-rc', action='store_true', help=('deliverables that do not have a release candidate, yet ' '(implies --model cycle-with-milestones)'), ) grp.add_argument( '--missing-final', action='store_true', help='deliverables that have pre-releases but no final releases, yet', ) args = parser.parse_args() # Deal with the inconsistency of the name for the independent # directory. series = args.series if series == 'independent': series = '_independent' if args.missing_milestone: model = 'cycle-with-milestones' version_ending = '.0b{}'.format(args.missing_milestone) elif args.missing_rc: model = 'cycle-with-milestones' version_ending = None elif args.missing_final: model = args.model version_ending = None else: model = args.model version_ending = None verbose_template = '{name:30} {team:20}' if not args.unreleased: verbose_template += ' {latest_release:15}' if not args.type: verbose_template += ' {type:15}' if not args.model: verbose_template += ' {model:15}' csvfile = None if args.csvfile: csvfile = open(args.csvfile, 'w') fieldnames = [ 'name', 'latest_release', 'repo', 'hash', 'team', 'type', 'model' ] writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() all_deliv = deliverable.Deliverables( root_dir=args.deliverables_dir, collapse_history=False, ) for entry in all_deliv.get_deliverables(args.team, series): deliv = deliverable.Deliverable(*entry) if args.deliverable and deliv.name != args.deliverable: continue if model and deliv.model != model: continue if args.cycle_based and not deliv.is_cycle_based: continue if args.type and deliv.type != args.type: continue if args.no_stable_branch: if deliv.get_branch_location('stable/' + series) is not None: continue if args.unreleased and deliv.versions: continue if version_ending and deliv.latest_release and deliv.latest_release.endswith( version_ending): continue if args.missing_rc and deliv.latest_release and 'rc' in deliv.latest_release: continue if args.tag: tags = deliv.tags for t in args.tag: if t not in tags: continue if args.missing_final and deliv.latest_release: if not ('rc' in deliv.latest_release or 'a' in deliv.latest_release or 'b' in deliv.latest_release): continue if csvfile: rel = (deliv.releases or [{}])[-1] for prj in rel.get('projects', [{}]): writer.writerow({ 'name': deliv.name, 'latest_release': rel.get('version', None), 'repo': prj.get('repo', None), 'hash': prj.get('hash', None), 'team': deliv.team, 'type': deliv.type, 'model': deliv.model, }) elif args.verbose: print( verbose_template.format( name=deliv.name, latest_release=deliv.latest_release, team=deliv.team, type=deliv.type, model=deliv.model, )) elif args.repos: for r in sorted(deliv.repos): print(r) else: print(deliv.name) if csvfile: csvfile.close()
def main(): parser = argparse.ArgumentParser() parser.add_argument( '--deliverables-dir', default=openstack_releases.deliverable_dir, help='location of deliverable files', ) parser.add_argument( '--no-cleanup', dest='cleanup', default=True, action='store_false', help='do not remove temporary files', ) parser.add_argument( 'repository_cache', help='location of existing copies of repositories', ) parser.add_argument( 'series', help='the release series, such as "newton" or "ocata"', ) parser.add_argument( 'deliverable', nargs='+', help='the deliverable name', ) args = parser.parse_args() workdir = tempfile.mkdtemp(prefix='releases-') def cleanup_workdir(): if args.cleanup: shutil.rmtree(workdir, True) atexit.register(cleanup_workdir) branch_name = 'origin/stable/' + args.series all_deliv = deliverable.Deliverables( root_dir=args.deliverables_dir, collapse_history=False, ) for deliv in all_deliv.get_deliverables(None, args.series): if deliv.name not in args.deliverable: continue if deliv.get_branch_location(branch_name) is not None: # the branch is already defined for this project sys.stderr.write('{} already has a branch {}\n'.format( deliv.name, branch_name)) continue # We're only importing stable branches, and those are # specified by the version number. We therefore only need one # repository, and it shouldn't matter which one. That said, we # might not actually find the branch in the first repo so loop # until we do. for r in deliv.repos: reporoot = os.path.join(args.repository_cache, r) version = _get_branch_base(reporoot, branch_name) if version: print(deliv.name, args.series, version) break else: sys.stderr.write('could not find {} in any repos for {}\n'.format( branch_name, deliv.name))
def main(): parser = argparse.ArgumentParser() parser.add_argument( '-v', '--verbose', action='store_true', default=False, help='show more than the deliverable name', ) parser.add_argument( '--team', help='the name of the project team, such as "Nova" or "Oslo"', ) parser.add_argument( '--deliverable', help='the name of the deliverable, such as "nova" or "oslo.config"', ) parser.add_argument( '--series', default=defaults.RELEASE, help='the release series, such as "newton" or "ocata"', ) model = parser.add_mutually_exclusive_group() model.add_argument( '--model', help=('the release model, such as "cycle-with-milestones"' ' or "independent"'), ) model.add_argument( '--cycle-based', action='store_true', default=False, help='include all cycle-based code repositories', ) parser.add_argument( '--type', help='deliverable type, such as "library" or "service"', ) parser.add_argument( '--deliverables-dir', default=openstack_releases.deliverable_dir, help='location of deliverable files', ) parser.add_argument( '--branch', default=None, help='branch name, defaults to stable/$series', ) parser.add_argument( '--no-cleanup', dest='cleanup', default=True, action='store_false', help='do not remove temporary files', ) args = parser.parse_args() if args.verbose: def verbose(msg): print(msg) else: def verbose(msg): pass # Deal with the inconsistency of the name for the independent # directory. series = args.series if series == 'independent': series = '_independent' branch = args.branch if not branch: branch = 'stable/{}'.format(series) workdir = tempfile.mkdtemp(prefix='releases-') verbose('creating temporary files in {}'.format(workdir)) def cleanup_workdir(): if args.cleanup: try: verbose('cleaning up temporary files in {}'.format(workdir)) shutil.rmtree(workdir) except: pass else: print('not cleaning up {}'.format(workdir)) atexit.register(cleanup_workdir) # Count any errors for our exit code. errors = 0 all_deliv = deliverable.Deliverables( root_dir=args.deliverables_dir, collapse_history=False, ) for entry in all_deliv.get_deliverables(args.team, series): deliv = deliverable.Deliverable(*entry) branch_loc = deliv.get_branch_location(branch) if branch_loc is None: verbose('No stable branch for {}'.format(deliv.name)) continue all_versions = deliv.versions if all_versions[-1] == branch_loc: verbose('Most recent release for {} ({}) is at {}'.format( deliv.name, branch_loc, branch)) continue idx = all_versions.index(branch_loc) late_releases = all_versions[idx + 1:] print('{} releases {} come after {}'.format(deliv.name, late_releases, branch)) for repo in sorted(deliv.repos): verbose('cloning {}'.format(repo)) gitutils.clone_repo( workdir, repo, ) for version in late_releases: containing_br = gitutils.branches_containing( workdir, repo, version, ) for cb in containing_br: if branch in cb: # allow for remote prefix verbose('{} version {} is on branch {}'.format( repo, version, branch)) break else: print('{} version {} is not on branch {} ({})'.format( repo, version, branch, containing_br)) errors += 1 return (1 if errors else 0)
def main(): if not sys.stdout.encoding: # Wrap sys.stdout with a writer that knows how to handle # encoding Unicode data. import codecs wrapped_stdout = codecs.getwriter('UTF-8')(sys.stdout) sys.stdout = wrapped_stdout parser = argparse.ArgumentParser() parser.add_argument( '--no-cleanup', dest='cleanup', default=True, action='store_false', help='do not remove temporary files', ) parser.add_argument( '--no-shortcut', '--force', '-f', dest='shortcut', default=True, action='store_false', help='if a tag has been applied, skip the repo', ) parser.add_argument( 'input', nargs='*', help=('YAML files to validate, defaults to ' 'files changed in the latest commit'), ) args = parser.parse_args() # Set up logging, including making some loggers quiet. logging.basicConfig( format='%(levelname)7s: %(message)s', stream=sys.stdout, level=logging.DEBUG, ) logging.getLogger('urllib3.connectionpool').setLevel(logging.WARNING) filenames = args.input or gitutils.find_modified_deliverable_files() if not filenames: print('no modified deliverable files, skipping report') return 0 workdir = tempfile.mkdtemp(prefix='releases-') print('creating temporary files in %s' % workdir) def cleanup_workdir(): if args.cleanup: shutil.rmtree(workdir, True) else: print('not cleaning up %s' % workdir) atexit.register(cleanup_workdir) gov_data = governance.Governance.from_remote_repo() official_repos = set(r.name for r in gov_data.get_repositories()) all_deliverables = deliverable.Deliverables( './deliverables', False, ) liaison_data = liaisons.get_liaisons() # Remove any inherited PAGER environment variable to avoid # blocking the output waiting for input. os.environ['PAGER'] = '' for filename in filenames: if not os.path.exists(filename): print('\n%s was removed, skipping' % filename) continue print('\n' + ('=' * 80)) print('\nChecking %s\n' % filename) deliv = deliverable.Deliverable.read_file(filename) stable_branch = deliv.series not in ['independent', defaults.RELEASE] # By default assume the project does not use milestones. header('Release model') print(deliv.model) header('Team details') if deliv.team: team_name = deliv.team try: team = gov_data.get_team(team_name) except ValueError: team = None if team: print('found team %s' % team_name) print(' PTL : %(name)s (%(irc)s)' % team.ptl) for liaison in liaison_data.get(team.name.lower(), []): print(' Liaison: %(name)s (%(irc)s)' % liaison) team_deliv = team.deliverables.get(deliv.name) if team_deliv: print('found deliverable %s' % deliv.name) for rn, repo in sorted(team_deliv.repositories.items()): follows_stable_policy = 'stable:follows-policy' in repo.tags print('\nrepo %s\ntags:' % repo.name) for t in repo.tags: print(' %s' % t) print('') if stable_branch and follows_stable_policy: banner('Needs Stable Policy Review') print() else: print(('no deliverable %r found for team %r, ' 'cannot report on governance status') % (deliv.name, team_name)) if not deliv.is_independent: # Show other deliverables owned by the team and # included in this series. team_deliv_in_series = all_deliverables.get_deliverables( team.name, deliv.series) if team_deliv_in_series: print('Other {} deliverables in {}:'.format( team.name, deliv.series)) for d in team_deliv_in_series: print(' {} ({})'.format(d.name, d.latest_release or None)) print() else: print('no team %r found, cannot report on governance status' % team_name) else: print('no team name given, cannot report on governance status') # If there are no releases listed, this is probably a new # deliverable file for initializing a new series. We don't # need to list its changes. if not deliv.is_released: header('No releases') print('no releases were found, assuming an initialization file') continue # assume the releases are in order and take the last one new_release = deliv.releases[-1] for project in new_release.projects: tag_exists = gitutils.tag_exists( project.repo.name, new_release.version, ) if tag_exists: print('%s %s exists on git server already' % (project.repo.name, new_release.version)) if args.shortcut: print('skipping further processing') continue if project.repo.is_retired: print('%s is retired' % (project.repo.name, )) if args.shortcut: print('skipping further processing') continue # Start by checking out master, always. We need the repo # checked out before we can tell if the stable branch # really exists. gitutils.clone_repo( workdir, project.repo.name, branch='master', ) # Set some git configuration values to allow us to perform # local operations like tagging. gitutils.ensure_basic_git_config( workdir, project.repo.name, { 'user.email': '*****@*****.**', 'user.name': 'OpenStack Proposal Bot' }, ) # Determine which branch we should actually be looking # at. Assume any series for which there is no stable # branch will be on 'master'. if gitutils.stable_branch_exists(workdir, project.repo.name, deliv.series): branch = 'stable/' + deliv.series else: branch = 'master' if branch != 'master': # Check out the repo again to the right branch if we # didn't get it the first time. gitutils.clone_repo( workdir, project.repo.name, branch=branch, ) # look at the previous tag for the parent of the commit # getting the new release previous_tag = gitutils.get_latest_tag( workdir, project.repo.name, '{}^'.format(project.hash), always=False, ) try: previous_release = deliv.get_release(previous_tag) except ValueError: previous_release = None start_range = previous_tag if previous_release: previous_project = { x.repo.name: x for x in previous_release.projects }.get(project.repo.name) if previous_project is not None: start_range = previous_tag if start_range: git_range = '%s..%s' % (start_range, project.hash) else: git_range = project.hash # Show details about the commit being tagged. header('Details for commit receiving new tag %s' % new_release.version) print('\ngit describe %s\n' % project.hash) try: subprocess.check_call( ['git', 'describe', project.hash], cwd=os.path.join(workdir, project.repo.name), ) except subprocess.CalledProcessError as e: print('WARNING: Could not run git describe: %s' % e) git_show( workdir=workdir, repo=project.repo.name, title='Check existing tags', ref=project.hash, ) git_list_existing_branches( workdir=workdir, repo=project.repo.name, ) branches = git_branch_contains( workdir=workdir, repo=project.repo.name, title='Branches containing commit', commit=project.hash, ) header('Relationship to HEAD') if deliv.is_independent: if branches: tag_branch = branches[0] else: tag_branch = branch head_sha = gitutils.sha_for_tag( workdir, project.repo.name, tag_branch, ) print('HEAD of {} is {}'.format(tag_branch, head_sha)) else: if (branch in branches) or (not branches): tag_branch = branch else: tag_branch = branches[0] head_sha = gitutils.sha_for_tag( workdir, project.repo.name, tag_branch, ) print('HEAD of {} is {}'.format(tag_branch, head_sha)) requested_sha = gitutils.sha_for_tag( workdir, project.repo.name, project.hash, ) # If the sha for HEAD and the requested release don't # match, show any unreleased changes on the branch. We ask # git to give us the real SHA for the requested release in # case the deliverables file has the short version of the # hash. if head_sha == requested_sha: print('\nRequest releases from HEAD on %s' % tag_branch) else: git_log(workdir, project.repo.name, 'Release will NOT include', '%s..%s' % (requested_sha, head_sha), extra_args=['--format=%h %ci %s']) show_watched_queries(branch, project.repo.name) # Show any requirements changes in the upcoming release. # Include setup.cfg, in case the project uses "extras". if start_range: git_diff(workdir, project.repo.name, git_range, '*requirements*.txt', 'Requirements Changes %s' % git_range) git_diff(workdir, project.repo.name, git_range, 'doc/requirements.txt', 'Doc Requirements Changes %s' % git_range) git_diff(workdir, project.repo.name, git_range, 'setup.cfg', 'setup.cfg Changes %s' % git_range) git_diff(workdir, project.repo.name, git_range, 'bindep.txt', 'bindep.txt Changes %s' % git_range) # Before we try to determine if the previous release # is an ancestor or produce the release notes we need # the tag to exist in the local repository. if not tag_exists: header('Applying Temporary Tag') print('\ngit tag {version} {hash}'.format( version=new_release.version, hash=project.hash, )) subprocess.check_call( ['git', 'tag', new_release.version, project.hash], cwd=os.path.join(workdir, project.repo.name), ) # Show any changes in the previous release but not in this # release, in case someone picks an "early" SHA or a # regular commit instead of the appropriate merge commit. previous_tag_exists = False if previous_release: previous_tag_exists = gitutils.tag_exists( project.repo.name, previous_release.version, ) if previous_tag_exists: git_log( workdir, project.repo.name, 'Patches in previous release but not in this one', [previous_release.version, '--not', project.hash], extra_args=['--topo-order', '--oneline', '--no-merges'], ) # The tag will have been added as a local tag above if # it does not already exist. header('New release %s includes previous release %s' % (new_release.version, previous_release.version)) print('\ngit tag --contains %s\n' % previous_release.version) containing_tags = subprocess.check_output( ['git', 'tag', '--contains', previous_release.version], cwd=os.path.join(workdir, project.repo.name), ).decode('utf-8').split() print('Containing tags:', containing_tags) if new_release.version not in containing_tags: print('WARNING: Missing %s' % new_release.version) else: print('Found new version %s' % new_release.version) is_ancestor = gitutils.check_ancestry( workdir, project.repo.name, previous_release.version, project.hash, ) if is_ancestor: print('SHA found in descendants') else: print('SHA NOT FOUND in descendants') # Show the changes since the last release, first as a # graph view so we can check for bad merges, and then with # more detail. git_log(workdir, project.repo.name, 'Release %s will include' % new_release.version, git_range, extra_args=[ '--graph', '--oneline', '--decorate', '--topo-order' ]) git_log(workdir, project.repo.name, 'Details Contents', git_range, extra_args=['--no-merges', '--topo-order']) # The tag will have been added as a local tag above if it does # not already exist. header('Release Notes') try: notes = release_notes.generate_release_notes( repo=project.repo.name, repo_path=os.path.join(workdir, project.repo.name), start_revision=new_release.diff_start or start_range or '', end_revision=new_release.version, show_dates=True, skip_requirement_merges=True, is_stable=branch.startswith('stable/'), series=deliv.series, email='*****@*****.**', email_from='*****@*****.**', email_reply_to='*****@*****.**', email_tags='', include_pypi_link=False, changes_only=False, first_release=deliv.is_first_release, deliverable_file=filename, description='', publishing_dir_name=project.repo.name, ) except Exception as e: logging.exception('Failed to produce release notes') else: print('\n') print(notes) if 'library' in deliv.type: show_dependency_listings( project.guess_sdist_name(), official_repos, ) return 0
def main(): deliverable_schema = schema.Schema() parser = argparse.ArgumentParser() output_mode = parser.add_mutually_exclusive_group() output_mode.add_argument( '-v', '--verbose', action='store_true', default=False, help='show more than the deliverable name', ) output_mode.add_argument( '-r', '--repos', action='store_true', default=False, help='show the repository names not deliverable names', ) output_mode.add_argument( '-a', '--all-releases', action='store_true', default=False, help='show all of the releases for each deliverable', ) parser.add_argument( '--group-by', dest='group_key', default=None, choices=['team', 'type', 'model'], help='group output by the specified value', ) parser.add_argument( '--team', help='the name of the project team, such as "Nova" or "Oslo"', ) parser.add_argument( '--deliverable', help='the name of the deliverable, such as "nova" or "oslo.config"', ) parser.add_argument( '--series', default=defaults.RELEASE, help='the release series, such as "newton" or "ocata"', ) parser.add_argument( '--csvfile', help='Save results (same as when --verbose) to CSV file', ) parser.add_argument( '--show-dates', action='store_true', default=False, help='Show last release date (in verbose mode)', ) parser.add_argument( '--show-tags', action='store_true', default=False, help='Show tags associated with deliverable (in verbose mode)', ) model = parser.add_mutually_exclusive_group() model.add_argument( '--model', help=('the release model, such as "cycle-with-rc"' ' or "independent"'), choices=sorted(deliverable_schema.release_models + ['independent']), ) model.add_argument( '--cycle-based', action='store_true', default=False, help='include all cycle-based deliverables', ) model.add_argument( '--cycle-based-no-trailing', action='store_true', default=False, help='include all cycle-based deliverables, except trailing ones', ) parser.add_argument( '--type', default=[], action='append', choices=sorted(deliverable_schema.release_types), help='deliverable type, such as "library" or "service"', ) parser.add_argument( '--tag', default=[], action='append', help='look for one more more tags on the deliverable or team', ) parser.add_argument( '--deliverables-dir', default=openstack_releases.deliverable_dir, help='location of deliverable files', ) parser.add_argument( '--no-stable-branch', default=False, action='store_true', help='limit the list to deliverables without a stable branch', ) grp = parser.add_mutually_exclusive_group() grp.add_argument( '--unreleased', default=False, action='store_true', help='limit the list to deliverables not released in the cycle', ) grp.add_argument( '--unreleased-since', help=('limit the list to deliverables not released in the cycle ' 'since a given YYYY-MM-DD date'), ) grp.add_argument( '--missing-rc', action='store_true', help=('deliverables that do not have a release candidate, yet ' '(implies --model cycle-with-rc)'), ) grp.add_argument( '--missing-final', action='store_true', help='deliverables that have pre-releases but no final releases, yet', ) grp.add_argument( '--forced', action='store_true', help=('releases that have the "forced" flag applied ' '(implies --all-releases)'), ) args = parser.parse_args() series = args.series GET_REFS_API = 'https://opendev.org/api/v1/repos/{}/git/{}' GET_COMMIT_API = 'https://opendev.org/api/v1/repos/{}/git/commits/{}' if args.missing_rc: model = 'cycle-with-rc' version_ending = None elif args.missing_final: model = args.model version_ending = None else: model = args.model version_ending = None if args.unreleased_since: args.show_dates = True verbose_template = '{name:30} {team:20}' if not args.unreleased: verbose_template += ' {latest_release:12}' if args.show_dates: verbose_template += ' {last_release_date:11}' if len(args.type) != 1: verbose_template += ' {type:15}' if not args.model: verbose_template += ' {model:15}' if args.show_tags: verbose_template += ' {tags}' if args.forced: args.all_releases = True csvfile = None if args.csvfile: csvfile = open(args.csvfile, 'w') fieldnames = ['name', 'latest_release', 'repo', 'hash', 'team', 'type', 'model'] writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() all_deliv = deliverable.Deliverables( root_dir=args.deliverables_dir, collapse_history=False, ) deliv_iter = list(all_deliv.get_deliverables(args.team, series)) if args.group_key: deliv_iter = sorted(deliv_iter, key=operator.attrgetter(args.group_key)) name_fmt = ' {}' else: name_fmt = '{}' cur_group = None for deliv in deliv_iter: if args.group_key: deliv_group = getattr(deliv, args.group_key) if args.deliverable and deliv.name != args.deliverable: continue if model and deliv.model != model: continue if args.cycle_based and not deliv.is_cycle_based: continue if args.cycle_based_no_trailing and (not deliv.is_cycle_based or deliv.type == 'trailing'): continue if args.type and deliv.type not in args.type: continue if args.no_stable_branch: if deliv.is_branchless: continue if deliv.name == 'release-test': continue if deliv.stable_branch_type is None: continue if deliv.get_branch_location('stable/' + series) is not None: continue if args.unreleased and (deliv.is_released or not deliv.is_releasable): continue if version_ending and deliv.is_released: found = False for release in deliv.releases: if release.version.endswith(version_ending): found = True break if found: continue if args.missing_rc and deliv.is_released and 'rc' in deliv.latest_release: continue if args.tag: tags = deliv.tags ignore = False for t in args.tag: if t not in tags: ignore = True break if ignore: continue tag_str = '(' + ', '.join(deliv.tags) + ')' if args.missing_final and deliv.latest_release: if not ('rc' in deliv.latest_release or 'a' in deliv.latest_release or 'b' in deliv.latest_release): continue release_date = {} if (args.show_dates or args.unreleased_since) and deliv.is_released: if args.all_releases: versions = [a.version for a in deliv.releases] else: versions = [deliv.releases[-1].version] for ver in versions: ref = "refs/tags/{}".format(ver) api = GET_REFS_API.format(deliv.repos[0], ref) tagsjson = requests.get(api).json() # Gitea returns either a single tag object, or a list of # tag objects containing the provided string. So we need to # filter the list for the exact match. if isinstance(tagsjson, list): for release_tag in tagsjson: if release_tag['ref'] == ref: break else: release_tag = tagsjson release_sha = release_tag['object']['sha'] api = GET_COMMIT_API.format(deliv.repos[0], release_sha) release_commit = requests.get(api).json()['commit'] release_date[ver] = release_commit['author']['date'][0:10] if args.unreleased_since and deliv.is_released: if release_date[ver] >= args.unreleased_since: continue if csvfile: rel = (deliv.releases or [{}])[-1] for prj in rel.get('projects', [{}]): writer.writerow({ 'name': deliv.name, 'latest_release': rel.get('version', None), 'repo': prj.get('repo', None), 'hash': prj.get('hash', None), 'team': deliv.team, 'type': deliv.type, 'model': deliv.model, }) elif args.all_releases: for r in deliv.releases: if args.forced and not r.was_forced: continue print(verbose_template.format( name=deliv.name, latest_release=r.version, last_release_date=release_date.get(r.version, ''), team=deliv.team, type=deliv.type, model=deliv.model, tags=tag_str, )) elif args.verbose: print(verbose_template.format( name=deliv.name, latest_release=deliv.latest_release or '', last_release_date=release_date.get(deliv.latest_release, ''), team=deliv.team, type=deliv.type, model=deliv.model, tags=tag_str, )) elif args.repos: if args.group_key and cur_group != deliv_group: cur_group = deliv_group print(cur_group) for r in sorted(deliv.repos): print(name_fmt.format(r)) else: if args.group_key and cur_group != deliv_group: cur_group = deliv_group print(cur_group) print(name_fmt.format(deliv.name)) if csvfile: csvfile.close()
def main(): deliverable_schema = schema.Schema() parser = argparse.ArgumentParser() output_mode = parser.add_mutually_exclusive_group() output_mode.add_argument( '-v', '--verbose', action='store_true', default=False, help='show more than the deliverable name', ) output_mode.add_argument( '-r', '--repos', action='store_true', default=False, help='show the repository names not deliverable names', ) output_mode.add_argument( '-a', '--all-releases', action='store_true', default=False, help='show all of the releases for each deliverable', ) parser.add_argument( '--group-by', dest='group_key', default=None, choices=['team', 'type', 'model'], help='group output by the specified value', ) parser.add_argument( '--team', help='the name of the project team, such as "Nova" or "Oslo"', ) parser.add_argument( '--deliverable', help='the name of the deliverable, such as "nova" or "oslo.config"', ) parser.add_argument( '--series', default=defaults.RELEASE, help='the release series, such as "newton" or "ocata"', ) parser.add_argument( '--csvfile', help='Save results (same as when --verbose) to CSV file', ) model = parser.add_mutually_exclusive_group() model.add_argument( '--model', help=('the release model, such as "cycle-with-milestones"' ' or "independent"'), choices=sorted(deliverable_schema.release_models + ['independent']), ) model.add_argument( '--cycle-based', action='store_true', default=False, help='include all cycle-based code repositories', ) parser.add_argument( '--type', default=[], action='append', choices=sorted(deliverable_schema.release_types), help='deliverable type, such as "library" or "service"', ) parser.add_argument( '--tag', default=[], action='append', help='look for one more more tags on the deliverable or team', ) parser.add_argument( '--deliverables-dir', default=openstack_releases.deliverable_dir, help='location of deliverable files', ) parser.add_argument( '--no-stable-branch', default=False, action='store_true', help='limit the list to deliverables without a stable branch', ) grp = parser.add_mutually_exclusive_group() grp.add_argument( '--unreleased', default=False, action='store_true', help='limit the list to deliverables not released in the cycle', ) grp.add_argument( '--missing-milestone', help=('deliverables that do not have the specified milestone as ' 'the most current release; for example 2 would look for .0b2 ' 'in the version number (implies --model cycle-with-milestones)'), ) grp.add_argument( '--missing-rc', action='store_true', help=('deliverables that do not have a release candidate, yet ' '(implies --model cycle-with-milestones)'), ) grp.add_argument( '--missing-final', action='store_true', help='deliverables that have pre-releases but no final releases, yet', ) args = parser.parse_args() series = args.series if args.missing_milestone: model = 'cycle-with-milestones' version_ending = '.0b{}'.format(args.missing_milestone) elif args.missing_rc: model = 'cycle-with-milestones' version_ending = None elif args.missing_final: model = args.model version_ending = None else: model = args.model version_ending = None verbose_template = '{name:30} {team:20}' if not args.unreleased: verbose_template += ' {latest_release:15}' if len(args.type) != 1: verbose_template += ' {type:15}' if not args.model: verbose_template += ' {model:15}' verbose_template += ' {tags}' csvfile = None if args.csvfile: csvfile = open(args.csvfile, 'w') fieldnames = [ 'name', 'latest_release', 'repo', 'hash', 'team', 'type', 'model' ] writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() all_deliv = deliverable.Deliverables( root_dir=args.deliverables_dir, collapse_history=False, ) deliv_iter = list(all_deliv.get_deliverables(args.team, series)) if args.group_key: deliv_iter = sorted(deliv_iter, key=operator.attrgetter(args.group_key)) name_fmt = ' {}' else: name_fmt = '{}' cur_group = None for deliv in deliv_iter: if args.group_key: deliv_group = getattr(deliv, args.group_key) if args.deliverable and deliv.name != args.deliverable: continue if model and deliv.model != model: continue if args.cycle_based and not deliv.is_cycle_based: continue if args.type and deliv.type not in args.type: continue if args.no_stable_branch: if deliv.get_branch_location('stable/' + series) is not None: continue if args.unreleased and (deliv.is_released or not deliv.is_releasable): continue if version_ending and deliv.is_released and deliv.latest_release.endswith( version_ending): continue if args.missing_rc and deliv.is_released and 'rc' in deliv.latest_release: continue if args.tag: tags = deliv.tags ignore = False for t in args.tag: if t not in tags: ignore = True break if ignore: continue tag_str = '(' + ', '.join(deliv.tags) + ')' if args.missing_final and deliv.latest_release: if not ('rc' in deliv.latest_release or 'a' in deliv.latest_release or 'b' in deliv.latest_release): continue if csvfile: rel = (deliv.releases or [{}])[-1] for prj in rel.get('projects', [{}]): writer.writerow({ 'name': deliv.name, 'latest_release': rel.get('version', None), 'repo': prj.get('repo', None), 'hash': prj.get('hash', None), 'team': deliv.team, 'type': deliv.type, 'model': deliv.model, }) elif args.all_releases: for r in deliv.releases: print( verbose_template.format( name=deliv.name, latest_release=r.get('version', ''), team=deliv.team, type=deliv.type, model=deliv.model, tags=tag_str, )) elif args.verbose: print( verbose_template.format( name=deliv.name, latest_release=deliv.latest_release or '', team=deliv.team, type=deliv.type, model=deliv.model, tags=tag_str, )) elif args.repos: if args.group_key and cur_group != deliv_group: cur_group = deliv_group print(cur_group) for r in sorted(deliv.repos): print(name_fmt.format(r)) else: if args.group_key and cur_group != deliv_group: cur_group = deliv_group print(cur_group) print(name_fmt.format(deliv.name)) if csvfile: csvfile.close()
def _initialize_deliverable_data(app): global _deliverables _deliverables = deliverable.Deliverables('deliverables')
def main(): parser = argparse.ArgumentParser() parser.add_argument( '--verbose', '-v', action='store_true', default=False, ) parser.add_argument( '--deliverables-dir', default=openstack_releases.deliverable_dir, help='location of deliverable files', ) args = parser.parse_args() config_filename = os.path.join( appdirs.user_config_dir('openstack-release', 'openstack'), 'gerrit.ini', ) config = configparser.ConfigParser() config.read(config_filename, encoding='utf-8') if not config.has_option('DEFAULT', 'username'): parser.error('No username set in {}'.format(config_filename)) if not config.has_option('DEFAULT', 'password'): parser.error('No password set in {}'.format(config_filename)) team_data = governance.get_team_data() # Some deliverables were independent at one time but might not be # any more, so compare the independent list with the current # release series. all_independent_deliverables = set( name for team, series, name, deliv in deliverable.Deliverables( root_dir=args.deliverables_dir, collapse_history=True, ).get_deliverables(None, None)) current_deliverables = set( name for team, series, name, deliv in deliverable.Deliverables( root_dir=args.deliverables_dir, collapse_history=True, ).get_deliverables(None, defaults.RELEASE)) independent_deliverables = all_independent_deliverables.difference( current_deliverables) gerrit = GerritClient( config['DEFAULT']['username'], config['DEFAULT']['password'], ) for repo in governance.get_repositories(team_data, code_only=True): if repo.deliverable.team.name in IGNORED_TEAMS: if args.verbose: print('{}: ignoring {} team'.format( repo.name, repo.deliverable.team.name)) continue if repo.deliverable.name in independent_deliverables: if args.verbose: print('{}: ignoring independent deliverable'.format(repo.name)) continue acls = gerrit.get_access(repo.name) local_tag_acls = acls.get('local', {}).get('refs/tags/*', {}) if local_tag_acls: rules = local_tag_acls.get('permissions', {}).get('pushSignedTag', {}).get('rules', {}) if not rules and args.verbose: print('{}: OK'.format(repo.name)) for group_id, permissions in rules.items(): group_details = gerrit.get_group(group_id) group_name = group_details['name'] if group_name in ALLOWED: if args.verbose: print('{}: {} pushSignedTag OK'.format( repo.name, group_name)) continue if args.verbose: print('{}: {} pushSignedTag WARNING'.format( repo.name, group_name)) else: print('{}: {} pushSignedTag'.format(repo.name, group_name))