def find_change_log_section(change_log, version): # Find the first line that starts with '##'. Extract the version and # date from that line. The version must be the specified release # version OR the date must be the literal string 'unreleased'. # E.g.: ## 1.0.0 - unreleased change_log_header_re = r"^## (?P<version>.+) - (?P<date>.+)$" with change_log.open() as fp: for line_number, line in enumerate(fp): match = re.search(change_log_header_re, line) if match: found_version = match.group("version") found_date = match.group("date") if found_version == version: if found_date != "unreleased": printer.warning("Re-releasing", version) elif found_date == "unreleased": if found_version != version: printer.warning("Replacing", found_version, "with", version) else: msg = (f"Expected version {version} or release date " f'"unreleased"; got:\n\n {line}') abort(7, msg) return line_number abort(8, "Could not find section in change log")
def rmdir(path): if os.path.isdir(path): shutil.rmtree(path) else: printer.warning( 'Path does not exist or is not a directory: {path}'.format_map( locals()))
def rmdir(directory, quiet=False): if directory.is_dir(): shutil.rmtree(directory) if not quiet: printer.info('Removed directory:', directory) else: if not quiet: printer.warning('Directory does not exist:', directory)
def shell(): banner = 'byCycle Shell' try: import bpython except ImportError: printer.warning('bpython is not installed; falling back to python') code.interact(banner=banner) else: bpython.embed(banner=banner)
def rmdir(path): if os.path.isdir(path): shutil.rmtree(path) if not quiet: printer(f'Removed directory: {path}') else: if not quiet: printer.warning( f'Path does not exist or is not a directory: {path}')
def execute(engine, sql, condition=True): if not condition: return if isinstance(sql, (list, tuple)): sql = ' '.join(sql) printer.info('Running SQL:', sql) try: return engine.execute(sql) except ProgrammingError as exc: error = str(exc.orig) exc_str = str(exc) if 'already exists' in exc_str or 'does not exist' in exc_str: printer.warning(exc.statement.strip(), error.strip(), sep=': ') else: raise
def prepare_release(info): version = info.version print_step_header("Preparing release", version, "on", info.date) current_branch = get_current_branch() local(("git", "checkout", info.dev_branch)) if info.pyproject_file: quote = info.pyproject_version_quote update_line( info.pyproject_file, info.pyproject_version_line_number, f"version = {quote}{version}{quote}", ) if info.version_file: quote = info.version_quote update_line( info.version_file, info.version_line_number, f"__version__ = {quote}{version}{quote}", ) update_line( info.change_log, info.change_log_line_number, f"## {version} - {info.date}", ) commit_files = (info.pyproject_file, info.version_file, info.change_log) local(("git", "diff", *commit_files)) if info.confirmation_required: confirm("Commit these changes?", abort_on_unconfirmed=True) else: printer.warning("Committing changes") msg = f"Prepare {info.name} release {version}" msg = prompt("Commit message", default=msg) local(("git", "commit", commit_files, "-m", msg)) local(("git", "checkout", current_branch))
def merge_to_target_branch(info): print_step_header( "Merging", info.dev_branch, "into", info.target_branch, "for release", info.version, ) current_branch = get_current_branch() local(( "git", "log", "--oneline", "--reverse", f"{info.target_branch}..{info.dev_branch}", )) if info.confirmation_required: msg = (f"Merge these changes from {info.dev_branch} " f"into {info.target_branch} " f"for release {info.version}?") confirm(msg, abort_on_unconfirmed=True) else: printer.warning( "Merging changes from", info.dev_branch, "into", info.target_branch, "for release", info.release, ) local(("git", "checkout", info.target_branch)) msg = f"Merge branch '{info.dev_branch}' for {info.name} release {info.version}" msg = prompt("Commit message", default=msg) local(("git", "merge", "--no-ff", info.dev_branch, "-m", msg)) local(("git", "checkout", current_branch))
def virtualenv(config, where='.env', python='python3', overwrite=False): exists = os.path.exists(where) def create(): local(config, ('virtualenv', '-p', python, where)) printer.success( 'Virtualenv created; activate it by running `source {where}/bin/activate`' .format_map(locals())) if exists: if overwrite: printer.warning('Overwriting virtualenv', where, 'with', python) shutil.rmtree(where) create() else: printer.info('Virtualenv', where, 'exists; pass --overwrite to re-create it') else: printer.info('Creating virtualenv', where, 'with', python) create()
def clean(all_=False): """Clean up. - Remove build directory - Remove dist directory - Remove __pycache__ directories If --all: - Remove egg-info directory - Remove .venv directory """ def rmdir(directory, quiet=False): if directory.is_dir(): shutil.rmtree(directory) if not quiet: printer.info('Removed directory:', directory) else: if not quiet: printer.warning('Directory does not exist:', directory) cwd = Path.cwd() rmdir(Path('./build')) rmdir(Path('./dist')) pycache_dirs = tuple(Path('.').glob('**/__pycache__/')) num_pycache_dirs = len(pycache_dirs) for pycache_dir in pycache_dirs: rmdir(pycache_dir, quiet=True) if num_pycache_dirs: printer.info(f'Removed {num_pycache_dirs} __pycache__' f'director{"y" if num_pycache_dirs == 1 else "ies"}') else: printer.warning('No __pycache__ directories found') if all_: rmdir(cwd / '.venv') rmdir(cwd / f'{cwd.name}.egg-info')
def find_version_file(): # Try to find __version__ in: # # - package/__init__.py # - namespace_package/package/__init__.py # - src/package/__init__.py # - src/namespace_package/package/__init__.py cwd = pathlib.Path.cwd() candidates = [] candidates.extend(cwd.glob("*/__init__.py")) candidates.extend(cwd.glob("*/*/__init__.py")) candidates.extend(cwd.glob("src/*/__init__.py")) candidates.extend(cwd.glob("src/*/*/__init__.py")) for candidate in candidates: result = get_current_version(candidate, "__version__", False) if result is not None: return (candidate, ) + result candidates = "\n ".join(str(candidate) for candidate in candidates) printer.warning( f"Could not find file containing __version__; tried:\n {candidates}", ) return None
def release(config, version=None, date=None, tag_name=None, next_version=None, prepare=True, merge=True, create_tag=True, resume=True, yes=False): def update_line(file_name, line_number, content): with open(file_name) as fp: lines = fp.readlines() lines[line_number] = content with open(file_name, 'w') as fp: fp.writelines(lines) result = local(config, 'git rev-parse --abbrev-ref HEAD', hide='stdout') current_branch = result.stdout.strip() if current_branch != 'develop': abort(1, 'Must be on develop branch to make a release') init_module = 'runcommands/__init__.py' changelog = 'CHANGELOG' # E.g.: __version__ = '1.0.dev0' version_re = r"^__version__ = '(?P<version>.+)(?P<dev_marker>\.dev\d+)'$" # E.g.: ## 1.0.0 - 2017-04-01 changelog_header_re = r'^## (?P<version>.+) - (?P<date>.+)$' with open(init_module) as fp: for init_line_number, line in enumerate(fp): if line.startswith('__version__'): match = re.search(version_re, line) if match: current_version = match.group('version') if not version: version = current_version break else: abort( 1, 'Could not find __version__ in {init_module}'.format_map( locals())) date = date or datetime.date.today().isoformat() tag_name = tag_name or version if next_version is None: next_version_re = r'^(?P<major>\d+)\.(?P<minor>\d+)(?P<rest>.*)$' match = re.search(next_version_re, version) if match: major = match.group('major') minor = match.group('minor') major = int(major) minor = int(minor) rest = match.group('rest') patch_re = r'^\.(?P<patch>\d+)$' match = re.search(patch_re, rest) if match: # X.Y.Z minor += 1 patch = match.group('patch') next_version = '{major}.{minor}.{patch}'.format_map(locals()) else: pre_re = r'^(?P<pre_marker>a|b|rc)(?P<pre_version>\d+)$' match = re.search(pre_re, rest) if match: # X.YaZ pre_marker = match.group('pre_marker') pre_version = match.group('pre_version') pre_version = int(pre_version) pre_version += 1 next_version = '{major}.{minor}{pre_marker}{pre_version}'.format_map( locals()) else: # X.Y or starts with X.Y (but is not X.Y.Z or X.YaZ) minor += 1 next_version = '{major}.{minor}'.format_map(locals()) if next_version is None: msg = 'Cannot automatically determine next version from {version}'.format_map( locals()) abort(3, msg) next_version_dev = '{next_version}.dev0'.format_map(locals()) # Find the first line that starts with '##'. Extract the version and # date from that line. The version must be the specified release # version OR the date must be the literal string 'unreleased'. with open(changelog) as fp: for changelog_line_number, line in enumerate(fp): if line.startswith('## '): match = re.search(changelog_header_re, line) if match: found_version = match.group('version') found_date = match.group('date') if found_version == version: if found_date != 'unreleased': printer.warning('Re-releasing', version) elif found_date == 'unreleased': if found_version != version: printer.warning('Replacing', found_version, 'with', version) else: msg = ( 'Expected version {version} or release date "unreleased"; got:\n\n' ' {line}').format_map(locals()) abort(4, msg) break else: abort(5, 'Could not find section in change log') printer.info('Version:', version) printer.info('Tag name:', tag_name) printer.info('Release date:', date) printer.info('Next version:', next_version) msg = 'Continue with release?: {version} - {date}'.format_map(locals()) yes or confirm(config, msg, abort_on_unconfirmed=True) printer.header('Testing...') tox(config) # Prepare if prepare: printer.header('Preparing release', version, 'on', date) updated_init_line = "__version__ = '{version}'\n".format_map(locals()) updated_changelog_line = '## {version} - {date}\n'.format_map(locals()) update_line(init_module, init_line_number, updated_init_line) update_line(changelog, changelog_line_number, updated_changelog_line) local(config, ('git diff', init_module, changelog)) yes or confirm( config, 'Commit these changes?', abort_on_unconfirmed=True) msg = prompt('Commit message', default='Prepare release {version}'.format_map(locals())) msg = '-m "{msg}"'.format_map(locals()) local(config, ('git commit', init_module, changelog, msg)) # Merge and tag if merge: printer.header('Merging develop into master for release', version) local(config, 'git log --oneline --reverse master..') msg = 'Merge these changes from develop into master for release {version}?' msg = msg.format_map(locals()) yes or confirm(config, msg, abort_on_unconfirmed=True) local(config, 'git checkout master') msg = '"Merge branch \'develop\' for release {version}"'.format_map( locals()) local(config, ('git merge --no-ff develop -m', msg)) if create_tag: printer.header('Tagging release', version) msg = '"Release {version}"'.format_map(locals()) local(config, ('git tag -a -m', msg, version)) local(config, 'git checkout develop') # Resume if resume: printer.header('Resuming development at', next_version) updated_init_line = "__version__ = '{next_version_dev}'\n".format_map( locals()) new_changelog_lines = [ '## {next_version} - unreleased\n\n'.format_map(locals()), 'In progress...\n\n', ] update_line(init_module, init_line_number, updated_init_line) with open(changelog) as fp: lines = fp.readlines() lines = lines[:changelog_line_number] + new_changelog_lines + lines[ changelog_line_number:] with open(changelog, 'w') as fp: fp.writelines(lines) local(config, ('git diff', init_module, changelog)) yes or confirm( config, 'Commit these changes?', abort_on_unconfirmed=True) msg = prompt('Commit message', default='Resume development at {next_version}'.format_map( locals())) msg = '-m "{msg}"'.format_map(locals()) local(config, ('git commit', init_module, changelog, msg))
def deploy(env, version=None, # Corresponding tags will be included unless unset *or* any # tags are specified via --tags. prepare: 'Run preparation tasks (local)' = True, dijkstar: 'Run Dijkstar tasks (remote)' = True, deploy: 'Run deployment tasks (remote)' = True, # Corresponding tags will be skipped unless set. clean: 'Remove local build directory' = False, overwrite: 'Remove remote build directory' = False, tags: 'Run *only* tasks corresponding to these tags' = (), skip_tags: 'Skip tasks corresponding to these tags' = (), yes: 'Deploy without confirmation' = False, echo=True): """Deploy the byCycle web API. Typical usage:: run deploy -c """ version = version or git_version() tags = (tags,) if isinstance(tags, str) else tags skip_tags = (skip_tags,) if isinstance(skip_tags, str) else skip_tags yes = False if env == 'production' else yes if tags: prepare = 'prepare' in tags dijkstar = 'dijkstar' in tags deploy = 'deploy' in tags else: if prepare: tags += ('prepare',) if dijkstar: tags += ('dijkstar',) if deploy: tags += ('deploy',) if not clean: skip_tags += ('remove-build-directory',) if not overwrite: skip_tags += ('overwrite',) if not prepare: printer.info('Not preparing') if not dijkstar: printer.info('Not running Dijkstar tasks') if not deploy: printer.info('Not deploying') if tags: printer.info('Selected tags: %s' % ', '.join(tags)) if skip_tags: printer.info('Skipping tags: %s' % ', '.join(skip_tags)) if clean: printer.warning('Local build directory will be removed first') if overwrite: printer.warning('Remote build directory will be overwritten') environ = {} if not yes: message = f'Deploy version {version} to {env}?' confirm(message, abort_on_unconfirmed=True) printer.header(f'Deploying {version} to {env}...') ansible_args = get_ansible_args(env, version=version, tags=tags, skip_tags=skip_tags) return local(ansible_args, environ=environ, echo=echo)
def deploy(env, host, version=None, build_=True, clean_=True, verbose=False, push=True, overwrite=False, chown=True, chmod=True, link=True, dry_run=False): if env == 'development': abort(1, 'Can\'t deploy to development environment') version = version or git_version() root_dir = f'/sites/{host}/webui' build_dir = f'{root_dir}/builds/{version}' link_path = f'{root_dir}/current' real_run = not dry_run printer.hr( f'{"[DRY RUN] " if dry_run else ""}Deploying version {version} to {env}', color='header') printer.header('Host:', host) printer.header('Remote root directory:', root_dir) printer.header('Remote build directory:', build_dir) printer.header('Remote link to current build:', link_path) printer.header('Steps:') printer.header(f' - {"Cleaning" if clean_ else "Not cleaning"}') printer.header(f' - {"Building" if build_ else "Not building"}') printer.header(f' - {f"Pushing" if push else "Not pushing"}') printer.header(f' - {f"Setting owner" if chown else "Not setting owner"}') printer.header( f' - {f"Setting permissions" if chmod else "Not setting permissions"}' ) if overwrite: printer.warning(f' - Overwriting {build_dir}') printer.header(f' - {"Linking" if link else "Not linking"}') confirm(f'Continue with deployment of version {version} to {env}?', abort_on_unconfirmed=True) if build_: build(env, clean_=clean_, verbose=verbose) if push: remote(f'test -d {build_dir} || mkdir -p {build_dir}') printer.info(f'Pushing public/ to {build_dir}...') sync('public/', f'{build_dir}/', host, delete=overwrite, dry_run=dry_run, echo=verbose) if chown: owner = 'bycycle:www-data' printer.info(f'Setting ownership of {build_dir} to {owner}...') if real_run: remote(('chown', '-R', owner, build_dir), sudo=True) if chmod: mode = 'u=rwX,g=rwX,o=' printer.info(f'Setting permissions on {build_dir} to {mode}...') if real_run: remote(('chmod', '-R', mode, build_dir), sudo=True) if link: printer.info(f'Linking {link_path} to {build_dir}') if real_run: remote(('ln', '-sfn', build_dir, link_path))
def make_release( # Steps test: arg( short_option="-e", help="Run tests first", ) = True, prepare: arg( short_option="-p", help="Run release preparation tasks", ) = True, merge: arg( short_option="-m", help="Run merge tasks", ) = True, tag: arg( short_option="-t", help="Create release tag", ) = True, resume: arg( short_option="-r", help="Run resume development tasks", ) = True, test_command: arg( short_option="-c", help="Test command", ) = None, # Step config name: arg( short_option="-n", help="Release/package name [base name of CWD]", ) = None, version: arg( short_option="-v", help="Version to release", ) = None, version_file: arg( short_option="-V", help="File __version__ is in [search typical files]", ) = None, date: arg( short_option="-d", help="Release date [today]", ) = None, dev_branch: arg( short_option="-b", help="Branch to merge from [current branch]", ) = None, target_branch: arg( short_option="-B", help="Branch to merge into [prod]", ) = "prod", tag_name: arg( short_option="-a", help=("Release tag name; {name} and {version} in the tag name " "will be substituted [version]"), ) = None, next_version: arg( short_option="-w", help="Anticipated version of next release", ) = None, # Other yes: arg( short_option="-y", no_inverse=True, help="Run without being prompted for any confirmations", ) = False, show_version: arg( short_option="-s", long_option="--show-version", no_inverse=True, help="Show make-release version and exit", ) = False, ): """Make a release. Tries to guess the release version based on the current version and the next version based on the release version. Steps: - Prepare release: - Update ``version`` in ``pyproject.toml`` (if present) - Update ``__version__`` in version file (if present; typically ``package/__init__.py`` or ``src/package/__init__.py``) - Update next version header in change log - Commit version file and change log with prepare message - Merge to target branch (``prod`` by default): - Merge current branch into target branch with merge message - Create tag: - Add annotated tag for latest version; when merging, the tag will point at the merge commit on the target branch; when not merging, the tag will point at the prepare release commit on the current branch - Resume development: - Update version in ``pyproject.toml`` to next version (if present) - Update version in version file to next version (if present) - Add in-progress section for next version to change log - Commit version file and change log with resume message Caveats: - The next version will have the dev marker ".dev0" appended to it - The change log must be in Markdown format; release section headers must be second-level (i.e., start with ##) - The change log must be named CHANGELOG or CHANGELOG.md - The first release section header in the change log will be updated, so there always needs to be an in-progress section for the next version - Building distributions and uploading to PyPI isn't handled; after creating a release, build distributions using ``python setup.py sdist`` or ``poetry build`` (for example) and then upload them with ``twine upload`` """ if show_version: from . import __version__ print(f"make-release version {__version__}") return cwd = pathlib.Path.cwd() name = name or cwd.name printer.hr("Releasing", name) print_step("Testing?", test) print_step("Preparing?", prepare) print_step("Merging?", merge) print_step("Tagging?", tag) print_step("Resuming development?", resume) if merge: if dev_branch is None: dev_branch = get_current_branch() if dev_branch == target_branch: abort(1, f"Dev branch and target branch are the same: {dev_branch}") pyproject_file = pathlib.Path("pyproject.toml") if pyproject_file.is_file(): pyproject_version_info = get_current_version(pyproject_file, "version") ( pyproject_version_line_number, pyproject_version_quote, pyproject_current_version, ) = pyproject_version_info else: pyproject_file = None pyproject_version_line_number = None pyproject_version_quote = None pyproject_current_version = None if version_file: version_file = pathlib.Path(version_file) version_info = get_current_version(version_file) version_line_number, version_quote, current_version = version_info else: version_info = find_version_file() if version_info is not None: ( version_file, version_line_number, version_quote, current_version, ) = version_info else: version_file = None version_line_number = None version_quote = None current_version = pyproject_current_version if (current_version and pyproject_current_version and current_version != pyproject_current_version): abort( 2, f"Version in pyproject.toml and " f"{version_file.relative_to(cwd)} don't match", ) if not version: if current_version: version = current_version else: message = ("Current version not set in version file, so release " "version needs to be passed explicitly") abort(3, message) if tag_name: tag_name = tag_name.format(name=name, version=version) else: tag_name = version date = date or datetime.date.today().isoformat() if not next_version: next_version = get_next_version(version) change_log = find_change_log() change_log_line_number = find_change_log_section(change_log, version) info = ReleaseInfo( name, dev_branch, target_branch, pyproject_file, pyproject_version_line_number, pyproject_version_quote, version_file, version_line_number, version_quote, version, tag_name, date, next_version, change_log, change_log_line_number, not yes, ) print_info("Version:", info.version) print_info("Release date:", info.date) if merge: print_info("Dev branch:", dev_branch) print_info("Target branch:", target_branch) if tag: print_info("Tag name:", tag_name) print_info("Next version:", info.next_version) if info.confirmation_required: msg = f"Continue with release?: {info.version} - {info.date}" confirm(msg, abort_on_unconfirmed=True) else: printer.warning( "Continuing with release: {info.version} - {info.date}") if test: print_step_header("Testing") if test_command is None: if (cwd / "tests").is_dir(): test_command = "python -m unittest discover tests" else: test_command = "python -m unittest discover ." local(test_command, echo=True) else: printer.warning("Skipping tests") if prepare: prepare_release(info) if merge: merge_to_target_branch(info) if tag: create_release_tag(info, merge) if resume: resume_development(info)