def find_change_log_section(change_log, version):
# Find the first line that starts with '##'. Extract the version and
# date from that line. The version must be the specified release
# version OR the date must be the literal string 'unreleased'.
# E.g.: ## 1.0.0 - unreleased
change_log_header_re = r"^## (?P<version>.+) - (?P<date>.+)$"
with change_log.open() as fp:
for line_number, line in enumerate(fp):
match = re.search(change_log_header_re, line)
if match:
found_version = match.group("version")
found_date = match.group("date")
if found_version == version:
if found_date != "unreleased":
printer.warning("Re-releasing", version)
elif found_date == "unreleased":
if found_version != version:
printer.warning("Replacing", found_version, "with",
version)
else:
msg = (f"Expected version {version} or release date "
f'"unreleased"; got:\n\n {line}')
abort(7, msg)
return line_number
abort(8, "Could not find section in change log")
def rmdir(path):
if os.path.isdir(path):
shutil.rmtree(path)
else:
printer.warning(
'Path does not exist or is not a directory: {path}'.format_map(
locals()))
def rmdir(directory, quiet=False):
if directory.is_dir():
shutil.rmtree(directory)
if not quiet:
printer.info('Removed directory:', directory)
else:
if not quiet:
printer.warning('Directory does not exist:', directory)
def shell():
banner = 'byCycle Shell'
try:
import bpython
except ImportError:
printer.warning('bpython is not installed; falling back to python')
code.interact(banner=banner)
else:
bpython.embed(banner=banner)
def rmdir(path):
if os.path.isdir(path):
shutil.rmtree(path)
if not quiet:
printer(f'Removed directory: {path}')
else:
if not quiet:
printer.warning(
f'Path does not exist or is not a directory: {path}')
def execute(engine, sql, condition=True):
if not condition:
return
if isinstance(sql, (list, tuple)):
sql = ' '.join(sql)
printer.info('Running SQL:', sql)
try:
return engine.execute(sql)
except ProgrammingError as exc:
error = str(exc.orig)
exc_str = str(exc)
if 'already exists' in exc_str or 'does not exist' in exc_str:
printer.warning(exc.statement.strip(), error.strip(), sep=': ')
else:
raise
def prepare_release(info):
version = info.version
print_step_header("Preparing release", version, "on", info.date)
current_branch = get_current_branch()
local(("git", "checkout", info.dev_branch))
if info.pyproject_file:
quote = info.pyproject_version_quote
update_line(
info.pyproject_file,
info.pyproject_version_line_number,
f"version = {quote}{version}{quote}",
)
if info.version_file:
quote = info.version_quote
update_line(
info.version_file,
info.version_line_number,
f"__version__ = {quote}{version}{quote}",
)
update_line(
info.change_log,
info.change_log_line_number,
f"## {version} - {info.date}",
)
commit_files = (info.pyproject_file, info.version_file, info.change_log)
local(("git", "diff", *commit_files))
if info.confirmation_required:
confirm("Commit these changes?", abort_on_unconfirmed=True)
else:
printer.warning("Committing changes")
msg = f"Prepare {info.name} release {version}"
msg = prompt("Commit message", default=msg)
local(("git", "commit", commit_files, "-m", msg))
local(("git", "checkout", current_branch))
def merge_to_target_branch(info):
print_step_header(
"Merging",
info.dev_branch,
"into",
info.target_branch,
"for release",
info.version,
)
current_branch = get_current_branch()
local((
"git",
"log",
"--oneline",
"--reverse",
f"{info.target_branch}..{info.dev_branch}",
))
if info.confirmation_required:
msg = (f"Merge these changes from {info.dev_branch} "
f"into {info.target_branch} "
f"for release {info.version}?")
confirm(msg, abort_on_unconfirmed=True)
else:
printer.warning(
"Merging changes from",
info.dev_branch,
"into",
info.target_branch,
"for release",
info.release,
)
local(("git", "checkout", info.target_branch))
msg = f"Merge branch '{info.dev_branch}' for {info.name} release {info.version}"
msg = prompt("Commit message", default=msg)
local(("git", "merge", "--no-ff", info.dev_branch, "-m", msg))
local(("git", "checkout", current_branch))
def virtualenv(config, where='.env', python='python3', overwrite=False):
exists = os.path.exists(where)
def create():
local(config, ('virtualenv', '-p', python, where))
printer.success(
'Virtualenv created; activate it by running `source {where}/bin/activate`'
.format_map(locals()))
if exists:
if overwrite:
printer.warning('Overwriting virtualenv', where, 'with', python)
shutil.rmtree(where)
create()
else:
printer.info('Virtualenv', where,
'exists; pass --overwrite to re-create it')
else:
printer.info('Creating virtualenv', where, 'with', python)
create()
def clean(all_=False):
"""Clean up.
- Remove build directory
- Remove dist directory
- Remove __pycache__ directories
If --all:
- Remove egg-info directory
- Remove .venv directory
"""
def rmdir(directory, quiet=False):
if directory.is_dir():
shutil.rmtree(directory)
if not quiet:
printer.info('Removed directory:', directory)
else:
if not quiet:
printer.warning('Directory does not exist:', directory)
cwd = Path.cwd()
rmdir(Path('./build'))
rmdir(Path('./dist'))
pycache_dirs = tuple(Path('.').glob('**/__pycache__/'))
num_pycache_dirs = len(pycache_dirs)
for pycache_dir in pycache_dirs:
rmdir(pycache_dir, quiet=True)
if num_pycache_dirs:
printer.info(f'Removed {num_pycache_dirs} __pycache__'
f'director{"y" if num_pycache_dirs == 1 else "ies"}')
else:
printer.warning('No __pycache__ directories found')
if all_:
rmdir(cwd / '.venv')
rmdir(cwd / f'{cwd.name}.egg-info')
def find_version_file():
# Try to find __version__ in:
#
# - package/__init__.py
# - namespace_package/package/__init__.py
# - src/package/__init__.py
# - src/namespace_package/package/__init__.py
cwd = pathlib.Path.cwd()
candidates = []
candidates.extend(cwd.glob("*/__init__.py"))
candidates.extend(cwd.glob("*/*/__init__.py"))
candidates.extend(cwd.glob("src/*/__init__.py"))
candidates.extend(cwd.glob("src/*/*/__init__.py"))
for candidate in candidates:
result = get_current_version(candidate, "__version__", False)
if result is not None:
return (candidate, ) + result
candidates = "\n ".join(str(candidate) for candidate in candidates)
printer.warning(
f"Could not find file containing __version__; tried:\n {candidates}",
)
return None
def release(config,
version=None,
date=None,
tag_name=None,
next_version=None,
prepare=True,
merge=True,
create_tag=True,
resume=True,
yes=False):
def update_line(file_name, line_number, content):
with open(file_name) as fp:
lines = fp.readlines()
lines[line_number] = content
with open(file_name, 'w') as fp:
fp.writelines(lines)
result = local(config, 'git rev-parse --abbrev-ref HEAD', hide='stdout')
current_branch = result.stdout.strip()
if current_branch != 'develop':
abort(1, 'Must be on develop branch to make a release')
init_module = 'runcommands/__init__.py'
changelog = 'CHANGELOG'
# E.g.: __version__ = '1.0.dev0'
version_re = r"^__version__ = '(?P<version>.+)(?P<dev_marker>\.dev\d+)'$"
# E.g.: ## 1.0.0 - 2017-04-01
changelog_header_re = r'^## (?P<version>.+) - (?P<date>.+)$'
with open(init_module) as fp:
for init_line_number, line in enumerate(fp):
if line.startswith('__version__'):
match = re.search(version_re, line)
if match:
current_version = match.group('version')
if not version:
version = current_version
break
else:
abort(
1, 'Could not find __version__ in {init_module}'.format_map(
locals()))
date = date or datetime.date.today().isoformat()
tag_name = tag_name or version
if next_version is None:
next_version_re = r'^(?P<major>\d+)\.(?P<minor>\d+)(?P<rest>.*)$'
match = re.search(next_version_re, version)
if match:
major = match.group('major')
minor = match.group('minor')
major = int(major)
minor = int(minor)
rest = match.group('rest')
patch_re = r'^\.(?P<patch>\d+)$'
match = re.search(patch_re, rest)
if match:
# X.Y.Z
minor += 1
patch = match.group('patch')
next_version = '{major}.{minor}.{patch}'.format_map(locals())
else:
pre_re = r'^(?P<pre_marker>a|b|rc)(?P<pre_version>\d+)$'
match = re.search(pre_re, rest)
if match:
# X.YaZ
pre_marker = match.group('pre_marker')
pre_version = match.group('pre_version')
pre_version = int(pre_version)
pre_version += 1
next_version = '{major}.{minor}{pre_marker}{pre_version}'.format_map(
locals())
else:
# X.Y or starts with X.Y (but is not X.Y.Z or X.YaZ)
minor += 1
next_version = '{major}.{minor}'.format_map(locals())
if next_version is None:
msg = 'Cannot automatically determine next version from {version}'.format_map(
locals())
abort(3, msg)
next_version_dev = '{next_version}.dev0'.format_map(locals())
# Find the first line that starts with '##'. Extract the version and
# date from that line. The version must be the specified release
# version OR the date must be the literal string 'unreleased'.
with open(changelog) as fp:
for changelog_line_number, line in enumerate(fp):
if line.startswith('## '):
match = re.search(changelog_header_re, line)
if match:
found_version = match.group('version')
found_date = match.group('date')
if found_version == version:
if found_date != 'unreleased':
printer.warning('Re-releasing', version)
elif found_date == 'unreleased':
if found_version != version:
printer.warning('Replacing', found_version, 'with',
version)
else:
msg = (
'Expected version {version} or release date "unreleased"; got:\n\n'
' {line}').format_map(locals())
abort(4, msg)
break
else:
abort(5, 'Could not find section in change log')
printer.info('Version:', version)
printer.info('Tag name:', tag_name)
printer.info('Release date:', date)
printer.info('Next version:', next_version)
msg = 'Continue with release?: {version} - {date}'.format_map(locals())
yes or confirm(config, msg, abort_on_unconfirmed=True)
printer.header('Testing...')
tox(config)
# Prepare
if prepare:
printer.header('Preparing release', version, 'on', date)
updated_init_line = "__version__ = '{version}'\n".format_map(locals())
updated_changelog_line = '## {version} - {date}\n'.format_map(locals())
update_line(init_module, init_line_number, updated_init_line)
update_line(changelog, changelog_line_number, updated_changelog_line)
local(config, ('git diff', init_module, changelog))
yes or confirm(
config, 'Commit these changes?', abort_on_unconfirmed=True)
msg = prompt('Commit message',
default='Prepare release {version}'.format_map(locals()))
msg = '-m "{msg}"'.format_map(locals())
local(config, ('git commit', init_module, changelog, msg))
# Merge and tag
if merge:
printer.header('Merging develop into master for release', version)
local(config, 'git log --oneline --reverse master..')
msg = 'Merge these changes from develop into master for release {version}?'
msg = msg.format_map(locals())
yes or confirm(config, msg, abort_on_unconfirmed=True)
local(config, 'git checkout master')
msg = '"Merge branch \'develop\' for release {version}"'.format_map(
locals())
local(config, ('git merge --no-ff develop -m', msg))
if create_tag:
printer.header('Tagging release', version)
msg = '"Release {version}"'.format_map(locals())
local(config, ('git tag -a -m', msg, version))
local(config, 'git checkout develop')
# Resume
if resume:
printer.header('Resuming development at', next_version)
updated_init_line = "__version__ = '{next_version_dev}'\n".format_map(
locals())
new_changelog_lines = [
'## {next_version} - unreleased\n\n'.format_map(locals()),
'In progress...\n\n',
]
update_line(init_module, init_line_number, updated_init_line)
with open(changelog) as fp:
lines = fp.readlines()
lines = lines[:changelog_line_number] + new_changelog_lines + lines[
changelog_line_number:]
with open(changelog, 'w') as fp:
fp.writelines(lines)
local(config, ('git diff', init_module, changelog))
yes or confirm(
config, 'Commit these changes?', abort_on_unconfirmed=True)
msg = prompt('Commit message',
default='Resume development at {next_version}'.format_map(
locals()))
msg = '-m "{msg}"'.format_map(locals())
local(config, ('git commit', init_module, changelog, msg))
def deploy(env,
version=None,
# Corresponding tags will be included unless unset *or* any
# tags are specified via --tags.
prepare: 'Run preparation tasks (local)' = True,
dijkstar: 'Run Dijkstar tasks (remote)' = True,
deploy: 'Run deployment tasks (remote)' = True,
# Corresponding tags will be skipped unless set.
clean: 'Remove local build directory' = False,
overwrite: 'Remove remote build directory' = False,
tags: 'Run *only* tasks corresponding to these tags' = (),
skip_tags: 'Skip tasks corresponding to these tags' = (),
yes: 'Deploy without confirmation' = False,
echo=True):
"""Deploy the byCycle web API.
Typical usage::
run deploy -c
"""
version = version or git_version()
tags = (tags,) if isinstance(tags, str) else tags
skip_tags = (skip_tags,) if isinstance(skip_tags, str) else skip_tags
yes = False if env == 'production' else yes
if tags:
prepare = 'prepare' in tags
dijkstar = 'dijkstar' in tags
deploy = 'deploy' in tags
else:
if prepare:
tags += ('prepare',)
if dijkstar:
tags += ('dijkstar',)
if deploy:
tags += ('deploy',)
if not clean:
skip_tags += ('remove-build-directory',)
if not overwrite:
skip_tags += ('overwrite',)
if not prepare:
printer.info('Not preparing')
if not dijkstar:
printer.info('Not running Dijkstar tasks')
if not deploy:
printer.info('Not deploying')
if tags:
printer.info('Selected tags: %s' % ', '.join(tags))
if skip_tags:
printer.info('Skipping tags: %s' % ', '.join(skip_tags))
if clean:
printer.warning('Local build directory will be removed first')
if overwrite:
printer.warning('Remote build directory will be overwritten')
environ = {}
if not yes:
message = f'Deploy version {version} to {env}?'
confirm(message, abort_on_unconfirmed=True)
printer.header(f'Deploying {version} to {env}...')
ansible_args = get_ansible_args(env, version=version, tags=tags, skip_tags=skip_tags)
return local(ansible_args, environ=environ, echo=echo)
def deploy(env,
host,
version=None,
build_=True,
clean_=True,
verbose=False,
push=True,
overwrite=False,
chown=True,
chmod=True,
link=True,
dry_run=False):
if env == 'development':
abort(1, 'Can\'t deploy to development environment')
version = version or git_version()
root_dir = f'/sites/{host}/webui'
build_dir = f'{root_dir}/builds/{version}'
link_path = f'{root_dir}/current'
real_run = not dry_run
printer.hr(
f'{"[DRY RUN] " if dry_run else ""}Deploying version {version} to {env}',
color='header')
printer.header('Host:', host)
printer.header('Remote root directory:', root_dir)
printer.header('Remote build directory:', build_dir)
printer.header('Remote link to current build:', link_path)
printer.header('Steps:')
printer.header(f' - {"Cleaning" if clean_ else "Not cleaning"}')
printer.header(f' - {"Building" if build_ else "Not building"}')
printer.header(f' - {f"Pushing" if push else "Not pushing"}')
printer.header(f' - {f"Setting owner" if chown else "Not setting owner"}')
printer.header(
f' - {f"Setting permissions" if chmod else "Not setting permissions"}'
)
if overwrite:
printer.warning(f' - Overwriting {build_dir}')
printer.header(f' - {"Linking" if link else "Not linking"}')
confirm(f'Continue with deployment of version {version} to {env}?',
abort_on_unconfirmed=True)
if build_:
build(env, clean_=clean_, verbose=verbose)
if push:
remote(f'test -d {build_dir} || mkdir -p {build_dir}')
printer.info(f'Pushing public/ to {build_dir}...')
sync('public/',
f'{build_dir}/',
host,
delete=overwrite,
dry_run=dry_run,
echo=verbose)
if chown:
owner = 'bycycle:www-data'
printer.info(f'Setting ownership of {build_dir} to {owner}...')
if real_run:
remote(('chown', '-R', owner, build_dir), sudo=True)
if chmod:
mode = 'u=rwX,g=rwX,o='
printer.info(f'Setting permissions on {build_dir} to {mode}...')
if real_run:
remote(('chmod', '-R', mode, build_dir), sudo=True)
if link:
printer.info(f'Linking {link_path} to {build_dir}')
if real_run:
remote(('ln', '-sfn', build_dir, link_path))
def make_release(
# Steps
test: arg(
short_option="-e",
help="Run tests first",
) = True,
prepare: arg(
short_option="-p",
help="Run release preparation tasks",
) = True,
merge: arg(
short_option="-m",
help="Run merge tasks",
) = True,
tag: arg(
short_option="-t",
help="Create release tag",
) = True,
resume: arg(
short_option="-r",
help="Run resume development tasks",
) = True,
test_command: arg(
short_option="-c",
help="Test command",
) = None,
# Step config
name: arg(
short_option="-n",
help="Release/package name [base name of CWD]",
) = None,
version: arg(
short_option="-v",
help="Version to release",
) = None,
version_file: arg(
short_option="-V",
help="File __version__ is in [search typical files]",
) = None,
date: arg(
short_option="-d",
help="Release date [today]",
) = None,
dev_branch: arg(
short_option="-b",
help="Branch to merge from [current branch]",
) = None,
target_branch: arg(
short_option="-B",
help="Branch to merge into [prod]",
) = "prod",
tag_name: arg(
short_option="-a",
help=("Release tag name; {name} and {version} in the tag name "
"will be substituted [version]"),
) = None,
next_version: arg(
short_option="-w",
help="Anticipated version of next release",
) = None,
# Other
yes: arg(
short_option="-y",
no_inverse=True,
help="Run without being prompted for any confirmations",
) = False,
show_version: arg(
short_option="-s",
long_option="--show-version",
no_inverse=True,
help="Show make-release version and exit",
) = False,
):
"""Make a release.
Tries to guess the release version based on the current version and
the next version based on the release version.
Steps:
- Prepare release:
- Update ``version`` in ``pyproject.toml`` (if present)
- Update ``__version__`` in version file (if present;
typically ``package/__init__.py`` or
``src/package/__init__.py``)
- Update next version header in change log
- Commit version file and change log with prepare message
- Merge to target branch (``prod`` by default):
- Merge current branch into target branch with merge message
- Create tag:
- Add annotated tag for latest version; when merging, the
tag will point at the merge commit on the target branch;
when not merging, the tag will point at the prepare
release commit on the current branch
- Resume development:
- Update version in ``pyproject.toml`` to next version (if
present)
- Update version in version file to next version (if
present)
- Add in-progress section for next version to change log
- Commit version file and change log with resume message
Caveats:
- The next version will have the dev marker ".dev0" appended to
it
- The change log must be in Markdown format; release section
headers must be second-level (i.e., start with ##)
- The change log must be named CHANGELOG or CHANGELOG.md
- The first release section header in the change log will be
updated, so there always needs to be an in-progress section
for the next version
- Building distributions and uploading to PyPI isn't handled;
after creating a release, build distributions using
``python setup.py sdist`` or ``poetry build`` (for example)
and then upload them with ``twine upload``
"""
if show_version:
from . import __version__
print(f"make-release version {__version__}")
return
cwd = pathlib.Path.cwd()
name = name or cwd.name
printer.hr("Releasing", name)
print_step("Testing?", test)
print_step("Preparing?", prepare)
print_step("Merging?", merge)
print_step("Tagging?", tag)
print_step("Resuming development?", resume)
if merge:
if dev_branch is None:
dev_branch = get_current_branch()
if dev_branch == target_branch:
abort(1,
f"Dev branch and target branch are the same: {dev_branch}")
pyproject_file = pathlib.Path("pyproject.toml")
if pyproject_file.is_file():
pyproject_version_info = get_current_version(pyproject_file, "version")
(
pyproject_version_line_number,
pyproject_version_quote,
pyproject_current_version,
) = pyproject_version_info
else:
pyproject_file = None
pyproject_version_line_number = None
pyproject_version_quote = None
pyproject_current_version = None
if version_file:
version_file = pathlib.Path(version_file)
version_info = get_current_version(version_file)
version_line_number, version_quote, current_version = version_info
else:
version_info = find_version_file()
if version_info is not None:
(
version_file,
version_line_number,
version_quote,
current_version,
) = version_info
else:
version_file = None
version_line_number = None
version_quote = None
current_version = pyproject_current_version
if (current_version and pyproject_current_version
and current_version != pyproject_current_version):
abort(
2,
f"Version in pyproject.toml and "
f"{version_file.relative_to(cwd)} don't match",
)
if not version:
if current_version:
version = current_version
else:
message = ("Current version not set in version file, so release "
"version needs to be passed explicitly")
abort(3, message)
if tag_name:
tag_name = tag_name.format(name=name, version=version)
else:
tag_name = version
date = date or datetime.date.today().isoformat()
if not next_version:
next_version = get_next_version(version)
change_log = find_change_log()
change_log_line_number = find_change_log_section(change_log, version)
info = ReleaseInfo(
name,
dev_branch,
target_branch,
pyproject_file,
pyproject_version_line_number,
pyproject_version_quote,
version_file,
version_line_number,
version_quote,
version,
tag_name,
date,
next_version,
change_log,
change_log_line_number,
not yes,
)
print_info("Version:", info.version)
print_info("Release date:", info.date)
if merge:
print_info("Dev branch:", dev_branch)
print_info("Target branch:", target_branch)
if tag:
print_info("Tag name:", tag_name)
print_info("Next version:", info.next_version)
if info.confirmation_required:
msg = f"Continue with release?: {info.version} - {info.date}"
confirm(msg, abort_on_unconfirmed=True)
else:
printer.warning(
"Continuing with release: {info.version} - {info.date}")
if test:
print_step_header("Testing")
if test_command is None:
if (cwd / "tests").is_dir():
test_command = "python -m unittest discover tests"
else:
test_command = "python -m unittest discover ."
local(test_command, echo=True)
else:
printer.warning("Skipping tests")
if prepare:
prepare_release(info)
if merge:
merge_to_target_branch(info)
if tag:
create_release_tag(info, merge)
if resume:
resume_development(info)
