def lint(
show_errors: arg(help="Show errors") = True,
disable_ignore: arg(no_inverse=True,
help="Don't ignore any errors") = False,
disable_noqa: arg(no_inverse=True, help="Ignore noqa directives") = False,
):
printer.header("Checking for lint...")
result = local(
(
"flake8",
".",
"--ignore=" if disable_ignore else None,
"--disable-noqa" if disable_noqa else None,
),
stdout="capture",
raise_on_error=False,
)
pieces_of_lint = len(result.stdout_lines)
if pieces_of_lint:
ess = "" if pieces_of_lint == 1 else "s"
colon = ":" if show_errors else ""
message = [
f"{pieces_of_lint} piece{ess} of lint found{colon}",
]
if show_errors:
message.append(result.stdout.rstrip())
message = "\n".join(message)
abort(1, message)
else:
printer.success("No lint found")
def format_code(check=False, where="./"):
if check:
printer.header("Checking code formatting...")
check_arg = "--check"
raise_on_error = False
else:
printer.header("Formatting code...")
check_arg = None
raise_on_error = True
result = local(("black", check_arg, where), raise_on_error=raise_on_error)
return result
def build(env, clean_=True, verbose=False):
printer.header(f'Building for environment: {env}')
quiet = not verbose
end = None if verbose else ' '
clean_ and do_done(f'Cleaning...', clean, quiet=quiet, _end=end)
do_done(f'Compiling SCSS to CSS...', sass, quiet=quiet, _end=end)
do_done(f'Rolling up...',
rollup,
env,
live_reload=False,
quiet=quiet,
_end=end)
def make_dist(
version: arg(help="Tag/version to release [latest tag]") = None,
formats=("sdist", "wheel"),
quiet=False,
):
"""Make a distribution for upload to PyPI.
Switches to the specified tag or branch, makes the distribution,
then switches back to the original branch.
Intended to be run from the develop branch. If a tag is already
checked out, the develop branch will be checked out first and then
switched back to after making the distribution.
"""
current_branch = get_current_branch()
original_branch = "develop" if current_branch == "HEAD" else current_branch
version = version or get_latest_tag()
stdout = "hide" if quiet else None
printer.header(f"Making dist for {version}")
if version != current_branch:
if current_branch == "HEAD":
printer.warning("Not on a branch; checking out develop first")
else:
printer.info("Currently on branch", current_branch)
printer.info("Checking out", version)
# XXX: Hide warning about detached HEAD state
result = local(("git", "checkout", version),
stdout=stdout,
stderr="capture")
if result.failed:
print(result.stderr, file=sys.stderr)
printer.info("Removing dist directory")
rmdir("dist", verbose=not quiet)
printer.info("Making dists for", version)
for format_ in formats:
local(("poetry", "build", "--format", format_), stdout=stdout)
if version != current_branch:
printer.info("Switching back to", original_branch)
# XXX: Hide message about previous HEAD state and branch info
result = local(("git", "checkout", original_branch),
stdout="hide",
stderr="capture")
if result.failed:
print(result.stderr, file=sys.stderr)
def upload_dists(
make: arg(help="Make dist first? [yes]") = True,
version: arg(help="Version/tag to release [latest tag]") = None,
quiet: arg(help="Make dist quietly? [no]") = False,
username: arg(help="Twine username [$USER]") = None,
password_command: arg(help="Command to retrieve twine password "
"(e.g. `password-manager show-password PyPI`) "
"[twine prompt]") = None,
):
"""Upload distributions in ./dist using ``twine``."""
if make:
printer.header("Making and uploading distributions")
make_dist(quiet=quiet)
else:
printer.header("Uploading distributions")
dists = os.listdir("dist")
if not dists:
abort(1, "No distributions found in dist directory")
paths = [os.path.join("dist", file) for file in dists]
printer.info("Found distributions:")
for path in paths:
printer.info(" -", path)
if not confirm("Continue?"):
abort()
if not username:
username = getpass.getuser()
environ = {"TWINE_USERNAME": username}
if password_command:
printer.info(f"Retrieving password via `{password_command}`...")
result = local(password_command, stdout="capture")
password = result.stdout.strip()
environ["TWINE_PASSWORD"] = password
printer.warning("TWINE_USERNAME:"******"TWINE_PASSWORD:"******"*" * len(password))
for path in paths:
if confirm(f"Upload dist?: {path}"):
local(("twine", "upload", path), environ=environ)
else:
printer.warning("Skipped dist:", path)
def dev_server(default_args, host='localhost', port=5000, directory='public'):
printer.header('Running dev server')
printer.hr('Cleaning', color='info')
clean()
printer.info('Running scss watcher in background')
sass_args = ['sass', '--watch']
for source in default_args['sass']['sources']:
name = os.path.basename(source)
root, ext = os.path.splitext(name)
destination = os.path.join('public', 'build', f'{root}.css')
sass_args.append(f'{source}:{destination}')
local(sass_args, background=True, environ={
'NODE_ENV': 'development',
})
wait_for_file(destination)
printer.hr('Running rollup watcher in background', color='info')
local(['rollup', '--config', '--watch'],
background=True,
environ={
'NODE_ENV': 'development',
'LIVE_RELOAD': 'true',
})
wait_for_file('public/build/bundle.js')
printer.hr(f'Serving {directory} directory at http://{host}:{port}/',
color='info')
class RequestHandler(SimpleHTTPRequestHandler):
def __init__(self, *args, **kwargs):
super().__init__(*args, directory=directory, **kwargs)
def translate_path(self, path):
path = super().translate_path(path)
if not os.path.exists(path):
path = os.path.join(self.directory, 'index.html')
return path
server = ThreadingHTTPServer((host, port), RequestHandler)
try:
server.serve_forever()
except KeyboardInterrupt:
abort(message='Shut down dev server')
def deploy(host,
to='/sites/bycycle.org/current/frontend/',
build_=True,
clean_=False,
overwrite=False,
dry_run=False):
printer.header(f'Deploying to {host}...')
if build_:
build(clean_=clean_)
printer.info(f'Pushing build/ to {to}...')
sync('build/',
to,
host,
run_as='bycycle',
delete=overwrite,
dry_run=dry_run,
echo=True)
printer.info(f'Setting ownership of {to}...')
remote(('chown -R bycycle:www-data', to), sudo=True)
def load_osm_data(db,
bbox: arg(type=float, nargs=4),
directory='../osm',
graph_path='../graph.marshal',
streets=True,
places=True,
actions: arg(container=tuple, type=int) = (),
show_actions: arg(short_option='-a') = False,
log_to=None):
"""Read OSM data from file and load into database."""
importer = OSMImporter(bbox, directory, graph_path, db, streets, places,
actions)
if show_actions:
printer.header('Available actions:')
for action in importer.all_actions:
printer.info(action)
return
importer.run()
if log_to:
message = f'Loaded OSM data from {importer.data_directory} to {importer.engine.url}'
log_to_file(log_to, message)
def test(config,
tests=(),
fail_fast=False,
with_coverage=True,
with_lint=True):
if tests:
num_tests = len(tests)
s = '' if num_tests == 1 else 's'
printer.header('Running {num_tests} test{s}...'.format_map(locals()))
else:
coverage_message = ' with coverage' if with_coverage else ''
printer.header('Running tests{coverage_message}...'.format_map(
locals()))
runner = unittest.TextTestRunner(failfast=fail_fast)
loader = unittest.TestLoader()
if with_coverage:
from coverage import Coverage
coverage = Coverage(source=['runcommands'])
coverage.start()
if tests:
for name in tests:
runner.run(loader.loadTestsFromName(name))
else:
tests = loader.discover('.')
result = runner.run(tests)
if not result.errors:
if with_coverage:
coverage.stop()
coverage.report()
if with_lint:
printer.header('Checking for lint...')
lint(config)
def test(*tests,
fail_fast=False,
verbosity=1,
with_coverage=True,
with_lint=True):
original_working_directory = os.getcwd()
if tests:
num_tests = len(tests)
s = "" if num_tests == 1 else "s"
printer.header(f"Running {num_tests} test{s}...")
else:
coverage_message = " with coverage" if with_coverage else ""
printer.header(f"Running tests{coverage_message}...")
runner = unittest.TextTestRunner(failfast=fail_fast, verbosity=verbosity)
loader = unittest.TestLoader()
if with_coverage:
from coverage import Coverage
coverage = Coverage(source=["src/runcommands"])
coverage.start()
if tests:
runner.run(loader.loadTestsFromNames(tests))
else:
tests = loader.discover("./tests", top_level_dir=".")
result = runner.run(tests)
if not result.errors:
if with_coverage:
coverage.stop()
coverage.report()
if with_lint:
# XXX: The test runner apparently changes CWD.
os.chdir(original_working_directory)
format_code(check=True)
lint()
def release(config,
version=None,
date=None,
tag_name=None,
next_version=None,
prepare=True,
merge=True,
create_tag=True,
resume=True,
yes=False):
def update_line(file_name, line_number, content):
with open(file_name) as fp:
lines = fp.readlines()
lines[line_number] = content
with open(file_name, 'w') as fp:
fp.writelines(lines)
result = local(config, 'git rev-parse --abbrev-ref HEAD', hide='stdout')
current_branch = result.stdout.strip()
if current_branch != 'develop':
abort(1, 'Must be on develop branch to make a release')
init_module = 'runcommands/__init__.py'
changelog = 'CHANGELOG'
# E.g.: __version__ = '1.0.dev0'
version_re = r"^__version__ = '(?P<version>.+)(?P<dev_marker>\.dev\d+)'$"
# E.g.: ## 1.0.0 - 2017-04-01
changelog_header_re = r'^## (?P<version>.+) - (?P<date>.+)$'
with open(init_module) as fp:
for init_line_number, line in enumerate(fp):
if line.startswith('__version__'):
match = re.search(version_re, line)
if match:
current_version = match.group('version')
if not version:
version = current_version
break
else:
abort(
1, 'Could not find __version__ in {init_module}'.format_map(
locals()))
date = date or datetime.date.today().isoformat()
tag_name = tag_name or version
if next_version is None:
next_version_re = r'^(?P<major>\d+)\.(?P<minor>\d+)(?P<rest>.*)$'
match = re.search(next_version_re, version)
if match:
major = match.group('major')
minor = match.group('minor')
major = int(major)
minor = int(minor)
rest = match.group('rest')
patch_re = r'^\.(?P<patch>\d+)$'
match = re.search(patch_re, rest)
if match:
# X.Y.Z
minor += 1
patch = match.group('patch')
next_version = '{major}.{minor}.{patch}'.format_map(locals())
else:
pre_re = r'^(?P<pre_marker>a|b|rc)(?P<pre_version>\d+)$'
match = re.search(pre_re, rest)
if match:
# X.YaZ
pre_marker = match.group('pre_marker')
pre_version = match.group('pre_version')
pre_version = int(pre_version)
pre_version += 1
next_version = '{major}.{minor}{pre_marker}{pre_version}'.format_map(
locals())
else:
# X.Y or starts with X.Y (but is not X.Y.Z or X.YaZ)
minor += 1
next_version = '{major}.{minor}'.format_map(locals())
if next_version is None:
msg = 'Cannot automatically determine next version from {version}'.format_map(
locals())
abort(3, msg)
next_version_dev = '{next_version}.dev0'.format_map(locals())
# Find the first line that starts with '##'. Extract the version and
# date from that line. The version must be the specified release
# version OR the date must be the literal string 'unreleased'.
with open(changelog) as fp:
for changelog_line_number, line in enumerate(fp):
if line.startswith('## '):
match = re.search(changelog_header_re, line)
if match:
found_version = match.group('version')
found_date = match.group('date')
if found_version == version:
if found_date != 'unreleased':
printer.warning('Re-releasing', version)
elif found_date == 'unreleased':
if found_version != version:
printer.warning('Replacing', found_version, 'with',
version)
else:
msg = (
'Expected version {version} or release date "unreleased"; got:\n\n'
' {line}').format_map(locals())
abort(4, msg)
break
else:
abort(5, 'Could not find section in change log')
printer.info('Version:', version)
printer.info('Tag name:', tag_name)
printer.info('Release date:', date)
printer.info('Next version:', next_version)
msg = 'Continue with release?: {version} - {date}'.format_map(locals())
yes or confirm(config, msg, abort_on_unconfirmed=True)
printer.header('Testing...')
tox(config)
# Prepare
if prepare:
printer.header('Preparing release', version, 'on', date)
updated_init_line = "__version__ = '{version}'\n".format_map(locals())
updated_changelog_line = '## {version} - {date}\n'.format_map(locals())
update_line(init_module, init_line_number, updated_init_line)
update_line(changelog, changelog_line_number, updated_changelog_line)
local(config, ('git diff', init_module, changelog))
yes or confirm(
config, 'Commit these changes?', abort_on_unconfirmed=True)
msg = prompt('Commit message',
default='Prepare release {version}'.format_map(locals()))
msg = '-m "{msg}"'.format_map(locals())
local(config, ('git commit', init_module, changelog, msg))
# Merge and tag
if merge:
printer.header('Merging develop into master for release', version)
local(config, 'git log --oneline --reverse master..')
msg = 'Merge these changes from develop into master for release {version}?'
msg = msg.format_map(locals())
yes or confirm(config, msg, abort_on_unconfirmed=True)
local(config, 'git checkout master')
msg = '"Merge branch \'develop\' for release {version}"'.format_map(
locals())
local(config, ('git merge --no-ff develop -m', msg))
if create_tag:
printer.header('Tagging release', version)
msg = '"Release {version}"'.format_map(locals())
local(config, ('git tag -a -m', msg, version))
local(config, 'git checkout develop')
# Resume
if resume:
printer.header('Resuming development at', next_version)
updated_init_line = "__version__ = '{next_version_dev}'\n".format_map(
locals())
new_changelog_lines = [
'## {next_version} - unreleased\n\n'.format_map(locals()),
'In progress...\n\n',
]
update_line(init_module, init_line_number, updated_init_line)
with open(changelog) as fp:
lines = fp.readlines()
lines = lines[:changelog_line_number] + new_changelog_lines + lines[
changelog_line_number:]
with open(changelog, 'w') as fp:
fp.writelines(lines)
local(config, ('git diff', init_module, changelog))
yes or confirm(
config, 'Commit these changes?', abort_on_unconfirmed=True)
msg = prompt('Commit message',
default='Resume development at {next_version}'.format_map(
locals()))
msg = '-m "{msg}"'.format_map(locals())
local(config, ('git commit', init_module, changelog, msg))
def deploy(env,
version=None,
# Corresponding tags will be included unless unset *or* any
# tags are specified via --tags.
prepare: 'Run preparation tasks (local)' = True,
dijkstar: 'Run Dijkstar tasks (remote)' = True,
deploy: 'Run deployment tasks (remote)' = True,
# Corresponding tags will be skipped unless set.
clean: 'Remove local build directory' = False,
overwrite: 'Remove remote build directory' = False,
tags: 'Run *only* tasks corresponding to these tags' = (),
skip_tags: 'Skip tasks corresponding to these tags' = (),
yes: 'Deploy without confirmation' = False,
echo=True):
"""Deploy the byCycle web API.
Typical usage::
run deploy -c
"""
version = version or git_version()
tags = (tags,) if isinstance(tags, str) else tags
skip_tags = (skip_tags,) if isinstance(skip_tags, str) else skip_tags
yes = False if env == 'production' else yes
if tags:
prepare = 'prepare' in tags
dijkstar = 'dijkstar' in tags
deploy = 'deploy' in tags
else:
if prepare:
tags += ('prepare',)
if dijkstar:
tags += ('dijkstar',)
if deploy:
tags += ('deploy',)
if not clean:
skip_tags += ('remove-build-directory',)
if not overwrite:
skip_tags += ('overwrite',)
if not prepare:
printer.info('Not preparing')
if not dijkstar:
printer.info('Not running Dijkstar tasks')
if not deploy:
printer.info('Not deploying')
if tags:
printer.info('Selected tags: %s' % ', '.join(tags))
if skip_tags:
printer.info('Skipping tags: %s' % ', '.join(skip_tags))
if clean:
printer.warning('Local build directory will be removed first')
if overwrite:
printer.warning('Remote build directory will be overwritten')
environ = {}
if not yes:
message = f'Deploy version {version} to {env}?'
confirm(message, abort_on_unconfirmed=True)
printer.header(f'Deploying {version} to {env}...')
ansible_args = get_ansible_args(env, version=version, tags=tags, skip_tags=skip_tags)
return local(ansible_args, environ=environ, echo=echo)
def build_static(self):
printer.header('Building static files (EcoRoofs custom)...')
build_static(self.config, static_root='{path.build.static_root}')
def deploy(env,
host,
version=None,
build_=True,
clean_=True,
verbose=False,
push=True,
overwrite=False,
chown=True,
chmod=True,
link=True,
dry_run=False):
if env == 'development':
abort(1, 'Can\'t deploy to development environment')
version = version or git_version()
root_dir = f'/sites/{host}/webui'
build_dir = f'{root_dir}/builds/{version}'
link_path = f'{root_dir}/current'
real_run = not dry_run
printer.hr(
f'{"[DRY RUN] " if dry_run else ""}Deploying version {version} to {env}',
color='header')
printer.header('Host:', host)
printer.header('Remote root directory:', root_dir)
printer.header('Remote build directory:', build_dir)
printer.header('Remote link to current build:', link_path)
printer.header('Steps:')
printer.header(f' - {"Cleaning" if clean_ else "Not cleaning"}')
printer.header(f' - {"Building" if build_ else "Not building"}')
printer.header(f' - {f"Pushing" if push else "Not pushing"}')
printer.header(f' - {f"Setting owner" if chown else "Not setting owner"}')
printer.header(
f' - {f"Setting permissions" if chmod else "Not setting permissions"}'
)
if overwrite:
printer.warning(f' - Overwriting {build_dir}')
printer.header(f' - {"Linking" if link else "Not linking"}')
confirm(f'Continue with deployment of version {version} to {env}?',
abort_on_unconfirmed=True)
if build_:
build(env, clean_=clean_, verbose=verbose)
if push:
remote(f'test -d {build_dir} || mkdir -p {build_dir}')
printer.info(f'Pushing public/ to {build_dir}...')
sync('public/',
f'{build_dir}/',
host,
delete=overwrite,
dry_run=dry_run,
echo=verbose)
if chown:
owner = 'bycycle:www-data'
printer.info(f'Setting ownership of {build_dir} to {owner}...')
if real_run:
remote(('chown', '-R', owner, build_dir), sudo=True)
if chmod:
mode = 'u=rwX,g=rwX,o='
printer.info(f'Setting permissions on {build_dir} to {mode}...')
if real_run:
remote(('chmod', '-R', mode, build_dir), sudo=True)
if link:
printer.info(f'Linking {link_path} to {build_dir}')
if real_run:
remote(('ln', '-sfn', build_dir, link_path))