def rmdir(name): if os.path.isdir(name): shutil.rmtree(name) if verbose: printer.info('Removed directory:', name) else: if verbose: printer.info('Directory not present:', name)
def rmdir(directory, quiet=False): if directory.is_dir(): shutil.rmtree(directory) if not quiet: printer.info('Removed directory:', directory) else: if not quiet: printer.warning('Directory does not exist:', directory)
def rmfile(name, verbose=False): if os.path.isfile(name): os.remove(name) if verbose: printer.info("Removed file:", name) else: if verbose: printer.info("File not present:", name)
def build_docs(source="docs", destination="docs/_build", builder="html", clean=False): if clean: printer.info(f"Removing {destination}...") shutil.rmtree(destination) local(("sphinx-build", "-b", builder, source, destination))
def rm(name): if os.path.isfile(name): os.remove(name) if verbose: printer.info('Removed file:', name) else: if verbose: printer.info('File not present:', name)
def rmdir(name, verbose=False): if os.path.isdir(name): shutil.rmtree(name) if verbose: printer.info("Removed directory:", name) else: if verbose: printer.info("Directory not present:", name)
def load_usps_street_suffixes(db): """Load USPS street suffixes into database.""" module = import_module('bycycle.core.model') base_path = os.path.dirname(module.__file__) path = os.path.join(base_path, 'usps_street_suffixes.csv') engine = create_engine(**db) session_factory = sessionmaker(bind=engine) session = session_factory() printer.info('Deleting existing USPS street suffixes...', end=' ') count = session.query(USPSStreetSuffix).delete() session.commit() printer.info(count, 'deleted') printer.info('Adding USPS street suffixes...', end=' ') with open(path) as fp: reader = csv.DictReader(fp) records = [USPSStreetSuffix(**row) for row in reader] count = len(records) session.add_all(records) session.commit() printer.info(count, 'added') session.close() engine.dispose()
def load_usps_street_suffixes(user, password, database, host='localhost', port=5432): """Load USPS street suffixes into database.""" file_name = '{model.__tablename__}.csv'.format(model=USPSStreetSuffix) path = asset_path('bycycle.core.model', file_name) engine = create_engine(user, password, host, port, database) session_factory = sessionmaker(bind=engine) session = session_factory() printer.info('Deleting existing USPS street suffixes...', end=' ') count = session.query(USPSStreetSuffix).delete() session.commit() printer.info(count, 'deleted') printer.info('Adding USPS street suffixes...', end=' ') with open(path) as fp: reader = csv.DictReader(fp) records = [USPSStreetSuffix(**row) for row in reader] count = len(records) session.add_all(records) session.commit() printer.info(count, 'added') session.close() engine.dispose()
def make_dist( version: arg(help="Tag/version to release [latest tag]") = None, formats=("sdist", "wheel"), quiet=False, ): """Make a distribution for upload to PyPI. Switches to the specified tag or branch, makes the distribution, then switches back to the original branch. Intended to be run from the develop branch. If a tag is already checked out, the develop branch will be checked out first and then switched back to after making the distribution. """ current_branch = get_current_branch() original_branch = "develop" if current_branch == "HEAD" else current_branch version = version or get_latest_tag() stdout = "hide" if quiet else None printer.header(f"Making dist for {version}") if version != current_branch: if current_branch == "HEAD": printer.warning("Not on a branch; checking out develop first") else: printer.info("Currently on branch", current_branch) printer.info("Checking out", version) # XXX: Hide warning about detached HEAD state result = local(("git", "checkout", version), stdout=stdout, stderr="capture") if result.failed: print(result.stderr, file=sys.stderr) printer.info("Removing dist directory") rmdir("dist", verbose=not quiet) printer.info("Making dists for", version) for format_ in formats: local(("poetry", "build", "--format", format_), stdout=stdout) if version != current_branch: printer.info("Switching back to", original_branch) # XXX: Hide message about previous HEAD state and branch info result = local(("git", "checkout", original_branch), stdout="hide", stderr="capture") if result.failed: print(result.stderr, file=sys.stderr)
def execute(engine, sql, condition=True): if not condition: return if isinstance(sql, (list, tuple)): sql = ' '.join(sql) printer.info('Running SQL:', sql) try: return engine.execute(sql) except ProgrammingError as exc: error = str(exc.orig) exc_str = str(exc) if 'already exists' in exc_str or 'does not exist' in exc_str: printer.warning(exc.statement.strip(), error.strip(), sep=': ') else: raise
def dev_server(default_args, host='localhost', port=5000, directory='public'): printer.header('Running dev server') printer.hr('Cleaning', color='info') clean() printer.info('Running scss watcher in background') sass_args = ['sass', '--watch'] for source in default_args['sass']['sources']: name = os.path.basename(source) root, ext = os.path.splitext(name) destination = os.path.join('public', 'build', f'{root}.css') sass_args.append(f'{source}:{destination}') local(sass_args, background=True, environ={ 'NODE_ENV': 'development', }) wait_for_file(destination) printer.hr('Running rollup watcher in background', color='info') local(['rollup', '--config', '--watch'], background=True, environ={ 'NODE_ENV': 'development', 'LIVE_RELOAD': 'true', }) wait_for_file('public/build/bundle.js') printer.hr(f'Serving {directory} directory at http://{host}:{port}/', color='info') class RequestHandler(SimpleHTTPRequestHandler): def __init__(self, *args, **kwargs): super().__init__(*args, directory=directory, **kwargs) def translate_path(self, path): path = super().translate_path(path) if not os.path.exists(path): path = os.path.join(self.directory, 'index.html') return path server = ThreadingHTTPServer((host, port), RequestHandler) try: server.serve_forever() except KeyboardInterrupt: abort(message='Shut down dev server')
def deploy(host, to='/sites/bycycle.org/current/frontend/', build_=True, clean_=False, overwrite=False, dry_run=False): printer.header(f'Deploying to {host}...') if build_: build(clean_=clean_) printer.info(f'Pushing build/ to {to}...') sync('build/', to, host, run_as='bycycle', delete=overwrite, dry_run=dry_run, echo=True) printer.info(f'Setting ownership of {to}...') remote(('chown -R bycycle:www-data', to), sudo=True)
def virtualenv(config, where='.env', python='python3', overwrite=False): exists = os.path.exists(where) def create(): local(config, ('virtualenv', '-p', python, where)) printer.success( 'Virtualenv created; activate it by running `source {where}/bin/activate`' .format_map(locals())) if exists: if overwrite: printer.warning('Overwriting virtualenv', where, 'with', python) shutil.rmtree(where) create() else: printer.info('Virtualenv', where, 'exists; pass --overwrite to re-create it') else: printer.info('Creating virtualenv', where, 'with', python) create()
def load_osm_data(db, bbox: arg(type=float, nargs=4), directory='../osm', graph_path='../graph.marshal', streets=True, places=True, actions: arg(container=tuple, type=int) = (), show_actions: arg(short_option='-a') = False, log_to=None): """Read OSM data from file and load into database.""" importer = OSMImporter(bbox, directory, graph_path, db, streets, places, actions) if show_actions: printer.header('Available actions:') for action in importer.all_actions: printer.info(action) return importer.run() if log_to: message = f'Loaded OSM data from {importer.data_directory} to {importer.engine.url}' log_to_file(log_to, message)
def install_completion(config, shell='bash', to='~/.bashrc.d', overwrite=True): """Install command line completion script. Currently, only Bash is supported. The script will be copied to the directory ``~/.bashrc.d`` by default. If the script already exists at that location, it will be overwritten by default. """ source = 'runcommands:completion/{shell}/runcommands.rc'.format( shell=shell) source = asset_path(source) destination = os.path.expanduser(to) if os.path.isdir(destination): to = os.path.join(to, 'runcommands.rc') destination = os.path.join(destination, 'runcommands.rc') printer.info('Installing', shell, 'completion script to', to) if os.path.exists(destination): if not overwrite: overwrite = confirm(config, 'Overwrite?', abort_on_unconfirmed=True) if overwrite: printer.info('Overwriting', to) shutil.copyfile(source, destination) printer.info('Installed; remember to `source {to}`'.format(to=to))
def clean(all_=False): """Clean up. - Remove build directory - Remove dist directory - Remove __pycache__ directories If --all: - Remove egg-info directory - Remove .venv directory """ def rmdir(directory, quiet=False): if directory.is_dir(): shutil.rmtree(directory) if not quiet: printer.info('Removed directory:', directory) else: if not quiet: printer.warning('Directory does not exist:', directory) cwd = Path.cwd() rmdir(Path('./build')) rmdir(Path('./dist')) pycache_dirs = tuple(Path('.').glob('**/__pycache__/')) num_pycache_dirs = len(pycache_dirs) for pycache_dir in pycache_dirs: rmdir(pycache_dir, quiet=True) if num_pycache_dirs: printer.info(f'Removed {num_pycache_dirs} __pycache__' f'director{"y" if num_pycache_dirs == 1 else "ies"}') else: printer.warning('No __pycache__ directories found') if all_: rmdir(cwd / '.venv') rmdir(cwd / f'{cwd.name}.egg-info')
def upload_dists( make: arg(help="Make dist first? [yes]") = True, version: arg(help="Version/tag to release [latest tag]") = None, quiet: arg(help="Make dist quietly? [no]") = False, username: arg(help="Twine username [$USER]") = None, password_command: arg(help="Command to retrieve twine password " "(e.g. `password-manager show-password PyPI`) " "[twine prompt]") = None, ): """Upload distributions in ./dist using ``twine``.""" if make: printer.header("Making and uploading distributions") make_dist(quiet=quiet) else: printer.header("Uploading distributions") dists = os.listdir("dist") if not dists: abort(1, "No distributions found in dist directory") paths = [os.path.join("dist", file) for file in dists] printer.info("Found distributions:") for path in paths: printer.info(" -", path) if not confirm("Continue?"): abort() if not username: username = getpass.getuser() environ = {"TWINE_USERNAME": username} if password_command: printer.info(f"Retrieving password via `{password_command}`...") result = local(password_command, stdout="capture") password = result.stdout.strip() environ["TWINE_PASSWORD"] = password printer.warning("TWINE_USERNAME:"******"TWINE_PASSWORD:"******"*" * len(password)) for path in paths: if confirm(f"Upload dist?: {path}"): local(("twine", "upload", path), environ=environ) else: printer.warning("Skipped dist:", path)
def deploy(env, version=None, # Corresponding tags will be included unless unset *or* any # tags are specified via --tags. prepare: 'Run preparation tasks (local)' = True, dijkstar: 'Run Dijkstar tasks (remote)' = True, deploy: 'Run deployment tasks (remote)' = True, # Corresponding tags will be skipped unless set. clean: 'Remove local build directory' = False, overwrite: 'Remove remote build directory' = False, tags: 'Run *only* tasks corresponding to these tags' = (), skip_tags: 'Skip tasks corresponding to these tags' = (), yes: 'Deploy without confirmation' = False, echo=True): """Deploy the byCycle web API. Typical usage:: run deploy -c """ version = version or git_version() tags = (tags,) if isinstance(tags, str) else tags skip_tags = (skip_tags,) if isinstance(skip_tags, str) else skip_tags yes = False if env == 'production' else yes if tags: prepare = 'prepare' in tags dijkstar = 'dijkstar' in tags deploy = 'deploy' in tags else: if prepare: tags += ('prepare',) if dijkstar: tags += ('dijkstar',) if deploy: tags += ('deploy',) if not clean: skip_tags += ('remove-build-directory',) if not overwrite: skip_tags += ('overwrite',) if not prepare: printer.info('Not preparing') if not dijkstar: printer.info('Not running Dijkstar tasks') if not deploy: printer.info('Not deploying') if tags: printer.info('Selected tags: %s' % ', '.join(tags)) if skip_tags: printer.info('Skipping tags: %s' % ', '.join(skip_tags)) if clean: printer.warning('Local build directory will be removed first') if overwrite: printer.warning('Remote build directory will be overwritten') environ = {} if not yes: message = f'Deploy version {version} to {env}?' confirm(message, abort_on_unconfirmed=True) printer.header(f'Deploying {version} to {env}...') ansible_args = get_ansible_args(env, version=version, tags=tags, skip_tags=skip_tags) return local(ansible_args, environ=environ, echo=echo)
def deploy(env, host, version=None, build_=True, clean_=True, verbose=False, push=True, overwrite=False, chown=True, chmod=True, link=True, dry_run=False): if env == 'development': abort(1, 'Can\'t deploy to development environment') version = version or git_version() root_dir = f'/sites/{host}/webui' build_dir = f'{root_dir}/builds/{version}' link_path = f'{root_dir}/current' real_run = not dry_run printer.hr( f'{"[DRY RUN] " if dry_run else ""}Deploying version {version} to {env}', color='header') printer.header('Host:', host) printer.header('Remote root directory:', root_dir) printer.header('Remote build directory:', build_dir) printer.header('Remote link to current build:', link_path) printer.header('Steps:') printer.header(f' - {"Cleaning" if clean_ else "Not cleaning"}') printer.header(f' - {"Building" if build_ else "Not building"}') printer.header(f' - {f"Pushing" if push else "Not pushing"}') printer.header(f' - {f"Setting owner" if chown else "Not setting owner"}') printer.header( f' - {f"Setting permissions" if chmod else "Not setting permissions"}' ) if overwrite: printer.warning(f' - Overwriting {build_dir}') printer.header(f' - {"Linking" if link else "Not linking"}') confirm(f'Continue with deployment of version {version} to {env}?', abort_on_unconfirmed=True) if build_: build(env, clean_=clean_, verbose=verbose) if push: remote(f'test -d {build_dir} || mkdir -p {build_dir}') printer.info(f'Pushing public/ to {build_dir}...') sync('public/', f'{build_dir}/', host, delete=overwrite, dry_run=dry_run, echo=verbose) if chown: owner = 'bycycle:www-data' printer.info(f'Setting ownership of {build_dir} to {owner}...') if real_run: remote(('chown', '-R', owner, build_dir), sudo=True) if chmod: mode = 'u=rwX,g=rwX,o=' printer.info(f'Setting permissions on {build_dir} to {mode}...') if real_run: remote(('chmod', '-R', mode, build_dir), sudo=True) if link: printer.info(f'Linking {link_path} to {build_dir}') if real_run: remote(('ln', '-sfn', build_dir, link_path))
def do_done(message, cmd, *args, _end=' ', **kwargs): printer.info(message, end=_end, flush=True) cmd(*args, **kwargs) printer.success('Done')
def release(config, version=None, date=None, tag_name=None, next_version=None, prepare=True, merge=True, create_tag=True, resume=True, yes=False): def update_line(file_name, line_number, content): with open(file_name) as fp: lines = fp.readlines() lines[line_number] = content with open(file_name, 'w') as fp: fp.writelines(lines) result = local(config, 'git rev-parse --abbrev-ref HEAD', hide='stdout') current_branch = result.stdout.strip() if current_branch != 'develop': abort(1, 'Must be on develop branch to make a release') init_module = 'runcommands/__init__.py' changelog = 'CHANGELOG' # E.g.: __version__ = '1.0.dev0' version_re = r"^__version__ = '(?P<version>.+)(?P<dev_marker>\.dev\d+)'$" # E.g.: ## 1.0.0 - 2017-04-01 changelog_header_re = r'^## (?P<version>.+) - (?P<date>.+)$' with open(init_module) as fp: for init_line_number, line in enumerate(fp): if line.startswith('__version__'): match = re.search(version_re, line) if match: current_version = match.group('version') if not version: version = current_version break else: abort( 1, 'Could not find __version__ in {init_module}'.format_map( locals())) date = date or datetime.date.today().isoformat() tag_name = tag_name or version if next_version is None: next_version_re = r'^(?P<major>\d+)\.(?P<minor>\d+)(?P<rest>.*)$' match = re.search(next_version_re, version) if match: major = match.group('major') minor = match.group('minor') major = int(major) minor = int(minor) rest = match.group('rest') patch_re = r'^\.(?P<patch>\d+)$' match = re.search(patch_re, rest) if match: # X.Y.Z minor += 1 patch = match.group('patch') next_version = '{major}.{minor}.{patch}'.format_map(locals()) else: pre_re = r'^(?P<pre_marker>a|b|rc)(?P<pre_version>\d+)$' match = re.search(pre_re, rest) if match: # X.YaZ pre_marker = match.group('pre_marker') pre_version = match.group('pre_version') pre_version = int(pre_version) pre_version += 1 next_version = '{major}.{minor}{pre_marker}{pre_version}'.format_map( locals()) else: # X.Y or starts with X.Y (but is not X.Y.Z or X.YaZ) minor += 1 next_version = '{major}.{minor}'.format_map(locals()) if next_version is None: msg = 'Cannot automatically determine next version from {version}'.format_map( locals()) abort(3, msg) next_version_dev = '{next_version}.dev0'.format_map(locals()) # Find the first line that starts with '##'. Extract the version and # date from that line. The version must be the specified release # version OR the date must be the literal string 'unreleased'. with open(changelog) as fp: for changelog_line_number, line in enumerate(fp): if line.startswith('## '): match = re.search(changelog_header_re, line) if match: found_version = match.group('version') found_date = match.group('date') if found_version == version: if found_date != 'unreleased': printer.warning('Re-releasing', version) elif found_date == 'unreleased': if found_version != version: printer.warning('Replacing', found_version, 'with', version) else: msg = ( 'Expected version {version} or release date "unreleased"; got:\n\n' ' {line}').format_map(locals()) abort(4, msg) break else: abort(5, 'Could not find section in change log') printer.info('Version:', version) printer.info('Tag name:', tag_name) printer.info('Release date:', date) printer.info('Next version:', next_version) msg = 'Continue with release?: {version} - {date}'.format_map(locals()) yes or confirm(config, msg, abort_on_unconfirmed=True) printer.header('Testing...') tox(config) # Prepare if prepare: printer.header('Preparing release', version, 'on', date) updated_init_line = "__version__ = '{version}'\n".format_map(locals()) updated_changelog_line = '## {version} - {date}\n'.format_map(locals()) update_line(init_module, init_line_number, updated_init_line) update_line(changelog, changelog_line_number, updated_changelog_line) local(config, ('git diff', init_module, changelog)) yes or confirm( config, 'Commit these changes?', abort_on_unconfirmed=True) msg = prompt('Commit message', default='Prepare release {version}'.format_map(locals())) msg = '-m "{msg}"'.format_map(locals()) local(config, ('git commit', init_module, changelog, msg)) # Merge and tag if merge: printer.header('Merging develop into master for release', version) local(config, 'git log --oneline --reverse master..') msg = 'Merge these changes from develop into master for release {version}?' msg = msg.format_map(locals()) yes or confirm(config, msg, abort_on_unconfirmed=True) local(config, 'git checkout master') msg = '"Merge branch \'develop\' for release {version}"'.format_map( locals()) local(config, ('git merge --no-ff develop -m', msg)) if create_tag: printer.header('Tagging release', version) msg = '"Release {version}"'.format_map(locals()) local(config, ('git tag -a -m', msg, version)) local(config, 'git checkout develop') # Resume if resume: printer.header('Resuming development at', next_version) updated_init_line = "__version__ = '{next_version_dev}'\n".format_map( locals()) new_changelog_lines = [ '## {next_version} - unreleased\n\n'.format_map(locals()), 'In progress...\n\n', ] update_line(init_module, init_line_number, updated_init_line) with open(changelog) as fp: lines = fp.readlines() lines = lines[:changelog_line_number] + new_changelog_lines + lines[ changelog_line_number:] with open(changelog, 'w') as fp: fp.writelines(lines) local(config, ('git diff', init_module, changelog)) yes or confirm( config, 'Commit these changes?', abort_on_unconfirmed=True) msg = prompt('Commit message', default='Resume development at {next_version}'.format_map( locals())) msg = '-m "{msg}"'.format_map(locals()) local(config, ('git commit', init_module, changelog, msg))
def install_completion( shell: arg( choices=("bash", "fish"), help="Shell to install completion for", ), to: arg( help="~/.bashrc.d/runcommands.rc or ~/.config/fish/runcommands.fish", ) = None, base_command: arg(help="Dotted path to base command", ) = None, base_command_name: arg( short_option="-B", help="Name of base command (if different from implementation name)", ) = None, overwrite: arg(help="Overwrite if exists", ) = False, ): """Install command line completion script. Currently, bash and fish are supported. The corresponding script will be copied to an appropriate directory. If the script already exists at that location, it will be overwritten by default. """ if base_command: if not base_command_name: _, base_command_name = base_command.rsplit(".", 1) source_base_name = "runcommands-base-command" template_type = "string" template_context = { "base_command_path": base_command, "base_command_name": base_command_name, } else: source_base_name = "runcommands" template_type = None template_context = {} if shell == "bash": ext = "rc" to = to or "~/.bashrc.d" elif shell == "fish": ext = "fish" to = to or "~/.config/fish" if base_command: to = f"{to}/{base_command_name}.{ext}" source = asset_path( f"runcommands:completion/{shell}/{source_base_name}.{ext}") destination = os.path.expanduser(to) if os.path.isdir(destination): destination = os.path.join(destination, os.path.basename(source)) printer.info("Installing", shell, "completion script to:\n ", destination) if os.path.exists(destination): if overwrite: printer.info(f"Overwriting:\n {destination}") else: confirm(f"File exists. Overwrite?", abort_on_unconfirmed=True) _copy_file(source, destination, template=template_type, context=template_context) printer.info(f"Installed; remember to:\n source {destination}")
def print_step(message, flag): printer.info(message, end=" ", flush=True) printer.print("yes" if flag else "no", color="green" if flag else "red")
def deploy(config, version=None, overwrite=False, overwrite_venv=False, install=True, push=True, link=True, reload=True): # Setup ---------------------------------------------------------- if version: config = config.copy(version=version) elif config.get('version'): printer.info('Using default version:', config.version) else: abort(1, 'Version must be specified via config or passed as an option') # Local ---------------------------------------------------------- build_dir = config.build.dir if overwrite and os.path.exists(build_dir): shutil.rmtree(build_dir) os.makedirs(build_dir, exist_ok=True) # Add config files copy_file(config, 'application.wsgi', build_dir, template=True) copy_file(config, 'base.ini', build_dir) copy_file(config, '{env}.ini', build_dir, template=True) copy_file(config, 'commands.py', build_dir) copy_file(config, 'commands.cfg', build_dir) # Create source distributions dist_dir = os.path.abspath(config.build.dist_dir) sdist_command = ('python setup.py sdist --dist-dir', dist_dir) local(config, sdist_command, hide='stdout') for path in config.deploy.sdists: local(config, sdist_command, hide='stdout', cd=path) tarball_name = '{config.version}.tar.gz'.format(config=config) tarball_path = os.path.join(build_dir, tarball_name) with tarfile.open(tarball_path, 'w:gz') as tarball: tarball.add(build_dir, config.version) if push: local(config, ( 'rsync -rltvz', '--rsync-path "sudo -u {deploy.user} rsync"', tarball_path, '{remote.host}:{deploy.root}', )) # Remote ---------------------------------------------------------- deploy_dir_exists = remote(config, 'test -d {deploy.dir}', abort_on_failure=False) if deploy_dir_exists and overwrite: remote(config, 'rm -r {deploy.dir}') remote(config, ('tar -xvzf', tarball_name), cd='{deploy.root}') # Create virtualenv for this version venv_exists = remote(config, 'test -d {deploy.venv}', abort_on_failure=False) if venv_exists and overwrite_venv: remote(config, 'rm -r {deploy.venv}') venv_exists = False if not venv_exists: remote(config, ( 'python{python.version} -m venv {deploy.venv} &&', '{deploy.pip.exe} install', '--cache-dir {deploy.pip.cache_dir}', '--upgrade setuptools pip wheel', )) # Build source if install: remote(config, ( '{deploy.pip.exe}', 'install', '--find-links {deploy.pip.find_links}', '--cache-dir {deploy.pip.cache_dir}', '--disable-pip-version-check', '{package}', ), cd='{deploy.root}', timeout=120) # Make this version the current version if link: remote(config, 'ln -sfn {deploy.dir} {deploy.link}') # Set permissions remote(config, 'chmod -R ug=rwX,o= {deploy.root}') if reload: reload_uwsgi(config)
def print_info(label, arg, *args): printer.info(label, end=" ", flush=True) printer.print(arg, *args)