def git_merge(base: str, head: str, no_ff: bool = False):
""" Merge *head* into *base*.
Args:
base (str):
The base branch. *head* will be merged into this branch.
head (str):
The branch that will be merged into *base*.
no_ff (bool):
If set to **True** it will force git to create merge commit. If set
to **False** (default) it will do a fast-forward merge if possible.
"""
pretend = context.get('pretend', False)
branch = git.current_branch(refresh=True)
if branch.name != base and not pretend:
git_checkout(base)
args = []
if no_ff:
args.append('--no-ff')
log.info("Merging <33>{}<32> into <33>{}<32>", head, base)
shell.run('git merge {args} {branch}'.format(
args=' '.join(args),
branch=head,
))
if branch.name != base and not pretend:
git_checkout(branch.name)
def exec_script_command(cmd: str, pretend: bool) -> int:
""" This will execute the already compiled script command.
This function serves the purpose of encapsulating the low level code of
spawning a subprocess from the rest of the logic.
"""
if not pretend:
with conf.within_proj_dir():
# God knows why, if we run the command using `shell.run()` and it
# exists with non-zero code it will also kill the parent peltak
# process. The Popen arguments are the same, to my knowledge
# everything is the same but the way it behaves is completely
# different. If we just use Popen directly, everything works as
# expected ¯\_(ツ)_/¯
p = subprocess.Popen(cmd, shell=True)
try:
p.communicate()
return p.returncode
except KeyboardInterrupt:
p.kill()
return -1
else:
log.info(
"<90>{bar}<0>\n{script}\n<90>{bar}",
bar='=' * 80,
script=shell.highlight(cmd, 'bash'),
)
return 0
def init(quick: bool, blank: bool, force: bool):
""" Create an empty pelconf.yaml from template """
config_file = 'pelconf.yaml'
prompt = "-- <35>{} <32>already exists. Wipe it?<0>".format(config_file)
if not force and exists(config_file) and not click.confirm(
shell.fmt(prompt)):
log.info("Canceled")
return
ctx = dict(blank=blank)
if not blank:
form = InitForm().run(quick=quick)
ctx.update(form.values)
config_content = templates.Engine().render_file('pelconf.yaml', ctx)
log.info('Writing <35>{}'.format(config_file))
fs.write_file(config_file, config_content)
if context.get('verbose') > 0:
print(
f"{'- ' * 40}\n{shell.highlight(config_content, 'yaml')}{'- ' * 40}"
)
def _manage_cmd(cmd, settings=None):
# type: () -> None
""" Run django ./manage.py command manually.
This function eliminates the need for having ``manage.py`` (reduces file
clutter).
"""
import sys
from os import environ
from peltak.core import conf
from peltak.core import context
from peltak.core import log
sys.path.insert(0, conf.get('src_dir'))
settings = settings or conf.get('django.settings', None)
environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
args = sys.argv[0:-1] + cmd
if context.get('pretend', False):
log.info("Would run the following manage command:\n<90>{}", args)
else:
from django.core.management import execute_from_command_line
execute_from_command_line(args)
def deploy(self, promote=False, quiet=False):
# type: (bool, bool, bool) -> None
""" Deploy the code to AppEngine.
Args:
promote (bool):
Migrate the traffic to the deployed version.
quiet (bool):
Pass ``--quiet`` flag to gcloud command
"""
args = [
'--promote' if promote else '--no-promote',
'--version {}'.format(self.app_version),
'--project {}'.format(self.app_id),
]
if quiet:
args += ['--quiet']
cmd = 'gcloud app deploy {args} {deployables}'.format(
deployables=fs.wrap_paths(self.deployables), args=' '.join(args))
if context.get('pretend', False):
log.info("Would deploy version <35>{ver}<32> to <35>{app}".format(
ver=self.app_version, app=self.app_id))
shell.cprint('<90>{}', cmd)
else:
log.info("Deploying version <35>{ver}<32> to <35>{app}".format(
ver=self.app_version,
app=self.app_id,
))
shell.run(cmd)
def finish(fast_forward: bool):
""" Merge current feature branch into develop. """
pretend = context.get('pretend', False)
if not pretend and (git.staged() or git.unstaged()):
log.err("You have uncommitted changes in your repo!\n"
"You need to stash them before you merge the hotfix branch")
sys.exit(1)
branch = git.current_branch(refresh=True)
base = common.get_base_branch()
prompt = "<32>Merge <33>{}<32> into <33>{}<0>?".format(branch.name, base)
if not click.confirm(shell.fmt(prompt)):
log.info("Cancelled")
return
common.assert_branch_type('task')
hooks.register.call('pre-task-finish', branch, base)
# Merge task into it's base feature branch
common.git_checkout(base)
common.git_pull(base)
common.git_merge(base, branch.name, no_ff=not fast_forward)
# Cleanup
common.git_branch_delete(branch.name)
common.git_prune()
common.git_checkout(base)
hooks.register.call('post-task-finish', branch, base)
def git_pull(branch_name: str):
""" Pull from remote branch.
Args:
branch_name (str):
The remote branch to pull.
"""
log.info("Pulling latest changes on <33>{}", branch_name)
shell.run('git pull origin {}'.format(branch_name))
def git_branch_delete(branch_name: str):
""" Delete the given branch.
Args:
branch_name (str):
Name of the branch to delete.
"""
if branch_name not in git.protected_branches():
log.info("Deleting branch <33>{}", branch_name)
shell.run('git branch -d {}'.format(branch_name))
def git_checkout(branch_name: str, create: bool = False):
""" Checkout or create a given branch
Args:
branch_name (str):
The name of the branch to checkout or create.
create (bool):
If set to **True** it will create the branch instead of checking it
out.
"""
log.info("Checking out <33>{}".format(branch_name))
shell.run('git checkout {} {}'.format('-b' if create else '', branch_name))
def tag(message: str):
""" Tag the current commit with the current version. """
release_ver = versioning.current()
message = message or 'v{} release'.format(release_ver)
with conf.within_proj_dir():
log.info("Creating release tag")
git.tag(
author=git.latest_commit().author,
name='v{}'.format(release_ver),
message=message,
)
def upload(target: str):
""" Upload the release to a pypi server.
TODO: Make sure the git directory is clean before allowing a release.
Args:
target (str):
pypi target as defined in ~/.pypirc
"""
log.info("Uploading to pypi server <33>{}".format(target))
with conf.within_proj_dir():
shell.run('python setup.py sdist register -r "{}"'.format(target))
shell.run('python setup.py sdist upload -r "{}"'.format(target))
def git_branch_rename(new_name: str):
""" Rename the current branch
Args:
new_name (str):
New name for the current branch.
"""
curr_name = git.current_branch(refresh=True).name
if curr_name not in git.protected_branches():
log.info("Renaming branch from <33>{}<32> to <33>{}".format(
curr_name, new_name))
shell.run('git branch -m {}'.format(new_name))
def get_base_branch() -> str:
""" Return the base branch for the current branch.
This function will first try to guess the base branch and if it can't it
will let the user choose the branch from the list of all local branches.
Returns:
str: The name of the branch the current branch is based on.
"""
base_branch = git.guess_base_branch()
if base_branch is None:
log.info("Can't guess the base branch, you have to pick one yourself:")
base_branch = choose_branch()
return base_branch
def assert_on_branch(branch_name: str):
""" Print error and exit if *branch_name* is not the current branch.
Args:
branch_name (str):
The supposed name of the current branch.
"""
branch = git.current_branch(refresh=True)
if branch.name != branch_name:
if context.get('pretend', False):
log.info("Would assert that you're on a <33>{}<32> branch",
branch_name)
else:
log.err("You're not on a <33>{}<31> branch!", branch_name)
sys.exit(1)
def start(component: str, exact: str):
""" Create a new release branch.
Args:
component (str):
Version component to bump when creating the release. Can be *major*,
*minor* or *patch*.
exact (str):
The exact version to set for the release. Overrides the component
argument. This allows to re-release a version if something went
wrong with the release upload.
"""
version_files = versioning.get_version_files()
develop = conf.get('git.devel_branch', 'develop')
common.assert_on_branch(develop)
with conf.within_proj_dir():
out = shell.run('git status --porcelain', capture=True).stdout
lines = out.split(os.linesep)
has_changes = any(not line.startswith('??') for line in lines
if line.strip())
if has_changes:
log.info("Cannot release: there are uncommitted changes")
exit(1)
old_ver, new_ver = versioning.bump(component, exact)
log.info("Bumping package version")
log.info(" old version: <35>{}".format(old_ver))
log.info(" new version: <35>{}".format(new_ver))
with conf.within_proj_dir():
branch = 'release/' + new_ver
hooks.register.call('pre-release-start', branch, old_ver, new_ver)
common.git_checkout(branch, create=True)
log.info("Creating commit for the release")
shell.run('git add {files} && git commit -m "{msg}"'.format(
files=' '.join(f'"{v.path}"' for v in version_files),
msg="Releasing v{}".format(new_ver)))
hooks.register.call('post-release-start', branch, old_ver, new_ver)
def add_hooks(pre_commit: str, pre_push: str):
""" Add git hooks for commit and push to run linting and tests. """
# Detect virtualenv the hooks should use
# Detect virtualenv
virtual_env = conf.get_env('VIRTUAL_ENV')
if virtual_env is None:
log.err("You are not inside a virtualenv")
confirm_msg = (
"Are you sure you want to use global python installation "
"to run your git hooks? [y/N] "
)
click.prompt(confirm_msg, default='')
if not click.confirm(confirm_msg):
log.info("Cancelling")
return
load_venv = ''
else:
load_venv = 'source "{}/bin/activate"'.format(virtual_env)
commit_hook = conf.proj_path('.git/hooks/pre-commit')
push_hook = conf.proj_path('.git/hooks/pre-push')
# Write pre-commit hook
log.info("Adding pre-commit hook <33>{}", commit_hook)
fs.write_file(commit_hook, util.remove_indent('''
#!/bin/bash
PATH="/opt/local/libexec/gnubin:$PATH"
{load_venv}
{command}
'''.format(load_venv=load_venv, command=pre_commit)))
# Write pre-push hook
log.info("Adding pre-push hook: <33>{}", push_hook)
fs.write_file(push_hook, util.remove_indent('''
#!/bin/bash
PATH="/opt/local/libexec/gnubin:$PATH"
{load_venv}
peltak test --allow-empty
{command}
'''.format(load_venv=load_venv, command=pre_push)))
log.info("Making hooks executable")
if not context.get('pretend', False):
os.chmod(conf.proj_path('.git/hooks/pre-commit'), 0o755)
os.chmod(conf.proj_path('.git/hooks/pre-push'), 0o755)
def _render_todos(todos: List[Todo]) -> None:
print('\n')
for file_path, file_todos in itertools.groupby(todos,
key=lambda x: x.file):
shell.cprint(f"<92>{file_path}\n")
for todo in sorted(file_todos, key=lambda x: x.lines.start):
if context.get('verbose') >= 1:
shell.cprint(
f"<36>{todo.pretty_timestamp} <33>{todo.author}<0>\n"
f"<95>{todo.file}:{todo.lines} <90>{todo.sha1}<0>\n\n"
f"{textwrap.indent(todo.color_text, ' ')}\n\n")
else:
shell.cprint(
f" <95>:{todo.lines} <36>{todo.pretty_timestamp} "
f"<33>{todo.author_email} <90>{todo.sha1}<0><0>\n\n"
f"{textwrap.indent(todo.color_text, ' ')}\n")
print()
log.info(f"Found <33>{len(todos)}<32> TODOs")
def assert_branch_type(branch_type: str):
""" Print error and exit if the current branch is not of a given type.
Args:
branch_type (str):
The branch type. This assumes the branch is in the '<type>/<title>`
format.
"""
branch = git.current_branch(refresh=True)
if branch.type != branch_type:
if context.get('pretend', False):
log.info("Would assert that you're on a <33>{}/*<32> branch",
branch_type)
else:
log.err("Not on a <33>{}<31> branch!", branch_type)
fmt = ("The branch must follow <33>{required_type}/<name><31>"
"format and your branch is called <33>{name}<31>.")
log.err(fmt, required_type=branch_type, name=branch.name)
sys.exit(1)
def write_file(path: str, content: Union[str, bytes], mode: str = 'w') -> None:
""" --pretend aware file writing.
You can always write files manually but you should always handle the
--pretend case.
Args:
path (str):
content (str):
mode (str):
"""
from peltak.core import context
from peltak.core import log
if context.get('pretend', False):
log.info("Would overwrite <34>{path}<32> with:\n<90>{content}",
path=path,
content=content)
else:
with open(path, mode) as fp:
fp.write(content)
def version_cli(ctx: click.Context, porcelain: bool) -> None:
""" Show project version. Has sub commands.
For this command to work you must specify where the project version is
stored. You can do that with version.file conf variable. peltak supports
multiple ways to store the project version. Right now you can store it in a
python file using built-in __version__ variable. You can use node.js
package.json and keep the version there or you can just use a plain text
file that just holds the raw project version. The appropriate storage is
guessed based on the file type and name.
Example Configuration::
version:
file: 'src/mypackage/__init__.py'
Examples:
\b
$ peltak version # Pretty print current version
$ peltak version --porcelain # Print version as raw string
$ peltak version bump patch # Bump patch version component
$ peltak version bump minor # Bump minor version component
$ peltak version bump major # Bump major version component
$ peltak version bump release # same as version bump patch
$ peltak version bump --exact=1.2.1 # Set project version to 1.2.1
"""
if ctx.invoked_subcommand:
return
from peltak.core import log
from peltak.core import versioning
current = versioning.current()
if porcelain:
print(current)
else:
log.info("Version: <35>{}".format(current))
def run_script(script: Script, options: CliOptions) -> None:
""" Run the script with the given (command line) options. """
template_ctx = build_template_context(script, options)
verbose = RunContext().get('verbose')
pretend = RunContext().get('pretend')
if verbose >= 3:
log.info('Compiling script <35>{name}\n{script}'.format(
name=script.name, script=shell.highlight(script.command, 'jinja')))
yaml_str = yaml.dump(template_ctx, default_flow_style=False)
log.info('with context:\n{}\n'.format(shell.highlight(
yaml_str, 'yaml')))
# Command is either specified directly in pelconf.yaml or lives in a
# separate file.
command = script.command
if script.command_file:
with open(conf.proj_path(script.command_file)) as fp:
command = fp.read()
if not command:
raise ValueError(
"Scripts must have 'command' or 'command_file' specified.")
cmd = templates.Engine().render(command, template_ctx)
retcode = exec_script_command(cmd, pretend)
if verbose:
log.info("Script exited with code: <33>{}", retcode)
if retcode not in script.success_exit_codes:
sys.exit(retcode)
def collect_files(files: types.FilesCollection) -> List[str]:
""" Collect files using the given configuration. """
paths = [conf.proj_path(p) for p in files.paths]
if context.RunContext().get('verbose', 0) >= 3:
log.info("<35>Files:")
log.info("only_staged: <33>{}".format(files.only_staged))
log.info("untracked: <33>{}".format(files.untracked))
log.info("whitelist: <33>\n{}".format('\n'.join(files.whitelist())))
log.info("blacklist: <33>\n{}".format('\n'.join(files.blacklist())))
if files.only_staged and files.include and not files.whitelist():
# include will be empty if none of the staged files match include
# and thus the final fs walk will pick up everything. We want
# to preserve the include patterns defined in `pelconf.yaml`
# so nothing is picked if none of the staged files match.
return []
return list(
itertools.chain.from_iterable(
filtered_walk(path, files.whitelist(), files.blacklist())
for path in paths))
def gen_pypirc(username: Optional[str] = None, password: Optional[str] = None):
""" Generate ~/.pypirc with the given credentials.
Useful for CI builds. Can also get credentials through env variables
``PYPI_USER`` and ``PYPI_PASS``.
Args:
username (str):
pypi username. If not given it will try to take it from the
`` PYPI_USER`` env variable.
password (str):
pypi password. If not given it will try to take it from the
`` PYPI_PASS`` env variable.
"""
path = join(conf.get_env('HOME'), '.pypirc')
username = username or conf.get_env('PYPI_USER', None)
password = password or conf.get_env('PYPI_PASS', None)
if username is None or password is None:
log.err("You must provide $PYPI_USER and $PYPI_PASS")
sys.exit(1)
log.info("Generating <94>{}".format(path))
fs.write_file(
path,
util.remove_indent('''
[distutils]
index-servers = pypi
[pypi]
repository: https://upload.pypi.org/legacy/
username: {username}
password: {password}
'''.format(username=username, password=password)))
def check(paths, include, exclude, only_staged, untracked):
# type: (str, Sequence[str], Sequence[str], bool, bool) -> None
""" Run mypy and pylint against the current directory."""
files = types.FilesCollection(
paths=paths,
include=['*.py'] + list(include), # We only want to lint python files.
exclude=exclude,
only_staged=only_staged,
untracked=untracked,
)
paths = fs.collect_files(files)
wrapped_paths = fs.wrap_paths(paths)
log.info("Paths: <33>{}", paths)
log.info("Wrapped paths: <33>{}", wrapped_paths)
log.info("Running <35>mypy")
shell.run('mypy --ignore-missing-imports {}'.format(wrapped_paths))
log.info("Running <35>pylint")
shell.run('pylint {}'.format(wrapped_paths))
def setup_ci():
# type: () -> None
""" Setup AppEngine SDK on CircleCI """
gcloud_path = shell.run('which gcloud', capture=True).stdout.strip()
sdk_path = normpath(join(gcloud_path, '../../platform/google_appengine'))
gcloud_cmd = gcloud_path + ' --quiet'
if not exists(sdk_path):
log.info("Installing AppEngine SDK")
shell.run(
'sudo {} components install app-engine-python'.format(gcloud_cmd))
else:
# Only initialise once. To reinitialise, just build without cache.
log.info("AppEngine SDK already initialised")
log.info("Using service account authentication")
shell.run('{} auth activate-service-account --key-file {}'.format(
gcloud_cmd, conf.proj_path('ops/client_secret.json')))
def bump_version(component: str = 'patch', exact: Optional[str] = None):
""" Bump current project version without committing anything.
No tags are created either.
Examples:
\b
$ peltak version bump patch # Bump patch version component
$ peltak version bump minor # Bump minor version component
$ peltak version bump major # Bump major version component
$ peltak version bump release # same as version bump patch
$ peltak version bump --exact=1.2.1 # Set project version to 1.2.1
"""
from peltak.core import log
from peltak.core import versioning
old_ver, new_ver = versioning.bump(component, exact)
log.info("Project version bumped")
log.info(" old version: <35>{}".format(old_ver))
log.info(" new version: <35>{}".format(new_ver))
def lint(paths, include, exclude, only_staged, ignore_untracked):
""" Run code checks (pylint + mypy) """
from peltak.core import log
from custom_commands_logic import check
log.info('<0><1>{}', '-' * 60)
log.info('paths: {}', paths)
log.info('include: {}', include)
log.info('exclude: {}', exclude)
log.info('only_staged: {}', only_staged)
log.info('ignore_untracked: {}', ignore_untracked)
log.info('<0><1>{}', '-' * 60)
check(
paths=paths,
include=include,
exclude=exclude,
only_staged=only_staged,
untracked=not ignore_untracked,
)
def test_formatting_works(p_cprint, msg, args, kw, expected):
log.info(msg, *args, **kw)
p_cprint.assert_called_once_with(expected)
def test_decorates_the_message(p_cprint):
log.info('hello')
p_cprint.assert_called_once_with('-- <32>hello<0>')
def git_prune():
""" Prune dead branches. """
log.info("Pruning")
shell.run('git fetch --prune origin')
