def test_commander_custom_environment(tmp_directory):
Path('workspace').mkdir()
cmdr = Commander(workspace='workspace',
templates_path=('test_pkg', 'templates'),
environment_kwargs=dict(variable_start_string='[[',
variable_end_string=']]'))
cmdr.copy_template('square-brackets.sql', placeholder='value')
assert Path('workspace',
'square-brackets.sql').read_text() == 'value {{another}}'
def _add(cfg, env_name):
"""Export Ploomber project to Airflow
Generates a .py file that exposes a dag variable
"""
click.echo('Exporting to Airflow...')
project_root = Path('.').resolve()
project_name = project_root.name
# TODO: modify Dockerfile depending on package or non-package
with Commander(workspace=env_name,
templates_path=('soopervisor', 'assets')) as e:
e.copy_template('airflow/dag.py',
project_name=project_name,
env_name=env_name)
path_out = str(Path(env_name, project_name + '.py'))
os.rename(Path(env_name, 'dag.py'), path_out)
e.copy_template('airflow/Dockerfile',
conda=Path('environment.lock.yml').exists())
click.echo(
f'Airflow DAG declaration saved to {path_out!r}, you may '
'edit the file to change the configuration if needed, '
'(e.g., set the execution period)')
def _export(cfg, env_name, mode, until, skip_tests):
with Commander(workspace=env_name,
templates_path=('soopervisor', 'assets')) as e:
tasks, args = commons.load_tasks(mode=mode)
if not tasks:
raise CommanderStop(f'Loaded DAG in {mode!r} mode has no '
'tasks to submit. Try "--mode force" to '
'submit all tasks regardless of status')
pkg_name, remote_name = docker.build(e,
cfg,
env_name,
until=until,
skip_tests=skip_tests)
e.info('Submitting jobs to AWS Batch')
submit_dag(tasks=tasks,
args=args,
job_def=pkg_name,
remote_name=remote_name,
job_queue=cfg.job_queue,
container_properties=cfg.container_properties,
region_name=cfg.region_name,
cmdr=e)
e.success('Done. Submitted to AWS Batch')
def _export(cfg, env_name, mode, until, skip_tests):
"""
Build and upload Docker image. Export Argo YAML spec.
"""
with Commander(workspace=env_name,
templates_path=('soopervisor', 'assets')) as e:
tasks, args = commons.load_tasks(mode=mode)
if not tasks:
raise CommanderStop(f'Loaded DAG in {mode!r} mode has no '
'tasks to submit. Try "--mode force" to '
'submit all tasks regardless of status')
pkg_name, target_image = docker.build(e,
cfg,
env_name,
until=until,
skip_tests=skip_tests)
e.info('Generating Argo Workflows YAML spec')
_make_argo_spec(tasks=tasks,
args=args,
env_name=env_name,
cfg=cfg,
pkg_name=pkg_name,
target_image=target_image)
e.info('Submitting jobs to Argo Workflows')
e.success('Done. Submitted to Argo Workflows')
def _add(cfg, env_name):
try:
pkg_name = default.find_package_name()
except ValueError as e:
raise ClickException(
'AWS Lambda is only supported in packaged projects. '
'See the documentation for an example.') from e
with Commander(workspace=env_name,
templates_path=('soopervisor', 'assets')) as e:
e.copy_template('aws-lambda/README.md')
e.copy_template('aws-lambda/Dockerfile')
e.copy_template('aws-lambda/test_aws_lambda.py',
package_name=pkg_name)
e.copy_template('aws-lambda/app.py', package_name=pkg_name)
e.copy_template('aws-lambda/template.yaml', package_name=pkg_name)
e.success('Done.')
e.print(
'Next steps:\n1. Add an input example to '
f'{env_name}/test_aws_lambda.py\n'
f'2. Add the input parsing logic to {env_name}/app.py\n'
f'3. Submit to AWS Lambda with: soopervisor export {env_name}')
# TODO: use e.warn_on_exit
for name in ['docker', 'aws', 'sam']:
warn_if_not_installed(name)
def test_commander_stop(capsys):
msg = 'Stopping because of reasons'
with Commander():
raise CommanderStop(msg)
captured = capsys.readouterr()
assert msg in captured.out
def test_hide_command_on_error():
with pytest.raises(CommanderException) as excinfo:
with Commander() as cmdr:
cmdr.run('pip', 'something', show_cmd=False)
lines = str(excinfo.value).splitlines()
assert lines[0] == 'An error occurred.'
assert 'returned non-zero exit status 1.' in lines[1]
assert len(lines) == 2
def test_get_template_nested(tmp_directory):
Path('workspace').mkdir()
Path('workspace', 'template').touch()
with Commander('workspace',
templates_path=('test_pkg', 'templates')) as cmdr:
cmdr.copy_template('nested/simple.sql')
assert Path('workspace', 'simple.sql').read_text() == 'SELECT * FROM data'
def _add(cfg, env_name):
"""
Add Dockerfile
"""
with Commander(workspace=env_name,
templates_path=('soopervisor', 'assets')) as e:
e.copy_template('argo-workflows/Dockerfile',
conda=Path('environment.lock.yml').exists())
e.success('Done')
def _add(cfg, env_name):
with Commander(workspace=env_name,
templates_path=('soopervisor', 'assets')) as e:
e.copy_template('aws-batch/Dockerfile',
conda=Path('environment.lock.yml').exists())
e.success('Done')
e.print(
f'Fill in the configuration in the {env_name!r} '
'section in soopervisor.yaml then submit to AWS Batch with: '
f'soopervisor export {env_name}')
def test_warns_if_fails_to_get_git_tracked_files(tmp_empty, capsys):
Path('file').touch()
Path('secrets.txt').touch()
with Commander() as cmdr:
source.copy(cmdr, '.', 'dist')
captured = capsys.readouterr()
assert 'Unable to get git tracked files' in captured.out
def main_pip(start_time, use_lock):
"""
Install pip-based project (uses venv), looks for requirements.txt files
Parameters
----------
start_time : datetime
The initial runtime of the function.
use_lock : bool
If True Uses requirements.txt and requirements.dev.lock.txt files
"""
reqs_txt = _REQS_LOCK_TXT if use_lock else _REQS_TXT
reqs_dev_txt = ('requirements.dev.lock.txt'
if use_lock else 'requirements.dev.txt')
cmdr = Commander()
# TODO: modify readme to add how to activate env? probably also in conda
name = Path('.').resolve().name
venv_dir = f'venv-{name}'
cmdr.run('python', '-m', 'venv', venv_dir, description='Creating venv')
# add venv_dir to .gitignore if it doesn't exist
if Path('.gitignore').exists():
with open('.gitignore') as f:
if venv_dir not in f.read():
cmdr.append_inline(venv_dir, '.gitignore')
else:
cmdr.append_inline(venv_dir, '.gitignore')
folder, bin_name = _get_pip_folder_and_bin_name()
pip = str(Path(venv_dir, folder, bin_name))
if Path(_SETUP_PY).exists():
_pip_install_setup_py_pip(cmdr, pip)
_pip_install(cmdr, pip, lock=not use_lock, requirements=reqs_txt)
if Path(reqs_dev_txt).exists():
_pip_install(cmdr, pip, lock=not use_lock, requirements=reqs_dev_txt)
if os.name == 'nt':
cmd_activate = f'{venv_dir}\\Scripts\\Activate.ps1'
else:
cmd_activate = f'source {venv_dir}/bin/activate'
_next_steps(cmdr, cmd_activate, start_time)
def test_warns_on_dirty_git(tmp_empty, capsys):
Path('file').touch()
Path('secrets.txt').touch()
Path('.gitignore').write_text('secrets.txt')
git_init()
Path('new-file').touch()
with Commander() as cmdr:
source.copy(cmdr, '.', 'dist')
captured = capsys.readouterr()
assert 'Your git repository contains untracked' in captured.out
def _export(cfg, env_name, mode, until, skip_tests):
"""
Copies the current source code to the target environment folder.
The code along with the DAG declaration file can be copied to
AIRFLOW_HOME for execution
"""
with Commander(workspace=env_name,
templates_path=('soopervisor', 'assets')) as e:
tasks, args = commons.load_tasks(mode=mode)
if not tasks:
raise CommanderStop(f'Loaded DAG in {mode!r} mode has no '
'tasks to submit. Try "--mode force" to '
'submit all tasks regardless of status')
pkg_name, target_image = commons.docker.build(
e, cfg, env_name, until=until, skip_tests=skip_tests)
dag_dict = generate_airflow_spec(tasks, args, target_image)
path_dag_dict_out = Path(pkg_name + '.json')
path_dag_dict_out.write_text(json.dumps(dag_dict))
def main_pip():
if not Path('requirements.txt').exists():
raise exceptions.ClickException(
'"ploomber install" requires a pip '
'requirements.txt file. Use "ploomber scaffold" to create one '
'from a template or create one manually')
cmdr = Commander()
# TODO: modify readme to add how to activate env? probably also in conda
# TODO: add to gitignore, create if it doesn't exist
name = Path('.').resolve().name
venv_dir = f'venv-{name}'
cmdr.run('python', '-m', 'venv', venv_dir, description='Creating venv')
cmdr.append_inline(venv_dir, '.gitignore')
folder = 'Scripts' if os.name == 'nt' else 'bin'
bin_name = 'pip.EXE' if os.name == 'nt' else 'pip'
pip = str(Path(venv_dir, folder, bin_name))
_try_pip_install_setup_py(cmdr, pip)
_pip_install_and_lock(cmdr, pip, requirements='requirements.txt')
if Path('requirements.dev.txt').exists():
_pip_install_and_lock(cmdr, pip, requirements='requirements.dev.txt')
if os.name == 'nt':
cmd_activate = (
f'\nIf using cmd.exe: {venv_dir}\\Scripts\\activate.bat'
f'\nIf using PowerShell: {venv_dir}\\Scripts\\Activate.ps1')
else:
cmd_activate = f'source {venv_dir}/bin/activate'
_next_steps(cmdr, cmd_activate)
def test_show_command(capsys):
with Commander() as cmdr:
cmdr.run('echo', 'hello', show_cmd=False, description='Do something')
captured = capsys.readouterr()
assert '==Do something: echo hello==' not in captured.out
def cmdr():
with Commander() as cmdr:
yield cmdr
def test_creates_workpace(tmp_directory):
with Commander('workspace'):
pass
assert Path('workspace').is_dir()
def test_empty_workspace():
Commander(workspace=None)
def main_conda():
if not Path('environment.yml').exists():
raise exceptions.ClickException(
'"ploomber install" requires a conda '
'environment.yml file. Use "ploomber scaffold" to create one '
'from a template or create one manually')
# TODO: ensure ploomber-scaffold includes dependency file (including
# lock files in MANIFEST.in
cmdr = Commander()
# TODO: provide helpful error messages on each command
with open('environment.yml') as f:
env_name = yaml.safe_load(f)['name']
current_env = Path(shutil.which('python')).parents[1].name
if env_name == current_env:
raise RuntimeError('environment.yaml will create an environment '
f'named {env_name!r}, which is the current active '
'environment. Move to a different one and try '
'again (e.g., "conda activate base")')
# get current installed envs
envs = cmdr.run('conda', 'env', 'list', '--json', capture_output=True)
already_installed = any([
env for env in json.loads(envs)['envs']
# only check in the envs folder, ignore envs in other locations
if 'envs' in env and env_name in env
])
# if already installed and running on windows, ask to delete first,
# otherwise it might lead to an intermitent error (permission denied
# on vcruntime140.dll)
if already_installed and os.name == 'nt':
raise ValueError(f'Environemnt {env_name!r} already exists, '
f'delete it and try again '
f'(conda env remove --name {env_name})')
pkg_manager = 'mamba' if shutil.which('mamba') else 'conda'
cmdr.run(pkg_manager,
'env',
'create',
'--file',
'environment.yml',
'--force',
description='Creating env')
pip = _locate_pip_inside_conda(env_name)
_try_pip_install_setup_py(cmdr, pip)
env_lock = cmdr.run('conda',
'env',
'export',
'--no-build',
'--name',
env_name,
description='Locking dependencies',
capture_output=True)
Path('environment.lock.yml').write_text(env_lock)
_try_conda_install_and_lock_dev(cmdr, pkg_manager, env_name)
cmd_activate = f'conda activate {env_name}'
_next_steps(cmdr, cmd_activate)
def _export(cfg, env_name, until, skip_tests):
# TODO: validate project structure: src/*/model.*, etc...
# TODO: build is required to run this, perhaps use python setup.py
# bdist?
# TODO: warn if deploying from a dirty commit, maybe ask for
# confirmation
# and show the version?
# TODO: support for OnlineDAG in app.py
# TODO: check if OnlineModel can be initialized from the package
with Commander(workspace=env_name,
templates_path=('soopervisor', 'assets')) as e:
if not Path('requirements.lock.txt').exists():
if Path('environment.lock.yml').exists():
e.warn_on_exit(
'Missing requirements.lock.txt file. '
'Once was created from the pip '
'section in the environment.lock.yml file but this '
'may not work if there are missing dependencies. Add '
'one or ensure environment.lock.yml includes all pip '
'dependencies.')
generate_reqs_txt_from_env_yml(
'environment.lock.yml', output='requirements.lock.txt')
e.rm_on_exit('requirements.lock.txt')
else:
raise ClickException('Expected environment.lock.yml or '
'requirements.txt.lock at the root '
'directory. Add one.')
e.cp('requirements.lock.txt')
# TODO: ensure user has pytest before running
if not skip_tests:
e.run('pytest', env_name, description='Testing')
e.rm('dist', 'build')
e.run('python',
'-m',
'build',
'--wheel',
'.',
description='Packaging')
e.cp('dist')
e.cd(env_name)
# TODO: template.yaml with version number
e.run('sam', 'build', description='Building Docker image')
if until == 'build':
e.tw.write(
'Done.\nRun "docker images" to see your image.'
'\nRun "sam local start-api" to test your API locally')
return
args = ['sam', 'deploy']
if Path('samconfig.toml').exists():
e.warn('samconfig.toml already exists. Skipping '
'guided deployment...')
else:
args.append('--guided')
e.info('Starting guided deployment...')
e.run(*args, description='Deploying')
e.success('Deployed to AWS Lambda')
def main_conda(start_time, use_lock):
"""
Install conda-based project, looks for environment.yml files
Parameters
----------
start_time : datetime
The initial runtime of the function.
use_lock : bool
If True Uses environment.lock.yml and environment.dev.lock.yml files
"""
env_yml = _ENV_LOCK_YML if use_lock else _ENV_YML
# TODO: ensure ploomber-scaffold includes dependency file (including
# lock files in MANIFEST.in
cmdr = Commander()
# TODO: provide helpful error messages on each command
with open(env_yml) as f:
env_name = yaml.safe_load(f)['name']
current_env = Path(shutil.which('python')).parents[1].name
if env_name == current_env:
err = (f'{env_yml} will create an environment '
f'named {env_name!r}, which is the current active '
'environment. Move to a different one and try '
'again (e.g., "conda activate base")')
telemetry.log_api("install-error",
metadata={
'type': 'env_running_conflict',
'exception': err
})
raise RuntimeError(err)
# get current installed envs
conda = shutil.which('conda')
mamba = shutil.which('mamba')
# if already installed and running on windows, ask to delete first,
# otherwise it might lead to an intermittent error (permission denied
# on vcruntime140.dll)
if os.name == 'nt':
envs = cmdr.run(conda, 'env', 'list', '--json', capture_output=True)
already_installed = any([
env for env in json.loads(envs)['envs']
# only check in the envs folder, ignore envs in other locations
if 'envs' in env and env_name in env
])
if already_installed:
err = (f'Environment {env_name!r} already exists, '
f'delete it and try again '
f'(conda env remove --name {env_name})')
telemetry.log_api("install-error",
metadata={
'type': 'duplicate_env',
'exception': err
})
raise ValueError(err)
pkg_manager = mamba if mamba else conda
cmdr.run(pkg_manager,
'env',
'create',
'--file',
env_yml,
'--force',
description='Creating env')
if Path(_SETUP_PY).exists():
_pip_install_setup_py_conda(cmdr, env_name)
if not use_lock:
env_lock = cmdr.run(conda,
'env',
'export',
'--no-build',
'--name',
env_name,
description='Locking dependencies',
capture_output=True)
Path(_ENV_LOCK_YML).write_text(env_lock)
_try_conda_install_and_lock_dev(cmdr,
pkg_manager,
env_name,
use_lock=use_lock)
cmd_activate = f'conda activate {env_name}'
_next_steps(cmdr, cmd_activate, start_time)
