def test_yaml(runner, logged_in_and_linked):
# Try to generate YAML from random .py source code
source_path = os.path.join(get_project().directory, 'mysnake.py')
with open(source_path, 'w') as python_fp:
python_fp.write(PYTHON_SOURCE)
args = ([source_path])
rv = runner.invoke(step, args, catch_exceptions=True)
assert isinstance(rv.exception, ValueError)
# Generate YAML from .py source code that is using valohai-utils
source_path = os.path.join(get_project().directory, 'mysnake.py')
with open(source_path, 'w') as python_fp:
python_fp.write(PYTHON_SOURCE_USING_VALOHAI_UTILS)
args = ([source_path])
rv = runner.invoke(step, args, catch_exceptions=False)
assert "valohai.yaml generated." in rv.output
# Update existing YAML from source code
config_path = os.path.join(get_project().directory, 'valohai.yaml')
with open(config_path, 'w') as yaml_fp:
yaml_fp.write(CONFIG_YAML)
args = ([source_path])
rv = runner.invoke(step, args, catch_exceptions=False)
assert "valohai.yaml updated." in rv.output
# Try the same update again
rv = runner.invoke(step, args, catch_exceptions=False)
assert "valohai.yaml already up-to-date." in rv.output
def run(ctx: Context, name: Optional[str], commit: Optional[str],
title: Optional[str]) -> None:
"""
Start a pipeline run.
"""
# Having to explicitly compare to `--help` is slightly weird, but it's because of the nested command thing.
if name == '--help' or not name:
click.echo(ctx.get_help(), color=ctx.color)
try:
project = get_project(require=True)
assert project
config = project.get_config(commit_identifier=commit)
if config.pipelines:
click.secho(
'\nThese pipelines are available in the selected commit:\n',
color=ctx.color,
bold=True)
for pipeline in sorted(config.pipelines):
click.echo(f' * {pipeline}', color=ctx.color)
except: # If we fail to extract the pipeline list, it's not that big of a deal.
pass
ctx.exit()
project = get_project(require=True)
assert project
commit = commit or project.resolve_commit()['identifier']
config = project.get_config()
matched_pipeline = match_pipeline(config, name)
pipeline = config.pipelines[matched_pipeline]
start_pipeline(config, pipeline, project.id, commit, title)
def unlink(yes: bool) -> None:
"""
Unlink a linked Valohai project.
"""
dir = get_project_directory()
project = get_project()
if not project:
warn(f'{dir} or its parents do not seem linked to a project.')
return
if not yes:
click.confirm(
'Unlink {dir} from {name}?'.format(
dir=click.style(project.directory, bold=True),
name=click.style(project.name, bold=True),
),
abort=True,
)
links = settings.links.copy()
links.pop(dir)
settings.persistence.set('links', links)
settings.persistence.save()
success('Unlinked {dir} from {name}.'.format(dir=click.style(dir,
bold=True),
name=click.style(project.name,
bold=True)))
def test_init(runner, logged_in):
name = get_random_string()
dir = get_project_directory()
with open(os.path.join(dir, 'my_script.py'), 'w') as script_fp:
script_fp.write('# Hello')
with requests_mock.mock() as m:
m.post('https://app.valohai.com/api/v0/projects/', json={
'id': str(uuid4()),
'name': name,
})
result = runner.invoke(init, input='\n'.join([
'y', # correct directory
'echo hello', # command
'y', # yes, that's right
'1', # image number 1
'n', # no, actually
'', # erm what
'docker', # image called docker, yes
'y', # okay, that's better
'y', # confirm write
'c', # create new
name, # that's a nice name
]), catch_exceptions=False)
assert result.exit_code == 0
assert 'my_script.py' in result.output
assert 'Happy (machine) learning!' in result.output
assert os.path.exists(os.path.join(dir, 'valohai.yaml'))
assert get_project(dir)
def test_run(runner, logged_in_and_linked, monkeypatch, pass_param, pass_input,
pass_env, adhoc):
project_id = PROJECT_DATA['id']
commit_id = 'f' * 40
monkeypatch.setattr(git, 'get_current_commit', lambda dir: commit_id)
with open(get_project().get_config_filename(), 'w') as yaml_fp:
yaml_fp.write(CONFIG_YAML)
args = ['train']
if adhoc:
args.insert(0, '--adhoc')
values = {}
if pass_param:
args.append('--max-steps=1801')
values['parameters'] = {'max_steps': 1801}
if pass_input:
args.append('--in1=http://url')
values['inputs'] = {'in1': 'http://url'}
if pass_env:
args.append('--environment=015dbd56-2670-b03e-f37c-dc342714f1b5')
values['environment'] = '015dbd56-2670-b03e-f37c-dc342714f1b5'
with RunAPIMock(project_id, commit_id, values):
output = runner.invoke(run, args, catch_exceptions=False).output
if adhoc:
assert 'Uploaded ad-hoc code' in output
else:
# Making sure that non-adhoc executions don't turn adhoc.
assert 'Uploaded ad-hoc code' not in output
assert '#{counter}'.format(counter=EXECUTION_DATA['counter']) in output
def open(counter):
"""
Open an execution in a web browser.
"""
execution = get_project(require=True).get_execution_from_counter(
counter=counter)
open_browser(execution)
def step(filenames: List[str]) -> None:
"""
Update a step config(s) in valohai.yaml based on Python source file(s).
Example:
vh yaml step hello.py
:param filenames: Path(s) of the Python source code files.
"""
project = get_project()
assert project
config_path = project.get_config_filename()
for source_path in filenames:
if not os.path.isfile(config_path):
update_yaml_from_source(source_path, project)
info("valohai.yaml generated.")
create_or_update_requirements(project)
elif yaml_needs_update(source_path, project):
update_yaml_from_source(source_path, project)
info("valohai.yaml updated.")
create_or_update_requirements(project)
else:
info("valohai.yaml already up-to-date.")
def list(count: int) -> None:
"""
Show a list of data in the project.
"""
params = {
'limit': count,
'ordering': '-ctime',
'deleted': 'false',
'no_count': 'true'
}
project = get_project(require=True)
assert project
if project:
params['project'] = project.id
data = request('get', '/api/v0/data/', params=params).json()['results']
if settings.output_format == 'json':
return print_json(data)
if not data:
info(f'{project}: No data.')
return
for datum in data:
datum['url'] = f'datum://{datum["id"]}'
datum['execution_string'] = 'Not from exec' if not datum['output_execution'] else \
f'#{datum["output_execution"]["counter"]}'
datum['size'] = convert_size(datum['size'])
datum['uri'] = 'No URI' if not datum['uri'] else datum['uri']
print_table(
data,
columns=['name', 'size', 'execution_string', 'ctime', 'url', 'uri'],
headers=['Name', 'Size', 'Output of Exec', 'Created At', 'URL', 'URI'],
)
def test_yaml(runner, logged_in_and_linked, default_run_api_mock, yaml_path):
# Try to generate YAML from random .py source code
project = get_project()
source_path = os.path.join(project.directory, 'mysnake.py')
yaml_path = yaml_path or project.get_yaml_path()
with open(source_path, 'w') as python_fp:
python_fp.write(PYTHON_SOURCE)
args = build_args(source_path, yaml_path)
rv = runner.invoke(step, args, catch_exceptions=True)
assert isinstance(rv.exception, ValueError)
# Generate YAML from .py source code that is using valohai-utils
with open(source_path, 'w') as python_fp:
python_fp.write(PYTHON_SOURCE_USING_VALOHAI_UTILS)
args = build_args(source_path, yaml_path)
rv = runner.invoke(step, args, catch_exceptions=False)
assert f"{yaml_path} generated." in rv.output
# Update existing YAML from source code
config_path = project.get_config_filename(yaml_path=yaml_path)
with open(config_path, 'w') as yaml_fp:
yaml_fp.write(CONFIG_YAML)
args = build_args(source_path, yaml_path)
rv = runner.invoke(step, args, catch_exceptions=False)
assert f"{yaml_path} updated." in rv.output
# Try the same update again
rv = runner.invoke(step, args, catch_exceptions=False)
assert f"{yaml_path} already up-to-date." in rv.output
def test_typo_check(runner, logged_in_and_linked):
with open(get_project().get_config_filename(), 'w') as yaml_fp:
yaml_fp.write(CONFIG_YAML)
args = ['train', '--max-setps=80'] # Oopsy!
output = runner.invoke(run, args, catch_exceptions=False).output
assert '(Possible options:' in output or 'Did you mean' in output
assert '--max-steps' in output
def parcel(destination, commit, code, valohai_local_run, docker_images,
unparcel_script):
project = get_project(require=True)
if not destination:
destination = sanitize_filename('{}-parcel-{}'.format(
project.name,
time.strftime('%Y%m%d-%H%M%S'),
))
click.echo('Packing {} to directory {}'.format(
click.style(project.name, bold=True, fg='blue'),
click.style(destination, bold=True, fg='green'),
))
ensure_makedirs(destination)
extra_docker_images = []
if code in ('bundle', 'archive', 'tarball'):
export_code(project, destination, mode=code)
if valohai_local_run:
export_valohai_local_run(project, destination)
if docker_images:
export_docker_images(project, destination, commit, extra_docker_images)
if unparcel_script:
write_unparcel_script(destination)
success('Parcel {} created!'.format(destination))
def watch(counter: str, force: bool, filter_download: Optional[str],
download_directory: Optional[str]) -> None:
if download_directory:
info(
f"Downloading to: {download_directory}\nWaiting for new outputs..."
)
else:
warn('Target folder is not set. Use --download to set it.')
return
project = get_project(require=True)
execution = project.get_execution_from_counter(
counter=counter,
params={'exclude': 'outputs'},
)
while True:
outputs = get_execution_outputs(execution)
outputs = filter_outputs(outputs, download_directory, filter_download,
force)
if outputs:
download_outputs(outputs,
download_directory,
show_success_message=False)
if execution['status'] in complete_execution_statuses:
info('Execution has finished.')
return
time.sleep(1)
def outputs(counter: str, download_directory: Optional[str],
filter_download: Optional[str], force: bool, sync: bool) -> None:
"""
List and download execution outputs.
"""
if download_directory:
download_directory = download_directory.replace(
"{counter}", str(counter))
if sync:
watch(counter, force, filter_download, download_directory)
return
project = get_project(require=True)
assert project
execution = project.get_execution_from_counter(
counter=counter,
params={'exclude': 'outputs'},
)
outputs = get_execution_outputs(execution)
if not outputs:
warn('The execution has no outputs.')
return
for output in outputs:
output['datum_url'] = f"datum://{output['id']}"
print_table(outputs, ('name', 'datum_url', 'size'))
if download_directory:
outputs = filter_outputs(outputs, download_directory, filter_download,
force)
download_outputs(outputs,
download_directory,
show_success_message=True)
def unlink(yes):
"""
Unlink a linked Valohai project.
"""
dir = get_project_directory()
project = get_project()
if not project:
click.echo('{dir} or its parents do not seem linked to a project.'.format(dir=dir))
return 1
if not yes:
click.confirm(
'Unlink {dir} from {name}?'.format(
dir=click.style(project.directory, bold=True),
name=click.style(project.name, bold=True),
),
abort=True,
)
links = settings.get('links', {})
links.pop(dir)
settings['links'] = links
settings.save()
success('Unlinked {dir} from {name}.'.format(
dir=click.style(dir, bold=True),
name=click.style(project.name, bold=True)
))
def test_command_help(runner, logged_in_and_linked, patch_git, default_run_api_mock):
with open(get_project().get_config_filename(), 'w') as yaml_fp:
yaml_fp.write(CONFIG_YAML)
output = runner.invoke(run, ['Train model', '--help'], catch_exceptions=False).output
assert 'Parameter Options' in output
assert 'Input Options' in output
def info(counter: str) -> None:
"""
Show execution info.
"""
project = get_project(require=True)
assert project
execution = project.get_execution_from_counter(
counter=counter,
params={
'exclude': 'metadata,events',
},
)
if settings.output_format == 'json':
return print_json(execution)
data = {humanize_identifier(key): str(value) for (key, value) in execution.items() if key not in ignored_keys}
data['project name'] = execution['project']['name']
data['environment name'] = execution['environment']['name']
print_table(data)
print()
print_table(
{input['name']: '; '.join(input['urls']) for input in execution.get('inputs', ())},
headers=('input', 'URLs'),
)
print()
print_table(
execution.get('parameters', {}),
headers=('parameter', 'value'),
)
print()
def info(counter):
"""
Show execution info.
"""
execution = get_project(require=True).get_execution_from_counter(
counter=counter, detail=True)
data = dict((humanize_identifier(key), str(value))
for (key, value) in execution.items()
if key not in ignored_keys)
data['project name'] = execution['project']['name']
data['environment name'] = execution['environment']['name']
print_table(data)
print()
print_table(
{
input['name']: '; '.join(input['urls'])
for input in execution.get('inputs', ())
},
headers=('input', 'URLs'),
)
print()
print_table(
execution.get('parameters', {}),
headers=('parameter', 'value'),
)
print()
def open():
"""
Open the project's view in a web browser.
"""
project = get_project(require=True)
project_data = request('get', '/api/v0/projects/{id}/'.format(id=project.id)).json()
open_browser(project_data)
def stop(
counters: Optional[Union[List[str], Tuple[str]]] = None,
all: bool = False,
) -> None:
"""
Stop one or more in-progress executions.
"""
project = get_project(require=True)
assert project
if counters and len(
counters) == 1 and counters[0] == 'all': # pragma: no cover
# Makes sense to support this spelling too.
counters = None
all = True
if counters and all:
# If we spell out latest and ranges in the error message, it becomes kinda
# unwieldy, so let's just do this.
raise click.UsageError(
'Pass execution counter(s), or `--all`, not both.')
counters = list(counters or [])
executions = get_executions_for_stop(
project,
counters=counters,
all=all,
)
for execution in executions:
progress(f"Stopping #{execution['counter']}... ")
resp = request('post', execution['urls']['stop'])
info(resp.text)
success('Done.')
def parcel(
destination: Optional[str],
commit: Optional[str],
code: str,
valohai_local_run: bool,
docker_images: bool,
unparcel_script: bool,
) -> None:
project = get_project(require=True)
if not destination:
destination = sanitize_filename(
f'{project.name}-parcel-{time.strftime("%Y%m%d-%H%M%S")}')
click.echo(
f'Packing {click.style(project.name, bold=True, fg="blue")} '
f'to directory {click.style(destination, bold=True, fg="green")}')
ensure_makedirs(destination)
extra_docker_images: List[str] = []
if code in ('bundle', 'archive', 'tarball'):
export_code(project, destination, mode=code)
if valohai_local_run:
export_valohai_local_run(destination)
if docker_images:
export_docker_images(project, destination, commit, extra_docker_images)
if unparcel_script:
write_unparcel_script(destination)
success(f'Parcel {destination} created!')
def stop(counters, all=False):
"""
Stop one or more in-progress executions.
"""
project = get_project(require=True)
params = {'project': project.id}
if counters and all:
raise click.UsageError(
'Pass either an execution # or `--all`, not both.')
elif counters:
params['counter'] = sorted(IntegerRange.parse(counters).as_set())
elif all:
params['status'] = 'incomplete'
else:
warn('Nothing to stop (pass #s or `--all`)')
return 1
for execution in request('get', '/api/v0/executions/',
params=params).json()['results']:
click.echo(
'Stopping #{counter}... '.format(counter=execution['counter']),
nl=False)
resp = request('post', execution['urls']['stop'])
click.echo(resp.text)
success('Done.')
def step(filenames: List[str], yaml: Optional[str]) -> None:
"""
Update a step config(s) in valohai.yaml based on Python source file(s).
Example:
vh yaml step hello.py
:param filenames: Path(s) of the Python source code files.
"""
project = get_project()
if project is None:
info("no project linked - assuming files are in current directory.")
project = Project(data={}, directory=os.getcwd())
project.name = "YAML command simulated project"
config_path = project.get_config_filename(yaml_path=yaml)
yaml = yaml or project.get_yaml_path()
for source_path in filenames:
if not os.path.isfile(config_path):
update_yaml_from_source(source_path, project, yaml)
info(f"{yaml} generated.")
create_or_update_requirements(project)
elif yaml_needs_update(source_path, project, yaml):
update_yaml_from_source(source_path, project, yaml)
info(f"{yaml} updated.")
create_or_update_requirements(project)
else:
info(f"{yaml} already up-to-date.")
def create_project(directory: str,
name: str,
description: str = '',
owner: Optional[str] = None,
link: bool = True,
yes: bool = False) -> None:
"""
Internal API for creating a project.
"""
project_data = request('post',
'/api/v0/projects/',
data=compact_dict({
'name': name,
'description': description,
'owner': owner,
})).json()
long_name = '{}/{}'.format(
project_data["owner"]["username"],
project_data["name"],
)
success(f'Project {long_name} created.')
if link:
current_project = get_project(directory)
if current_project and not yes:
if not click.confirm(
'The directory is already linked to {project}. Override that?'
.format(project=current_project.name, )):
return
set_project_link(directory, project_data, inform=True)
else:
info('Links left alone.')
def lint(filenames: List[str]) -> None:
"""
Lint (syntax-check) a valohai.yaml file.
The return code of this command will be the total number of errors found in all the files.
"""
if not filenames:
project = get_project()
if project:
project.refresh_details()
yaml_path = project.get_yaml_path()
else:
yaml_path = 'valohai.yaml'
directory = (project.directory if project else get_project_directory())
config_file = os.path.join(directory, yaml_path)
if not os.path.exists(config_file):
raise CLIException(
f'There is no {config_file} file. Pass in the names of configuration files to lint?'
)
filenames = [config_file]
total_errors = 0
for filename in filenames:
total_errors += validate_file(filename)
if total_errors:
warn(f'There were {total_errors} total errors.')
click.get_current_context().exit(total_errors)
def run(ctx, step, commit, environment, watch, adhoc, image, args):
"""
Start an execution of a step.
"""
if step == '--help': # This is slightly weird, but it's because of the nested command thing
click.echo(ctx.get_help(), color=ctx.color)
ctx.exit()
project = get_project(require=True)
if adhoc and commit:
raise click.UsageError('--commit and --adhoc are mutually exclusive.')
# We need to pass commit=None when adhoc=True to `get_config`, but
# the further steps do need the real commit identifier from remote,
# so this is done before `commit` is mangled by `create_adhoc_commit`.
config = project.get_config(commit=commit)
matched_step = match_step(config, step)
step = config.steps[matched_step]
if adhoc:
commit = create_adhoc_commit(project)['identifier']
rc = RunCommand(project,
step,
commit=commit,
environment=environment,
watch=watch,
image=image)
with rc.make_context(rc.name, list(args), parent=ctx) as ctx:
return rc.invoke(ctx)
def summarize(counters: List[str]) -> None:
"""
Summarize execution metadata.
Use the global `--table-format` switch to output JSON/TSV/CSV/...
"""
project = get_project(require=True)
assert project
executions = download_execution_data(project, counters)
all_metadata_keys = set()
all_metadata = {}
for execution in executions.values():
if execution['status'] in ('created', 'queued'):
continue
cmeta = (execution.get('cumulative_metadata') or {})
all_metadata_keys.update(set(cmeta.keys()))
all_metadata[execution['counter']] = (execution, cmeta)
table_data = []
for counter, (execution, metadata) in sorted(all_metadata.items()):
row = subset_keys(execution, {'counter', 'id', 'duration'})
row.update(metadata)
table_data.append(row)
columns = ['counter', 'duration'] + list(sorted(all_metadata_keys))
headers = ['Execution', 'Duration'] + list(sorted(all_metadata_keys))
print_table(table_data, columns=columns, headers=headers)
def list(count: int) -> None:
"""
Show a list of data aliases in the project.
"""
params = {
'limit': count,
'ordering': '-ctime',
'deleted': 'false',
'no_count': 'true'
}
project = get_project(require=True)
assert project
if project:
params['project'] = project.id
aliases = request('get', '/api/v0/datum-aliases/',
params=params).json()['results']
if settings.output_format == 'json':
return print_json(aliases)
if not aliases:
info(f'{project}: No data aliases.')
return
for alias in aliases:
alias['url'] = f'datum://{alias["name"]}'
alias['datum'] = 'No target' if not alias['datum'] else alias['datum'][
'name']
print_table(
aliases,
columns=['name', 'datum', 'mtime', 'url'],
headers=['Name', 'Data', 'Last Modified', 'URL'],
)
def run(ctx: Context, name: Optional[str], commit: Optional[str],
title: Optional[str], adhoc: bool, yaml: Optional[str]) -> None:
"""
Start a pipeline run.
"""
# Having to explicitly compare to `--help` is slightly weird, but it's because of the nested command thing.
if name == '--help' or not name:
click.echo(ctx.get_help(), color=ctx.color)
print_pipeline_list(ctx, commit)
ctx.exit()
return
project = get_project(require=True)
assert project
if yaml and not adhoc:
raise click.UsageError('--yaml is only valid with --adhoc')
commit = create_or_resolve_commit(project,
commit=commit,
adhoc=adhoc,
yaml_path=yaml)
config = project.get_config()
matched_pipeline = match_pipeline(config, name)
pipeline = config.pipelines[matched_pipeline]
start_pipeline(config, pipeline, project.id, commit, title)
def run(ctx, step, commit, environment, watch, adhoc, args):
"""
Start an execution of a step.
"""
if step == '--help': # This is slightly weird, but it's because of the nested command thing
click.echo(ctx.get_help(), color=ctx.color)
ctx.exit()
project = get_project(require=True)
if adhoc:
commit = create_adhoc_commit(project)['identifier']
config = project.get_config()
step = match_prefix(config.steps, step)
if not step:
raise BadParameter(
'{step} is not a known step (try one of {steps})'.format(
step=step,
steps=', '.join(
click.style(t, bold=True) for t in sorted(config.steps))))
step = config.steps[step]
rc = RunCommand(project,
step,
commit=commit,
environment=environment,
watch=watch)
with rc.make_context(rc.name, list(args), parent=ctx) as ctx:
return rc.invoke(ctx)
def open() -> None:
"""
Open the project's view in a web browser.
"""
project = get_project(require=True)
project_data = request('get', f'/api/v0/projects/{project.id}/').json()
open_browser(project_data)
