def test_yaml(runner, logged_in_and_linked): # Try to generate YAML from random .py source code source_path = os.path.join(get_project().directory, 'mysnake.py') with open(source_path, 'w') as python_fp: python_fp.write(PYTHON_SOURCE) args = ([source_path]) rv = runner.invoke(step, args, catch_exceptions=True) assert isinstance(rv.exception, ValueError) # Generate YAML from .py source code that is using valohai-utils source_path = os.path.join(get_project().directory, 'mysnake.py') with open(source_path, 'w') as python_fp: python_fp.write(PYTHON_SOURCE_USING_VALOHAI_UTILS) args = ([source_path]) rv = runner.invoke(step, args, catch_exceptions=False) assert "valohai.yaml generated." in rv.output # Update existing YAML from source code config_path = os.path.join(get_project().directory, 'valohai.yaml') with open(config_path, 'w') as yaml_fp: yaml_fp.write(CONFIG_YAML) args = ([source_path]) rv = runner.invoke(step, args, catch_exceptions=False) assert "valohai.yaml updated." in rv.output # Try the same update again rv = runner.invoke(step, args, catch_exceptions=False) assert "valohai.yaml already up-to-date." in rv.output
def run(ctx: Context, name: Optional[str], commit: Optional[str], title: Optional[str]) -> None: """ Start a pipeline run. """ # Having to explicitly compare to `--help` is slightly weird, but it's because of the nested command thing. if name == '--help' or not name: click.echo(ctx.get_help(), color=ctx.color) try: project = get_project(require=True) assert project config = project.get_config(commit_identifier=commit) if config.pipelines: click.secho( '\nThese pipelines are available in the selected commit:\n', color=ctx.color, bold=True) for pipeline in sorted(config.pipelines): click.echo(f' * {pipeline}', color=ctx.color) except: # If we fail to extract the pipeline list, it's not that big of a deal. pass ctx.exit() project = get_project(require=True) assert project commit = commit or project.resolve_commit()['identifier'] config = project.get_config() matched_pipeline = match_pipeline(config, name) pipeline = config.pipelines[matched_pipeline] start_pipeline(config, pipeline, project.id, commit, title)
def unlink(yes: bool) -> None: """ Unlink a linked Valohai project. """ dir = get_project_directory() project = get_project() if not project: warn(f'{dir} or its parents do not seem linked to a project.') return if not yes: click.confirm( 'Unlink {dir} from {name}?'.format( dir=click.style(project.directory, bold=True), name=click.style(project.name, bold=True), ), abort=True, ) links = settings.links.copy() links.pop(dir) settings.persistence.set('links', links) settings.persistence.save() success('Unlinked {dir} from {name}.'.format(dir=click.style(dir, bold=True), name=click.style(project.name, bold=True)))
def test_init(runner, logged_in): name = get_random_string() dir = get_project_directory() with open(os.path.join(dir, 'my_script.py'), 'w') as script_fp: script_fp.write('# Hello') with requests_mock.mock() as m: m.post('https://app.valohai.com/api/v0/projects/', json={ 'id': str(uuid4()), 'name': name, }) result = runner.invoke(init, input='\n'.join([ 'y', # correct directory 'echo hello', # command 'y', # yes, that's right '1', # image number 1 'n', # no, actually '', # erm what 'docker', # image called docker, yes 'y', # okay, that's better 'y', # confirm write 'c', # create new name, # that's a nice name ]), catch_exceptions=False) assert result.exit_code == 0 assert 'my_script.py' in result.output assert 'Happy (machine) learning!' in result.output assert os.path.exists(os.path.join(dir, 'valohai.yaml')) assert get_project(dir)
def test_run(runner, logged_in_and_linked, monkeypatch, pass_param, pass_input, pass_env, adhoc): project_id = PROJECT_DATA['id'] commit_id = 'f' * 40 monkeypatch.setattr(git, 'get_current_commit', lambda dir: commit_id) with open(get_project().get_config_filename(), 'w') as yaml_fp: yaml_fp.write(CONFIG_YAML) args = ['train'] if adhoc: args.insert(0, '--adhoc') values = {} if pass_param: args.append('--max-steps=1801') values['parameters'] = {'max_steps': 1801} if pass_input: args.append('--in1=http://url') values['inputs'] = {'in1': 'http://url'} if pass_env: args.append('--environment=015dbd56-2670-b03e-f37c-dc342714f1b5') values['environment'] = '015dbd56-2670-b03e-f37c-dc342714f1b5' with RunAPIMock(project_id, commit_id, values): output = runner.invoke(run, args, catch_exceptions=False).output if adhoc: assert 'Uploaded ad-hoc code' in output else: # Making sure that non-adhoc executions don't turn adhoc. assert 'Uploaded ad-hoc code' not in output assert '#{counter}'.format(counter=EXECUTION_DATA['counter']) in output
def open(counter): """ Open an execution in a web browser. """ execution = get_project(require=True).get_execution_from_counter( counter=counter) open_browser(execution)
def step(filenames: List[str]) -> None: """ Update a step config(s) in valohai.yaml based on Python source file(s). Example: vh yaml step hello.py :param filenames: Path(s) of the Python source code files. """ project = get_project() assert project config_path = project.get_config_filename() for source_path in filenames: if not os.path.isfile(config_path): update_yaml_from_source(source_path, project) info("valohai.yaml generated.") create_or_update_requirements(project) elif yaml_needs_update(source_path, project): update_yaml_from_source(source_path, project) info("valohai.yaml updated.") create_or_update_requirements(project) else: info("valohai.yaml already up-to-date.")
def list(count: int) -> None: """ Show a list of data in the project. """ params = { 'limit': count, 'ordering': '-ctime', 'deleted': 'false', 'no_count': 'true' } project = get_project(require=True) assert project if project: params['project'] = project.id data = request('get', '/api/v0/data/', params=params).json()['results'] if settings.output_format == 'json': return print_json(data) if not data: info(f'{project}: No data.') return for datum in data: datum['url'] = f'datum://{datum["id"]}' datum['execution_string'] = 'Not from exec' if not datum['output_execution'] else \ f'#{datum["output_execution"]["counter"]}' datum['size'] = convert_size(datum['size']) datum['uri'] = 'No URI' if not datum['uri'] else datum['uri'] print_table( data, columns=['name', 'size', 'execution_string', 'ctime', 'url', 'uri'], headers=['Name', 'Size', 'Output of Exec', 'Created At', 'URL', 'URI'], )
def test_yaml(runner, logged_in_and_linked, default_run_api_mock, yaml_path): # Try to generate YAML from random .py source code project = get_project() source_path = os.path.join(project.directory, 'mysnake.py') yaml_path = yaml_path or project.get_yaml_path() with open(source_path, 'w') as python_fp: python_fp.write(PYTHON_SOURCE) args = build_args(source_path, yaml_path) rv = runner.invoke(step, args, catch_exceptions=True) assert isinstance(rv.exception, ValueError) # Generate YAML from .py source code that is using valohai-utils with open(source_path, 'w') as python_fp: python_fp.write(PYTHON_SOURCE_USING_VALOHAI_UTILS) args = build_args(source_path, yaml_path) rv = runner.invoke(step, args, catch_exceptions=False) assert f"{yaml_path} generated." in rv.output # Update existing YAML from source code config_path = project.get_config_filename(yaml_path=yaml_path) with open(config_path, 'w') as yaml_fp: yaml_fp.write(CONFIG_YAML) args = build_args(source_path, yaml_path) rv = runner.invoke(step, args, catch_exceptions=False) assert f"{yaml_path} updated." in rv.output # Try the same update again rv = runner.invoke(step, args, catch_exceptions=False) assert f"{yaml_path} already up-to-date." in rv.output
def test_typo_check(runner, logged_in_and_linked): with open(get_project().get_config_filename(), 'w') as yaml_fp: yaml_fp.write(CONFIG_YAML) args = ['train', '--max-setps=80'] # Oopsy! output = runner.invoke(run, args, catch_exceptions=False).output assert '(Possible options:' in output or 'Did you mean' in output assert '--max-steps' in output
def parcel(destination, commit, code, valohai_local_run, docker_images, unparcel_script): project = get_project(require=True) if not destination: destination = sanitize_filename('{}-parcel-{}'.format( project.name, time.strftime('%Y%m%d-%H%M%S'), )) click.echo('Packing {} to directory {}'.format( click.style(project.name, bold=True, fg='blue'), click.style(destination, bold=True, fg='green'), )) ensure_makedirs(destination) extra_docker_images = [] if code in ('bundle', 'archive', 'tarball'): export_code(project, destination, mode=code) if valohai_local_run: export_valohai_local_run(project, destination) if docker_images: export_docker_images(project, destination, commit, extra_docker_images) if unparcel_script: write_unparcel_script(destination) success('Parcel {} created!'.format(destination))
def watch(counter: str, force: bool, filter_download: Optional[str], download_directory: Optional[str]) -> None: if download_directory: info( f"Downloading to: {download_directory}\nWaiting for new outputs..." ) else: warn('Target folder is not set. Use --download to set it.') return project = get_project(require=True) execution = project.get_execution_from_counter( counter=counter, params={'exclude': 'outputs'}, ) while True: outputs = get_execution_outputs(execution) outputs = filter_outputs(outputs, download_directory, filter_download, force) if outputs: download_outputs(outputs, download_directory, show_success_message=False) if execution['status'] in complete_execution_statuses: info('Execution has finished.') return time.sleep(1)
def outputs(counter: str, download_directory: Optional[str], filter_download: Optional[str], force: bool, sync: bool) -> None: """ List and download execution outputs. """ if download_directory: download_directory = download_directory.replace( "{counter}", str(counter)) if sync: watch(counter, force, filter_download, download_directory) return project = get_project(require=True) assert project execution = project.get_execution_from_counter( counter=counter, params={'exclude': 'outputs'}, ) outputs = get_execution_outputs(execution) if not outputs: warn('The execution has no outputs.') return for output in outputs: output['datum_url'] = f"datum://{output['id']}" print_table(outputs, ('name', 'datum_url', 'size')) if download_directory: outputs = filter_outputs(outputs, download_directory, filter_download, force) download_outputs(outputs, download_directory, show_success_message=True)
def unlink(yes): """ Unlink a linked Valohai project. """ dir = get_project_directory() project = get_project() if not project: click.echo('{dir} or its parents do not seem linked to a project.'.format(dir=dir)) return 1 if not yes: click.confirm( 'Unlink {dir} from {name}?'.format( dir=click.style(project.directory, bold=True), name=click.style(project.name, bold=True), ), abort=True, ) links = settings.get('links', {}) links.pop(dir) settings['links'] = links settings.save() success('Unlinked {dir} from {name}.'.format( dir=click.style(dir, bold=True), name=click.style(project.name, bold=True) ))
def test_command_help(runner, logged_in_and_linked, patch_git, default_run_api_mock): with open(get_project().get_config_filename(), 'w') as yaml_fp: yaml_fp.write(CONFIG_YAML) output = runner.invoke(run, ['Train model', '--help'], catch_exceptions=False).output assert 'Parameter Options' in output assert 'Input Options' in output
def info(counter: str) -> None: """ Show execution info. """ project = get_project(require=True) assert project execution = project.get_execution_from_counter( counter=counter, params={ 'exclude': 'metadata,events', }, ) if settings.output_format == 'json': return print_json(execution) data = {humanize_identifier(key): str(value) for (key, value) in execution.items() if key not in ignored_keys} data['project name'] = execution['project']['name'] data['environment name'] = execution['environment']['name'] print_table(data) print() print_table( {input['name']: '; '.join(input['urls']) for input in execution.get('inputs', ())}, headers=('input', 'URLs'), ) print() print_table( execution.get('parameters', {}), headers=('parameter', 'value'), ) print()
def info(counter): """ Show execution info. """ execution = get_project(require=True).get_execution_from_counter( counter=counter, detail=True) data = dict((humanize_identifier(key), str(value)) for (key, value) in execution.items() if key not in ignored_keys) data['project name'] = execution['project']['name'] data['environment name'] = execution['environment']['name'] print_table(data) print() print_table( { input['name']: '; '.join(input['urls']) for input in execution.get('inputs', ()) }, headers=('input', 'URLs'), ) print() print_table( execution.get('parameters', {}), headers=('parameter', 'value'), ) print()
def open(): """ Open the project's view in a web browser. """ project = get_project(require=True) project_data = request('get', '/api/v0/projects/{id}/'.format(id=project.id)).json() open_browser(project_data)
def stop( counters: Optional[Union[List[str], Tuple[str]]] = None, all: bool = False, ) -> None: """ Stop one or more in-progress executions. """ project = get_project(require=True) assert project if counters and len( counters) == 1 and counters[0] == 'all': # pragma: no cover # Makes sense to support this spelling too. counters = None all = True if counters and all: # If we spell out latest and ranges in the error message, it becomes kinda # unwieldy, so let's just do this. raise click.UsageError( 'Pass execution counter(s), or `--all`, not both.') counters = list(counters or []) executions = get_executions_for_stop( project, counters=counters, all=all, ) for execution in executions: progress(f"Stopping #{execution['counter']}... ") resp = request('post', execution['urls']['stop']) info(resp.text) success('Done.')
def parcel( destination: Optional[str], commit: Optional[str], code: str, valohai_local_run: bool, docker_images: bool, unparcel_script: bool, ) -> None: project = get_project(require=True) if not destination: destination = sanitize_filename( f'{project.name}-parcel-{time.strftime("%Y%m%d-%H%M%S")}') click.echo( f'Packing {click.style(project.name, bold=True, fg="blue")} ' f'to directory {click.style(destination, bold=True, fg="green")}') ensure_makedirs(destination) extra_docker_images: List[str] = [] if code in ('bundle', 'archive', 'tarball'): export_code(project, destination, mode=code) if valohai_local_run: export_valohai_local_run(destination) if docker_images: export_docker_images(project, destination, commit, extra_docker_images) if unparcel_script: write_unparcel_script(destination) success(f'Parcel {destination} created!')
def stop(counters, all=False): """ Stop one or more in-progress executions. """ project = get_project(require=True) params = {'project': project.id} if counters and all: raise click.UsageError( 'Pass either an execution # or `--all`, not both.') elif counters: params['counter'] = sorted(IntegerRange.parse(counters).as_set()) elif all: params['status'] = 'incomplete' else: warn('Nothing to stop (pass #s or `--all`)') return 1 for execution in request('get', '/api/v0/executions/', params=params).json()['results']: click.echo( 'Stopping #{counter}... '.format(counter=execution['counter']), nl=False) resp = request('post', execution['urls']['stop']) click.echo(resp.text) success('Done.')
def step(filenames: List[str], yaml: Optional[str]) -> None: """ Update a step config(s) in valohai.yaml based on Python source file(s). Example: vh yaml step hello.py :param filenames: Path(s) of the Python source code files. """ project = get_project() if project is None: info("no project linked - assuming files are in current directory.") project = Project(data={}, directory=os.getcwd()) project.name = "YAML command simulated project" config_path = project.get_config_filename(yaml_path=yaml) yaml = yaml or project.get_yaml_path() for source_path in filenames: if not os.path.isfile(config_path): update_yaml_from_source(source_path, project, yaml) info(f"{yaml} generated.") create_or_update_requirements(project) elif yaml_needs_update(source_path, project, yaml): update_yaml_from_source(source_path, project, yaml) info(f"{yaml} updated.") create_or_update_requirements(project) else: info(f"{yaml} already up-to-date.")
def create_project(directory: str, name: str, description: str = '', owner: Optional[str] = None, link: bool = True, yes: bool = False) -> None: """ Internal API for creating a project. """ project_data = request('post', '/api/v0/projects/', data=compact_dict({ 'name': name, 'description': description, 'owner': owner, })).json() long_name = '{}/{}'.format( project_data["owner"]["username"], project_data["name"], ) success(f'Project {long_name} created.') if link: current_project = get_project(directory) if current_project and not yes: if not click.confirm( 'The directory is already linked to {project}. Override that?' .format(project=current_project.name, )): return set_project_link(directory, project_data, inform=True) else: info('Links left alone.')
def lint(filenames: List[str]) -> None: """ Lint (syntax-check) a valohai.yaml file. The return code of this command will be the total number of errors found in all the files. """ if not filenames: project = get_project() if project: project.refresh_details() yaml_path = project.get_yaml_path() else: yaml_path = 'valohai.yaml' directory = (project.directory if project else get_project_directory()) config_file = os.path.join(directory, yaml_path) if not os.path.exists(config_file): raise CLIException( f'There is no {config_file} file. Pass in the names of configuration files to lint?' ) filenames = [config_file] total_errors = 0 for filename in filenames: total_errors += validate_file(filename) if total_errors: warn(f'There were {total_errors} total errors.') click.get_current_context().exit(total_errors)
def run(ctx, step, commit, environment, watch, adhoc, image, args): """ Start an execution of a step. """ if step == '--help': # This is slightly weird, but it's because of the nested command thing click.echo(ctx.get_help(), color=ctx.color) ctx.exit() project = get_project(require=True) if adhoc and commit: raise click.UsageError('--commit and --adhoc are mutually exclusive.') # We need to pass commit=None when adhoc=True to `get_config`, but # the further steps do need the real commit identifier from remote, # so this is done before `commit` is mangled by `create_adhoc_commit`. config = project.get_config(commit=commit) matched_step = match_step(config, step) step = config.steps[matched_step] if adhoc: commit = create_adhoc_commit(project)['identifier'] rc = RunCommand(project, step, commit=commit, environment=environment, watch=watch, image=image) with rc.make_context(rc.name, list(args), parent=ctx) as ctx: return rc.invoke(ctx)
def summarize(counters: List[str]) -> None: """ Summarize execution metadata. Use the global `--table-format` switch to output JSON/TSV/CSV/... """ project = get_project(require=True) assert project executions = download_execution_data(project, counters) all_metadata_keys = set() all_metadata = {} for execution in executions.values(): if execution['status'] in ('created', 'queued'): continue cmeta = (execution.get('cumulative_metadata') or {}) all_metadata_keys.update(set(cmeta.keys())) all_metadata[execution['counter']] = (execution, cmeta) table_data = [] for counter, (execution, metadata) in sorted(all_metadata.items()): row = subset_keys(execution, {'counter', 'id', 'duration'}) row.update(metadata) table_data.append(row) columns = ['counter', 'duration'] + list(sorted(all_metadata_keys)) headers = ['Execution', 'Duration'] + list(sorted(all_metadata_keys)) print_table(table_data, columns=columns, headers=headers)
def list(count: int) -> None: """ Show a list of data aliases in the project. """ params = { 'limit': count, 'ordering': '-ctime', 'deleted': 'false', 'no_count': 'true' } project = get_project(require=True) assert project if project: params['project'] = project.id aliases = request('get', '/api/v0/datum-aliases/', params=params).json()['results'] if settings.output_format == 'json': return print_json(aliases) if not aliases: info(f'{project}: No data aliases.') return for alias in aliases: alias['url'] = f'datum://{alias["name"]}' alias['datum'] = 'No target' if not alias['datum'] else alias['datum'][ 'name'] print_table( aliases, columns=['name', 'datum', 'mtime', 'url'], headers=['Name', 'Data', 'Last Modified', 'URL'], )
def run(ctx: Context, name: Optional[str], commit: Optional[str], title: Optional[str], adhoc: bool, yaml: Optional[str]) -> None: """ Start a pipeline run. """ # Having to explicitly compare to `--help` is slightly weird, but it's because of the nested command thing. if name == '--help' or not name: click.echo(ctx.get_help(), color=ctx.color) print_pipeline_list(ctx, commit) ctx.exit() return project = get_project(require=True) assert project if yaml and not adhoc: raise click.UsageError('--yaml is only valid with --adhoc') commit = create_or_resolve_commit(project, commit=commit, adhoc=adhoc, yaml_path=yaml) config = project.get_config() matched_pipeline = match_pipeline(config, name) pipeline = config.pipelines[matched_pipeline] start_pipeline(config, pipeline, project.id, commit, title)
def run(ctx, step, commit, environment, watch, adhoc, args): """ Start an execution of a step. """ if step == '--help': # This is slightly weird, but it's because of the nested command thing click.echo(ctx.get_help(), color=ctx.color) ctx.exit() project = get_project(require=True) if adhoc: commit = create_adhoc_commit(project)['identifier'] config = project.get_config() step = match_prefix(config.steps, step) if not step: raise BadParameter( '{step} is not a known step (try one of {steps})'.format( step=step, steps=', '.join( click.style(t, bold=True) for t in sorted(config.steps)))) step = config.steps[step] rc = RunCommand(project, step, commit=commit, environment=environment, watch=watch) with rc.make_context(rc.name, list(args), parent=ctx) as ctx: return rc.invoke(ctx)
def open() -> None: """ Open the project's view in a web browser. """ project = get_project(require=True) project_data = request('get', f'/api/v0/projects/{project.id}/').json() open_browser(project_data)