def prompt_for_value(self, ctx: Context) -> Optional[str]: try: options: List[str] = request( 'get', '/api/v0/projects/ownership_options/').json() except APINotFoundError: # Endpoint not there, ah well! return None except APIError as ae: warn(f'Unable to retrieve ownership options: {ae}') return None if not options: return None if len(options) == 1: return options[0] print('Who should own this project? The options available to you are:') for option in options: print(f' * {option}') return str( prompt( self.prompt, default=options[0], type=click.Choice(options), show_choices=False, value_proc=lambda x: self.process_value(ctx, x), ))
def stop(counters, all=False): """ Stop one or more in-progress executions. """ project = get_project(require=True) params = {'project': project.id} if counters and all: raise click.UsageError( 'Pass either an execution # or `--all`, not both.') elif counters: params['counter'] = sorted(IntegerRange.parse(counters).as_set()) elif all: params['status'] = 'incomplete' else: warn('Nothing to stop (pass #s or `--all`)') return 1 for execution in request('get', '/api/v0/executions/', params=params).json()['results']: click.echo( 'Stopping #{counter}... '.format(counter=execution['counter']), nl=False) resp = request('post', execution['urls']['stop']) click.echo(resp.text) success('Done.')
def lint(filenames: List[str]) -> None: """ Lint (syntax-check) a valohai.yaml file. The return code of this command will be the total number of errors found in all the files. """ if not filenames: project = get_project() if project: project.refresh_details() yaml_path = project.get_yaml_path() else: yaml_path = 'valohai.yaml' directory = (project.directory if project else get_project_directory()) config_file = os.path.join(directory, yaml_path) if not os.path.exists(config_file): raise CLIException( f'There is no {config_file} file. Pass in the names of configuration files to lint?' ) filenames = [config_file] total_errors = 0 for filename in filenames: total_errors += validate_file(filename) if total_errors: warn(f'There were {total_errors} total errors.') click.get_current_context().exit(total_errors)
def download_outputs(outputs, output_path): if not outputs: warn('Nothing to download.') return total_size = sum(o['size'] for o in outputs) num_width = len(str(len( outputs))) # How many digits required to print the number of outputs start_time = time.time() with \ click.progressbar(length=total_size, show_pos=True, item_show_func=force_text) as prog, \ requests.Session() as dl_sess: for i, output in enumerate(outputs, 1): url = output['url'] out_path = os.path.join(output_path, output['name']) out_dir = os.path.dirname(out_path) if not os.path.isdir(out_dir): os.makedirs(out_dir) resp = dl_sess.get(url, stream=True) resp.raise_for_status() prog.current_item = '(%*d/%-*d) %s' % ( num_width, i, num_width, len(outputs), output['name']) with open(out_path, 'wb') as outf: for chunk in resp.iter_content(chunk_size=131072): prog.update(len(chunk)) outf.write(chunk) duration = time.time() - start_time success( 'Downloaded {n} outputs ({size} bytes) in {duration} seconds'.format( n=len(outputs), size=total_size, duration=round(duration, 2), ))
def delete_execution(project, counter, purge_outputs=False): execution_url = '/api/v0/executions/{project_id}:{counter}/'.format( project_id=project.id, counter=counter) try: execution = request('get', execution_url).json() except APIError as ae: # pragma: no cover if ae.response.status_code == 404: return False raise if purge_outputs: for output_datum in execution.get('outputs', ()): if not output_datum.get('purged'): click.echo('#{counter}: Purging output {name}... '.format( counter=execution['counter'], name=output_datum['name'], )) purge_url = '/api/v0/data/{datum_id}/purge/'.format( datum_id=output_datum['id']) resp = request('post', purge_url, handle_errors=False) if resp.status_code >= 400: # pragma: no cover warn( 'Error purging output: {error}; leaving this execution alone!' .format(error=resp.text)) return False click.echo('Deleting #{counter}... '.format(counter=execution['counter'])) resp = request('delete', execution_url, handle_errors=False) if resp.status_code >= 400: # pragma: no cover warn('Error deleting execution: {error}'.format(error=resp.text)) return False return True
def package_adhoc_commit(project: Project, validate: bool = True) -> Dict[str, Any]: """ Create an ad-hoc tarball and commit of the project directory. :return: Commit response object from API """ directory = project.directory tarball = None try: description = '' try: description = describe_current_commit(directory) except (NoGitRepo, NoCommit): pass except Exception as exc: warn(f'Unable to derive Git description: {exc}') if description: click.echo(f'Packaging {directory} ({description})...') else: click.echo(f'Packaging {directory}...') tarball = package_directory(directory, progress=True, validate=validate) return create_adhoc_commit_from_tarball(project, tarball, description) finally: if tarball: try: os.unlink(tarball) except OSError as err: # pragma: no cover warn(f'Unable to remove temporary file: {err}')
def choose_project(dir, spec=None): """ Choose a project, possibly interactively. :param dir: Directory (only used for prompts) :param spec: An optional search string :return: project object or None """ projects = request('get', '/api/v0/projects/', params={'count': '1000'}).json()['results'] if not projects: if click.confirm('You don\'t have any projects. Create one instead?'): raise NewProjectInstead() return None if spec: projects = filter_projects(projects, spec) if not projects: warn('No projects match %s' % spec) return None if len(projects) == 1: return projects[0] def nonlist_validator(answer): if answer.startswith('n'): raise NewProjectInstead() prompt = 'Which project would you like to link with {dir}?\nEnter [n] to create a new project.'.format( dir=click.style(dir, bold=True), ) return prompt_from_list(projects, prompt, nonlist_validator)
def delete_execution(project: Project, counter: int, purge_outputs: bool = False) -> bool: execution_url = f'/api/v0/executions/{project.id}:{counter}/' try: execution = request('get', execution_url).json() except APIError as ae: # pragma: no cover if ae.response.status_code == 404: return False raise if purge_outputs: for output_datum in execution.get('outputs', ()): if not output_datum.get('purged'): progress('#{counter}: Purging output {name}... '.format( counter=execution['counter'], name=output_datum['name'], )) purge_url = f"/api/v0/data/{output_datum['id']}/purge/" resp = request('post', purge_url, handle_errors=False) if resp.status_code >= 400: # pragma: no cover warn( f'Error purging output: {resp.text}; leaving this execution alone!' ) return False progress(f"Deleting #{execution['counter']}... ") resp = request('delete', execution_url, handle_errors=False) if resp.status_code >= 400: # pragma: no cover warn(f'Error deleting execution: {resp.text}') return False return True
def watch(counter: str, force: bool, filter_download: Optional[str], download_directory: Optional[str]) -> None: if download_directory: info( f"Downloading to: {download_directory}\nWaiting for new outputs..." ) else: warn('Target folder is not set. Use --download to set it.') return project = get_project(require=True) execution = project.get_execution_from_counter( counter=counter, params={'exclude': 'outputs'}, ) while True: outputs = get_execution_outputs(execution) outputs = filter_outputs(outputs, download_directory, filter_download, force) if outputs: download_outputs(outputs, download_directory, show_success_message=False) if execution['status'] in complete_execution_statuses: info('Execution has finished.') return time.sleep(1)
def outputs(counter: str, download_directory: Optional[str], filter_download: Optional[str], force: bool, sync: bool) -> None: """ List and download execution outputs. """ if download_directory: download_directory = download_directory.replace( "{counter}", str(counter)) if sync: watch(counter, force, filter_download, download_directory) return project = get_project(require=True) assert project execution = project.get_execution_from_counter( counter=counter, params={'exclude': 'outputs'}, ) outputs = get_execution_outputs(execution) if not outputs: warn('The execution has no outputs.') return for output in outputs: output['datum_url'] = f"datum://{output['id']}" print_table(outputs, ('name', 'datum_url', 'size')) if download_directory: outputs = filter_outputs(outputs, download_directory, filter_download, force) download_outputs(outputs, download_directory, show_success_message=True)
def _get_files_inner( dir: str, allow_git: bool = True) -> Tuple[GitUsage, Iterable[Tuple[str, str]]]: # Inner, pre-vhignore-supporting generator function... gitignore_path = os.path.join(dir, '.gitignore') if allow_git: if os.path.exists(os.path.join(dir, '.git')): # We have .git, so we can try to use Git to figure out a file list of nonignored files try: return (GitUsage.GIT_LS_FILES, _get_files_with_git(dir) ) # return the generator except subprocess.CalledProcessError as cpe: warn( f'.git exists, but we could not use git ls-files (error {cpe.returncode}), falling back to non-git' ) # Limited support of .gitignore even without git if os.path.exists(gitignore_path): with open(gitignore_path, "r") as gitignore_file: gitignore_rules = list( gitignorant.parse_gitignore_file(gitignore_file)) if gitignore_rules: return ( GitUsage.GITIGNORE_WITHOUT_GIT, (p for p in _get_files_walk(dir) if not gitignorant.check_path_match(gitignore_rules, p[0])), ) return (GitUsage.NONE, _get_files_walk(dir)) # return the generator
def unlink(yes: bool) -> None: """ Unlink a linked Valohai project. """ dir = get_project_directory() project = get_project() if not project: warn(f'{dir} or its parents do not seem linked to a project.') return if not yes: click.confirm( 'Unlink {dir} from {name}?'.format( dir=click.style(project.directory, bold=True), name=click.style(project.name, bold=True), ), abort=True, ) links = settings.links.copy() links.pop(dir) settings.persistence.set('links', links) settings.persistence.save() success('Unlinked {dir} from {name}.'.format(dir=click.style(dir, bold=True), name=click.style(project.name, bold=True)))
def login( username: str, password: str, token: Optional[str], host: Optional[str], yes: bool, verify_ssl: bool, ) -> None: """Log in into Valohai.""" if settings.user and settings.token: current_username = settings.user['username'] current_host = settings.host if not yes: click.confirm(( f'You are already logged in as {current_username} on {current_host}.\n' 'Are you sure you wish to acquire a new token?'), abort=True) else: info( f'--yes set: ignoring pre-existing login for {current_username} on {current_host}' ) if not (token or username or password or host): # Don't show the banner if this seems like a non-interactive login. click.secho(f'Welcome to Valohai CLI {__version__}!', bold=True) host = validate_host(host) if token: if username or password: error('Token is mutually exclusive with username/password') raise Exit(1) click.echo(f'Using token {token[:5]}... to log in.') else: token = do_user_pass_login( host=host, username=username, password=password, verify_ssl=verify_ssl, ) click.echo(f'Verifying API token on {host}...') with APISession(host, token, verify_ssl=verify_ssl) as sess: user_data = sess.get('/api/v0/users/me/').json() settings.persistence.update( host=host, user=user_data, token=token, verify_ssl=verify_ssl, ) settings.persistence.save() success(f"Logged in. Hey {user_data.get('username', 'there')}!") if not verify_ssl: warn( "SSL verification is off. This may leave you vulnerable to man-in-the-middle attacks." )
def get_git_commit(project: Project) -> Optional[str]: try: return git.get_current_commit(project.directory) except NoGitRepo: warn( 'The directory is not a Git repository. \n' 'Would you like to just run using the latest commit known by Valohai?' ) if not click.confirm('Use latest commit?', default=True): raise click.Abort() return None
def _get_files( dir: str, allow_git: bool = True) -> Tuple[bool, Iterable[Tuple[str, str]]]: if allow_git and os.path.exists(os.path.join(dir, '.git')): # We have .git, so we can try to use Git to figure out a file list of nonignored files try: return (True, _get_files_with_git(dir)) # return the generator except subprocess.CalledProcessError as cpe: warn( f'.git exists, but we could not use git ls-files (error {cpe.returncode}), falling back to non-git' ) return (False, _get_files_walk(dir)) # return the generator
def choose_project(dir: str, spec: Optional[str] = None) -> Optional[dict]: """ Choose a project, possibly interactively. :param dir: Directory (only used for prompts) :param spec: An optional search string :return: project object or None """ projects: List[dict] = request('get', '/api/v0/projects/', params={ 'limit': '1000' }).json()['results'] if not projects: if click.confirm('You don\'t have any projects. Create one instead?'): raise NewProjectInstead() return None if spec: projects = filter_projects(projects, spec) if not projects: warn(f'No projects match {spec}') return None if len(projects) == 1: return projects[0] def nonlist_validator(answer: str) -> Any: if answer.startswith('n'): raise NewProjectInstead() prompt = 'Which project would you like to link with {dir}?\nEnter [n] to create a new project.'.format( dir=click.style(dir, bold=True), ) has_multiple_owners = (len( {p.get('owner', {}).get('id') for p in projects}) > 1) def project_name_formatter(project: dict) -> str: name: str = project['name'] try: if has_multiple_owners: dim_owner = click.style(project['owner']['username'] + '/', dim=True) return f'{dim_owner}{name}' except Exception: pass return name projects.sort(key=lambda project: project_name_formatter(project).lower()) return prompt_from_list(projects, prompt, nonlist_validator, name_formatter=project_name_formatter)
def outputs(counter, download): """ List and download execution outputs. """ execution = get_project(require=True).get_execution_from_counter( counter=counter, detail=True) outputs = execution.get('outputs', ()) if not outputs: warn('The execution has no outputs.') return print_table(outputs, ('name', 'url', 'size')) if download: download_outputs(outputs, download)
def determine_upgrade_status(current_version: str, latest_version: str) -> Optional[str]: try: from distutils.version import LooseVersion parsed_current_version = LooseVersion(current_version) parsed_latest_version = LooseVersion(latest_version) if parsed_latest_version > parsed_current_version: return 'upgrade' elif parsed_latest_version < parsed_current_version: return 'delorean' elif parsed_latest_version == parsed_current_version: return 'current' except Exception as exc: warn(f'Unable to determine whether the version is older or newer ({exc})') return None
def resolve_commit(self, commit): if not commit: commit = git.get_current_commit(self.project.directory) commits = request( 'get', '/api/v0/projects/{id}/commits/'.format( id=self.project.id)).json() by_identifier = {c['identifier']: c for c in commits} if commit not in by_identifier: warn( 'Commit {commit} is not known for the project. Have you pushed it?' .format(commit=commit)) raise click.Abort() return commit
def logs(counter: str, status: bool, stderr: bool, stdout: bool, stream: bool, all: bool) -> None: """ Show or stream execution event log. """ project = get_project(require=True) assert project execution = project.get_execution_from_counter(counter=counter) accepted_streams = {v for v in [ 'status' if status else None, 'stderr' if stderr else None, 'stdout' if stdout else None, ] if v} lm = LogManager(execution) limit = (0 if all else None) while True: events_response = lm.fetch_events(limit=limit) events = events_response['events'] if not stream and events_response.get('truncated'): warn( 'There are {total} events, but only the last {n} are shown. Use `--all` to fetch everything.'.format( total=events_response['total'], n=len(events), ) ) for event in events: if event['stream'] not in accepted_streams: continue message = '{short_time} {text}'.format( short_time=(event['time'].split('T')[1][:-4]), text=clean_log_line(event['message']), ) style = stream_styles.get(event['stream'], {}) click.echo(click.style(message, **style)) # type: ignore[arg-type] if stream: lm.update_execution() if lm.execution['status'] in complete_execution_statuses: click.echo( 'The execution has finished (status {status}); stopping stream.'.format( status=execution['status'], ), err=True ) break # Fetch less on subsequent queries limit = 100 time.sleep(1) else: break
def get_image_suggestions() -> List[dict]: try: resp = requests.get( 'https://raw.githubusercontent.com/valohai/images/master/images.v2.yaml' ) resp.raise_for_status() images = [{ 'name': image, 'description': info['description'], } for image, info in yaml.safe_load(resp.content).items() if info.get("isRecommended")] images.sort(key=lambda i: str(i.get('name')).lower()) return images except Exception as exc: warn(f'Could not load online image suggestions: {exc}') return []
def delete(counters: Sequence[str], purge_outputs: bool = False) -> None: """ Delete one or more executions, optionally purging their outputs as well. """ project = get_project(require=True) assert project n = 0 for counter in sorted(IntegerRange.parse(counters).as_set()): if delete_execution(project, counter, purge_outputs): n += 1 if n: success(f'Deleted {n} executions.') else: warn('Nothing was deleted.') sys.exit(1)
def delete(counters, purge_outputs=False): """ Delete one or more executions, optionally purging their outputs as well. """ project = get_project(require=True) counters = IntegerRange.parse(counters).as_set() n = 0 for counter in sorted(counters): if delete_execution(project, counter, purge_outputs): n += 1 if n: success('Deleted {n} executions.'.format(n=n)) else: warn('Nothing was deleted.') sys.exit(1)
def get_image_suggestions() -> List[dict]: try: resp = requests.get('https://raw.githubusercontent.com/valohai/images/master/images.yaml') resp.raise_for_status() data = yaml.safe_load(resp.content) description_map = data.get('descriptions', {}) return [ { 'name': image, 'description': description_map.get(image), } for image in data.get('suggestions', []) ] except Exception as exc: warn(f'Could not load online image suggestions: {exc}') return []
def outputs(counter, download, filter_download): """ List and download execution outputs. """ execution = get_project(require=True).get_execution_from_counter( counter=counter) outputs = execution.get('outputs', ()) if not outputs: warn('The execution has no outputs.') return print_table(outputs, ('name', 'url', 'size')) if download: if filter_download: outputs = [ output for output in outputs if fnmatch(output['name'], filter_download) ] download_outputs(outputs, download)
def link_or_create_prompt(cwd): while True: response = click.prompt( 'Do you want to link this directory to a pre-existing project, or create a new one? [L/C]' ).lower().strip() if response.startswith('l'): link.main(prog_name='vh-link', args=[], standalone_mode=False) elif response.startswith('c'): create.main(prog_name='vh-create', args=[], standalone_mode=False) else: warn('Sorry, I couldn\'t understand that.') continue project = get_project(cwd) if not project: error('Oops, looks like something went wrong.') sys.exit(2) break return project
def get_executions_for_stop(project: Project, counters: Optional[List[str]], *, all: bool) -> List[dict]: params: Dict[str, Any] = {'project': project.id} if counters == ['latest']: return [project.get_execution_from_counter('latest')] if counters: params['counter'] = sorted(IntegerRange.parse(counters).as_set()) elif all: params['status'] = 'incomplete' else: warn('Nothing to stop (pass #s or `--all`)') return [] data = request('get', '/api/v0/executions/', params=params).json()['results'] assert isinstance(data, list) return data
def resolve_commit(commit_identifier: Optional[str], project: Project) -> str: if commit_identifier and commit_identifier.startswith('~'): # Assume ad-hoc commits are qualified already return commit_identifier try: commit_obj = project.resolve_commit(commit_identifier=commit_identifier) except KeyError: warn(f'Commit {commit_identifier} is not known for the project. Have you pushed it?') raise click.Abort() except IndexError: warn('No commits are known for the project.') raise click.Abort() resolved_commit_identifier: str = commit_obj['identifier'] if not commit_identifier: click.echo(f'Resolved to commit {resolved_commit_identifier}', err=True) return resolved_commit_identifier
def lint(filenames): """ Lint (syntax-check) a valohai.yaml file. The return code of this command will be the total number of errors found in all the files. """ if not filenames: project = get_project() directory = (project.directory if project else get_project_directory()) config_file = os.path.join(directory, 'valohai.yaml') if not os.path.exists(config_file): raise CLIException( 'There is no %s file. Pass in the names of configuration files to lint?' % config_file) filenames = [config_file] total_errors = 0 for filename in filenames: total_errors += validate_file(filename) if total_errors: warn('There were %d total errors.' % total_errors) click.get_current_context().exit(total_errors)
def _process_parameters(self, parameters: Dict[str, Any], parameter_file: Optional[str]) -> None: if parameter_file: parameter_file_data = read_data_file(parameter_file) if not isinstance(parameter_file_data, dict): raise CLIException( 'Parameter file could not be parsed as a dictionary') for name, parameter in self.step.parameters.items(): # See if we can match the name or the sanitized name to an option for key in (name, sanitize_option_name(name)): if key not in parameter_file_data: continue value = parameter_file_data.pop(key) type = self.parameter_type_map.get(parameter.type, click.STRING) value = type.convert(value, param=None, ctx=None) parameters[name] = value if parameter_file_data: # Not everything was popped off unparsed_parameter_names = ', '.join( sorted(str(k) for k in parameter_file_data)) warn( f'Parameters ignored in parameter file: {unparsed_parameter_names}' ) missing_required_parameters = set() for name, parameter in self.step.parameters.items(): if name in parameters: # Clean out default-less flag parameters whose value would be None if parameter.type == 'flag' and parameters[name] is None: del parameters[name] else: required = (parameter.default is None and not parameter.optional) if required: missing_required_parameters.add(name) if missing_required_parameters: raise CLIException( f'Required parameters missing: {missing_required_parameters}')