def prompt_for_value(self, ctx: Context) -> Optional[str]:
try:
options: List[str] = request(
'get', '/api/v0/projects/ownership_options/').json()
except APINotFoundError: # Endpoint not there, ah well!
return None
except APIError as ae:
warn(f'Unable to retrieve ownership options: {ae}')
return None
if not options:
return None
if len(options) == 1:
return options[0]
print('Who should own this project? The options available to you are:')
for option in options:
print(f' * {option}')
return str(
prompt(
self.prompt,
default=options[0],
type=click.Choice(options),
show_choices=False,
value_proc=lambda x: self.process_value(ctx, x),
))
def stop(counters, all=False):
"""
Stop one or more in-progress executions.
"""
project = get_project(require=True)
params = {'project': project.id}
if counters and all:
raise click.UsageError(
'Pass either an execution # or `--all`, not both.')
elif counters:
params['counter'] = sorted(IntegerRange.parse(counters).as_set())
elif all:
params['status'] = 'incomplete'
else:
warn('Nothing to stop (pass #s or `--all`)')
return 1
for execution in request('get', '/api/v0/executions/',
params=params).json()['results']:
click.echo(
'Stopping #{counter}... '.format(counter=execution['counter']),
nl=False)
resp = request('post', execution['urls']['stop'])
click.echo(resp.text)
success('Done.')
def lint(filenames: List[str]) -> None:
"""
Lint (syntax-check) a valohai.yaml file.
The return code of this command will be the total number of errors found in all the files.
"""
if not filenames:
project = get_project()
if project:
project.refresh_details()
yaml_path = project.get_yaml_path()
else:
yaml_path = 'valohai.yaml'
directory = (project.directory if project else get_project_directory())
config_file = os.path.join(directory, yaml_path)
if not os.path.exists(config_file):
raise CLIException(
f'There is no {config_file} file. Pass in the names of configuration files to lint?'
)
filenames = [config_file]
total_errors = 0
for filename in filenames:
total_errors += validate_file(filename)
if total_errors:
warn(f'There were {total_errors} total errors.')
click.get_current_context().exit(total_errors)
def download_outputs(outputs, output_path):
if not outputs:
warn('Nothing to download.')
return
total_size = sum(o['size'] for o in outputs)
num_width = len(str(len(
outputs))) # How many digits required to print the number of outputs
start_time = time.time()
with \
click.progressbar(length=total_size, show_pos=True, item_show_func=force_text) as prog, \
requests.Session() as dl_sess:
for i, output in enumerate(outputs, 1):
url = output['url']
out_path = os.path.join(output_path, output['name'])
out_dir = os.path.dirname(out_path)
if not os.path.isdir(out_dir):
os.makedirs(out_dir)
resp = dl_sess.get(url, stream=True)
resp.raise_for_status()
prog.current_item = '(%*d/%-*d) %s' % (
num_width, i, num_width, len(outputs), output['name'])
with open(out_path, 'wb') as outf:
for chunk in resp.iter_content(chunk_size=131072):
prog.update(len(chunk))
outf.write(chunk)
duration = time.time() - start_time
success(
'Downloaded {n} outputs ({size} bytes) in {duration} seconds'.format(
n=len(outputs),
size=total_size,
duration=round(duration, 2),
))
def delete_execution(project, counter, purge_outputs=False):
execution_url = '/api/v0/executions/{project_id}:{counter}/'.format(
project_id=project.id, counter=counter)
try:
execution = request('get', execution_url).json()
except APIError as ae: # pragma: no cover
if ae.response.status_code == 404:
return False
raise
if purge_outputs:
for output_datum in execution.get('outputs', ()):
if not output_datum.get('purged'):
click.echo('#{counter}: Purging output {name}... '.format(
counter=execution['counter'],
name=output_datum['name'],
))
purge_url = '/api/v0/data/{datum_id}/purge/'.format(
datum_id=output_datum['id'])
resp = request('post', purge_url, handle_errors=False)
if resp.status_code >= 400: # pragma: no cover
warn(
'Error purging output: {error}; leaving this execution alone!'
.format(error=resp.text))
return False
click.echo('Deleting #{counter}... '.format(counter=execution['counter']))
resp = request('delete', execution_url, handle_errors=False)
if resp.status_code >= 400: # pragma: no cover
warn('Error deleting execution: {error}'.format(error=resp.text))
return False
return True
def package_adhoc_commit(project: Project, validate: bool = True) -> Dict[str, Any]:
"""
Create an ad-hoc tarball and commit of the project directory.
:return: Commit response object from API
"""
directory = project.directory
tarball = None
try:
description = ''
try:
description = describe_current_commit(directory)
except (NoGitRepo, NoCommit):
pass
except Exception as exc:
warn(f'Unable to derive Git description: {exc}')
if description:
click.echo(f'Packaging {directory} ({description})...')
else:
click.echo(f'Packaging {directory}...')
tarball = package_directory(directory, progress=True, validate=validate)
return create_adhoc_commit_from_tarball(project, tarball, description)
finally:
if tarball:
try:
os.unlink(tarball)
except OSError as err: # pragma: no cover
warn(f'Unable to remove temporary file: {err}')
def choose_project(dir, spec=None):
"""
Choose a project, possibly interactively.
:param dir: Directory (only used for prompts)
:param spec: An optional search string
:return: project object or None
"""
projects = request('get', '/api/v0/projects/', params={'count': '1000'}).json()['results']
if not projects:
if click.confirm('You don\'t have any projects. Create one instead?'):
raise NewProjectInstead()
return None
if spec:
projects = filter_projects(projects, spec)
if not projects:
warn('No projects match %s' % spec)
return None
if len(projects) == 1:
return projects[0]
def nonlist_validator(answer):
if answer.startswith('n'):
raise NewProjectInstead()
prompt = 'Which project would you like to link with {dir}?\nEnter [n] to create a new project.'.format(
dir=click.style(dir, bold=True),
)
return prompt_from_list(projects, prompt, nonlist_validator)
def delete_execution(project: Project,
counter: int,
purge_outputs: bool = False) -> bool:
execution_url = f'/api/v0/executions/{project.id}:{counter}/'
try:
execution = request('get', execution_url).json()
except APIError as ae: # pragma: no cover
if ae.response.status_code == 404:
return False
raise
if purge_outputs:
for output_datum in execution.get('outputs', ()):
if not output_datum.get('purged'):
progress('#{counter}: Purging output {name}... '.format(
counter=execution['counter'],
name=output_datum['name'],
))
purge_url = f"/api/v0/data/{output_datum['id']}/purge/"
resp = request('post', purge_url, handle_errors=False)
if resp.status_code >= 400: # pragma: no cover
warn(
f'Error purging output: {resp.text}; leaving this execution alone!'
)
return False
progress(f"Deleting #{execution['counter']}... ")
resp = request('delete', execution_url, handle_errors=False)
if resp.status_code >= 400: # pragma: no cover
warn(f'Error deleting execution: {resp.text}')
return False
return True
def watch(counter: str, force: bool, filter_download: Optional[str],
download_directory: Optional[str]) -> None:
if download_directory:
info(
f"Downloading to: {download_directory}\nWaiting for new outputs..."
)
else:
warn('Target folder is not set. Use --download to set it.')
return
project = get_project(require=True)
execution = project.get_execution_from_counter(
counter=counter,
params={'exclude': 'outputs'},
)
while True:
outputs = get_execution_outputs(execution)
outputs = filter_outputs(outputs, download_directory, filter_download,
force)
if outputs:
download_outputs(outputs,
download_directory,
show_success_message=False)
if execution['status'] in complete_execution_statuses:
info('Execution has finished.')
return
time.sleep(1)
def outputs(counter: str, download_directory: Optional[str],
filter_download: Optional[str], force: bool, sync: bool) -> None:
"""
List and download execution outputs.
"""
if download_directory:
download_directory = download_directory.replace(
"{counter}", str(counter))
if sync:
watch(counter, force, filter_download, download_directory)
return
project = get_project(require=True)
assert project
execution = project.get_execution_from_counter(
counter=counter,
params={'exclude': 'outputs'},
)
outputs = get_execution_outputs(execution)
if not outputs:
warn('The execution has no outputs.')
return
for output in outputs:
output['datum_url'] = f"datum://{output['id']}"
print_table(outputs, ('name', 'datum_url', 'size'))
if download_directory:
outputs = filter_outputs(outputs, download_directory, filter_download,
force)
download_outputs(outputs,
download_directory,
show_success_message=True)
def _get_files_inner(
dir: str,
allow_git: bool = True) -> Tuple[GitUsage, Iterable[Tuple[str, str]]]:
# Inner, pre-vhignore-supporting generator function...
gitignore_path = os.path.join(dir, '.gitignore')
if allow_git:
if os.path.exists(os.path.join(dir, '.git')):
# We have .git, so we can try to use Git to figure out a file list of nonignored files
try:
return (GitUsage.GIT_LS_FILES, _get_files_with_git(dir)
) # return the generator
except subprocess.CalledProcessError as cpe:
warn(
f'.git exists, but we could not use git ls-files (error {cpe.returncode}), falling back to non-git'
)
# Limited support of .gitignore even without git
if os.path.exists(gitignore_path):
with open(gitignore_path, "r") as gitignore_file:
gitignore_rules = list(
gitignorant.parse_gitignore_file(gitignore_file))
if gitignore_rules:
return (
GitUsage.GITIGNORE_WITHOUT_GIT,
(p for p in _get_files_walk(dir) if
not gitignorant.check_path_match(gitignore_rules, p[0])),
)
return (GitUsage.NONE, _get_files_walk(dir)) # return the generator
def unlink(yes: bool) -> None:
"""
Unlink a linked Valohai project.
"""
dir = get_project_directory()
project = get_project()
if not project:
warn(f'{dir} or its parents do not seem linked to a project.')
return
if not yes:
click.confirm(
'Unlink {dir} from {name}?'.format(
dir=click.style(project.directory, bold=True),
name=click.style(project.name, bold=True),
),
abort=True,
)
links = settings.links.copy()
links.pop(dir)
settings.persistence.set('links', links)
settings.persistence.save()
success('Unlinked {dir} from {name}.'.format(dir=click.style(dir,
bold=True),
name=click.style(project.name,
bold=True)))
def login(
username: str,
password: str,
token: Optional[str],
host: Optional[str],
yes: bool,
verify_ssl: bool,
) -> None:
"""Log in into Valohai."""
if settings.user and settings.token:
current_username = settings.user['username']
current_host = settings.host
if not yes:
click.confirm((
f'You are already logged in as {current_username} on {current_host}.\n'
'Are you sure you wish to acquire a new token?'),
abort=True)
else:
info(
f'--yes set: ignoring pre-existing login for {current_username} on {current_host}'
)
if not (token or username or password or host):
# Don't show the banner if this seems like a non-interactive login.
click.secho(f'Welcome to Valohai CLI {__version__}!', bold=True)
host = validate_host(host)
if token:
if username or password:
error('Token is mutually exclusive with username/password')
raise Exit(1)
click.echo(f'Using token {token[:5]}... to log in.')
else:
token = do_user_pass_login(
host=host,
username=username,
password=password,
verify_ssl=verify_ssl,
)
click.echo(f'Verifying API token on {host}...')
with APISession(host, token, verify_ssl=verify_ssl) as sess:
user_data = sess.get('/api/v0/users/me/').json()
settings.persistence.update(
host=host,
user=user_data,
token=token,
verify_ssl=verify_ssl,
)
settings.persistence.save()
success(f"Logged in. Hey {user_data.get('username', 'there')}!")
if not verify_ssl:
warn(
"SSL verification is off. This may leave you vulnerable to man-in-the-middle attacks."
)
def get_git_commit(project: Project) -> Optional[str]:
try:
return git.get_current_commit(project.directory)
except NoGitRepo:
warn(
'The directory is not a Git repository. \n'
'Would you like to just run using the latest commit known by Valohai?'
)
if not click.confirm('Use latest commit?', default=True):
raise click.Abort()
return None
def _get_files(
dir: str,
allow_git: bool = True) -> Tuple[bool, Iterable[Tuple[str, str]]]:
if allow_git and os.path.exists(os.path.join(dir, '.git')):
# We have .git, so we can try to use Git to figure out a file list of nonignored files
try:
return (True, _get_files_with_git(dir)) # return the generator
except subprocess.CalledProcessError as cpe:
warn(
f'.git exists, but we could not use git ls-files (error {cpe.returncode}), falling back to non-git'
)
return (False, _get_files_walk(dir)) # return the generator
def choose_project(dir: str, spec: Optional[str] = None) -> Optional[dict]:
"""
Choose a project, possibly interactively.
:param dir: Directory (only used for prompts)
:param spec: An optional search string
:return: project object or None
"""
projects: List[dict] = request('get',
'/api/v0/projects/',
params={
'limit': '1000'
}).json()['results']
if not projects:
if click.confirm('You don\'t have any projects. Create one instead?'):
raise NewProjectInstead()
return None
if spec:
projects = filter_projects(projects, spec)
if not projects:
warn(f'No projects match {spec}')
return None
if len(projects) == 1:
return projects[0]
def nonlist_validator(answer: str) -> Any:
if answer.startswith('n'):
raise NewProjectInstead()
prompt = 'Which project would you like to link with {dir}?\nEnter [n] to create a new project.'.format(
dir=click.style(dir, bold=True), )
has_multiple_owners = (len(
{p.get('owner', {}).get('id')
for p in projects}) > 1)
def project_name_formatter(project: dict) -> str:
name: str = project['name']
try:
if has_multiple_owners:
dim_owner = click.style(project['owner']['username'] + '/',
dim=True)
return f'{dim_owner}{name}'
except Exception:
pass
return name
projects.sort(key=lambda project: project_name_formatter(project).lower())
return prompt_from_list(projects,
prompt,
nonlist_validator,
name_formatter=project_name_formatter)
def outputs(counter, download):
"""
List and download execution outputs.
"""
execution = get_project(require=True).get_execution_from_counter(
counter=counter, detail=True)
outputs = execution.get('outputs', ())
if not outputs:
warn('The execution has no outputs.')
return
print_table(outputs, ('name', 'url', 'size'))
if download:
download_outputs(outputs, download)
def determine_upgrade_status(current_version: str, latest_version: str) -> Optional[str]:
try:
from distutils.version import LooseVersion
parsed_current_version = LooseVersion(current_version)
parsed_latest_version = LooseVersion(latest_version)
if parsed_latest_version > parsed_current_version:
return 'upgrade'
elif parsed_latest_version < parsed_current_version:
return 'delorean'
elif parsed_latest_version == parsed_current_version:
return 'current'
except Exception as exc:
warn(f'Unable to determine whether the version is older or newer ({exc})')
return None
def resolve_commit(self, commit):
if not commit:
commit = git.get_current_commit(self.project.directory)
commits = request(
'get', '/api/v0/projects/{id}/commits/'.format(
id=self.project.id)).json()
by_identifier = {c['identifier']: c for c in commits}
if commit not in by_identifier:
warn(
'Commit {commit} is not known for the project. Have you pushed it?'
.format(commit=commit))
raise click.Abort()
return commit
def logs(counter: str, status: bool, stderr: bool, stdout: bool, stream: bool, all: bool) -> None:
"""
Show or stream execution event log.
"""
project = get_project(require=True)
assert project
execution = project.get_execution_from_counter(counter=counter)
accepted_streams = {v for v in [
'status' if status else None,
'stderr' if stderr else None,
'stdout' if stdout else None,
] if v}
lm = LogManager(execution)
limit = (0 if all else None)
while True:
events_response = lm.fetch_events(limit=limit)
events = events_response['events']
if not stream and events_response.get('truncated'):
warn(
'There are {total} events, but only the last {n} are shown. Use `--all` to fetch everything.'.format(
total=events_response['total'],
n=len(events),
)
)
for event in events:
if event['stream'] not in accepted_streams:
continue
message = '{short_time} {text}'.format(
short_time=(event['time'].split('T')[1][:-4]),
text=clean_log_line(event['message']),
)
style = stream_styles.get(event['stream'], {})
click.echo(click.style(message, **style)) # type: ignore[arg-type]
if stream:
lm.update_execution()
if lm.execution['status'] in complete_execution_statuses:
click.echo(
'The execution has finished (status {status}); stopping stream.'.format(
status=execution['status'],
),
err=True
)
break
# Fetch less on subsequent queries
limit = 100
time.sleep(1)
else:
break
def get_image_suggestions() -> List[dict]:
try:
resp = requests.get(
'https://raw.githubusercontent.com/valohai/images/master/images.v2.yaml'
)
resp.raise_for_status()
images = [{
'name': image,
'description': info['description'],
} for image, info in yaml.safe_load(resp.content).items()
if info.get("isRecommended")]
images.sort(key=lambda i: str(i.get('name')).lower())
return images
except Exception as exc:
warn(f'Could not load online image suggestions: {exc}')
return []
def delete(counters: Sequence[str], purge_outputs: bool = False) -> None:
"""
Delete one or more executions, optionally purging their outputs as well.
"""
project = get_project(require=True)
assert project
n = 0
for counter in sorted(IntegerRange.parse(counters).as_set()):
if delete_execution(project, counter, purge_outputs):
n += 1
if n:
success(f'Deleted {n} executions.')
else:
warn('Nothing was deleted.')
sys.exit(1)
def delete(counters, purge_outputs=False):
"""
Delete one or more executions, optionally purging their outputs as well.
"""
project = get_project(require=True)
counters = IntegerRange.parse(counters).as_set()
n = 0
for counter in sorted(counters):
if delete_execution(project, counter, purge_outputs):
n += 1
if n:
success('Deleted {n} executions.'.format(n=n))
else:
warn('Nothing was deleted.')
sys.exit(1)
def get_image_suggestions() -> List[dict]:
try:
resp = requests.get('https://raw.githubusercontent.com/valohai/images/master/images.yaml')
resp.raise_for_status()
data = yaml.safe_load(resp.content)
description_map = data.get('descriptions', {})
return [
{
'name': image,
'description': description_map.get(image),
}
for image
in data.get('suggestions', [])
]
except Exception as exc:
warn(f'Could not load online image suggestions: {exc}')
return []
def outputs(counter, download, filter_download):
"""
List and download execution outputs.
"""
execution = get_project(require=True).get_execution_from_counter(
counter=counter)
outputs = execution.get('outputs', ())
if not outputs:
warn('The execution has no outputs.')
return
print_table(outputs, ('name', 'url', 'size'))
if download:
if filter_download:
outputs = [
output for output in outputs
if fnmatch(output['name'], filter_download)
]
download_outputs(outputs, download)
def link_or_create_prompt(cwd):
while True:
response = click.prompt(
'Do you want to link this directory to a pre-existing project, or create a new one? [L/C]'
).lower().strip()
if response.startswith('l'):
link.main(prog_name='vh-link', args=[], standalone_mode=False)
elif response.startswith('c'):
create.main(prog_name='vh-create', args=[], standalone_mode=False)
else:
warn('Sorry, I couldn\'t understand that.')
continue
project = get_project(cwd)
if not project:
error('Oops, looks like something went wrong.')
sys.exit(2)
break
return project
def get_executions_for_stop(project: Project, counters: Optional[List[str]], *,
all: bool) -> List[dict]:
params: Dict[str, Any] = {'project': project.id}
if counters == ['latest']:
return [project.get_execution_from_counter('latest')]
if counters:
params['counter'] = sorted(IntegerRange.parse(counters).as_set())
elif all:
params['status'] = 'incomplete'
else:
warn('Nothing to stop (pass #s or `--all`)')
return []
data = request('get', '/api/v0/executions/',
params=params).json()['results']
assert isinstance(data, list)
return data
def resolve_commit(commit_identifier: Optional[str], project: Project) -> str:
if commit_identifier and commit_identifier.startswith('~'):
# Assume ad-hoc commits are qualified already
return commit_identifier
try:
commit_obj = project.resolve_commit(commit_identifier=commit_identifier)
except KeyError:
warn(f'Commit {commit_identifier} is not known for the project. Have you pushed it?')
raise click.Abort()
except IndexError:
warn('No commits are known for the project.')
raise click.Abort()
resolved_commit_identifier: str = commit_obj['identifier']
if not commit_identifier:
click.echo(f'Resolved to commit {resolved_commit_identifier}', err=True)
return resolved_commit_identifier
def lint(filenames):
"""
Lint (syntax-check) a valohai.yaml file.
The return code of this command will be the total number of errors found in all the files.
"""
if not filenames:
project = get_project()
directory = (project.directory if project else get_project_directory())
config_file = os.path.join(directory, 'valohai.yaml')
if not os.path.exists(config_file):
raise CLIException(
'There is no %s file. Pass in the names of configuration files to lint?'
% config_file)
filenames = [config_file]
total_errors = 0
for filename in filenames:
total_errors += validate_file(filename)
if total_errors:
warn('There were %d total errors.' % total_errors)
click.get_current_context().exit(total_errors)
def _process_parameters(self, parameters: Dict[str, Any],
parameter_file: Optional[str]) -> None:
if parameter_file:
parameter_file_data = read_data_file(parameter_file)
if not isinstance(parameter_file_data, dict):
raise CLIException(
'Parameter file could not be parsed as a dictionary')
for name, parameter in self.step.parameters.items():
# See if we can match the name or the sanitized name to an option
for key in (name, sanitize_option_name(name)):
if key not in parameter_file_data:
continue
value = parameter_file_data.pop(key)
type = self.parameter_type_map.get(parameter.type,
click.STRING)
value = type.convert(value, param=None, ctx=None)
parameters[name] = value
if parameter_file_data: # Not everything was popped off
unparsed_parameter_names = ', '.join(
sorted(str(k) for k in parameter_file_data))
warn(
f'Parameters ignored in parameter file: {unparsed_parameter_names}'
)
missing_required_parameters = set()
for name, parameter in self.step.parameters.items():
if name in parameters:
# Clean out default-less flag parameters whose value would be None
if parameter.type == 'flag' and parameters[name] is None:
del parameters[name]
else:
required = (parameter.default is None
and not parameter.optional)
if required:
missing_required_parameters.add(name)
if missing_required_parameters:
raise CLIException(
f'Required parameters missing: {missing_required_parameters}')
