def step(filenames: List[str], yaml: Optional[str]) -> None:
"""
Update a step config(s) in valohai.yaml based on Python source file(s).
Example:
vh yaml step hello.py
:param filenames: Path(s) of the Python source code files.
"""
project = get_project()
if project is None:
info("no project linked - assuming files are in current directory.")
project = Project(data={}, directory=os.getcwd())
project.name = "YAML command simulated project"
config_path = project.get_config_filename(yaml_path=yaml)
yaml = yaml or project.get_yaml_path()
for source_path in filenames:
if not os.path.isfile(config_path):
update_yaml_from_source(source_path, project, yaml)
info(f"{yaml} generated.")
create_or_update_requirements(project)
elif yaml_needs_update(source_path, project, yaml):
update_yaml_from_source(source_path, project, yaml)
info(f"{yaml} updated.")
create_or_update_requirements(project)
else:
info(f"{yaml} already up-to-date.")
def package_adhoc_commit(project: Project, validate: bool = True, yaml_path: Optional[str] = None) -> Dict[str, Any]:
"""
Create an ad-hoc tarball and commit of the project directory.
:return: Commit response object from API
"""
project.refresh_details()
directory = project.directory
tarball = None
try:
description = ''
try:
description = describe_current_commit(directory)
except (NoGitRepo, NoCommit):
pass
except Exception as exc:
warn(f'Unable to derive Git description: {exc}')
if description:
click.echo(f'Packaging {directory} ({description})...')
else:
click.echo(f'Packaging {directory}...')
yaml_path = yaml_path or project.get_yaml_path()
tarball = package_directory(directory=directory, progress=True, validate=validate, yaml_path=yaml_path)
return create_adhoc_commit_from_tarball(project=project, tarball=tarball, yaml_path=yaml_path, description=description)
finally:
if tarball:
try:
os.unlink(tarball)
except OSError as err: # pragma: no cover
warn(f'Unable to remove temporary file: {err}')
def export_docker_images(
project: Project,
destination: str,
commit: Optional[str],
extra_docker_images: Iterable[str] = (),
) -> None:
commit = expand_commit_id(project.directory, commit=(commit or 'HEAD'))
docker_images = {
step.image
for step in project.get_config(commit).steps.values()
}
docker_images |= set(extra_docker_images)
for i, image in enumerate(docker_images, 1):
output_path = os.path.join(destination,
sanitize_filename(f'docker-{image}.tar'))
if image not in extra_docker_images:
print_parcel_progress('::: Pulling image {}/{} ({})'.format(
i,
len(docker_images),
image,
))
subprocess.check_call(['docker', 'pull', image])
print_parcel_progress(
f'::: Exporting image {i}/{len(docker_images)} ({image})')
export_docker_image(image, output_path)
def update_yaml_from_source(source_path: str, project: Project, yaml_file: Optional[str]) -> bool:
"""Updates valohai.yaml by parsing the source code file for a call to valohai.prepare()
Call to valohai.prepare() will contain step name, parameters and inputs.
We use the AST parser to parse those from the Python source code file and
update (or generate) valohai.yaml accordingly.
:param source_path: Path of the Python source code file
:param project: Currently linked Valohai project
"""
old_config = get_current_config(project, yaml_file)
new_config = get_updated_config(source_path, project, yaml_file)
if old_config != new_config:
project.refresh_details()
with open(project.get_config_filename(yaml_path=yaml_file), 'w') as out_file:
out_file.write(config_to_yaml(new_config))
return True
return False
def download_execution_data(project: Project,
counters: Sequence[str]) -> Dict[str, dict]:
executions = {}
with click.progressbar(IntegerRange.parse(counters).as_set(),
label='fetching information') as counter_iter:
for counter in counter_iter:
execution = project.get_execution_from_counter(
counter=counter, params={
'exclude': 'metadata,events,tags',
})
executions[execution['id']] = execution
return executions
def get_updated_config(source_path: str, project: Project) -> Config:
"""Opens the old valohai.yaml, parses source Python file and merges the resulting config to the old
Call to valohai.prepare() will contain step name, parameters and inputs.
We use the AST parser to parse those from the Python source code file and
return the merged config.
:param source_path: Path of the Python source code file
:param project: Currently linked Valohai project
"""
old_config = get_current_config(project)
new_config = parse_config_from_source(source_path,
project.get_config_filename())
if old_config:
return old_config.merge_with(new_config)
return new_config
def create_adhoc_commit_from_tarball(*, project: Project, tarball: str, yaml_path: Optional[str] = None, description: str = '') -> Dict[str, Any]:
"""
Using a precreated ad-hoc tarball, create or retrieve an ad-hoc commit of it on the Valohai host.
:param project: Project
:param tarball: Tgz tarball path, likely created by the packager
:param yaml_path: Optional custom yaml path attached to the command.
:param description: Optional description for the commit
:return: Commit response object from API
"""
yaml_path = yaml_path or project.get_yaml_path()
commit_obj = _get_pre_existing_commit(tarball)
if commit_obj:
success(f"Ad-hoc code {commit_obj['identifier']} already uploaded")
else:
commit_obj = _upload_commit_code(project=project, tarball=tarball, yaml_path=yaml_path, description=description)
return commit_obj
def get_executions_for_stop(project: Project, counters: Optional[List[str]], *,
all: bool) -> List[dict]:
params: Dict[str, Any] = {'project': project.id}
if counters == ['latest']:
return [project.get_execution_from_counter('latest')]
if counters:
params['counter'] = sorted(IntegerRange.parse(counters).as_set())
elif all:
params['status'] = 'incomplete'
else:
warn('Nothing to stop (pass #s or `--all`)')
return []
data = request('get', '/api/v0/executions/',
params=params).json()['results']
assert isinstance(data, list)
return data
def resolve_commit(commit_identifier: Optional[str], project: Project) -> str:
if commit_identifier and commit_identifier.startswith('~'):
# Assume ad-hoc commits are qualified already
return commit_identifier
try:
commit_obj = project.resolve_commit(commit_identifier=commit_identifier)
except KeyError:
warn(f'Commit {commit_identifier} is not known for the project. Have you pushed it?')
raise click.Abort()
except IndexError:
warn('No commits are known for the project.')
raise click.Abort()
resolved_commit_identifier: str = commit_obj['identifier']
if not commit_identifier:
click.echo(f'Resolved to commit {resolved_commit_identifier}', err=True)
return resolved_commit_identifier
def get_project(self, directory: str) -> Optional[Union['RemoteProject', 'Project']]:
"""
Get the Valohai project object for a directory context.
The directory tree is walked upwards to find an actual linked directory.
If a project override is active, it is always returned.
:param dir: Directory
:return: Project object, or None.
"""
if self.override_project:
return self.override_project
links = self.links
if not links:
return None
for directory in walk_directory_parents(directory): # noqa: B020
project_obj = links.get(directory)
if project_obj:
from valohai_cli.models.project import Project
return Project(data=project_obj, directory=directory)
return None # No project.
def get_project(dir=None, require=False):
"""
Get the Valohai project object for a directory context.
The object is augmented with the `dir` key.
:param dir: Directory (defaults to cwd)
:param require: Raise an exception if no project is found
:return: Project object, or None.
:rtype: valohai_cli.models.project.Project|None
"""
links = settings.get('links') or {}
if not links:
if require:
raise NoProject('No projects are configured')
return None
orig_dir = dir or get_project_directory()
for dir in walk_directory_parents(orig_dir):
project_obj = links.get(dir)
if project_obj:
return Project(data=project_obj, directory=dir)
if require:
raise NoProject('No project is linked to %s' % orig_dir)
return None
def get_current_config(project: Project) -> Optional[Config]:
try:
return project.get_config()
except FileNotFoundError:
return None
def get_current_config(project: Project, yaml_file: Optional[str] = None) -> Optional[Config]:
try:
return project.get_config(yaml_path=yaml_file)
except FileNotFoundError:
return None