def version(self, dataset_id: UUID,
version_id: UUID) -> Union[str, DatasetVersion]:
dataset_versions = valfilter(
lambda _: _.parent_id == ensure_uuid(dataset_id), self._versions)
if not len(dataset_versions):
return f'dataset: {dataset_id} not found'
print(dataset_versions.keys())
print(f'version id: {version_id}')
version = dataset_versions.get(ensure_uuid(version_id))
return version if version else f'version {version_id} not found in dataset {dataset_id}'
def from_dict(data: dict):
return Result(
id=ensure_uuid(data['id']),
method=data['method'],
storage_path=Path(data['storage_path']),
archive_path=data.get('archive_path', Path("")),
input_data_path=Path(data['input_data_path']),
code_directory=data.get('code_directory', Path(data['method'].model_snapshot.code_directory)),
pre_processing=data.get('pre_processing', {}),
post_processing=data.get('pre_processing', {}),
container_id=data.get('container_id', None),
container_name=data['container_name'],
container_image_name=data['method'].model_snapshot.new_container_image_name,
container_pull_logs=data.get('container_pull_logs', {}),
container_performance_statistics=data.get('statistics', ''),
container_pre_processing_logs=data.get('container_pre_processing_logs', ""),
container_run_logs=data.get('container_run_logs', ''),
container_info=data.get('container_info', dict()),
container_ports=data.get('container_ports', {}),
status=data.get('status', 'not started yet'),
container_mount_volumes=data.get('container_mount_volumes', {}),
container_environment_variables=data.get('container_environment_variables', {}),
container_archive_path=Path(
data.get('container_archive_path', Path(data['storage_path']) / 'container_archive.gz')),
success=data['success'],
running=data['running'],
issuer=data['issuer'],
created=datetime.fromisoformat(data['created']),
finished=datetime.fromisoformat(data.get('finished', str(datetime(1, 1, 1, 1, 1, 1, 1)))),
runtime_logs=data.get('runtime_logs', dict()),
performance_metrics=data.get('performance_metrics', dict())
)
def from_dict(data: dict):
return Method(id=ensure_uuid(data['id']),
name=data['name'],
created=datetime.fromisoformat(data['created']),
description=data['description'],
model_snapshot=data['model_snapshot'],
pre_processing=data.get('pre_processing', {}),
post_processing=data.get('pre_processing', {}))
def from_dict(data: dict):
return ModelVersion(id=ensure_uuid(data['id']),
name=data['name'],
git_commit_id=data['git_commit_id'],
model=data['model'],
author=data['author'],
description=data['description'],
created=datetime.fromisoformat(data['created']))
def delete_version(self, dataset_id: UUID,
version_id: UUID) -> Union[Exception, DatasetVersion]:
return excepting_pipe(
version_id,
self.get_version(ensure_uuid(dataset_id)),
self.check_version_delete,
# Remove the version directory
curry(do)(lambda v: shutil.rmtree(v.storage_path)),
self.unload_version)
def from_dict(data: dict):
return Dataset(id=ensure_uuid(data['id']),
name=data['name'],
description=data['description'],
license=data['license'],
origin=data['origin'],
maintainer=data['maintainer'],
storage_path=str(Path(data.get('storage_path', ''))),
created=datetime.fromisoformat(data['created']))
def from_dict(data: dict):
return DatasetVersion(
id=ensure_uuid(data['id']),
name=data.get('name', 'no name provided'),
parent_id=data['parent_id']
if type(data['parent_id']) == UUID else UUID(data['parent_id']),
description=data['description'],
created=datetime.fromisoformat(data['created']),
storage_path=Path(data.get('storage_path', '')))
def get_version(self, dataset_id: UUID,
version_id: UUID) -> Union[Exception, DatasetVersion]:
version = self._versions.get(ensure_uuid(version_id))
if dataset_id != version.parent_id:
return Exception(
f'version {version_id} does not belong to dataset {dataset_id}'
)
return version if version else Exception(
f'version: {version_id} of dataset {dataset_id} not found')
def delete_dataset(self, dataset_id: UUID) -> Union[Exception, Dataset]:
return excepting_pipe(
ensure_uuid(dataset_id),
self.get_dataset,
# Remove complete dataset directory
curry(do)(lambda ds: shutil.rmtree(ds.storage_path)),
# Unload dataset from data manager
self.unload_dataset,
# Unload all dataset versions from data manager
self.unload_dataset_versions)
def list_snapshots_filtered(self, model_id: UUID, git_commit_id: str,
query: dict) -> Mapping[UUID, ModelSnapshot]:
return excepting_pipe(
self._snapshots.values(),
curry(filter)(lambda v: v.model.id == ensure_uuid(model_id)),
curry(filter)(lambda v: v.model_git_commit == git_commit_id),
curry(sorted,
key=lambda snap: getattr(snap, query['sortby']),
reverse=query['order']),
curry(partition_all)(query['limit'] if query['limit'] > 0 else len(
self._snapshots.values())), list,
curry(get, default=[])(query['offset']))
def from_dict(data: dict):
return Model(id=ensure_uuid(data['id']),
source_url=data['source_url'],
name=data.get('name', "No name provided."),
description=data.get('description',
"No description provided."),
license=data.get('license', "No license provided."),
origin=data.get('origin', "No origin provided."),
maintainer=data.get('maintainer',
"No maintainer provided."),
storage_path=Path(data['storage_path']),
code_path=Path(data['code_path']),
created=datetime.fromisoformat(data['created']))
def from_dict(data: dict):
config = Config.Config.from_dict()
return ModelSnapshot(
id=ensure_uuid(data.get('id', uuid4())),
model=data['model'],
model_git_commit=data['model_git_commit'],
parent_id=data.get('parent_id', None),
dataset_version=data['dataset_version'],
input_data_path=data['dataset_version'].storage_path,
parameters=data['parameters'],
code_directory=data.get('code_directory',
Path(data['storage_path']) /
"source_code"),
pre_processing=data.get('pre_processing', {}),
post_processing=data.get('pre_processing', {}),
storage_path=Path(data['storage_path']),
archive_path=data.get('archive_path', Path("")),
container_id=data.get('container_id', None),
container_name=data['container_name'],
container_image_id=data.get('container_image_id', None),
container_image_name=data.get(
'container_image_name',
f"{config.docker_registry_address}model_{data['model_git_commit']}"
),
container_mount_volumes=data.get('container_mount_volumes', {}),
container_environment_variables=data.get(
'container_environment_variables', {}),
container_ports=data.get('container_ports', {}),
container_build_logs=data.get('container_build_logs',
{"info": "no build required."}),
container_pre_processing_logs=data.get(
'container_pre_processing_logs', ""),
container_pull_logs=data.get('container_pull_logs', {}),
container_push_logs=data.get('container_push_logs', {}),
container_run_logs=data.get('container_run_logs', ''),
container_archive_path=Path(
data.get(
'container_archive_path',
Path(data['storage_path']) / 'container_archive.tar.gz')),
container_performance_statistics=data.get('statistics', ''),
container_info=data.get('container_info', dict()),
new_container_image_name=data.get('new_container_image_name', ''),
success=data.get('success', 'not finished'),
status=data.get('status', 'not started yet'),
created=datetime.fromisoformat(data['created']),
finished=datetime.fromisoformat(
data.get('finished', str(datetime(1, 1, 1, 1, 1, 1, 1)))),
runtime_logs=data.get('runtime_logs', 'not set yet'),
performance_metrics=data.get('performance_metrics', 'not set yet'))
def versions(self, dataset_id: UUID,
query: dict) -> Iterator[DatasetVersion]:
"""
Lists all versions of a given dataset
:param dataset_id:
:param query:
:return:
"""
# TODO: Not completely pure pipeline
return excepting_pipe(
self._versions.values(),
curry(filter)(lambda v: v.parent_id == ensure_uuid(dataset_id)),
curry(sorted,
key=lambda ds: getattr(ds, query['sortby']),
reverse=query['order']), partition_versions(query), list,
curry(get, default=[])(query['offset']))
def get_snapshot(self, snap_id: UUID) -> Union[Exception, ModelSnapshot]:
snap = self._snapshots.get(ensure_uuid(snap_id))
return snap if snap else Exception(
f'Model Snapshot: {snap_id} not found')
def get_result(self, result_id: UUID) -> Union[Exception, Result]:
result = self._results.get(ensure_uuid(result_id))
return result if result else Exception(
f'Result: {result_id} not found')
def dataset_version_count(self, dataset_id: UUID) -> Union[Exception, int]:
return excepting_pipe(
self._versions.values(),
curry(filter)(
lambda version: version.parent_id == ensure_uuid(dataset_id)),
list, len)
def get_dataset_version(self, version_id: UUID) -> DatasetVersion:
dataset_version = self._versions.get(ensure_uuid(version_id))
return dataset_version if dataset_version else Exception(
f'Dataset Version: {version_id} not found')
def get_dataset(self, dataset_id: UUID) -> Union[Exception, Dataset]:
dataset = self._datasets.get(ensure_uuid(dataset_id))
return dataset if dataset else Exception(
f'dataset: {dataset_id} not found')
