def store_from_dict(self, dyct: dict = None):
to_compress = []
to_store = []
work = None if not self.compressed else Workpath.get_tmp()
try:
for file_spec in self.infer_schema_from_dict(dyct):
file_content = dyct.get(file_spec.name)
if file_content is None:
if file_spec.required:
raise NhaStorageError("File '{}' is required".format(file_spec.name))
else:
continue
elif self.compressed:
work.deploy_text_file(name=file_spec.name, content=file_content)
to_compress.append(file_spec)
else:
file_spec.content = file_content
to_store.append(file_spec)
if self.compressed:
self._compress_and_store(work, to_compress=to_compress)
else:
self.validate_file_sizes(to_store)
self.warehouse.store_files(
hierarchy=self.make_hierarchy(),
file_schema=to_store
)
finally:
if work is not None:
work.dispose()
def _verify_schema(self, path: str):
if self.schema:
for file_spec in self.schema:
file_exists = os.path.isfile(os.path.join(path, file_spec.name))
assert file_exists or not file_spec.required,\
NhaStorageError("Required file '{}' is missing from {}".format(file_spec.name, self.subject))
def store_from_path(self, path):
path, schema = self.infer_schema_from_path(path)
to_compress = []
to_store = []
for file_spec in schema:
file_path = os.path.join(path, file_spec.alias)
if not os.path.exists(file_path) or not os.path.isfile(file_path):
if file_spec.required:
raise NhaStorageError("File '{}' not found in path: {}".format(file_spec.name, path))
else:
self.LOG.info('Ignoring absent file: {}'.format(file_spec.name))
continue
elif self.compressed:
to_compress.append(file_spec)
else:
file_spec.set_path(path)
to_store.append(file_spec)
if self.compressed:
self._compress_and_store(path, to_compress=to_compress)
else:
self.validate_file_sizes(to_store)
self.warehouse.store_files(
hierarchy=self.make_hierarchy(),
file_schema=to_store
)
def download(self, path_from, path_to):
url = self.format_nexus_path(path_from)
try:
self.client.download_file(url, path_to)
except Exception as e:
raise NhaStorageError("Download failed. Check if the remote artifact exists in the repository") from e
def _raise_not_found(self, hierarchy: StoreHierarchy):
raise NhaStorageError(
"Nothing found in lightweight storage with key '{key}'"
.format(
key=hierarchy.child
)
)
def download(self, path_from, path_to):
uri = self.format_artif_path(path_from)
try:
with uri.open() as src:
with open(path_to, "wb") as out:
out.write(src.read())
except Exception as e:
raise NhaStorageError("Download failed. Check if the remote artifact exists in the repository") from e
def _save_output(self, note_path, output_path):
try:
LOG.info("Saving output notebook: {}".format(output_path))
# TODO: convert to pdf (find a light-weight lib for that)
NotebookBarrel(proj=self.proj,
notebook=note_path,
file_name=output_path).store_from_path(os.getcwd())
except Exception as e:
err = NhaStorageError(
"Failed to save output notebook '{}'".format(output_path))
e.__cause__ = e
LOG.error(err)
def __init__(self, ds: Dataset, section: str, **kwargs):
assert ds.stored, NhaStorageError(
"""Dataset '{}' is not stored by the framework, so it cannot be mounted in a container"""
.format(ds.show()))
subdir = ds.get_dir_name()
dyr = OnBoard.SHARED_DATA_DIR
super().__init__(alias='dataset-{}'.format(ds.get_pk()),
mount_to=os.path.join(dyr, subdir),
mode='ro',
barrel=DatasetBarrel(ds, **kwargs),
section=section,
lightweight=ds.lightweight)
def infer_schema_from_path(self, path):
assert os.path.exists(path), NhaStorageError("Path not found: {}".format(path))
if self.schema is None:
if os.path.isdir(path):
schema = [FileSpec(name=name) for name in os.listdir(path)]
else:
schema = [FileSpec(name=os.path.basename(path))]
path = os.path.dirname(path)
self._print_files(schema)
else:
schema = self.schema
if os.path.isfile(path):
if len(schema) == 1:
schema = [FileSpec(name=schema[0].name, alias=os.path.basename(path))]
path = os.path.dirname(path)
self.LOG.warn("File '{}' will be renamed to '{}'".format(schema[0].alias, schema[0].name))
else:
n_reqs = len(list(filter(lambda f: f.required, schema)))
assert n_reqs == 1, NhaStorageError("Cannot find all required files in path {}".format(path))
return path, schema
def delete(self, hierarchy: StoreHierarchy, ignore=False):
path = hierarchy.join_as_path()
uri = os.path.join(self.repo, self.section, path) # TODO use format_nexus_path function
del_count = self.client.delete(uri)
if del_count == 0:
message = "Delete from Nexus failed. Check if the path exists: {}".format(uri)
if ignore:
self.LOG.warn(message)
return False
else:
raise NhaStorageError(message)
else:
return True
def move(self, path_from, path_to):
path_from, schema = self.infer_schema_from_path(path_from)
if not os.path.isdir(path_to):
pathlib.Path(path_to).mkdir(parents=True, exist_ok=True)
for file_spec in schema:
file_path = os.path.join(path_from, file_spec.name)
if not os.path.exists(file_path) or not os.path.isfile(file_path):
if file_spec.required:
raise NhaStorageError("File '{}' not found in path: {}".format(file_spec.name, path_from))
else:
continue
else:
shutil.move(file_path, os.path.join(path_to, file_spec.name))
def upload(self, path_to, path_from: (str, None) = None, content: (str, None) = None):
work = None
try:
if path_from is None:
work = Workpath.get_tmp()
file_name = os.path.basename(path_to)
work.deploy_text_file(name=file_name, content=content)
path_from = work.join(file_name)
dest_path = os.path.join(self.repo, self.section, path_to)
self.client.upload(path_from, dest_path)
except Exception as e:
raise NhaStorageError("Upload failed. Check if the artifact´s path is correct") from e
finally:
if work is not None:
work.dispose()
def upload(self, path_to, path_from=None, content=None):
work = None
try:
if content is not None:
work = Workpath.get_tmp()
file_name = os.path.basename(path_to)
work.deploy_text_file(name=file_name, content=content)
path_from = work.join(file_name)
dest_path = self.format_artif_path(path_to)
dest_path.deploy_file(path_from)
except Exception as e:
raise NhaStorageError("Upload failed. Check if the artifact´s path is correct") from e
finally:
if work is not None:
work.dispose()
def delete(self, hierarchy: StoreHierarchy, ignore=False):
path = hierarchy.join_as_path()
uri = self.format_artif_path(path)
try:
if uri.is_dir():
uri.rmdir()
else:
uri.unlink()
return True
except FileNotFoundError as e:
message = "Delete from Artifactory failed. Check if the path exists: {}".format(uri)
if ignore:
self.LOG.warn(message)
return False
else:
raise NhaStorageError(message) from e
def assert_repo_exists(self):
repositories = self.client.repositories.raw_list()
nexus_repo = list(filter(lambda d: d['name'] == self.repo, repositories))
assert len(nexus_repo) > 0, NhaStorageError("""The {} repository does not exist""".format(self.repo))
def raise_for_file_size(self, file_spec: FileSpec, actual_size_mb: int):
raise NhaStorageError(
"File {} in version '{}' of model '{}' is too large (expected size={} MB / actual size={} MB)"
.format(file_spec.name, self.mv_name, self.model_name, file_spec.max_mb, actual_size_mb)
)
def assert_repo_exists(self):
assert self.client.exists(), NhaStorageError("""The {} repository does not exist""".format(self.repo))
def raise_for_file_size(self, file_spec: FileSpec, actual_size_mb: int):
raise NhaStorageError(
"File {} is too large: {} MB"
.format(file_spec.name, actual_size_mb)
)