def store_from_dict(self, dyct: dict = None):
to_compress = []
to_store = []
work = None if not self.compressed else Workpath.get_tmp()
try:
for file_spec in self.infer_schema_from_dict(dyct):
file_content = dyct.get(file_spec.name)
if file_content is None:
if file_spec.required:
raise NhaStorageError("File '{}' is required".format(file_spec.name))
else:
continue
elif self.compressed:
work.deploy_text_file(name=file_spec.name, content=file_content)
to_compress.append(file_spec)
else:
file_spec.content = file_content
to_store.append(file_spec)
if self.compressed:
self._compress_and_store(work, to_compress=to_compress)
else:
self.validate_file_sizes(to_store)
self.warehouse.store_files(
hierarchy=self.make_hierarchy(),
file_schema=to_store
)
finally:
if work is not None:
work.dispose()
def upload(self, path_to, path_from: (str, None) = None, content: (str, None) = None):
work = None
try:
if path_from is None:
work = Workpath.get_tmp()
file_name = os.path.basename(path_to)
work.deploy_text_file(name=file_name, content=content)
path_from = work.join(file_name)
dest_path = os.path.join(self.repo, self.section, path_to)
self.client.upload(path_from, dest_path)
except Exception as e:
raise NhaStorageError("Upload failed. Check if the artifact´s path is correct") from e
finally:
if work is not None:
work.dispose()
def upload(self, path_to, path_from=None, content=None):
work = None
try:
if content is not None:
work = Workpath.get_tmp()
file_name = os.path.basename(path_to)
work.deploy_text_file(name=file_name, content=content)
path_from = work.join(file_name)
dest_path = self.format_artif_path(path_to)
dest_path.deploy_file(path_from)
except Exception as e:
raise NhaStorageError("Upload failed. Check if the artifact´s path is correct") from e
finally:
if work is not None:
work.dispose()
def _compress_and_store(self, path: str, to_compress: List[FileSpec] = None):
work = Workpath.get_tmp()
try:
target = work.join(self.compressed)
with tarfile.open(target, 'w:gz') as f:
for file_spec in to_compress:
file_path = os.path.join(path, file_spec.alias)
f.add(file_path, arcname=file_spec.name)
file_spec = FileSpec(name=self.compressed)
self.validate_file_sizes([file_spec])
self.warehouse.store_files(
hierarchy=self.make_hierarchy(),
file_schema=[file_spec]
)
finally:
work.dispose()
def load_vol(self, cargo: Cargo, mule_alias: str = None):
work_path, mule, error = None, None, None
if not isinstance(cargo, MappedCargo):
self.assert_vol(cargo)
if isinstance(cargo, EmptyCargo) or len(cargo.contents) == 0:
return False
try:
self.LOG.debug("Loading volume '{}'".format(cargo.name))
mule = self.get_mule(cargo, mule_alias)
self.clear_mule(mule)
work_path = Workpath.get_tmp()
kwargs = dict(include_heavy_cargos=True) if isinstance(
cargo, SharedCargo) else {}
cargo.deploy(work_path, **kwargs)
for file_name in os.listdir(work_path):
self.copy_to(src=work_path.join(file_name),
dest=DockerConst.STG_MOUNT,
cont=mule)
except Exception as e:
error = e
else:
return True
finally:
if work_path is not None:
work_path.dispose()
if mule is not None:
self.rm_cont(mule)
if error is not None:
self.rm_vol(cargo, ignore=True)
raise error
def load_vol(self, cargo: Cargo, mule_alias: str = None):
if isinstance(cargo, EmptyCargo):
self.assert_vol(cargo)
return
elif isinstance(cargo, MappedCargo):
return
work_path, error = None, None
vol_path = os.path.join(DockerConst.STG_MOUNT, cargo.name)
try:
self.prepare_mule(mule_alias)
self.LOG.debug("Creating volume '{}'".format(cargo.name))
self.clear_mule(self.mule, vol_path)
work_path = Workpath.get_tmp()
if not isinstance(cargo, HeavyCargo):
cargo.deploy(work_path)
for file_name in os.listdir(work_path):
self.copy_to(src=work_path.join(file_name),
dest=vol_path,
pod=self.mule)
if isinstance(cargo, (HeavyCargo, SharedCargo)):
for msg, cmd in cargo.get_deployables(vol_path):
self.LOG.info(msg)
self._exec_in_pod(self.mule, cmd)
except Exception as e:
self.rm_vol(cargo, ignore=True)
raise e
finally:
if work_path is not None:
work_path.dispose()
def make_work_path(self) -> Workpath:
return Workpath.get_tmp()
def deploy_source(self, work_path: Workpath) -> str:
self.repo.clone(work_path)
return work_path.join(self.repo.name)
def make_work_path(self) -> Workpath:
return Workpath.get_fixed(path=self.repo.address)
def make_local_file(self, basename, content):
work = Workpath.get_tmp()
work.deploy_text_file(name=basename, content=content)
path_from = work.join(basename)
return path_from