def get(self, request, *args, **kwargs):
try:
store_manager = stores.get_outputs_store(
persistence_outputs=self.experiment.persistence_outputs)
except (PolyaxonStoresException, VolumeNotFoundError) as e:
raise ValidationError(e)
experiment_outputs_path = stores.get_experiment_outputs_path(
persistence=self.experiment.persistence_outputs,
experiment_name=self.experiment.unique_name,
original_name=self.experiment.original_unique_name,
cloning_strategy=self.experiment.cloning_strategy)
if request.query_params.get('path'):
experiment_outputs_path = os.path.join(
experiment_outputs_path, request.query_params.get('path'))
try:
data = store_manager.ls(experiment_outputs_path)
except VolumeNotFoundError:
raise ValidationError(
'Store manager could not load the volume requested,'
' to get the outputs data.')
except Exception:
raise ValidationError(
'Experiment outputs path does not exists or bad configuration.'
)
return Response(data=data, status=200)
def archive_outputs(outputs_path: str, namepath: str,
persistence_outputs: str) -> Tuple[str, str]:
archive_root = conf.get(ARCHIVES_ROOT_ARTIFACTS)
check_or_create_path(archive_root)
check_or_create_path(conf.get(DOWNLOADS_ROOT_ARTIFACTS))
download_path = os.path.join(conf.get(DOWNLOADS_ROOT_ARTIFACTS),
namepath.replace('.', '/'))
download_dir = '/'.join(download_path.split('/'))
check_or_create_path(download_dir)
try:
store_manager = stores.get_outputs_store(
persistence_outputs=persistence_outputs)
store_manager.download_dir(outputs_path, download_path)
except (PolyaxonStoresException, StoreNotFoundError) as e:
raise ValidationError(e)
if store_manager.store.is_local_store:
outputs_files = get_files_in_path(outputs_path)
else:
outputs_files = get_files_in_path(download_path)
tar_name = "{}.tar.gz".format(namepath.replace('.', '_'))
create_tarfile(files=outputs_files,
tar_path=os.path.join(archive_root, tar_name))
return archive_root, tar_name
def archive_outputs_file(persistence_outputs, outputs_path, namepath,
filepath):
check_archive_path(conf.get('OUTPUTS_DOWNLOAD_ROOT'))
namepath = namepath.replace('.', '/')
download_filepath = os.path.join(conf.get('OUTPUTS_DOWNLOAD_ROOT'),
namepath, filepath)
download_dir = '/'.join(download_filepath.split('/')[:-1])
check_archive_path(download_dir)
store_manager = stores.get_outputs_store(
persistence_outputs=persistence_outputs)
outputs_filepath = os.path.join(outputs_path, filepath)
store_manager.download_file(outputs_filepath, download_filepath)
if store_manager.store.is_local_store:
return outputs_filepath
return download_filepath
def archive_outputs_file(outputs_path: str, namepath: str, filepath: str,
persistence_outputs: str) -> str:
check_or_create_path(conf.get(DOWNLOADS_ROOT_ARTIFACTS))
download_filepath = os.path.join(conf.get(DOWNLOADS_ROOT_ARTIFACTS),
namepath.replace('.', '/'), filepath)
download_dir = '/'.join(download_filepath.split('/')[:-1])
check_or_create_path(download_dir)
try:
store_manager = stores.get_outputs_store(
persistence_outputs=persistence_outputs)
outputs_filepath = os.path.join(outputs_path, filepath)
store_manager.download_file(outputs_filepath, download_filepath)
except (PolyaxonStoresException, StoreNotFoundError) as e:
raise ValidationError(e)
if store_manager.store.is_local_store:
return outputs_filepath
return download_filepath
def get(self, request, *args, **kwargs):
store_manager = stores.get_outputs_store(
persistence_outputs=self.job.persistence_outputs)
job_outputs_path = get_job_outputs_path(
persistence_outputs=self.job.persistence_outputs,
job_name=self.job.unique_name)
if request.query_params.get('path'):
job_outputs_path = os.path.join(job_outputs_path,
request.query_params.get('path'))
try:
data = store_manager.ls(job_outputs_path)
except VolumeNotFoundError:
raise ValidationError(
'Store manager could not load the volume requested,'
' to get the outputs data.')
except Exception:
raise ValidationError(
'Experiment outputs path does not exists or bad configuration.'
)
return Response(data=data, status=200)
