def archive_outputs(outputs_path: str, name: str) -> Tuple[str, str]:
archive_root = conf.get('OUTPUTS_ARCHIVE_ROOT')
check_or_create_path(archive_root)
outputs_files = get_files_in_path(outputs_path)
tar_name = "{}.tar.gz".format(name.replace('.', '_'))
create_tarfile(files=outputs_files, tar_path=os.path.join(archive_root, tar_name))
return archive_root, tar_name
def create_job_logs_path(cls, job_name, temp, persistence='default'):
import conf
if temp:
check_or_create_path(conf.get('LOGS_ARCHIVE_ROOT'))
return create_job_path(job_name, conf.get('LOGS_ARCHIVE_ROOT'))
persistence_logs = cls.get_logs_path(persistence=persistence)
return create_job_path(job_name, persistence_logs)
def archive_repo(repo_git: Any, repo_name: str, commit: str = None) -> Tuple[str, str]:
archive_root = conf.get('REPOS_ARCHIVE_ROOT')
check_or_create_path(archive_root)
archive_name = '{}-{}.tar.gz'.format(repo_name, commit or 'master')
with open(os.path.join(archive_root, archive_name), 'wb') as fp:
repo_git.archive(fp, format='tgz', treeish=commit)
return archive_root, archive_name
def create_experiment_logs_path(cls,
experiment_name,
temp,
persistence='default'):
import conf
if temp:
check_or_create_path(conf.get(ARCHIVES_ROOT_LOGS))
return create_experiment_path(experiment_name,
conf.get(ARCHIVES_ROOT_LOGS))
persistence_logs = cls.get_logs_path(persistence=persistence)
return create_experiment_path(experiment_name, persistence_logs)
def archive_logs_file(log_path: str, namepath: str, persistence_logs: str = 'default') -> str:
check_or_create_path(conf.get('LOGS_DOWNLOAD_ROOT'))
namepath = namepath.replace('.', '/')
download_filepath = os.path.join(conf.get('LOGS_DOWNLOAD_ROOT'), namepath)
download_dir = '/'.join(download_filepath.split('/')[:-1])
check_or_create_path(download_dir)
try:
store_manager = stores.get_logs_store(persistence_logs=persistence_logs)
store_manager.download_file(log_path, download_filepath)
except (PolyaxonStoresException, VolumeNotFoundError) as e:
raise ValidationError(e)
if store_manager.store.is_local_store:
return log_path
return download_filepath
def archive_outputs_file(outputs_path: str, namepath: str, filepath: str,
persistence_outputs: str) -> str:
check_or_create_path(conf.get(DOWNLOADS_ROOT_ARTIFACTS))
download_filepath = os.path.join(conf.get(DOWNLOADS_ROOT_ARTIFACTS),
namepath.replace('.', '/'), filepath)
download_dir = '/'.join(download_filepath.split('/')[:-1])
check_or_create_path(download_dir)
try:
store_manager = stores.get_outputs_store(
persistence_outputs=persistence_outputs)
outputs_filepath = os.path.join(outputs_path, filepath)
store_manager.download_file(outputs_filepath, download_filepath)
except (PolyaxonStoresException, StoreNotFoundError) as e:
raise ValidationError(e)
if store_manager.store.is_local_store:
return outputs_filepath
return download_filepath
def archive_outputs(outputs_path: str, namepath: str,
persistence_outputs: str) -> Tuple[str, str]:
archive_root = conf.get(ARCHIVES_ROOT_ARTIFACTS)
check_or_create_path(archive_root)
check_or_create_path(conf.get(DOWNLOADS_ROOT_ARTIFACTS))
download_path = os.path.join(conf.get(DOWNLOADS_ROOT_ARTIFACTS),
namepath.replace('.', '/'))
download_dir = '/'.join(download_path.split('/'))
check_or_create_path(download_dir)
try:
store_manager = stores.get_outputs_store(
persistence_outputs=persistence_outputs)
store_manager.download_dir(outputs_path, download_path)
except (PolyaxonStoresException, StoreNotFoundError) as e:
raise ValidationError(e)
if store_manager.store.is_local_store:
outputs_files = get_files_in_path(outputs_path)
else:
outputs_files = get_files_in_path(download_path)
tar_name = "{}.tar.gz".format(namepath.replace('.', '_'))
create_tarfile(files=outputs_files,
tar_path=os.path.join(archive_root, tar_name))
return archive_root, tar_name
