def create_experiment_job_logs_path(experiment_job_name, temp):
if temp:
check_archive_path(settings.LOGS_ARCHIVE_ROOT)
return create_experiment_job_path(experiment_job_name,
settings.LOGS_ARCHIVE_ROOT)
return create_experiment_job_path(experiment_job_name,
settings.LOGS_MOUNT_PATH)
def create_experiment_job_logs_path(experiment_job_name, temp):
if temp:
check_archive_path(settings.LOGS_ARCHIVE_ROOT)
return create_experiment_job_path(experiment_job_name,
settings.LOGS_ARCHIVE_ROOT)
persistence_logs = stores.get_logs_paths()
return create_experiment_job_path(experiment_job_name, persistence_logs)
def archive_outputs(outputs_path: str, name: str) -> Tuple[str, str]:
archive_root = conf.get('OUTPUTS_ARCHIVE_ROOT')
check_archive_path(archive_root)
outputs_files = get_files_in_path(outputs_path)
tar_name = "{}.tar.gz".format(name.replace('.', '_'))
create_tarfile(files=outputs_files,
tar_path=os.path.join(archive_root, tar_name))
return archive_root, tar_name
def archive_outputs(outputs_path, name):
check_archive_path(settings.OUTPUTS_ARCHIVE_ROOT)
outputs_files = get_files_in_path(outputs_path)
tar_name = "{}.tar.gz".format(name.replace('.', '_'))
create_tarfile(files=outputs_files,
tar_path=os.path.join(settings.OUTPUTS_ARCHIVE_ROOT,
tar_name))
return settings.OUTPUTS_ARCHIVE_ROOT, tar_name
def archive_repo(repo_git, repo_name, commit=None):
check_archive_path(settings.REPOS_ARCHIVE_ROOT)
archive_name = '{}-{}.tar.gz'.format(repo_name, commit or 'master')
with open(os.path.join(settings.REPOS_ARCHIVE_ROOT, archive_name),
'wb') as fp:
repo_git.archive(fp, format='tgz', treeish=commit)
return settings.REPOS_ARCHIVE_ROOT, archive_name
def create_job_logs_path(cls, job_name, temp, persistence='default'):
import conf
if temp:
check_archive_path(conf.get('LOGS_ARCHIVE_ROOT'))
return create_job_path(job_name, conf.get('LOGS_ARCHIVE_ROOT'))
persistence_logs = cls.get_logs_path(persistence=persistence)
return create_job_path(job_name, persistence_logs)
def archive_repo(repo_git, repo_name, commit=None):
archive_root = conf.get('REPOS_ARCHIVE_ROOT')
check_archive_path(archive_root)
archive_name = '{}-{}.tar.gz'.format(repo_name, commit or 'master')
with open(os.path.join(archive_root, archive_name), 'wb') as fp:
repo_git.archive(fp, format='tgz', treeish=commit)
return archive_root, archive_name
def archive_outputs_file(persistence_outputs, outputs_path, namepath,
filepath):
check_archive_path(settings.OUTPUTS_DOWNLOAD_ROOT)
namepath = namepath.replace('.', '/')
download_filepath = os.path.join(settings.OUTPUTS_DOWNLOAD_ROOT, namepath,
filepath)
download_dir = '/'.join(download_filepath.split('/')[:-1])
check_archive_path(download_dir)
store_manager = get_outputs_store(persistence_outputs=persistence_outputs)
outputs_filepath = os.path.join(outputs_path, filepath)
store_manager.download_file(outputs_filepath, download_filepath)
if store_manager.store.is_local_store:
return outputs_filepath
return download_filepath
def archive_logs_file(log_path: str,
namepath: str,
persistence_logs: str = 'default') -> str:
check_archive_path(conf.get('LOGS_DOWNLOAD_ROOT'))
namepath = namepath.replace('.', '/')
download_filepath = os.path.join(conf.get('LOGS_DOWNLOAD_ROOT'), namepath)
download_dir = '/'.join(download_filepath.split('/')[:-1])
check_archive_path(download_dir)
try:
store_manager = stores.get_logs_store(
persistence_logs=persistence_logs)
store_manager.download_file(log_path, download_filepath)
except (PolyaxonStoresException, VolumeNotFoundError) as e:
raise ValidationError(e)
if store_manager.store.is_local_store:
return log_path
return download_filepath