def archive_outputs(outputs_path: str, name: str) -> Tuple[str, str]: archive_root = conf.get('OUTPUTS_ARCHIVE_ROOT') check_or_create_path(archive_root) outputs_files = get_files_in_path(outputs_path) tar_name = "{}.tar.gz".format(name.replace('.', '_')) create_tarfile(files=outputs_files, tar_path=os.path.join(archive_root, tar_name)) return archive_root, tar_name
def create_job_logs_path(cls, job_name, temp, persistence='default'): import conf if temp: check_or_create_path(conf.get('LOGS_ARCHIVE_ROOT')) return create_job_path(job_name, conf.get('LOGS_ARCHIVE_ROOT')) persistence_logs = cls.get_logs_path(persistence=persistence) return create_job_path(job_name, persistence_logs)
def archive_repo(repo_git: Any, repo_name: str, commit: str = None) -> Tuple[str, str]: archive_root = conf.get('REPOS_ARCHIVE_ROOT') check_or_create_path(archive_root) archive_name = '{}-{}.tar.gz'.format(repo_name, commit or 'master') with open(os.path.join(archive_root, archive_name), 'wb') as fp: repo_git.archive(fp, format='tgz', treeish=commit) return archive_root, archive_name
def create_experiment_logs_path(cls, experiment_name, temp, persistence='default'): import conf if temp: check_or_create_path(conf.get(ARCHIVES_ROOT_LOGS)) return create_experiment_path(experiment_name, conf.get(ARCHIVES_ROOT_LOGS)) persistence_logs = cls.get_logs_path(persistence=persistence) return create_experiment_path(experiment_name, persistence_logs)
def archive_logs_file(log_path: str, namepath: str, persistence_logs: str = 'default') -> str: check_or_create_path(conf.get('LOGS_DOWNLOAD_ROOT')) namepath = namepath.replace('.', '/') download_filepath = os.path.join(conf.get('LOGS_DOWNLOAD_ROOT'), namepath) download_dir = '/'.join(download_filepath.split('/')[:-1]) check_or_create_path(download_dir) try: store_manager = stores.get_logs_store(persistence_logs=persistence_logs) store_manager.download_file(log_path, download_filepath) except (PolyaxonStoresException, VolumeNotFoundError) as e: raise ValidationError(e) if store_manager.store.is_local_store: return log_path return download_filepath
def archive_outputs_file(outputs_path: str, namepath: str, filepath: str, persistence_outputs: str) -> str: check_or_create_path(conf.get(DOWNLOADS_ROOT_ARTIFACTS)) download_filepath = os.path.join(conf.get(DOWNLOADS_ROOT_ARTIFACTS), namepath.replace('.', '/'), filepath) download_dir = '/'.join(download_filepath.split('/')[:-1]) check_or_create_path(download_dir) try: store_manager = stores.get_outputs_store( persistence_outputs=persistence_outputs) outputs_filepath = os.path.join(outputs_path, filepath) store_manager.download_file(outputs_filepath, download_filepath) except (PolyaxonStoresException, StoreNotFoundError) as e: raise ValidationError(e) if store_manager.store.is_local_store: return outputs_filepath return download_filepath
def archive_outputs(outputs_path: str, namepath: str, persistence_outputs: str) -> Tuple[str, str]: archive_root = conf.get(ARCHIVES_ROOT_ARTIFACTS) check_or_create_path(archive_root) check_or_create_path(conf.get(DOWNLOADS_ROOT_ARTIFACTS)) download_path = os.path.join(conf.get(DOWNLOADS_ROOT_ARTIFACTS), namepath.replace('.', '/')) download_dir = '/'.join(download_path.split('/')) check_or_create_path(download_dir) try: store_manager = stores.get_outputs_store( persistence_outputs=persistence_outputs) store_manager.download_dir(outputs_path, download_path) except (PolyaxonStoresException, StoreNotFoundError) as e: raise ValidationError(e) if store_manager.store.is_local_store: outputs_files = get_files_in_path(outputs_path) else: outputs_files = get_files_in_path(download_path) tar_name = "{}.tar.gz".format(namepath.replace('.', '_')) create_tarfile(files=outputs_files, tar_path=os.path.join(archive_root, tar_name)) return archive_root, tar_name