def s3(connection_name, path_from, path_to, is_file, workers): """Create s3 path context.""" from polyaxon.stores.manager import download_file_or_dir download_file_or_dir( connection_type=V1ConnectionType(name=connection_name, kind=V1ConnectionKind.S3), path_from=path_from, path_to=path_to, workers=workers, is_file=is_file, ) Printer.print_success("S3 path is initialized, path: `{}`".format(path_to))
async def download_dir(subpath: str, to_tar: bool = False) -> str: path_from = get_path(settings.AGENT_CONFIG.artifacts_store.store_path, subpath) path_to = os.path.join(settings.CLIENT_CONFIG.archive_root, subpath) check_or_create_path(path_to, is_dir=True) return manager.download_file_or_dir( connection_type=settings.AGENT_CONFIG.artifacts_store, path_from=path_from, path_to=path_to, is_file=False, workers=5, to_tar=to_tar, )
async def download_file(subpath: str, check_cache=True) -> str: path_from = get_path(settings.AGENT_CONFIG.artifacts_store.store_path, subpath) path_to = os.path.join(settings.CLIENT_CONFIG.archive_root, subpath) if check_cache and os.path.exists(path_to): # file already exists return path_to check_or_create_path(path_to, is_dir=False) return manager.download_file_or_dir( connection_type=settings.AGENT_CONFIG.artifacts_store, path_from=path_from, path_to=path_to, is_file=True, )
async def download_dir(subpath: str, to_tar: bool = False) -> Optional[str]: path_from = get_path(settings.AGENT_CONFIG.artifacts_store.store_path, subpath) path_to = os.path.join(settings.CLIENT_CONFIG.archive_root, subpath) check_or_create_path(path_to, is_dir=True) try: return manager.download_file_or_dir( connection_type=settings.AGENT_CONFIG.artifacts_store, path_from=path_from, path_to=path_to, is_file=False, workers=5, to_tar=to_tar, ) except (OSError, PolyaxonException) as e: logger.warning("Could not download %s. Error %s" % (path_from, e)) return None
async def download_file(subpath: str, check_cache=True) -> Optional[str]: path_from = get_path(settings.AGENT_CONFIG.artifacts_store.store_path, subpath) path_to = os.path.join(settings.CLIENT_CONFIG.archive_root, subpath) if os.path.exists(path_to): if check_cache: # file already exists return path_to else: os.remove(path_to) check_or_create_path(path_to, is_dir=False) try: return manager.download_file_or_dir( connection_type=settings.AGENT_CONFIG.artifacts_store, path_from=path_from, path_to=path_to, is_file=True, ) except PolyaxonException: return None