Python download_file_or_dirの例

コード例 #1

ファイル: initializer.py プロジェクト: mort8Q/polyaxon

def s3(connection_name, path_from, path_to, is_file, workers):
    """Create s3 path context."""
    from polyaxon.stores.manager import download_file_or_dir

    download_file_or_dir(
        connection_type=V1ConnectionType(name=connection_name,
                                         kind=V1ConnectionKind.S3),
        path_from=path_from,
        path_to=path_to,
        workers=workers,
        is_file=is_file,
    )

    Printer.print_success("S3 path is initialized, path: `{}`".format(path_to))

コード例 #2

ファイル: async_manager.py プロジェクト: savvihub/polyaxon

async def download_dir(subpath: str, to_tar: bool = False) -> str:
    path_from = get_path(settings.AGENT_CONFIG.artifacts_store.store_path,
                         subpath)
    path_to = os.path.join(settings.CLIENT_CONFIG.archive_root, subpath)
    check_or_create_path(path_to, is_dir=True)
    return manager.download_file_or_dir(
        connection_type=settings.AGENT_CONFIG.artifacts_store,
        path_from=path_from,
        path_to=path_to,
        is_file=False,
        workers=5,
        to_tar=to_tar,
    )

コード例 #3

ファイル: async_manager.py プロジェクト: savvihub/polyaxon

async def download_file(subpath: str, check_cache=True) -> str:
    path_from = get_path(settings.AGENT_CONFIG.artifacts_store.store_path,
                         subpath)
    path_to = os.path.join(settings.CLIENT_CONFIG.archive_root, subpath)

    if check_cache and os.path.exists(path_to):
        # file already exists
        return path_to

    check_or_create_path(path_to, is_dir=False)
    return manager.download_file_or_dir(
        connection_type=settings.AGENT_CONFIG.artifacts_store,
        path_from=path_from,
        path_to=path_to,
        is_file=True,
    )

コード例 #4

ファイル: async_manager.py プロジェクト: opentechfn/polyaxon

async def download_dir(subpath: str, to_tar: bool = False) -> Optional[str]:
    path_from = get_path(settings.AGENT_CONFIG.artifacts_store.store_path,
                         subpath)
    path_to = os.path.join(settings.CLIENT_CONFIG.archive_root, subpath)
    check_or_create_path(path_to, is_dir=True)
    try:
        return manager.download_file_or_dir(
            connection_type=settings.AGENT_CONFIG.artifacts_store,
            path_from=path_from,
            path_to=path_to,
            is_file=False,
            workers=5,
            to_tar=to_tar,
        )
    except (OSError, PolyaxonException) as e:
        logger.warning("Could not download %s. Error %s" % (path_from, e))
        return None

コード例 #5

ファイル: async_manager.py プロジェクト: zhaohb/polyaxon

async def download_file(subpath: str, check_cache=True) -> Optional[str]:
    path_from = get_path(settings.AGENT_CONFIG.artifacts_store.store_path,
                         subpath)
    path_to = os.path.join(settings.CLIENT_CONFIG.archive_root, subpath)

    if os.path.exists(path_to):
        if check_cache:
            # file already exists
            return path_to
        else:
            os.remove(path_to)

    check_or_create_path(path_to, is_dir=False)
    try:
        return manager.download_file_or_dir(
            connection_type=settings.AGENT_CONFIG.artifacts_store,
            path_from=path_from,
            path_to=path_to,
            is_file=True,
        )
    except PolyaxonException:
        return None