Python get_connection_from_type Examples

Example #1

def download_file_or_dir(
    path_from: str,
    path_to: str,
    is_file: bool,
    workers: int = 0,
    connection_type: V1ConnectionType = None,
    to_tar: bool = False,
) -> Optional[str]:
    connection_type = connection_type or get_artifacts_connection()

    validate_store(connection_type)
    store_manager = get_connection_from_type(connection_type=connection_type)

    if is_file:
        store_manager.download_file(path_from, path_to, use_basename=False)
    else:
        store_manager.download_dir(path_from,
                                   path_to,
                                   use_basename=False,
                                   workers=workers)
    if not os.path.exists(path_to):
        return None
    if to_tar:
        return tar_dir(path_to)
    return path_to

Example #2

def upload_file_or_dir(
    path_from: str,
    path_to: str,
    is_file: bool,
    workers: int = 0,
    last_time: datetime = None,
    connection_type: V1ConnectionType = None,
    exclude: List[str] = None,
):
    connection_type = connection_type or get_artifacts_connection()

    validate_store(connection_type)
    store_manager = get_connection_from_type(connection_type=connection_type)

    if is_file:
        store_manager.upload_file(path_from, path_to, use_basename=False)
    else:
        store_manager.upload_dir(
            path_from,
            path_to,
            use_basename=False,
            workers=workers,
            last_time=last_time,
            exclude=exclude,
        )

Example #3

def delete_path(
    subpath: str, workers: int = 0, connection_type: V1ConnectionType = None
):
    connection_type = connection_type or get_artifacts_connection()

    validate_store(connection_type)

    store_path = get_path(connection_type.store_path, subpath)

    store_manager = get_connection_from_type(connection_type=connection_type)
    store_manager.delete(store_path, workers=workers)

Example #4

def delete_file_or_dir(
    subpath: str,
    is_file: bool = False,
    workers: int = 0,
    connection_type: V1ConnectionType = None,
):
    connection_type = connection_type or get_artifacts_connection()

    validate_store(connection_type)

    store_manager = get_connection_from_type(connection_type=connection_type)
    if is_file:
        store_manager.delete_file(subpath)
    else:
        store_manager.delete(subpath, workers=workers)

Example #5

File: manager.py Project: stjordanis/polyaxon

def delete_file_or_dir(
    subpath: Union[str, List[str]],
    is_file: bool = False,
    workers: int = 0,
    connection_type: V1ConnectionType = None,
):
    connection_type = connection_type or get_artifacts_connection()

    validate_store(connection_type)

    store_manager = get_connection_from_type(connection_type=connection_type)
    subpath = to_list(subpath, check_none=True)
    for sp in subpath:
        if is_file:
            store_manager.delete_file(sp)
        else:
            store_manager.delete(sp, workers=workers)

Example #6

def list_files(subpath: str,
               filepath: str = None,
               connection_type: V1ConnectionType = None):
    connection_type = connection_type or get_artifacts_connection()

    validate_store(connection_type)

    store_path = get_path(connection_type.store_path, subpath)
    if filepath:
        store_path = get_path(store_path, filepath)

    store_manager = get_connection_from_type(connection_type=connection_type)

    try:
        results = store_manager.ls(store_path)
        results["files"] = {f[0]: f[1] for f in results["files"]}
        return results
    except Exception:
        raise PolyaxonStoresException(
            "Run store path does not exists or bad configuration.")