Python get_cloud_tree Examples

Example #1

File: test_remote.py Project: vignesh1997/dvc

def test_remote_with_hash_jobs(dvc):
    dvc.config["remote"]["with_hash_jobs"] = {
        "url": "s3://bucket/name",
        "hash_jobs": 100,
    }
    dvc.config["core"]["hash_jobs"] = 200

    tree = get_cloud_tree(dvc, name="with_hash_jobs")
    assert tree.hash_jobs == 100

Example #2

File: cache.py Project: surono24/dvc

    def getter(self):
        from dvc.remote import get_cloud_tree
        from dvc.remote.base import CloudCache

        remote = self.config.get(name)
        if not remote:
            return None

        tree = get_cloud_tree(self.repo, name=remote)
        return CloudCache(tree)

Example #3

File: cache.py Project: surono24/dvc

    def __init__(self, repo):
        from dvc.remote import get_cloud_tree
        from dvc.remote.local import LocalCache

        self.repo = repo
        self.config = config = repo.config["cache"]

        local = config.get("local")

        if local:
            settings = {"name": local}
        else:
            settings = {**config, "url": config["dir"]}

        tree = get_cloud_tree(repo, **settings)
        self.local = LocalCache(tree)

Example #4

def _get(stage, p, info):
    parsed = urlparse(p) if p else None
    if parsed and parsed.scheme == "remote":
        tree = get_cloud_tree(stage.repo, name=parsed.netloc)
        return DEP_MAP[tree.scheme](stage, p, info, tree=tree)

    if info and info.get(RepoDependency.PARAM_REPO):
        repo = info.pop(RepoDependency.PARAM_REPO)
        return RepoDependency(repo, stage, p, info)

    if info and info.get(ParamsDependency.PARAM_PARAMS):
        params = info.pop(ParamsDependency.PARAM_PARAMS)
        return ParamsDependency(stage, p, params)

    for d in DEPS:
        if d.supported(p):
            return d(stage, p, info)
    return LocalDependency(stage, p, info)

Example #5

File: __init__.py Project: gbiagomba/dvc

def _get(stage,
         p,
         info=None,
         cache=True,
         metric=False,
         plot=False,
         persist=False):
    parsed = urlparse(p)

    if parsed.scheme == "remote":
        tree = get_cloud_tree(stage.repo, name=parsed.netloc)
        return OUTS_MAP[tree.scheme](
            stage,
            p,
            info,
            cache=cache,
            tree=tree,
            metric=metric,
            plot=plot,
            persist=persist,
        )

    for o in OUTS:
        if o.supported(p):
            return o(
                stage,
                p,
                info,
                cache=cache,
                tree=None,
                metric=metric,
                plot=plot,
                persist=persist,
            )
    return LocalOutput(
        stage,
        p,
        info,
        cache=cache,
        tree=None,
        metric=metric,
        plot=plot,
        persist=persist,
    )

Example #6

File: test_remote.py Project: vignesh1997/dvc

def test_remote_without_hash_jobs_default(dvc):
    dvc.config["remote"]["without_hash_jobs"] = {"url": "s3://bucket/name"}

    tree = get_cloud_tree(dvc, name="without_hash_jobs")
    assert tree.hash_jobs == tree.HASH_JOBS

Example #7

def test_remote_without_checksum_jobs_default(dvc):
    dvc.config["remote"]["without_checksum_jobs"] = {"url": "s3://bucket/name"}

    tree = get_cloud_tree(dvc, name="without_checksum_jobs")
    assert tree.checksum_jobs == tree.CHECKSUM_JOBS

Example #8

def test_remote_without_checksum_jobs(dvc):
    dvc.config["remote"]["without_checksum_jobs"] = {"url": "s3://bucket/name"}
    dvc.config["core"]["checksum_jobs"] = 200

    tree = get_cloud_tree(dvc, name="without_checksum_jobs")
    assert tree.checksum_jobs == 200