def _get(stage, p, info):
parsed = urlparse(p) if p else None
if parsed and parsed.scheme == "remote":
remote = get_remote(stage.repo, name=parsed.netloc)
return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
if info and info.get(RepoDependency.PARAM_REPO):
repo = info.pop(RepoDependency.PARAM_REPO)
return RepoDependency(repo, stage, p, info)
if info and info.get(ParamsDependency.PARAM_PARAMS):
params = info.pop(ParamsDependency.PARAM_PARAMS)
return ParamsDependency(stage, p, params)
for d in DEPS:
if d.supported(p):
return d(stage, p, info)
return LocalDependency(stage, p, info)
def _get(stage,
p,
info=None,
cache=True,
metric=False,
plot=False,
persist=False):
parsed = urlparse(p)
if parsed.scheme == "remote":
remote = get_remote(stage.repo, name=parsed.netloc)
return OUTS_MAP[remote.scheme](
stage,
p,
info,
cache=cache,
remote=remote,
metric=metric,
plot=plot,
persist=persist,
)
for o in OUTS:
if o.supported(p):
return o(
stage,
p,
info,
cache=cache,
remote=None,
metric=metric,
plot=plot,
persist=persist,
)
return LocalOutput(
stage,
p,
info,
cache=cache,
remote=None,
metric=metric,
plot=plot,
persist=persist,
)
def test_copy_preserve_etag_across_buckets(cloud, dvc):
cloud.gen(FILE_WITH_CONTENTS)
rem = get_remote(dvc, **cloud.config)
s3 = rem.fs
s3.fs.mkdir("another/")
config = cloud.config.copy()
config["url"] = "s3://another"
config["region"] = "us-east-1"
another = S3FileSystem(**config)
from_info = rem.fs.path_info / "foo"
to_info = another.path_info / "foo"
rem.fs.copy(from_info, to_info)
from_hash = rem.fs.info(from_info)["etag"]
to_hash = another.info(to_info)["etag"]
assert from_hash == to_hash
def remote(request, dvc):
cloud = request.param
cloud.gen(FILE_WITH_CONTENTS)
return get_remote(dvc, **cloud.config)
def _init_remote(self, name):
return get_remote(self.repo, name=name)
def _init_remote(self, name):
from dvc.remote import get_remote
return get_remote(self.repo, name=name)
