def _get(stage, p, info): parsed = urlparse(p) if p else None if parsed and parsed.scheme == "remote": remote = get_remote(stage.repo, name=parsed.netloc) return DEP_MAP[remote.scheme](stage, p, info, remote=remote) if info and info.get(RepoDependency.PARAM_REPO): repo = info.pop(RepoDependency.PARAM_REPO) return RepoDependency(repo, stage, p, info) if info and info.get(ParamsDependency.PARAM_PARAMS): params = info.pop(ParamsDependency.PARAM_PARAMS) return ParamsDependency(stage, p, params) for d in DEPS: if d.supported(p): return d(stage, p, info) return LocalDependency(stage, p, info)
def _get(stage, p, info=None, cache=True, metric=False, plot=False, persist=False): parsed = urlparse(p) if parsed.scheme == "remote": remote = get_remote(stage.repo, name=parsed.netloc) return OUTS_MAP[remote.scheme]( stage, p, info, cache=cache, remote=remote, metric=metric, plot=plot, persist=persist, ) for o in OUTS: if o.supported(p): return o( stage, p, info, cache=cache, remote=None, metric=metric, plot=plot, persist=persist, ) return LocalOutput( stage, p, info, cache=cache, remote=None, metric=metric, plot=plot, persist=persist, )
def test_copy_preserve_etag_across_buckets(cloud, dvc): cloud.gen(FILE_WITH_CONTENTS) rem = get_remote(dvc, **cloud.config) s3 = rem.fs s3.fs.mkdir("another/") config = cloud.config.copy() config["url"] = "s3://another" config["region"] = "us-east-1" another = S3FileSystem(**config) from_info = rem.fs.path_info / "foo" to_info = another.path_info / "foo" rem.fs.copy(from_info, to_info) from_hash = rem.fs.info(from_info)["etag"] to_hash = another.info(to_info)["etag"] assert from_hash == to_hash
def remote(request, dvc): cloud = request.param cloud.gen(FILE_WITH_CONTENTS) return get_remote(dvc, **cloud.config)
def _init_remote(self, name): return get_remote(self.repo, name=name)
def _init_remote(self, name): from dvc.remote import get_remote return get_remote(self.repo, name=name)