def test(self):
# Import
with StaticFileServer() as httpd:
import_url = urljoin(self.get_remote(httpd.server_port), self.FOO)
import_output = "imported_file"
import_stage = self.dvc.imp_url(import_url, import_output)
self.assertTrue(os.path.exists(import_output))
self.assertTrue(filecmp.cmp(import_output, self.FOO, shallow=False))
self.dvc.remove("imported_file.dvc")
with StaticFileServer(handler_class=ContentMD5Handler) as httpd:
import_url = urljoin(self.get_remote(httpd.server_port), self.FOO)
import_output = "imported_file"
import_stage = self.dvc.imp_url(import_url, import_output)
self.assertTrue(os.path.exists(import_output))
self.assertTrue(filecmp.cmp(import_output, self.FOO, shallow=False))
# Run --deps
with StaticFileServer() as httpd:
remote = self.get_remote(httpd.server_port)
cache_id = str(uuid.uuid4())
cache = urljoin(remote, cache_id)
ret1 = main(["remote", "add", "mycache", cache])
ret2 = main(["remote", "add", "myremote", remote])
self.assertEqual(ret1, 0)
self.assertEqual(ret2, 0)
self.dvc = DvcRepo(".")
run_dependency = urljoin(remote, self.BAR)
run_output = "remote_file"
cmd = 'open("{}", "w+")'.format(run_output)
with open("create-output.py", "w") as fd:
fd.write(cmd)
run_stage = self.dvc.run(
deps=[run_dependency],
outs=[run_output],
cmd="python create-output.py",
)
self.assertTrue(run_stage is not None)
self.assertTrue(os.path.exists(run_output))
# Pull
self.dvc.remove(import_stage.path, outs_only=True)
self.assertFalse(os.path.exists(import_output))
shutil.move(self.local_cache, cache_id)
self.assertFalse(os.path.exists(self.local_cache))
self.dvc.pull([import_stage.path], remote="mycache")
self.assertTrue(os.path.exists(import_output))
def test(self):
ret1 = main(["remote", "add", "mycache", self.external_cache])
ret2 = main(["remote", "add", "myremote", self.remote])
self.assertEqual(ret1, 0)
self.assertEqual(ret2, 0)
self.dvc = Project(".")
# Import
with StaticFileServer():
import_url = urljoin(self.remote, self.FOO)
import_output = "imported_file"
import_stage = self.dvc.imp(import_url, import_output)
self.assertTrue(os.path.exists(import_output))
self.assertTrue(filecmp.cmp(import_output, self.FOO, shallow=False))
self.dvc.remove("imported_file.dvc")
with StaticFileServer(handler="Content-MD5"):
import_url = urljoin(self.remote, self.FOO)
import_output = "imported_file"
import_stage = self.dvc.imp(import_url, import_output)
self.assertTrue(os.path.exists(import_output))
self.assertTrue(filecmp.cmp(import_output, self.FOO, shallow=False))
# Run --deps
with StaticFileServer():
run_dependency = urljoin(self.remote, self.BAR)
run_output = "remote_file"
cmd = 'open("{}", "w+")'.format(run_output)
with open("create-output.py", "w") as fd:
fd.write(cmd)
run_stage = self.dvc.run(
deps=[run_dependency],
outs=[run_output],
cmd="python create-output.py",
)
self.assertTrue(run_stage is not None)
self.assertTrue(os.path.exists(run_output))
# Pull
self.dvc.remove(import_stage.path, outs_only=True)
self.assertFalse(os.path.exists(import_output))
shutil.move(self.local_cache, self.external_cache_id)
self.assertFalse(os.path.exists(self.local_cache))
with StaticFileServer():
self.dvc.pull(import_stage.path, remote="mycache")
self.assertTrue(os.path.exists(import_output))
def _prepare_interrupted_download(self):
import_url = urljoin(self.remote, self.FOO)
import_output = "imported_file"
tmp_file_name = import_output + ".part"
tmp_file_path = os.path.join(self._root_dir, tmp_file_name)
self._import_with_interrupt(import_output, import_url)
self.assertTrue(os.path.exists(tmp_file_name))
self.assertFalse(os.path.exists(import_output))
return import_output, import_url, tmp_file_path
def __init__(self, stage, path, info=None, remote=None):
super(DependencyHTTP, self).__init__(stage,
path,
info=info,
remote=remote)
if path.startswith("remote"):
path = urljoin(self.remote.cache_dir, urlparse(path).path)
self.path_info = Path(self.scheme, url=self.url, path=path)
def test(self):
ret1 = main(['remote', 'add', 'mycache', self.external_cache])
ret2 = main(['remote', 'add', 'myremote', self.remote])
self.assertEqual(ret1, 0)
self.assertEqual(ret2, 0)
self.dvc = Project('.')
# Import
with StaticFileServer():
import_url = urljoin(self.remote, self.FOO)
import_output = 'imported_file'
import_stage = self.dvc.imp(import_url, import_output)
self.assertTrue(os.path.exists(import_output))
self.assertTrue(filecmp.cmp(import_output, self.FOO, shallow=False))
# Run --deps
with StaticFileServer():
run_dependency = urljoin(self.remote, self.BAR)
run_output = 'remote_file'
cmd = 'open("{}", "w+")'.format(run_output)
with open('create-output.py', 'w') as fd:
fd.write(cmd)
run_stage = self.dvc.run(deps=[run_dependency],
outs=[run_output],
cmd='python create-output.py')
self.assertTrue(run_stage is not None)
self.assertTrue(os.path.exists(run_output))
# Pull
self.dvc.remove(import_stage.path, outs_only=True)
self.assertFalse(os.path.exists(import_output))
shutil.move(self.local_cache, self.external_cache_id)
self.assertFalse(os.path.exists(self.local_cache))
with StaticFileServer():
self.dvc.pull(import_stage.path, remote='mycache')
self.assertTrue(os.path.exists(import_output))
def _prepare_interrupted_download(self, port):
import_url = urljoin("http://localhost:{}/".format(port), self.FOO)
import_output = "imported_file"
tmp_file_name = import_output + ".part"
tmp_file_path = os.path.realpath(
os.path.join(self._root_dir, tmp_file_name))
self._import_with_interrupt(import_output, import_url)
self.assertTrue(os.path.exists(tmp_file_name))
self.assertFalse(os.path.exists(import_output))
return import_output, import_url, tmp_file_path
def __init__(self, stage, path, info=None, remote=None):
super(DependencyHTTP, self).__init__(stage,
path,
info=info,
remote=remote)
if path.startswith('remote'):
path = urljoin(self.remote.cache_dir, urlparse(path).path)
self.path_info = {
'scheme': urlparse(path).scheme,
'path': path,
}
def external_cache(self):
return urljoin(self.remote, self.external_cache_id)
