def load_dir_cache(self, checksum):
path_info = self.checksum_to_path_info(checksum)
fobj = tempfile.NamedTemporaryFile(delete=False)
path = fobj.name
to_info = PathLOCAL(path=path)
self.cache.download([path_info], [to_info], no_progress_bar=True)
try:
with open(path, "r") as fobj:
d = json.load(fobj)
except ValueError:
logger.exception("Failed to load dir cache '{}'".format(path_info))
return []
finally:
os.unlink(path)
if not isinstance(d, list):
msg = "dir cache file format error '{}' [skipping the file]"
logger.error(msg.format(os.path.relpath(path)))
return []
for info in d:
info[self.PARAM_RELPATH] = self.to_ospath(info[self.PARAM_RELPATH])
return d
def __init__(
self,
stage,
path,
info=None,
remote=None,
cache=True,
metric=False,
persist=False,
tags=None,
):
super(OutputLOCAL, self).__init__(
stage,
path,
info,
remote=remote,
cache=cache,
metric=metric,
persist=persist,
tags=tags,
)
if remote:
p = os.path.join(remote.prefix,
urlparse(self.url).path.lstrip("/"))
else:
p = path
if not os.path.isabs(p):
p = self.remote.to_ospath(p)
p = os.path.join(stage.wdir, p)
p = os.path.abspath(os.path.normpath(p))
self.path_info = PathLOCAL(url=self.url, path=p)
def test_transforms_inode(self, get_inode_mock):
state = State(self.dvc, self.dvc.config.config)
inode = state.MAX_INT + 2
self.assertNotEqual(inode, state._to_sqlite(inode))
path = os.path.join(self.dvc.root_dir, self.FOO)
md5 = file_md5(path)[0]
get_inode_mock.side_effect = self.mock_get_inode(path, inode)
with state:
state.save(PathLOCAL(path=path), md5)
ret = state.get_state_record_for_inode(inode)
self.assertIsNotNone(ret)
def _get_dir_info_checksum(self, dir_info, path_info):
to_info = copy(path_info)
to_info.path = self.cache.ospath.join(self.cache.prefix, tmp_fname(""))
tmp = tempfile.NamedTemporaryFile(delete=False).name
with open(tmp, "w+") as fobj:
json.dump(dir_info, fobj, sort_keys=True)
from_info = PathLOCAL(path=tmp)
self.cache.upload([from_info], [to_info], no_progress_bar=True)
checksum = self.get_file_checksum(to_info) + self.CHECKSUM_DIR_SUFFIX
from_info = copy(to_info)
to_info.path = self.cache.checksum_to_path(checksum)
return checksum, from_info, to_info
def test_dir_checksum_should_be_key_order_agnostic(dvc):
data_dir = os.path.join(dvc.root_dir, "data")
file1 = os.path.join(data_dir, "1")
file2 = os.path.join(data_dir, "2")
os.mkdir(data_dir)
with open(file1, "w") as fobj:
fobj.write("1")
with open(file2, "w") as fobj:
fobj.write("2")
path_info = PathLOCAL(path=data_dir)
with dvc.state:
with patch.object(
RemoteBASE,
"_collect_dir",
return_value=[
{
"relpath": "1",
"md5": "1"
},
{
"relpath": "2",
"md5": "2"
},
],
):
checksum1 = dvc.cache.local.get_dir_checksum(path_info)
with patch.object(
RemoteBASE,
"_collect_dir",
return_value=[
{
"md5": "1",
"relpath": "1"
},
{
"md5": "2",
"relpath": "2"
},
],
):
checksum2 = dvc.cache.local.get_dir_checksum(path_info)
assert checksum1 == checksum2
def __init__(self, repo, config):
super(RemoteLOCAL, self).__init__(repo, config)
self.state = self.repo.state if self.repo else None
self.protected = config.get(Config.SECTION_CACHE_PROTECTED, False)
storagepath = config.get(Config.SECTION_AWS_STORAGEPATH, None)
self.cache_dir = config.get(Config.SECTION_REMOTE_URL, storagepath)
if self.cache_dir is not None and not os.path.isabs(self.cache_dir):
cwd = config[Config.PRIVATE_CWD]
self.cache_dir = os.path.abspath(os.path.join(cwd, self.cache_dir))
types = config.get(Config.SECTION_CACHE_TYPE, None)
if types:
if isinstance(types, str):
types = [t.strip() for t in types.split(",")]
self.cache_types = types
else:
self.cache_types = copy(self.DEFAULT_CACHE_TYPES)
if self.cache_dir is not None and not os.path.exists(self.cache_dir):
os.mkdir(self.cache_dir)
self._dir_info = {}
self.path_info = PathLOCAL()
def test_update(self):
path = os.path.join(self.dvc.root_dir, self.FOO)
path_info = PathLOCAL(path=path)
md5 = file_md5(path)[0]
state = State(self.dvc, self.dvc.config.config)
with state:
state.save(path_info, md5)
entry_md5 = state.get(path_info)
self.assertEqual(entry_md5, md5)
os.unlink(path)
with open(path, "a") as fd:
fd.write("1")
entry_md5 = state.get(path_info)
self.assertTrue(entry_md5 is None)
md5 = file_md5(path)[0]
state.save(path_info, md5)
entry_md5 = state.get(path_info)
self.assertEqual(entry_md5, md5)
def unprotect(self, target):
path_info = PathLOCAL(path=target)
return self.cache.local.unprotect(path_info)