def save_dir_info(cls, odb, dir_info, hash_info=None):
if hash_info and hash_info.name == odb.fs.PARAM_CHECKSUM:
try:
odb.check(hash_info)
assert hash_info.dir_info == dir_info
return hash_info
except (FileNotFoundError, ObjectFormatError):
pass
from dvc.fs.memory import MemoryFileSystem
from dvc.path_info import PathInfo
from dvc.utils import tmp_fname
fs = MemoryFileSystem(None, {})
path_info = PathInfo(tmp_fname(""))
with fs.open(path_info, "w") as fobj:
json.dump(dir_info.to_list(), fobj, sort_keys=True)
tmp_info = odb.fs.path_info / tmp_fname("")
with fs.open(path_info, "rb") as fobj:
odb.fs.upload_fobj(fobj, tmp_info)
hash_info = get_hash(tmp_info, odb.fs, odb.fs.PARAM_CHECKSUM)
hash_info.value += odb.fs.CHECKSUM_DIR_SUFFIX
hash_info.dir_info = dir_info
hash_info.nfiles = dir_info.nfiles
odb.add(tmp_info, odb.fs, hash_info)
return hash_info
def digest(self):
from dvc.fs.memory import MemoryFileSystem
from dvc.path_info import PathInfo
from dvc.utils import tmp_fname
memfs = MemoryFileSystem()
path_info = PathInfo(tmp_fname(""))
with memfs.open(path_info, "wb") as fobj:
fobj.write(self.as_bytes())
self.fs = memfs
self.path_info = path_info
self.hash_info = get_file_hash(path_info, memfs, "md5")
self.hash_info.value += ".dir"
self.hash_info.size = self.size
self.hash_info.nfiles = len(self)
def digest(self, hash_info: Optional["HashInfo"] = None):
from dvc.fs.memory import MemoryFileSystem
from dvc.utils import tmp_fname
memfs = MemoryFileSystem()
fs_path = "memory://{}".format(tmp_fname(""))
with memfs.open(fs_path, "wb") as fobj:
fobj.write(self.as_bytes())
self.fs = memfs
self.fs_path = fs_path
if hash_info:
self.hash_info = hash_info
else:
_, self.hash_info = get_file_hash(fs_path, memfs, "md5")
assert self.hash_info.value
self.hash_info.value += ".dir"
def digest(self):
from dvc.fs.memory import MemoryFileSystem
from dvc.path_info import CloudURLInfo
from dvc.utils import tmp_fname
memfs = MemoryFileSystem()
path_info = CloudURLInfo("memory://{}".format(tmp_fname("")))
with memfs.open(path_info, "wb") as fobj:
fobj.write(self.as_bytes())
self.fs = memfs
self.path_info = path_info
self.hash_info = get_file_hash(path_info, memfs, "md5")
self.hash_info.value += ".dir"
try:
self.hash_info.size = sum(obj.size for _, obj in self)
except TypeError:
self.hash_info.size = None
self.hash_info.nfiles = len(self)
def clean_staging():
from dvc.fs.memory import MemoryFileSystem
from dvc.objects.stage import _STAGING_MEMFS_PATH
try:
MemoryFileSystem().fs.rm(f"memory://{_STAGING_MEMFS_PATH}",
recursive=True)
except FileNotFoundError:
pass
def _get_staging(odb: "ObjectDB") -> "ObjectDB":
"""Return an ODB that can be used for staging objects.
Staging will be a reference ODB stored in the the global memfs.
"""
from dvc.fs.memory import MemoryFileSystem
fs = MemoryFileSystem()
fs_path = _make_staging_url(fs, odb, odb.fs_path)
state = odb.state
return ReferenceObjectDB(fs, fs_path, state=state)
def get_staging(odb: Optional["ObjectDB"] = None) -> "ObjectDB":
"""Return an ODB that can be used for staging objects.
If odb.fs is local, .dvc/tmp/staging will be returned. Otherwise
the the global (temporary) memfs ODB will be returned.
"""
from dvc.fs.memory import MemoryFileSystem
from dvc.path_info import CloudURLInfo, PathInfo
from dvc.scheme import Schemes
from .db import get_odb
if odb and odb.fs.scheme == Schemes.LOCAL and odb.tmp_dir:
fs = odb.fs
path_info: "DvcPath" = PathInfo(odb.tmp_dir) / _STAGING_DIR
config = odb.config
else:
fs = MemoryFileSystem()
path_info = CloudURLInfo(f"{Schemes.MEMORY}://{_STAGING_MEMFS_PATH}")
config = {}
return get_odb(fs, path_info, **config)