def test_KeyDBFactory_diff(tmp_Path):
with KeyDBWrapper(tmp_Path) as db:
window = KeyDBWindow("window", db)
factory = KeyDBFactory(window, str, lambda data, name: safetype(data))
assert factory.list() == []
factory.write("five", 5)
assert factory.list() == ["five"]
value = factory.read("five")
assert value == safetype(5)
factory.delete("five")
assert factory.list() == []
def test_KeyDBFactory_copy(tmp_Path):
with KeyDBWrapper(Path("db1", tmp_Path)) as db1:
window1 = KeyDBWindow("window", db1)
factory1 = KeyDBFactory(window1, str,
lambda data, name: safetype(data))
assert factory1.list() == []
factory1.write("five", 5)
with KeyDBWrapper(Path("db2", tmp_Path)) as db2:
window2 = KeyDBWindow("other", db2)
factory2 = KeyDBFactory(window2, str,
lambda data, name: safetype(data))
factory2.copy("five", window1)
value = factory2.read("five")
assert value == safetype(5)
def join(self, child):
child = safetype(child)
try:
output = Path( self._path + sep + child)
except UnicodeDecodeError as e:
raise ValueError(str(e) + "\nself path: "+ self._path + "\nchild: ", child)
return output
def checksum_from_link_fast(link):
m = r.search(safetype(link))
if (m):
csum = "".join(m.groups())
return csum
else:
raise ValueError("link %s checksum didn't parse" % (link))
def checksum_from_link(link):
"""Takes a path into the userdata, returns the matching csum."""
m = r.search(safetype(link))
if (m):
csum_slash = m.group()[1:]
csum = _remove_sep_(csum_slash)
return csum
else:
raise ValueError("link %s checksum didn't parse" % (link))
def fsck_frozen_ignored(vol, cwd):
'''Look for frozen links which are in the ignored file.'''
#TODO some of this logic could be moved to volume. Which files are members of the volume is a function of the volume.
ignore_mdd = partial(skip_ignored, [safetype(vol.mdd)])
ignored_frozen = pipeline(ftype_selector([LINK]),
ffilter(uncurry(vol.is_ignored)), fmap(first),
fmap(lambda p: p.relative_to(cwd)),
fmap(partial(print, "Ignored file frozen")),
count)(vol.root.entries(ignore_mdd))
return ignored_frozen
def write(self, key, value):
key = safetype(key)
value_json = JSONEncoder(ensure_ascii=False, sort_keys=True).encode(value)
value_bytes = egest(value_json)
value_hash = egest(checksum(value_bytes))
key_path = self.root.join(key)
with ensure_file(key_path, 'wb') as f:
f.write(value_bytes)
f.write(b"\n")
f.write(value_hash)
f.write(b"\n")
def list(self, query=None):
if query is None:
query = ""
query = safetype(query)
query_path = self.root.join(query)
assert self.root in query_path.parents(), "%s is not a parent of %s" % (self.root, query_path)
if query_path.exists and query_path.isdir():
return [ p.relative_to(self.root)
for (p,t) in query_path.entries()
if t == 'file' ]
else:
return []
def checksum(self):
"""
If self path is a file or a symlink to a file, compute a checksum returned as a string.
If self points to a missing file or a broken symlink, raises FileDoesNotExist.
If self points to a directory or a symlink facing directory, raises IsADirectory.
"""
hasher = md5()
with self.open('rb') as fd:
buf = fd.read(_BLOCKSIZE)
while len(buf) > 0:
hasher.update(buf)
buf = fd.read(_BLOCKSIZE)
digest = safetype(hasher.hexdigest())
return digest
def readraw(self, key):
key = safetype(key)
try:
with self.root.join(key).open('rb') as f:
obj_bytes = f.readline().strip()
obj_bytes_checksum = checksum(obj_bytes).encode('utf-8')
key_checksum = f.readline().strip()
if obj_bytes_checksum != key_checksum:
raise ValueError("Checksum mismatch for key %s. Expected %s, calculated %s" % (key, key_checksum, obj_bytes_checksum))
obj_str = egest(obj_bytes)
return obj_str
except IOError as e:
if e.errno == NoSuchFile or e.errno == IsDirectory:
return None
else:
raise e
O_TRUNC truncate size to 0
O_EXCL error if O_CREAT and the file exists
"""
def ensure_file(path, mode):
assert isinstance(path, Path)
parent = path.parent()
assert parent != path, "Path and parent were the same!"
ensure_dir(parent)
fd = path.open(mode)
return fd
ROOT = Path(sep)
PARENT_STR = safetype("..")
SELF_STR = safetype(".")
def walk(*roots, **kwargs):
skip = kwargs.get('skip', None)
dirs = [iter(sorted(roots))]
while len(dirs) > 0:
curDir = dirs[-1]
curPath = next(curDir, None)
if curPath is None:
dirs.pop()
else:
type = curPath.ftype()
if skip and skip(curPath, type):
continue
def __init__(self, window, keydb): window = safetype(window) assert isinstance(keydb, KeyDB) self.prefix = window + sep self.keydb = keydb
def delete(self, key): key = safetype(key) path = self.root.join(key) path.unlink(clean=self.root)
