def __init__(self, path, frame=None, fast=False):
# output = Path( self._path + sep + child)
if fast:
# Fast path is generated by walk. frame is already a Path and path is a single element from listdir.
self._path = frame._path + sep + path
self._parent = frame
elif isinstance(path, Path):
# Copy constructor from another Path.
assert frame is None
self._path = path._path
self._parent = path._parent
else:
if path is None:
raise ValueError("path must be defined")
path = ingest(path)
if frame is None:
self._path = normpath(path)
assert self._path.startswith(
sep
), "Frame is required when building relative paths: %s" % path
else:
assert isinstance(frame, Path)
assert not path.startswith(
sep
), "path %s is required to be relative when a frame %s is provided" % (
path, frame)
self._path = normpath(frame._path + sep + path)
self._parent = None
assert isinstance(self._path, safetype)
def test_ingest():
assert isinstance(ingest('abc'), safetype)
assert ingest('abc') == 'abc'
assert isinstance(ingest(b'abc'), safetype)
assert ingest(b'abc') == 'abc'
assert isinstance(ingest(u'abc'), safetype)
assert ingest(u'abc') == 'abc'
with pytest.raises(TypeError):
assert ingest(5)
def userPath2Path(arg, frame):
"""
Building paths using conventional POSIX systems will discard CWD if the
path is absolute. FarmFS makes passing of CWD explicit so that path APIs
are pure functions. Additionally FarmFS path construction doesn't allow
for absolute paths to be mixed with frames. This is useful for
spotting bugs and making sure that pathing has strong guarantees. However
this comes at the expense of user expectation. When dealing with user
input, there is an expecation that POSIX semantics are at play.
userPath2Path checks to see if the provided path is absolute, and if not,
adds the CWD frame.
"""
arg = ingest(arg)
if isabs(arg):
return Path(arg)
else:
return Path(arg, frame)
def __init__(self, path, frame=None):
if path is None:
raise ValueError("path must be defined")
elif isinstance(path, Path):
assert frame is None
self._path = path._path
self._parent = path._parent
else:
path = ingest(path)
if frame is None:
assert isabs(path), "Frame is required when building relative paths: %s" % path
self._path = normpath(path)
else:
assert isinstance(frame, Path)
assert not isabs(path), "path %s is required to be relative when a frame %s is provided" % (path, frame)
self._path = frame.join(path)._path
self._parent = first(split(self._path))
assert isinstance(self._path, safetype)
assert isinstance(self._parent, safetype)
def test_egest_ingest():
tst_str = u'abc'
b = egest(tst_str)
s = ingest(b)
assert tst_str == s
def test_ingest_egest():
byte_str = b'I\xc3\xb1t\xc3\xabrn\xc3\xa2ti\xc3\xb4n\xc3\xa0li\xc5\xbe\xc3\xa6ti\xc3\xb8n\n'
s = ingest(byte_str)
b = egest(s)
assert byte_str == b
from farmfs.volume import mkfs as make_volume
from farmfs.volume import FarmFSVolume
from farmfs.fs import Path
from farmfs.keydb import KeyDBWindow
from func_prototypes import typed, returned
from farmfs.util import take, ingest
try:
from os import getcwdu
getcwd_utf = lambda: ingest(getcwdu())
except ImportError:
from os import getcwdb
getcwd_utf = lambda: ingest(getcwdb())
try:
from itertools import imap
except ImportError:
# On python3 map is lazy.
imap = map
try:
from itertools import ifilter
except ImportError:
ifilter = filter
cwd = Path(getcwd_utf())
@returned(Path)
@typed(Path)
def _find_root_path(path):
candidates = imap(lambda x: x.join(".farmfs"), path.parents())
matches = ifilter(lambda x: x.isdir(), candidates)
root = next(take(1)(matches), None)
def dbg_ui(argv, cwd):
exitcode = 0
args = docopt(DBG_USAGE, argv)
vol = getvol(cwd)
if args['reverse']:
csum = args['<csum>']
if args['--all']:
trees = vol.trees()
elif args['--snap']:
trees = [vol.snapdb.read(args['--snap'])]
else:
trees = [vol.tree()]
tree_items = concatMap(lambda t: zipFrom(t,iter(t)))
tree_links = ffilter(uncurry(lambda snap, item: item.is_link()))
matching_links = ffilter(uncurry(lambda snap, item: item.csum() == csum))
def link_printr(snap_item):
(snap, item) = snap_item
print(snap.name, item.to_path(vol.root).relative_to(cwd))
links_printr = fmap(identify(link_printr))
pipeline(
tree_items,
tree_links,
matching_links,
links_printr,
consume)(trees)
elif args['key']:
db = vol.keydb
key = args['<key>']
if args['read']:
printNotNone(db.readraw(key))
elif args['delete']:
db.delete(key)
elif args['list']:
for v in db.list(key):
print(v)
elif args['write']:
value = args['<value>']
db.write(key, value)
elif args['walk']:
if args['root']:
printr = json_printr if args.get('--json') else snapshot_printr
printr(encode_snapshot(vol.tree()))
elif args['snap']:
#TODO could add a test for output encoding.
#TODO could add a test for snap format. Leading '/' on paths.
printr = json_printr if args.get('--json') else snapshot_printr
printr(encode_snapshot(vol.snapdb.read(args['<snapshot>'])))
elif args['userdata']:
blobs = vol.bs.blobs()
printr = json_printr if args.get('--json') else strs_printr
printr(blobs)
elif args['keys']:
printr = json_printr if args.get('--json') else strs_printr
printr(vol.keydb.list())
elif args['checksum']:
#TODO <checksum> <full path>
paths = empty_default(map(lambda x: Path(x, cwd), args['<path>']), [vol.root])
for p in paths:
print(p.checksum(), p.relative_to(cwd))
elif args['link']:
f = Path(args['<file>'], cwd)
b = ingest(args['<target>'])
if not vol.bs.exists(b):
print("blob %s doesn't exist" % b)
if args['--remote']:
remote = vol.remotedb.read(args['--remote'])
else:
raise(ValueError("aborting due to missing blob"))
vol.bs.fetch_blob(remote.bs, b)
else:
pass #b exists, can we check its checksum?
vol.bs.link_to_blob(f, b)
elif args['rewrite-links']:
target = Path(args['<target>'], cwd)
for item in vol.tree():
if not item.is_link():
continue
path = item.to_path(vol.root)
new = vol.repair_link(path)
if new is not None:
print("Relinked %s to %s" % (path.relative_to(cwd), new))
elif args['missing']:
tree_csums = pipeline(
ffilter(lambda item: item.is_link()),
fmap(lambda item: item.csum()),
set
)(iter(vol.tree()))
snapNames = args['<snap>']
def missing_printr(csum, pathStrs):
paths = sorted(imap(lambda pathStr: vol.root.join(pathStr), pathStrs))
for path in paths:
print("%s\t%s" % (csum, path.relative_to(cwd)))
missing_csum2pathStr = pipeline(
fmap(vol.snapdb.read),
concatMap(iter),
ffilter(lambda item: item.is_link()),
ffilter(lambda item: not vol.is_ignored(item.to_path(vol.root), None)),
ffilter(lambda item: item.csum() not in tree_csums),
partial(groupby, lambda item: item.csum()),
ffilter(uncurry(lambda csum, items: every(lambda item: not item.to_path(vol.root).exists(), items))),
fmap(uncurry(lambda csum, items: (csum, list(imap(lambda item: item.pathStr(), items))))),
fmap(uncurry(missing_printr)),
count
)(snapNames)
elif args['blobtype']:
for blob in args['<blob>']:
blob = ingest(blob)
#TODO here csum_to_path is really needed.
print(
blob,
maybe("unknown", vol.bs.csum_to_path(blob).filetype()))
elif args['blob']:
for csum in args['<blob>']:
csum = ingest(csum)
#TODO here csum_to_path is needed
print(csum,
vol.bs.csum_to_path(csum).relative_to(cwd))
elif args['s3']:
bucket = args['<bucket>']
prefix = args['<prefix>']
access_id, secret_key = load_s3_creds(None)
s3bs = S3Blobstore(bucket, prefix, access_id, secret_key)
if args['list']:
pipeline(fmap(print), consume)(s3bs.blobs()())
elif args['upload']:
quiet = args.get('--quiet')
print("Fetching remote blobs")
s3_blobs = set(tqdm(s3bs.blobs()(), disable=quiet, desc="Fetching remote blobs", smoothing=1.0, dynamic_ncols=True, maxinterval=1.0))
print("Remote Blobs: %s" % len(s3_blobs))
print("Fetching local blobs") #TODO we are looking at tree, so blobs in snaps won't be sent.
tree_blobs = set(tqdm(pipeline(
ffilter(lambda x: x.is_link()),
fmap(lambda x: x.csum()),
uniq,
)(iter(vol.tree())), disable=quiet, desc="Calculating local blobs", smoothing=1.0, dynamic_ncols=True, maxinterval=1.0))
print("Local Blobs: %s" % len(tree_blobs))
upload_blobs = tree_blobs - s3_blobs
print("Uploading %s blobs to s3" % len(upload_blobs))
with tqdm(desc="Uploading to S3", disable=quiet, total=len(upload_blobs), smoothing=1.0, dynamic_ncols=True, maxinterval=1.0) as pbar:
def update_pbar(blob):
pbar.update(1)
pbar.set_description("Uploaded %s" % blob)
def upload(blob):
s3bs.upload(blob, vol.bs.csum_to_path(blob))()
return blob
all_success = pipeline(
ffilter(lambda x: x not in s3_blobs),
pfmap(upload, workers=2),
fmap(identify(update_pbar)),
partial(every, identity),
)(upload_blobs)
if all_success:
print("Successfully uploaded")
else:
print("Failed to upload")
exitcode = 1
elif args['check']:
num_corrupt_blobs = pipeline(
ffilter(lambda obj: obj['ETag'][1:-1] != obj['blob']),
fmap(identify(lambda obj: print(obj['blob'], obj['ETag'][1:-1]))),
count
)(s3bs.blob_stats()())
if num_corrupt_blobs == 0:
print("All S3 blobs etags match")
else:
exitcode = 2
return exitcode
def dict_printr(keys, d):
print("\t".join([ingest(d.get(k, '')) for k in keys]))
