def materialize(src_uri, dst_path, config):
mpath, label, dpath = src_uri.split(',')
fs = filesystem.LocalFileSystem()
fs.mkdir(dst_path)
mf = list(manifest.read_manifest(get_backend_factory(mpath)(), label))
sq = storagequeue.StorageQueue(get_backend_factory(dpath), CONCURRENCY)
materialization.materialize(fs, dst_path, mf, sq)
def persist(src_path, dst_uri, config):
mpath, label, dpath = dst_uri.split(',')
fs = filesystem.LocalFileSystem()
traverser = traversal.traverse(fs, src_path)
sq = storagequeue.StorageQueue(get_backend_factory(dpath), CONCURRENCY)
mf = list(
persistence.persist(fs, traverser, None, src_path, sq, blocksize=2000))
manifest.write_manifest(get_backend_factory(mpath)(), label, mf)
def make_file_system(self):
return fs.LocalFileSystem()
log.info("checking old manifests...")
mfs = get_all_manifests(b_manifest)
if len(mfs) != 0:
mfs = zip(*mfs)[1]
uploaded.extend(get_all_blockhashes(mfs))
if conf.get_option('continue').get_required():
log.info("checking for previously upped blocks...")
for hash in bf_data().list():
hash = hash.strip()
if len(hash) == 512/4:
alg = 'sha512' # TODO: assume this?
uploaded.append( (alg, hash) )
# run persist
fs = filesystem.LocalFileSystem()
traverser = traversal.traverse(fs, src_path)
sq = storagequeue.StorageQueue(bf_data,
CONCURRENCY)
mf = list(persistence.persist(fs,
traverser,
None,
src_path,
sq,
blocksize=blocksize,
skip_blocks=uploaded))
manifest.write_manifest(b_manifest, label, mf)
def materialize(config, src_uri, dst_path, *files):
if len(files) == 0:
files = None