Ejemplo n.º 1
0
def persist(src_path, dst_uri, config):
    mpath, label, dpath = dst_uri.split(",")
    fs = filesystem.LocalFileSystem()
    traverser = traversal.traverse(fs, src_path)
    sq = storagequeue.StorageQueue(get_backend_factory(dpath), CONCURRENCY)
    mf = list(persistence.persist(fs, traverser, None, src_path, sq, blocksize=2000))
    manifest.write_manifest(get_backend_factory(mpath)(), label, mf)
Ejemplo n.º 2
0
    def test_basic(self):
        with self.make_backend() as b:
            def make_entry(n, contents):
                hashes = []
                while contents:
                    hashes.append(('sha512', contents[0:5])) # treat contents as hash, nevermind
                    contents = contents[5:]

                return ('%d' % (n,), md.FileMetaData.from_string('-rwxr-xr-x 5 6 7 8 9 10'), hashes)

            entries_in = [ make_entry(n, 'contents'.join([ unicode(m) for m in xrange(0, n)])) for n in xrange(0, 100) ]
            
            manifest.write_manifest(b, 'test_manifest', entries_in)
            entries_out = [ entry for entry in manifest.read_manifest(b, 'test_manifest') ]

            #print b.get('test_manifest')

            self.assertEqual(manifest.list_manifests(b), [ 'test_manifest' ])
            manifest.delete_manifest(b, 'test_manifest')
            self.assertEqual(manifest.list_manifests(b), [])

            def to_comparable(entry):
                path, md, algos = entry

                return (path, md.to_string(), algos)

            self.assertEqual([ to_comparable(entry) for entry in entries_in ],
                             [ to_comparable(entry) for entry in entries_out])
Ejemplo n.º 3
0
def persist(src_path, dst_uri, config):
    mpath, label, dpath = dst_uri.split(',')
    fs = filesystem.LocalFileSystem()
    traverser = traversal.traverse(fs, src_path)
    sq = storagequeue.StorageQueue(get_backend_factory(dpath), CONCURRENCY)
    mf = list(
        persistence.persist(fs, traverser, None, src_path, sq, blocksize=2000))
    manifest.write_manifest(get_backend_factory(mpath)(), label, mf)
Ejemplo n.º 4
0
    def test_basic(self):
        with self.make_backend() as b:

            def make_entry(n, contents):
                hashes = []
                while contents:
                    hashes.append(
                        ('sha512',
                         contents[0:5]))  # treat contents as hash, nevermind
                    contents = contents[5:]

                return ('%d' % (n, ),
                        md.FileMetaData.from_string('-rwxr-xr-x 5 6 7 8 9 10'),
                        hashes)

            entries_in = [
                make_entry(n,
                           'contents'.join([unicode(m) for m in xrange(0, n)]))
                for n in xrange(0, 100)
            ]

            manifest.write_manifest(b, 'test_manifest', entries_in)
            entries_out = [
                entry for entry in manifest.read_manifest(b, 'test_manifest')
            ]

            #print b.get('test_manifest')

            self.assertEqual(manifest.list_manifests(b), ['test_manifest'])
            manifest.delete_manifest(b, 'test_manifest')
            self.assertEqual(manifest.list_manifests(b), [])

            def to_comparable(entry):
                path, md, algos = entry

                return (path, md.to_string(), algos)

            self.assertEqual([to_comparable(entry) for entry in entries_in],
                             [to_comparable(entry) for entry in entries_out])
Ejemplo n.º 5
0
                alg = 'sha512' # TODO: assume this?
            uploaded.append( (alg, hash) )

    # run persist
    fs = filesystem.LocalFileSystem()
    traverser = traversal.traverse(fs, src_path)
    sq = storagequeue.StorageQueue(bf_data,
                                   CONCURRENCY)
    mf = list(persistence.persist(fs,
                                  traverser,
                                  None,
                                  src_path,
                                  sq,
                                  blocksize=blocksize,
                                  skip_blocks=uploaded))
    manifest.write_manifest(b_manifest, label, mf)

def materialize(config, src_uri, dst_path, *files):
    if len(files) == 0:
        files = None
    mpath, label, dpath = src_uri.split(',')
    fs = filesystem.LocalFileSystem()
    fs.mkdir(dst_path)
    mf = list(manifest.read_manifest(get_backend_factory(mpath, config)(),
                                     label))
    sq = storagequeue.StorageQueue(get_backend_factory(dpath, config),
                                   CONCURRENCY)
    materialization.materialize(fs, dst_path, mf, sq, files)

def get_backend_factory(uri, config):
    """get_backend_factory(uri, config)