def test_basic(self): with storagequeue.StorageQueue(lambda: self.make_backend(), CONCURRENCY) as sq: with self.fs.tempdir() as tdir: # populate self.fs.mkdir(self.path(tdir.path, 'testdir')) self.fs.open(self.path(tdir.path, 'testdir/testfile'), 'a').close() with self.fs.open(self.path(tdir.path, 'testdir/testfile2'), 'a') as f: f.write('this is the body of testfile2') self.fs.symlink(self.path(tdir.path, 'testdir/testfile2'), self.path(tdir.path, 'testdir/testfile2-symlink')) with self.fs.open(self.path(tdir.path, 'testdir/testfile3'), 'a') as f: f.write('testfile3 body') traverser = traversal.traverse(self.fs, tdir.path) manifest = [ elt for elt in persistence.persist(self.fs, traverser, None, tdir.path, sq, blocksize=20) ] #print meta.to_string() + ' ' + path + ' ' + unicode(hashes) self.assertEqual(len(manifest), 5) files = self.backend.list() self.assertEqual(len(files), 3) # two blocks for 2, one for 3 file_contents = [ self.backend.get(fname) for fname in files ] self.assertTrue('testfile3 body' in file_contents) self.assertTrue('this is the body of ' in file_contents) self.assertTrue('testfile2' in file_contents) for fname in files: self.assertEqual(fname, hash.make_hasher('sha512')(self.backend.get(fname))[1])
def test_sha512(self): sha512 = hash.make_hasher("sha512") self.assertEqual( sha512("test"), ( "sha512", "ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff", ), )
def test_basic(self): with storagequeue.StorageQueue(lambda: self.make_backend(), CONCURRENCY) as sq: with self.fs.tempdir() as tdir: # Populate a tree. self.fs.mkdir(self.path(tdir.path, 'testdir')) self.fs.open(self.path(tdir.path, 'testdir/testfile'), 'a').close() with self.fs.open(self.path(tdir.path, 'testdir/testfile2'), 'a') as f: f.write('this is the body of testfile2') self.fs.symlink(self.path(tdir.path, 'testdir/testfile2'), self.path(tdir.path, 'testdir/testfile2-symlink')) with self.fs.open(self.path(tdir.path, 'testdir/testfile3'), 'a') as f: f.write('testfile3 body') # Traverse it and persist to store. traverser = traversal.traverse(self.fs, tdir.path) manifest = [ elt for elt in persistence.persist(self.fs, traverser, None, tdir.path, sq, blocksize=20) ] #print meta.to_string() + ' ' + path + ' ' + unicode(hashes) self.assertEqual(len(manifest), 5) files = self.backend.list() self.assertEqual(len(files), 3) # two blocks for 2, one for 3 file_contents = [ self.backend.get(fname) for fname in files ] self.assertTrue('testfile3 body' in file_contents) self.assertTrue('this is the body of ' in file_contents) self.assertTrue('testfile2' in file_contents) for fname in files: self.assertEqual(fname, hash.make_hasher('sha512')(self.backend.get(fname))[1]) # Create another tempdir and materialize into it, and compare # the results. with self.fs.tempdir() as rdir: materialization.materialize(self.fs, rdir.path, manifest, sq) def rec(refdir, tstdir): reflst = sorted(self.fs.listdir(refdir)) tstlst = sorted(self.fs.listdir(tstdir)) self.assertEqual(tstlst, reflst) # todo: confirm file sizes # todo: confirm file contents # todo: confirm file meta for entry in reflst: entrypath = os.path.join(refdir, entry) if not self.fs.is_symlink(entrypath) and self.fs.is_dir(entrypath): rec(entrypath, os.path.join(tstdir, entry)) rec(tdir.path, rdir.path)
def test_generic(self): for algo in hash._our_supported_hashlib_algos: h = hash.make_hasher(algo) self.assertTrue(h is not None, 'should have been given a hasher') self.assertTrue(callable(h), 'hasher should be callable') name, digest = h('test data') self.assertEqual(name, algo) # PY3-TODO self.assertTrue(isinstance(digest, str) or isinstance(digest, unicode), 'digest does not look like one') h('')
def test_generic(self): for algo in hash._our_supported_hashlib_algos: h = hash.make_hasher(algo) self.assertTrue(h is not None, "should have been given a hasher") self.assertTrue(callable(h), "hasher should be callable") name, digest = h("test data") self.assertEqual(name, algo) # PY3-TODO self.assertTrue(isinstance(digest, str) or isinstance(digest, unicode), "digest does not look like one") h("")
def test_basic(self): with storagequeue.StorageQueue(lambda: self.make_backend(), CONCURRENCY) as sq: with self.fs.tempdir() as tdir: # populate self.fs.mkdir(self.path(tdir.path, 'testdir')) self.fs.open(self.path(tdir.path, 'testdir/testfile'), 'a').close() with self.fs.open(self.path(tdir.path, 'testdir/testfile2'), 'a') as f: f.write('this is the body of testfile2') self.fs.symlink( self.path(tdir.path, 'testdir/testfile2'), self.path(tdir.path, 'testdir/testfile2-symlink')) with self.fs.open(self.path(tdir.path, 'testdir/testfile3'), 'a') as f: f.write('testfile3 body') traverser = traversal.traverse(self.fs, tdir.path) manifest = [ elt for elt in persistence.persist( self.fs, traverser, None, tdir.path, sq, blocksize=20) ] #print meta.to_string() + ' ' + path + ' ' + unicode(hashes) self.assertEqual(len(manifest), 5) files = self.backend.list() self.assertEqual(len(files), 3) # two blocks for 2, one for 3 file_contents = [self.backend.get(fname) for fname in files] self.assertTrue('testfile3 body' in file_contents) self.assertTrue('this is the body of ' in file_contents) self.assertTrue('testfile2' in file_contents) for fname in files: self.assertEqual( fname, hash.make_hasher('sha512')(self.backend.get(fname))[1])
''' Takes stream(s) of (path, metadata) and persists to backends. ''' from __future__ import absolute_import from __future__ import with_statement import os.path import shastity.filesystem as filesystem import shastity.hash as hash import shastity.logging as logging import shastity.storagequeue as storagequeue DEFAULT_BLOCKSIZE = 1024*1024 DEFAULT_HASHER = hash.make_hasher('sha512') log = logging.get_logger(__name__) def _next_block(f, blocksize): parts = [] sofar = 0 while sofar < blocksize: part = f.read(blocksize - sofar) if len(part) == 0: break # eof parts += part sofar += len(part)
def __init__(self, next, cryptoKey): BackendWrapper.__init__(self, next) self.cryptoKey = hash.make_hasher('sha512')(cryptoKey)[1]
def fail(): hash.make_hasher('random non-existent algo')
def test_sha512(self): sha512 = hash.make_hasher('sha512') self.assertEqual(sha512('test'), ('sha512', 'ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff'))
def __init__(self, next, cryptoKey, hash_algorithm='sha512'): BackendWrapper.__init__(self, next) self.cryptoKey = hash.make_hasher(hash_algorithm)(cryptoKey)[1]
def fail(): hash.make_hasher("random non-existent algo")