def test_basic(self):
with storagequeue.StorageQueue(lambda: self.make_backend(), CONCURRENCY) as sq:
with self.fs.tempdir() as tdir:
# populate
self.fs.mkdir(self.path(tdir.path, 'testdir'))
self.fs.open(self.path(tdir.path, 'testdir/testfile'), 'a').close()
with self.fs.open(self.path(tdir.path, 'testdir/testfile2'), 'a') as f:
f.write('this is the body of testfile2')
self.fs.symlink(self.path(tdir.path, 'testdir/testfile2'),
self.path(tdir.path, 'testdir/testfile2-symlink'))
with self.fs.open(self.path(tdir.path, 'testdir/testfile3'), 'a') as f:
f.write('testfile3 body')
traverser = traversal.traverse(self.fs, tdir.path)
manifest = [ elt for elt in persistence.persist(self.fs,
traverser,
None,
tdir.path,
sq,
blocksize=20) ]
#print meta.to_string() + ' ' + path + ' ' + unicode(hashes)
self.assertEqual(len(manifest), 5)
files = self.backend.list()
self.assertEqual(len(files), 3) # two blocks for 2, one for 3
file_contents = [ self.backend.get(fname) for fname in files ]
self.assertTrue('testfile3 body' in file_contents)
self.assertTrue('this is the body of ' in file_contents)
self.assertTrue('testfile2' in file_contents)
for fname in files:
self.assertEqual(fname, hash.make_hasher('sha512')(self.backend.get(fname))[1])
def test_sha512(self):
sha512 = hash.make_hasher("sha512")
self.assertEqual(
sha512("test"),
(
"sha512",
"ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff",
),
)
def test_basic(self):
with storagequeue.StorageQueue(lambda: self.make_backend(), CONCURRENCY) as sq:
with self.fs.tempdir() as tdir:
# Populate a tree.
self.fs.mkdir(self.path(tdir.path, 'testdir'))
self.fs.open(self.path(tdir.path, 'testdir/testfile'), 'a').close()
with self.fs.open(self.path(tdir.path, 'testdir/testfile2'), 'a') as f:
f.write('this is the body of testfile2')
self.fs.symlink(self.path(tdir.path, 'testdir/testfile2'),
self.path(tdir.path, 'testdir/testfile2-symlink'))
with self.fs.open(self.path(tdir.path, 'testdir/testfile3'), 'a') as f:
f.write('testfile3 body')
# Traverse it and persist to store.
traverser = traversal.traverse(self.fs, tdir.path)
manifest = [ elt for elt in persistence.persist(self.fs,
traverser,
None,
tdir.path,
sq,
blocksize=20) ]
#print meta.to_string() + ' ' + path + ' ' + unicode(hashes)
self.assertEqual(len(manifest), 5)
files = self.backend.list()
self.assertEqual(len(files), 3) # two blocks for 2, one for 3
file_contents = [ self.backend.get(fname) for fname in files ]
self.assertTrue('testfile3 body' in file_contents)
self.assertTrue('this is the body of ' in file_contents)
self.assertTrue('testfile2' in file_contents)
for fname in files:
self.assertEqual(fname, hash.make_hasher('sha512')(self.backend.get(fname))[1])
# Create another tempdir and materialize into it, and compare
# the results.
with self.fs.tempdir() as rdir:
materialization.materialize(self.fs, rdir.path, manifest, sq)
def rec(refdir, tstdir):
reflst = sorted(self.fs.listdir(refdir))
tstlst = sorted(self.fs.listdir(tstdir))
self.assertEqual(tstlst, reflst)
# todo: confirm file sizes
# todo: confirm file contents
# todo: confirm file meta
for entry in reflst:
entrypath = os.path.join(refdir, entry)
if not self.fs.is_symlink(entrypath) and self.fs.is_dir(entrypath):
rec(entrypath, os.path.join(tstdir, entry))
rec(tdir.path, rdir.path)
def test_generic(self):
for algo in hash._our_supported_hashlib_algos:
h = hash.make_hasher(algo)
self.assertTrue(h is not None, 'should have been given a hasher')
self.assertTrue(callable(h), 'hasher should be callable')
name, digest = h('test data')
self.assertEqual(name, algo)
# PY3-TODO
self.assertTrue(isinstance(digest, str) or isinstance(digest, unicode), 'digest does not look like one')
h('')
def test_generic(self):
for algo in hash._our_supported_hashlib_algos:
h = hash.make_hasher(algo)
self.assertTrue(h is not None, "should have been given a hasher")
self.assertTrue(callable(h), "hasher should be callable")
name, digest = h("test data")
self.assertEqual(name, algo)
# PY3-TODO
self.assertTrue(isinstance(digest, str) or isinstance(digest, unicode), "digest does not look like one")
h("")
def test_basic(self):
with storagequeue.StorageQueue(lambda: self.make_backend(),
CONCURRENCY) as sq:
with self.fs.tempdir() as tdir:
# populate
self.fs.mkdir(self.path(tdir.path, 'testdir'))
self.fs.open(self.path(tdir.path, 'testdir/testfile'),
'a').close()
with self.fs.open(self.path(tdir.path, 'testdir/testfile2'),
'a') as f:
f.write('this is the body of testfile2')
self.fs.symlink(
self.path(tdir.path, 'testdir/testfile2'),
self.path(tdir.path, 'testdir/testfile2-symlink'))
with self.fs.open(self.path(tdir.path, 'testdir/testfile3'),
'a') as f:
f.write('testfile3 body')
traverser = traversal.traverse(self.fs, tdir.path)
manifest = [
elt for elt in persistence.persist(
self.fs, traverser, None, tdir.path, sq, blocksize=20)
]
#print meta.to_string() + ' ' + path + ' ' + unicode(hashes)
self.assertEqual(len(manifest), 5)
files = self.backend.list()
self.assertEqual(len(files), 3) # two blocks for 2, one for 3
file_contents = [self.backend.get(fname) for fname in files]
self.assertTrue('testfile3 body' in file_contents)
self.assertTrue('this is the body of ' in file_contents)
self.assertTrue('testfile2' in file_contents)
for fname in files:
self.assertEqual(
fname,
hash.make_hasher('sha512')(self.backend.get(fname))[1])
'''
Takes stream(s) of (path, metadata) and persists to backends.
'''
from __future__ import absolute_import
from __future__ import with_statement
import os.path
import shastity.filesystem as filesystem
import shastity.hash as hash
import shastity.logging as logging
import shastity.storagequeue as storagequeue
DEFAULT_BLOCKSIZE = 1024*1024
DEFAULT_HASHER = hash.make_hasher('sha512')
log = logging.get_logger(__name__)
def _next_block(f, blocksize):
parts = []
sofar = 0
while sofar < blocksize:
part = f.read(blocksize - sofar)
if len(part) == 0:
break # eof
parts += part
sofar += len(part)
def __init__(self, next, cryptoKey):
BackendWrapper.__init__(self, next)
self.cryptoKey = hash.make_hasher('sha512')(cryptoKey)[1]
def __init__(self, next, cryptoKey):
BackendWrapper.__init__(self, next)
self.cryptoKey = hash.make_hasher('sha512')(cryptoKey)[1]
def fail():
hash.make_hasher('random non-existent algo')
def test_sha512(self):
sha512 = hash.make_hasher('sha512')
self.assertEqual(sha512('test'), ('sha512', 'ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff'))
def __init__(self, next, cryptoKey, hash_algorithm='sha512'):
BackendWrapper.__init__(self, next)
self.cryptoKey = hash.make_hasher(hash_algorithm)(cryptoKey)[1]
def fail():
hash.make_hasher("random non-existent algo")
