def test_bulk_ops(self):
with storagequeue.StorageQueue(lambda: self.make_backend(), CONCURRENCY) as sq:
COUNT = 100
puts = [ storagequeue.PutOperation(prefix(str(n)), str(n)) for n in xrange(0, COUNT) ]
gets = [ storagequeue.GetOperation(prefix(str(n))) for n in xrange(0, COUNT) ]
dels = [ storagequeue.DeleteOperation(prefix(str(n))) for n in xrange(0, COUNT) ]
for p in puts:
sq.enqueue(p)
sq.barrier()
for g in gets:
sq.enqueue(g)
sq.barrier()
for d in dels:
sq.enqueue(d)
sq.wait()
for op in puts + gets + dels:
self.assertTrue(op.is_done())
self.assertTrue(op.succeeded())
self.assertEqual([g.value() for g in gets], [ str(n) for n in xrange(0, COUNT) ])
def _persist_file(fs,
path,
basepath,
meta,
sq,
blocksize,
hasher,
skip_blocks):
'''Persist a single file and return its entry to be yielded back
to the parent caller. Parameters match those of persist().'''
# TODO: fstat() after open to make sure we are not subject to
# races. This particular case is important because regardless of
# races in directory traversal, we definitely do not want to store
# actual data under incorrect permissions.
# TODO #2: Block devices and major/minor preservation. Bah.
assert len(basepath) > 0, 'basepath cannot be empty'
if not basepath.endswith('/'):
basepath += '/'
assert path.startswith(basepath), 'path %s not in basepath %s' % (path, basepath)
assert len(path) > len(basepath), ('basepath %s must be parent to path %s'
'' % (basepath, path))
stripped_path = path[len(basepath):]
blocks_upped = 0
blocks_skipped = 0
if meta.is_symlink:
return (stripped_path, meta, [])
elif meta.is_directory:
return (stripped_path, meta, [])
else:
hashes = []
with fs.open(path, "r") as f:
while True:
block = _next_block(f, blocksize)
if len(block) == 0:
break
algo, hash = hasher(block)
hashes.append((algo, hash))
if (algo,hash) not in skip_blocks:
#print "Putting block"
sq.enqueue(storagequeue.PutOperation(name=hash,
data=block))
skip_blocks.append( (algo,hash) )
blocks_upped += 1
else:
#print "Skipping block"
blocks_skipped += 1
pass
log.info("[%s] scheduled. Blocks up/skip: %d/%d",
stripped_path, blocks_upped, blocks_skipped)
return (stripped_path, meta, hashes)
def test_bad_get_fail(self):
with logging.FakeLogger(storagequeue, 'log'):
with storagequeue.StorageQueue(lambda: self.make_backend(), CONCURRENCY) as sq:
p1 = storagequeue.PutOperation(prefix('test1'), 'data')
g1 = storagequeue.GetOperation(prefix('test1'))
g2 = storagequeue.GetOperation(prefix('test2'))
sq.enqueue(p1)
sq.enqueue(g1)
sq.enqueue(g2)
self.assertRaises(storagequeue.OperationHasFailed, sq.wait)
def test_basic(self):
with storagequeue.StorageQueue(lambda: self.make_backend(), CONCURRENCY) as sq:
puts = [ storagequeue.PutOperation(prefix('test%s' % (n,)), str(n)) for n in xrange(0, 5) ]
gets = [ storagequeue.GetOperation(prefix('test%s' % (n,))) for n in xrange(0, 5) ]
dels = [ storagequeue.DeleteOperation(prefix('test%s' % (n,))) for n in xrange(0, 5) ]
# PUT
for p in puts:
self.assertFalse(p.is_done())
self.assertRaises(AssertionError, lambda: p.value())
self.assertRaises(AssertionError, lambda: p.succeeded())
sq.enqueue(p)
sq.wait()
for p in puts:
self.assertTrue(p.is_done())
self.assertTrue(p.succeeded())
# GET + DELETE
for g in gets:
self.assertFalse(g.is_done())
self.assertRaises(AssertionError, lambda: g.value())
self.assertRaises(AssertionError, lambda: g.succeeded())
sq.enqueue(g)
sq.barrier()
for d in dels:
self.assertFalse(d.is_done())
self.assertRaises(AssertionError, lambda: d.value())
self.assertRaises(AssertionError, lambda: d.succeeded())
sq.enqueue(d)
for d in dels:
d.wait() # test individual op waits
sq.wait()
for g in gets:
self.assertTrue(p.is_done())
self.assertTrue(p.succeeded())
self.assertEqual([g.value() for g in gets], [ '0', '1', '2', '3', '4' ])
for d in dels:
self.assertTrue(d.is_done())
self.assertTrue(d.succeeded())
self.assertEqual(d.value(), None)