def test_shard_cleanup(self):
from sigasync import spooler
_oldtime = time() - spooler.SHARD_SECONDS - 10
def oldtime():
return _oldtime
_time = spooler.time
spooler.time = oldtime
try:
s = Spool("test", directory=self._spool_dir, shard=True)
s._submit_datum("test1")
spooler.time = _time
s._submit_datum("test2")
shards = s._shards(s._in)
self.assertEqual(len(shards), 2)
self.assertEqual(len(os.listdir(shards[0])), 1)
self.assertEqual(len(os.listdir(shards[1])), 1)
s.process()
s.process()
shards = s._shards(s._in)
self.assertEqual(len(shards), 2)
self.assertEqual(len(os.listdir(shards[0])), 0)
self.assertEqual(len(os.listdir(shards[1])), 0)
oldshard = shards[0]
# Update mtime to far enough in the past
os.utime(oldshard, (_oldtime, _oldtime))
s.process()
self.assertFalse(oldshard in s._shards(s._in),
"Old empty shard didn't get cleaned up")
finally:
spooler.time = _time
def test_stress(self):
s = Spool("test", directory=self._spool_dir)
for i in xrange(1000):
s._submit_datum("test_submission")
#start = time()
s.process()
def test_submit_datum(self):
s = Spool("test", directory=self._spool_dir, shard=True)
s._submit_datum("test_submit_datum")
shards = s._shards(s._in)
self.assertEqual(len(shards), 1)
entries = os.listdir(shards[0])
self.assertEqual(len(entries), 1)
entry = os.path.join(shards[0], entries[0])
with open(entry) as fd:
content = fd.read()
self.assertEqual(content, "test_submit_datum")
def test_shard_processes_all(self):
"""Test that a sharded spool processes leftover unsharded entries"""
s1 = Spool("test", directory=self._spool_dir, shard=False)
s1._submit_datum("test1")
s2 = Spool("test", directory=self._spool_dir, shard=True)
s2._submit_datum("test1")
self.assertTrue(s2._sharded)
shards = s2._shards(s2._in)
self.assertEqual(len(shards), 1)
self.assertEqual(len(os.listdir(shards[0])), 1)
self.assertEqual(len(list(s2._incoming())), 1)
s2.process()
self.assertEqual(len(os.listdir(shards[0])), 0)
self.assertEqual(len(list(s2._incoming())), 0)
