def test_shard_cleanup(self): from sigasync import spooler _oldtime = time() - spooler.SHARD_SECONDS - 10 def oldtime(): return _oldtime _time = spooler.time spooler.time = oldtime try: s = Spool("test", directory=self._spool_dir, shard=True) s._submit_datum("test1") spooler.time = _time s._submit_datum("test2") shards = s._shards(s._in) self.assertEqual(len(shards), 2) self.assertEqual(len(os.listdir(shards[0])), 1) self.assertEqual(len(os.listdir(shards[1])), 1) s.process() s.process() shards = s._shards(s._in) self.assertEqual(len(shards), 2) self.assertEqual(len(os.listdir(shards[0])), 0) self.assertEqual(len(os.listdir(shards[1])), 0) oldshard = shards[0] # Update mtime to far enough in the past os.utime(oldshard, (_oldtime, _oldtime)) s.process() self.assertFalse(oldshard in s._shards(s._in), "Old empty shard didn't get cleaned up") finally: spooler.time = _time
def test_stress(self): s = Spool("test", directory=self._spool_dir) for i in xrange(1000): s._submit_datum("test_submission") #start = time() s.process()
def test_submit_datum(self): s = Spool("test", directory=self._spool_dir, shard=True) s._submit_datum("test_submit_datum") shards = s._shards(s._in) self.assertEqual(len(shards), 1) entries = os.listdir(shards[0]) self.assertEqual(len(entries), 1) entry = os.path.join(shards[0], entries[0]) with open(entry) as fd: content = fd.read() self.assertEqual(content, "test_submit_datum")
def test_shard_processes_all(self): """Test that a sharded spool processes leftover unsharded entries""" s1 = Spool("test", directory=self._spool_dir, shard=False) s1._submit_datum("test1") s2 = Spool("test", directory=self._spool_dir, shard=True) s2._submit_datum("test1") self.assertTrue(s2._sharded) shards = s2._shards(s2._in) self.assertEqual(len(shards), 1) self.assertEqual(len(os.listdir(shards[0])), 1) self.assertEqual(len(list(s2._incoming())), 1) s2.process() self.assertEqual(len(os.listdir(shards[0])), 0) self.assertEqual(len(list(s2._incoming())), 0)