def test_stress(self):
s = Spool("test", directory=self._spool_dir)
for i in xrange(1000):
s._submit_datum("test_submission")
#start = time()
s.process()
def test_submit_datum(self):
s = Spool("test", directory=self._spool_dir, shard=True)
s._submit_datum("test_submit_datum")
shards = s._shards(s._in)
self.assertEqual(len(shards), 1)
entries = os.listdir(shards[0])
self.assertEqual(len(entries), 1)
entry = os.path.join(shards[0], entries[0])
with open(entry) as fd:
content = fd.read()
self.assertEqual(content, "test_submit_datum")
def test_standard_submit(self):
"""This tests the submit function of the spooler.
The test make a file and then submits it to a spooler. The spooler
processes the file with the default (no-op) processing.
"""
# Make a file to submit
(fd, submit_filename) = tempfile.mkstemp(suffix="_spooler_test_file",
dir="/tmp")
try:
with os.fdopen(fd, "w+") as fh:
fh.write("hello!")
# Make a spool
s = Spool("test", directory=self._spool_dir)
# Submit the pre-existing file to it
s._submit_file(submit_filename)
dc = [os.path.join(s._in, d) for d in os.listdir(s._in)]
assert dc, "the incoming file didn't arrive"
filename = dc[0]
with open(filename) as fd:
content = fd.read()
assert content == "hello!"
# Process with the defaults
s.process()
# Now assert it's gone to the output
filename = filename.replace('/in/', '/out/')
# read the last file.
try:
with open(filename) as fd:
content = fd.read()
except IOError:
assert False, "test1: the processed file didn't arrive"
assert content == "hello!"
finally:
os.unlink(submit_filename)
try:
fd.close()
except Exception:
pass
def test_shard_cleanup(self):
from sigasync import spooler
_oldtime = time() - spooler.SHARD_SECONDS - 10
def oldtime():
return _oldtime
_time = spooler.time
spooler.time = oldtime
try:
s = Spool("test", directory=self._spool_dir, shard=True)
s._submit_datum("test1")
spooler.time = _time
s._submit_datum("test2")
shards = s._shards(s._in)
self.assertEqual(len(shards), 2)
self.assertEqual(len(os.listdir(shards[0])), 1)
self.assertEqual(len(os.listdir(shards[1])), 1)
s.process()
s.process()
shards = s._shards(s._in)
self.assertEqual(len(shards), 2)
self.assertEqual(len(os.listdir(shards[0])), 0)
self.assertEqual(len(os.listdir(shards[1])), 0)
oldshard = shards[0]
# Update mtime to far enough in the past
os.utime(oldshard, (_oldtime, _oldtime))
s.process()
self.assertFalse(oldshard in s._shards(s._in),
"Old empty shard didn't get cleaned up")
finally:
spooler.time = _time
def test_shard_processes_all(self):
"""Test that a sharded spool processes leftover unsharded entries"""
s1 = Spool("test", directory=self._spool_dir, shard=False)
s1._submit_datum("test1")
s2 = Spool("test", directory=self._spool_dir, shard=True)
s2._submit_datum("test1")
self.assertTrue(s2._sharded)
shards = s2._shards(s2._in)
self.assertEqual(len(shards), 1)
self.assertEqual(len(os.listdir(shards[0])), 1)
self.assertEqual(len(list(s2._incoming())), 1)
s2.process()
self.assertEqual(len(os.listdir(shards[0])), 0)
self.assertEqual(len(list(s2._incoming())), 0)
