def test_endpoint_error(self): trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE, timedelta(seconds=0), max_endpoint_errors=2) step = 10 oneTrSize = (MAX_QUEUE_SIZE / step) - 1 for i in xrange(step): trManager.append(memTransaction(oneTrSize, trManager)) trManager.flush() # There should be exactly step transaction in the list, # and only 2 of them with a flush count of 1 self.assertEqual(len(trManager._transactions), step) flush_count = 0 for tr in trManager._transactions: flush_count += tr._flush_count self.assertEqual(flush_count, 2) # If we retry to flush, two OTHER transactions should be tried trManager.flush() self.assertEqual(len(trManager._transactions), step) flush_count = 0 for tr in trManager._transactions: flush_count += tr._flush_count self.assertIn(tr._flush_count, [0, 1]) self.assertEqual(flush_count, 4) # Finally when it's possible to flush, everything should go smoothly for tr in trManager._transactions: tr.is_flushable = True trManager.flush() self.assertEqual(len(trManager._transactions), 0)
def test_no_parallelism(self): step = 2 trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE, timedelta(seconds=0), max_parallelism=1, max_endpoint_errors=100) for i in xrange(step): trManager.append(SleepingTransaction(trManager, delay=1)) trManager.flush() # Flushes should be sequential for i in xrange(step): self.assertEqual(trManager._running_flushes, 1) self.assertEqual(trManager._finished_flushes, i) self.assertEqual(len(trManager._trs_to_flush), step - (i + 1)) time.sleep(1)
def test_drop_repeated_error(self): trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE, timedelta(seconds=0), max_endpoint_errors=1) # Fail it once oneTrSize = 10 trManager.append(memTransaction(oneTrSize, trManager)) # It should still be there after flush trManager.flush() self.assertEqual(len(trManager._transactions), 1) #Try again, now it should be gone trManager.flush() self.assertEqual(len(trManager._transactions), 0)
def testThrottling(self): """Test throttling while flushing""" # No throttling, no delay for replay trManager = TransactionManager(timedelta(seconds = 0), MAX_QUEUE_SIZE, THROTTLING_DELAY) trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop # Add 3 transactions, make sure no memory limit is in the way oneTrSize = MAX_QUEUE_SIZE / 10 for i in xrange(3): tr = memTransaction(oneTrSize, trManager) trManager.append(tr) # Try to flush them, time it before = datetime.now() trManager.flush() after = datetime.now() self.assertTrue( (after-before) > 3 * THROTTLING_DELAY)
def testMemoryLimit(self): """Test memory limit as well as simple flush""" # No throttling, no delay for replay trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE, timedelta(seconds=0)) step = 10 oneTrSize = (MAX_QUEUE_SIZE / step) - 1 for i in xrange(step): tr = memTransaction(oneTrSize, trManager) trManager.append(tr) trManager.flush() # There should be exactly step transaction in the list, with # a flush count of 1 self.assertEqual(len(trManager._transactions), step) for tr in trManager._transactions: self.assertEqual(tr._flush_count, 1) # Try to add one more tr = memTransaction(oneTrSize + 10, trManager) trManager.append(tr) # At this point, transaction one (the oldest) should have been removed from the list self.assertEqual(len(trManager._transactions), step) for tr in trManager._transactions: self.assertNotEqual(tr._id, 1) trManager.flush() self.assertEqual(len(trManager._transactions), step) # Check and allow transactions to be flushed for tr in trManager._transactions: tr.is_flushable = True # Last transaction has been flushed only once if tr._id == step + 1: self.assertEqual(tr._flush_count, 1) else: self.assertEqual(tr._flush_count, 2) trManager.flush() self.assertEqual(len(trManager._transactions), 0)
def test_parallelism(self): step = 4 trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE, timedelta(seconds=0), max_parallelism=step, max_endpoint_errors=100) for i in xrange(step): trManager.append(SleepingTransaction(trManager)) trManager.flush() self.assertEqual(trManager._running_flushes, step) self.assertEqual(trManager._finished_flushes, 0) # If _trs_to_flush != None, it means that it's still running as it should be self.assertEqual(trManager._trs_to_flush, []) time.sleep(1) # It should be finished self.assertEqual(trManager._running_flushes, 0) self.assertEqual(trManager._finished_flushes, step) self.assertIs(trManager._trs_to_flush, None)