def test_requeue_triggers_requeue(self): """ _requeue_handler() sets the requeue event on child blocks --> child (zombie -->) parent -| --> child """ # parent has a zombie parent to have somewhere to requeue its objects parent = DummyProcessBlock(parent=ZombieBlock()) children = [ ZombieBlock(parent=parent), ZombieBlock(parent=parent), ] parent.events["requeue"].set() requeue_thr = Thread(target=parent._requeue_handler) requeue_thr.start() for child in children: self.assertTrue(child.events["requeue"].wait(timeout=1)) child.events["requeue"].clear() # Join requeue_thr requeue_thr.join(timeout=1) self.assertFalse(requeue_thr.is_alive())
def test_multiple_blocks(self): """ One ObjFactory with 2 blocks --> WaitingBlock (--> zombie) ObjFactory --| --> WaitingBlock (--> zombie) """ obj_factory = ObjFactory(range(10)) blocks = [ WaitingBlock(parent=obj_factory), WaitingBlock(parent=obj_factory), ] # To be able to get the blocks' objects for block in blocks: ZombieBlock(parent=block) obj_factory.start() # Internal check, to ensure the next loop's condition is correct self.assertFalse(len(obj_factory) % len(blocks)) blocks_cycle = cycle(blocks) for obj in obj_factory: block = next(blocks_cycle) block.consume.set() self.assertEqual(block.objs.get(timeout=1), obj) # Consume the end event for block in blocks: block.consume.set() self.assertIsNone(block.objs.get(timeout=1)) obj_factory.join(timeout=1) self.assertFalse(obj_factory.is_alive())
def test_stop_handler(self): """ _stop_handler() sends one 'end object' per child block """ parent = DummyProcessBlock() children = [ ZombieBlock(parent=parent), ZombieBlock(parent=parent), ] # Stop the block parent.events["stop"].set() parent._stop_handler() for _ in children: self.assertIsNone(parent.objs.get(timeout=1))
def test_publish_blocking(self): """ try_publish_obj() will block until there is space in the queue """ block = DummyProcessBlock(queue_size=1) ZombieBlock(parent=block) # Fill the queue block.objs.put = Mock(side_effect=Full()) # To test a blocking call, run it in a thread publish_thr = Thread(target=block.try_publish_obj, args=(0, )) publish_thr.start() while not block.objs.put.called: pass # The call is indeed blocking self.assertTrue(publish_thr.is_alive()) # Unblock it block.event.set() # Check it stops publish_thr.join(timeout=1) self.assertFalse(publish_thr.is_alive())
def test_publish_none(self): """ try_publish_obj() does nothing if obj is None """ block = DummyProcessBlock() ZombieBlock(parent=block) self.assertTrue(block.try_publish_obj(None)) self.assertTrue(block.objs.empty())
def test_publish_interrupted(self): """ try_publish_obj() will fail if an event occurs """ block = DummyProcessBlock() ZombieBlock(parent=block) block.event.set() self.assertFalse(block.try_publish_obj(0)) self.assertTrue(block.objs.empty())
def test_publish_obj(self): """ try_publish_obj() puts a object in the block's queue """ block = DummyProcessBlock() # Objects are only stored if there are children ZombieBlock(parent=block) self.assertTrue(block.try_publish_obj(0)) self.assertEqual(block.objs.get(timeout=1), 0)
def test_requeue_stopped_block(self): """ _requeue_handler() clears both requeue and stop events """ child = DummyProcessBlock(parent=ZombieBlock()) # Requeue child.events["requeue"].set() child._requeue_handler() self.assertFalse(child.events["requeue"].is_set()) self.assertFalse(child.events["stop"].is_set())
def test_cancel_triggers_requeue(self): """ _cancel_handler() sets the requeue event on child blocks """ parent = DummyProcessBlock() children = [ ZombieBlock(parent=parent), ZombieBlock(parent=parent), ] parent.events["cancel"].set() cancel_thr = Thread(target=parent._cancel_handler) cancel_thr.start() for child in children: self.assertTrue(child.events["requeue"].wait(timeout=1)) child.events["requeue"].clear() # Join cancel_th cancel_thr.join(timeout=1) self.assertFalse(cancel_thr.is_alive())
def test_run(self): """ Run a process block parent --> child (--> zombie) """ parent = ZombieBlock() # Zombie <= allow child to stop child = DummyProcessBlock(parent=parent) ZombieBlock(parent=child) child.process_obj = Mock(side_effect=lambda obj: obj + 1) child.start() self.assertTrue(child.is_alive()) # Simulate parent producing objects for i in range(10): parent.objs.put(i, timeout=1) self.assertEqual(child.objs.get(timeout=1), child.process_obj(i)) parent.objs.put(None, timeout=1) child.join(timeout=1) self.assertFalse(child.is_alive())
def test_dynamic_requeue(self): """ Upon setting the requeue event objects are sent back to parents ObjFactory --> WaitingBlock --> DummyProcessBlock (--> zombie) """ obj_factory = ObjFactory(tuple()) waiting_block = WaitingBlock(parent=obj_factory) block = DummyProcessBlock(parent=waiting_block) ZombieBlock(parent=block).die.set() obj_factory.start() # Manually add objects to block's queue for obj in range(10): block.objs.put(obj) # Wait for waiting_block to block self.assertTrue(waiting_block.waiting.wait(timeout=1)) waiting_block.waiting.clear() # Requeue from waiting_block waiting_block.events["requeue"].set() waiting_block.event.set() # Let waiting_block process the requeue event waiting_block.consume.set() # block should receive a requeue event and requeue its objects block.objs.join() requeued_objs = [] for _ in range(10): requeued_objs.append(obj_factory.objs.get(timeout=1)) obj_factory.objs.task_done() self.assertCountEqual(range(10), requeued_objs) # Wait until the event is entirely processed self.assertTrue(waiting_block.waiting.wait(timeout=1)) waiting_block.waiting.clear() # Publish "end object" manually obj_factory.objs.put(None) waiting_block.consume.set() obj_factory.join(timeout=1) self.assertFalse(obj_factory.is_alive())
def test_basic_pipeline(self): """ ObjFactory --> DummyProcessBlock (--> zombie) """ obj_factory = ObjFactory(range(10)) block = DummyProcessBlock(parent=obj_factory) # To be able to get the block's objects ZombieBlock(parent=block) obj_factory.start() for obj in obj_factory: self.assertEqual(block.objs.get(timeout=1), obj) obj_factory.join(timeout=1) self.assertFalse(obj_factory.is_alive())
def test_cancel_objs(self): """ Upon setting the cancel event, objs are re-processed ObjFactory --> WaitingBlock --> DummyProcessBlock (--> zombie) """ # This factory only produces the "end object" obj_factory = ObjFactory(tuple()) waiting_block = WaitingBlock(parent=obj_factory) block = DummyProcessBlock(parent=waiting_block) ZombieBlock(parent=block).die.set() obj_factory.start() # Manually add objects to block's queue for obj in range(10): block.objs.put(obj) # Wait for waiting_block to block self.assertTrue(waiting_block.waiting.wait(timeout=1)) waiting_block.waiting.clear() # Cancel waiting_block waiting_block.cancel() # Let waiting_block process the cancel event waiting_block.consume.set() # block should receive a requeue event and requeue its objects block.objs.join() for _ in range(11): # 10 + "end object" self.assertTrue(waiting_block.waiting.wait(timeout=1)) waiting_block.waiting.clear() waiting_block.consume.set() # Objects get reprocessed self.assertCountEqual( range(10), iter(block.objs.get(timeout=1) for _ in range(10))) obj_factory.join(timeout=1) self.assertFalse(obj_factory.is_alive())