def test_requeue_triggers_requeue(self):
"""
_requeue_handler() sets the requeue event on child blocks
--> child
(zombie -->) parent -|
--> child
"""
# parent has a zombie parent to have somewhere to requeue its objects
parent = DummyProcessBlock(parent=ZombieBlock())
children = [
ZombieBlock(parent=parent),
ZombieBlock(parent=parent),
]
parent.events["requeue"].set()
requeue_thr = Thread(target=parent._requeue_handler)
requeue_thr.start()
for child in children:
self.assertTrue(child.events["requeue"].wait(timeout=1))
child.events["requeue"].clear()
# Join requeue_thr
requeue_thr.join(timeout=1)
self.assertFalse(requeue_thr.is_alive())
def test_multiple_blocks(self):
"""
One ObjFactory with 2 blocks
--> WaitingBlock (--> zombie)
ObjFactory --|
--> WaitingBlock (--> zombie)
"""
obj_factory = ObjFactory(range(10))
blocks = [
WaitingBlock(parent=obj_factory),
WaitingBlock(parent=obj_factory),
]
# To be able to get the blocks' objects
for block in blocks:
ZombieBlock(parent=block)
obj_factory.start()
# Internal check, to ensure the next loop's condition is correct
self.assertFalse(len(obj_factory) % len(blocks))
blocks_cycle = cycle(blocks)
for obj in obj_factory:
block = next(blocks_cycle)
block.consume.set()
self.assertEqual(block.objs.get(timeout=1), obj)
# Consume the end event
for block in blocks:
block.consume.set()
self.assertIsNone(block.objs.get(timeout=1))
obj_factory.join(timeout=1)
self.assertFalse(obj_factory.is_alive())
def test_stop_handler(self):
"""
_stop_handler() sends one 'end object' per child block
"""
parent = DummyProcessBlock()
children = [
ZombieBlock(parent=parent),
ZombieBlock(parent=parent),
]
# Stop the block
parent.events["stop"].set()
parent._stop_handler()
for _ in children:
self.assertIsNone(parent.objs.get(timeout=1))
def test_publish_blocking(self):
"""
try_publish_obj() will block until there is space in the queue
"""
block = DummyProcessBlock(queue_size=1)
ZombieBlock(parent=block)
# Fill the queue
block.objs.put = Mock(side_effect=Full())
# To test a blocking call, run it in a thread
publish_thr = Thread(target=block.try_publish_obj, args=(0, ))
publish_thr.start()
while not block.objs.put.called:
pass
# The call is indeed blocking
self.assertTrue(publish_thr.is_alive())
# Unblock it
block.event.set()
# Check it stops
publish_thr.join(timeout=1)
self.assertFalse(publish_thr.is_alive())
def test_publish_none(self):
"""
try_publish_obj() does nothing if obj is None
"""
block = DummyProcessBlock()
ZombieBlock(parent=block)
self.assertTrue(block.try_publish_obj(None))
self.assertTrue(block.objs.empty())
def test_publish_interrupted(self):
"""
try_publish_obj() will fail if an event occurs
"""
block = DummyProcessBlock()
ZombieBlock(parent=block)
block.event.set()
self.assertFalse(block.try_publish_obj(0))
self.assertTrue(block.objs.empty())
def test_publish_obj(self):
"""
try_publish_obj() puts a object in the block's queue
"""
block = DummyProcessBlock()
# Objects are only stored if there are children
ZombieBlock(parent=block)
self.assertTrue(block.try_publish_obj(0))
self.assertEqual(block.objs.get(timeout=1), 0)
def test_requeue_stopped_block(self):
"""
_requeue_handler() clears both requeue and stop events
"""
child = DummyProcessBlock(parent=ZombieBlock())
# Requeue
child.events["requeue"].set()
child._requeue_handler()
self.assertFalse(child.events["requeue"].is_set())
self.assertFalse(child.events["stop"].is_set())
def test_cancel_triggers_requeue(self):
"""
_cancel_handler() sets the requeue event on child blocks
"""
parent = DummyProcessBlock()
children = [
ZombieBlock(parent=parent),
ZombieBlock(parent=parent),
]
parent.events["cancel"].set()
cancel_thr = Thread(target=parent._cancel_handler)
cancel_thr.start()
for child in children:
self.assertTrue(child.events["requeue"].wait(timeout=1))
child.events["requeue"].clear()
# Join cancel_th
cancel_thr.join(timeout=1)
self.assertFalse(cancel_thr.is_alive())
def test_run(self):
"""
Run a process block
parent --> child (--> zombie)
"""
parent = ZombieBlock() # Zombie <= allow child to stop
child = DummyProcessBlock(parent=parent)
ZombieBlock(parent=child)
child.process_obj = Mock(side_effect=lambda obj: obj + 1)
child.start()
self.assertTrue(child.is_alive())
# Simulate parent producing objects
for i in range(10):
parent.objs.put(i, timeout=1)
self.assertEqual(child.objs.get(timeout=1), child.process_obj(i))
parent.objs.put(None, timeout=1)
child.join(timeout=1)
self.assertFalse(child.is_alive())
def test_dynamic_requeue(self):
"""
Upon setting the requeue event objects are sent back to parents
ObjFactory --> WaitingBlock --> DummyProcessBlock (--> zombie)
"""
obj_factory = ObjFactory(tuple())
waiting_block = WaitingBlock(parent=obj_factory)
block = DummyProcessBlock(parent=waiting_block)
ZombieBlock(parent=block).die.set()
obj_factory.start()
# Manually add objects to block's queue
for obj in range(10):
block.objs.put(obj)
# Wait for waiting_block to block
self.assertTrue(waiting_block.waiting.wait(timeout=1))
waiting_block.waiting.clear()
# Requeue from waiting_block
waiting_block.events["requeue"].set()
waiting_block.event.set()
# Let waiting_block process the requeue event
waiting_block.consume.set()
# block should receive a requeue event and requeue its objects
block.objs.join()
requeued_objs = []
for _ in range(10):
requeued_objs.append(obj_factory.objs.get(timeout=1))
obj_factory.objs.task_done()
self.assertCountEqual(range(10), requeued_objs)
# Wait until the event is entirely processed
self.assertTrue(waiting_block.waiting.wait(timeout=1))
waiting_block.waiting.clear()
# Publish "end object" manually
obj_factory.objs.put(None)
waiting_block.consume.set()
obj_factory.join(timeout=1)
self.assertFalse(obj_factory.is_alive())
def test_basic_pipeline(self):
"""
ObjFactory --> DummyProcessBlock (--> zombie)
"""
obj_factory = ObjFactory(range(10))
block = DummyProcessBlock(parent=obj_factory)
# To be able to get the block's objects
ZombieBlock(parent=block)
obj_factory.start()
for obj in obj_factory:
self.assertEqual(block.objs.get(timeout=1), obj)
obj_factory.join(timeout=1)
self.assertFalse(obj_factory.is_alive())
def test_cancel_objs(self):
"""
Upon setting the cancel event, objs are re-processed
ObjFactory --> WaitingBlock --> DummyProcessBlock (--> zombie)
"""
# This factory only produces the "end object"
obj_factory = ObjFactory(tuple())
waiting_block = WaitingBlock(parent=obj_factory)
block = DummyProcessBlock(parent=waiting_block)
ZombieBlock(parent=block).die.set()
obj_factory.start()
# Manually add objects to block's queue
for obj in range(10):
block.objs.put(obj)
# Wait for waiting_block to block
self.assertTrue(waiting_block.waiting.wait(timeout=1))
waiting_block.waiting.clear()
# Cancel waiting_block
waiting_block.cancel()
# Let waiting_block process the cancel event
waiting_block.consume.set()
# block should receive a requeue event and requeue its objects
block.objs.join()
for _ in range(11): # 10 + "end object"
self.assertTrue(waiting_block.waiting.wait(timeout=1))
waiting_block.waiting.clear()
waiting_block.consume.set()
# Objects get reprocessed
self.assertCountEqual(
range(10), iter(block.objs.get(timeout=1) for _ in range(10)))
obj_factory.join(timeout=1)
self.assertFalse(obj_factory.is_alive())