def test_concurrency_under(self):
items = self._new_items(100)
item_queue = ItemQueue()
task = MyItemTask()
pipeline = Pipeline(MySource(items), [task], item_queue)
pipeline.concurrency = 2
yield from pipeline.process()
self._check_item_values(items)
self.assertEqual(2, task.peak_work)
def test_concurrency_under(self):
items = self._new_items(100)
item_queue = ItemQueue()
task = MyItemTask()
pipeline = Pipeline(MySource(items), [task], item_queue)
pipeline.concurrency = 2
yield from pipeline.process()
self._check_item_values(items)
self.assertEqual(2, task.peak_work)
def test_concurrency_zero(self):
items = self._new_items(100)
task = MyItemTask()
pipeline = Pipeline(MySource(items), [task], ItemQueue())
pipeline.concurrency = 5
def task_callback():
if task.item_count == 10:
_logger.debug('Set concurrency to 0')
pipeline.concurrency = 0
def callback():
_logger.debug('Set concurrency to 10')
pipeline.concurrency = 10
asyncio.get_event_loop().call_later(0.5, callback)
task.callback = task_callback
yield from pipeline.process()
self._check_item_values(items)
self.assertEqual(10, task.peak_work)
def test_concurrency_step_down(self):
items = self._new_items(100)
task = MyItemTask()
pipeline = Pipeline(MySource(items), [task], ItemQueue())
pipeline.concurrency = 10
def task_callback():
if task.item_count == 19:
self.assertEqual(10, task.peak_work)
if task.item_count == 20:
_logger.debug('Set concurrency 1')
pipeline.concurrency = 1
if task.item_count == 30:
task.reset_peak_work()
task.callback = task_callback
yield from pipeline.process()
self._check_item_values(items)
self.assertEqual(1, task.peak_work)
def test_concurrency_zero(self):
items = self._new_items(100)
task = MyItemTask()
pipeline = Pipeline(MySource(items), [task], ItemQueue())
pipeline.concurrency = 5
def task_callback():
if task.item_count == 10:
_logger.debug('Set concurrency to 0')
pipeline.concurrency = 0
def callback():
_logger.debug('Set concurrency to 10')
pipeline.concurrency = 10
asyncio.get_event_loop().call_later(0.5, callback)
task.callback = task_callback
yield from pipeline.process()
self._check_item_values(items)
self.assertEqual(10, task.peak_work)
def test_concurrency_step_down(self):
items = self._new_items(100)
task = MyItemTask()
pipeline = Pipeline(MySource(items), [task], ItemQueue())
pipeline.concurrency = 10
def task_callback():
if task.item_count == 19:
self.assertEqual(10, task.peak_work)
if task.item_count == 20:
_logger.debug('Set concurrency 1')
pipeline.concurrency = 1
if task.item_count == 30:
task.reset_peak_work()
task.callback = task_callback
yield from pipeline.process()
self._check_item_values(items)
self.assertEqual(1, task.peak_work)
