def test_concurrency_under(self):
items = self._new_items(100)
item_queue = ItemQueue()
task = MyItemTask()
pipeline = Pipeline(MySource(items), [task], item_queue)
pipeline.concurrency = 2
yield from pipeline.process()
self._check_item_values(items)
self.assertEqual(2, task.peak_work)
def test_concurrency_step_up(self):
items = self._new_items(100)
task = MyItemTask()
pipeline = Pipeline(MySource(items), [task], ItemQueue())
def task_callback():
if task.item_count == 20:
_logger.debug('Set concurrency 10')
pipeline.concurrency = 10
task.callback = task_callback
yield from pipeline.process()
self._check_item_values(items)
self.assertEqual(10, task.peak_work)
def test_pipeline_series(self):
items = self._new_items(100)
item_queue = ItemQueue()
task = MyItemTask()
pipeline_1 = Pipeline(MySource(items), [task], item_queue)
pipeline_2 = Pipeline(MySource(items), [task], item_queue)
series = PipelineSeries((pipeline_1, pipeline_2))
series.concurrency_pipelines.add(pipeline_2)
self.assertEqual(1, series.concurrency)
series.concurrency = 2
self.assertEqual(2, series.concurrency)
self.assertEqual(1, pipeline_1.concurrency)
self.assertEqual(2, pipeline_2.concurrency)
def test_concurrency_zero(self):
items = self._new_items(100)
task = MyItemTask()
pipeline = Pipeline(MySource(items), [task], ItemQueue())
pipeline.concurrency = 5
def task_callback():
if task.item_count == 10:
_logger.debug('Set concurrency to 0')
pipeline.concurrency = 0
def callback():
_logger.debug('Set concurrency to 10')
pipeline.concurrency = 10
asyncio.get_event_loop().call_later(0.5, callback)
task.callback = task_callback
yield from pipeline.process()
self._check_item_values(items)
self.assertEqual(10, task.peak_work)