def test_concurrency_under(self): items = self._new_items(100) item_queue = ItemQueue() task = MyItemTask() pipeline = Pipeline(MySource(items), [task], item_queue) pipeline.concurrency = 2 yield from pipeline.process() self._check_item_values(items) self.assertEqual(2, task.peak_work)
def test_concurrency_under(self): items = self._new_items(100) item_queue = ItemQueue() task = MyItemTask() pipeline = Pipeline(MySource(items), [task], item_queue) pipeline.concurrency = 2 yield from pipeline.process() self._check_item_values(items) self.assertEqual(2, task.peak_work)
def test_concurrency_zero(self): items = self._new_items(100) task = MyItemTask() pipeline = Pipeline(MySource(items), [task], ItemQueue()) pipeline.concurrency = 5 def task_callback(): if task.item_count == 10: _logger.debug('Set concurrency to 0') pipeline.concurrency = 0 def callback(): _logger.debug('Set concurrency to 10') pipeline.concurrency = 10 asyncio.get_event_loop().call_later(0.5, callback) task.callback = task_callback yield from pipeline.process() self._check_item_values(items) self.assertEqual(10, task.peak_work)
def test_concurrency_step_down(self): items = self._new_items(100) task = MyItemTask() pipeline = Pipeline(MySource(items), [task], ItemQueue()) pipeline.concurrency = 10 def task_callback(): if task.item_count == 19: self.assertEqual(10, task.peak_work) if task.item_count == 20: _logger.debug('Set concurrency 1') pipeline.concurrency = 1 if task.item_count == 30: task.reset_peak_work() task.callback = task_callback yield from pipeline.process() self._check_item_values(items) self.assertEqual(1, task.peak_work)
def test_concurrency_zero(self): items = self._new_items(100) task = MyItemTask() pipeline = Pipeline(MySource(items), [task], ItemQueue()) pipeline.concurrency = 5 def task_callback(): if task.item_count == 10: _logger.debug('Set concurrency to 0') pipeline.concurrency = 0 def callback(): _logger.debug('Set concurrency to 10') pipeline.concurrency = 10 asyncio.get_event_loop().call_later(0.5, callback) task.callback = task_callback yield from pipeline.process() self._check_item_values(items) self.assertEqual(10, task.peak_work)
def test_concurrency_step_down(self): items = self._new_items(100) task = MyItemTask() pipeline = Pipeline(MySource(items), [task], ItemQueue()) pipeline.concurrency = 10 def task_callback(): if task.item_count == 19: self.assertEqual(10, task.peak_work) if task.item_count == 20: _logger.debug('Set concurrency 1') pipeline.concurrency = 1 if task.item_count == 30: task.reset_peak_work() task.callback = task_callback yield from pipeline.process() self._check_item_values(items) self.assertEqual(1, task.peak_work)