def test_simple_items(self):
items = self._new_items(4)
pipeline = Pipeline(MySource(items), [MyItemTask()])
yield from pipeline.process()
self._check_item_values(items)
def test_simple_items(self):
items = self._new_items(4)
pipeline = Pipeline(MySource(items), [MyItemTask()])
yield from pipeline.process()
self._check_item_values(items)
def test_concurrency_under(self):
items = self._new_items(100)
item_queue = ItemQueue()
task = MyItemTask()
pipeline = Pipeline(MySource(items), [task], item_queue)
pipeline.concurrency = 2
yield from pipeline.process()
self._check_item_values(items)
self.assertEqual(2, task.peak_work)
def test_concurrency_under(self):
items = self._new_items(100)
item_queue = ItemQueue()
task = MyItemTask()
pipeline = Pipeline(MySource(items), [task], item_queue)
pipeline.concurrency = 2
yield from pipeline.process()
self._check_item_values(items)
self.assertEqual(2, task.peak_work)
def test_stopping(self):
items = self._new_items(10)
task = MyItemTask()
pipeline = Pipeline(MySource(items), [task])
def task_callback():
if task.item_count == 5:
pipeline.stop()
task.callback = task_callback
yield from pipeline.process()
self.assertIsNone(items[-1].processed_value)
def test_stopping(self):
items = self._new_items(10)
task = MyItemTask()
pipeline = Pipeline(MySource(items), [task])
def task_callback():
if task.item_count == 5:
pipeline.stop()
task.callback = task_callback
yield from pipeline.process()
self.assertIsNone(items[-1].processed_value)
def test_concurrency_step_up(self):
items = self._new_items(100)
task = MyItemTask()
pipeline = Pipeline(MySource(items), [task], ItemQueue())
def task_callback():
if task.item_count == 20:
_logger.debug('Set concurrency 10')
pipeline.concurrency = 10
task.callback = task_callback
yield from pipeline.process()
self._check_item_values(items)
self.assertEqual(10, task.peak_work)
def test_concurrency_step_up(self):
items = self._new_items(100)
task = MyItemTask()
pipeline = Pipeline(MySource(items), [task], ItemQueue())
def task_callback():
if task.item_count == 20:
_logger.debug('Set concurrency 10')
pipeline.concurrency = 10
task.callback = task_callback
yield from pipeline.process()
self._check_item_values(items)
self.assertEqual(10, task.peak_work)
def test_concurrency_zero(self):
items = self._new_items(100)
task = MyItemTask()
pipeline = Pipeline(MySource(items), [task], ItemQueue())
pipeline.concurrency = 5
def task_callback():
if task.item_count == 10:
_logger.debug('Set concurrency to 0')
pipeline.concurrency = 0
def callback():
_logger.debug('Set concurrency to 10')
pipeline.concurrency = 10
asyncio.get_event_loop().call_later(0.5, callback)
task.callback = task_callback
yield from pipeline.process()
self._check_item_values(items)
self.assertEqual(10, task.peak_work)
def test_concurrency_zero(self):
items = self._new_items(100)
task = MyItemTask()
pipeline = Pipeline(MySource(items), [task], ItemQueue())
pipeline.concurrency = 5
def task_callback():
if task.item_count == 10:
_logger.debug('Set concurrency to 0')
pipeline.concurrency = 0
def callback():
_logger.debug('Set concurrency to 10')
pipeline.concurrency = 10
asyncio.get_event_loop().call_later(0.5, callback)
task.callback = task_callback
yield from pipeline.process()
self._check_item_values(items)
self.assertEqual(10, task.peak_work)
def test_item_task_error(self):
items = self._new_items(4)
pipeline = Pipeline(MySource(items), [MyItemTask(test_error=True)])
with self.assertRaises(MyItemTaskError):
yield from pipeline.process()
def test_item_task_error(self):
items = self._new_items(4)
pipeline = Pipeline(MySource(items), [MyItemTask(test_error=True)])
with self.assertRaises(MyItemTaskError):
yield from pipeline.process()
