Esempio n. 1
0
    def test_concurrency_under(self):
        items = self._new_items(100)
        item_queue = ItemQueue()
        task = MyItemTask()
        pipeline = Pipeline(MySource(items), [task], item_queue)
        pipeline.concurrency = 2

        yield from pipeline.process()

        self._check_item_values(items)
        self.assertEqual(2, task.peak_work)
Esempio n. 2
0
    def test_concurrency_step_up(self):
        items = self._new_items(100)
        task = MyItemTask()
        pipeline = Pipeline(MySource(items), [task], ItemQueue())

        def task_callback():
            if task.item_count == 20:
                _logger.debug('Set concurrency 10')
                pipeline.concurrency = 10

        task.callback = task_callback

        yield from pipeline.process()

        self._check_item_values(items)
        self.assertEqual(10, task.peak_work)
Esempio n. 3
0
    def test_pipeline_series(self):
        items = self._new_items(100)
        item_queue = ItemQueue()
        task = MyItemTask()
        pipeline_1 = Pipeline(MySource(items), [task], item_queue)
        pipeline_2 = Pipeline(MySource(items), [task], item_queue)

        series = PipelineSeries((pipeline_1, pipeline_2))
        series.concurrency_pipelines.add(pipeline_2)

        self.assertEqual(1, series.concurrency)

        series.concurrency = 2

        self.assertEqual(2, series.concurrency)
        self.assertEqual(1, pipeline_1.concurrency)
        self.assertEqual(2, pipeline_2.concurrency)
Esempio n. 4
0
    def test_concurrency_zero(self):
        items = self._new_items(100)
        task = MyItemTask()
        pipeline = Pipeline(MySource(items), [task], ItemQueue())
        pipeline.concurrency = 5

        def task_callback():
            if task.item_count == 10:
                _logger.debug('Set concurrency to 0')
                pipeline.concurrency = 0

                def callback():
                    _logger.debug('Set concurrency to 10')
                    pipeline.concurrency = 10

                asyncio.get_event_loop().call_later(0.5, callback)

        task.callback = task_callback

        yield from pipeline.process()

        self._check_item_values(items)
        self.assertEqual(10, task.peak_work)