Пример #1
0
    def test_basic_iteration(self):
        def task(i, sleep_for=1):
            print("Task", i, "sleeping for", sleep_for)
            time.sleep(sleep_for)
            return i

        def tasks():
            for i in range(8):
                yield task, [i], {"sleep_for": i / 8.0}

        for i, res in enumerate(
                response_iterator(tasks(), get_futures_threadpool(3))):
            self.assertEqual(i, res)
        for i, res in enumerate(
                response_iterator(tasks(), get_futures_threadpool(5))):
            self.assertEqual(i, res)
        for i, res in enumerate(
                response_iterator(tasks(),
                                  get_futures_threadpool(5),
                                  max_active_tasks=2)):
            self.assertEqual(i, res)
        for i, res in enumerate(
                response_iterator(tasks(),
                                  get_futures_threadpool(5),
                                  max_active_tasks=6)):
            self.assertEqual(i, res)
Пример #2
0
    def test_basic_iteration(self):
        def task(i, sleep_for=1):
            print("Task", i, "sleeping for", sleep_for)
            time.sleep(sleep_for)
            return i

        def tasks():
            for i in range(8):
                yield task, [i], {"sleep_for": i/8.0}

        for i, res in enumerate(response_iterator(tasks(), get_futures_threadpool(3))):
            self.assertEqual(i, res)
        for i, res in enumerate(response_iterator(tasks(), get_futures_threadpool(5))):
            self.assertEqual(i, res)
        for i, res in enumerate(response_iterator(tasks(), get_futures_threadpool(5), max_active_tasks=2)):
            self.assertEqual(i, res)
        for i, res in enumerate(response_iterator(tasks(), get_futures_threadpool(5), max_active_tasks=6)):
            self.assertEqual(i, res)

        def tasks2():
            for i in range(8):
                yield task, [i], {"sleep_for": (8-i)/8.0}

        for i, res in enumerate(response_iterator(tasks2(), get_futures_threadpool(5), num_retries=2, retry_after=0.1)):
            self.assertEqual(i, res)
Пример #3
0
    def test_first_chunk_sequentially(self):
        pending_tasks = set()
        complete_tasks = set()

        def task(i):
            pending_tasks.add(i)
            if i >= 1:
                self.assertNotIn(0, pending_tasks,
                                 "Task 0 was pending during task %d" % i)
            time.sleep(1)
            complete_tasks.add(i)
            pending_tasks.remove(i)

        def tasks():
            for i in range(8):
                yield task, [i], {}

        #
        # |---0---|
        #           |---1---|
        #           |---2---|
        #           |---3---|
        #
        # Assert that 0 doesn't overlap with 1, 2, or 3
        list(
            response_iterator(tasks(),
                              get_futures_threadpool(8),
                              do_first_task_sequentially=True))
Пример #4
0
    def test_first_chunk_sequentially(self):
        pending_tasks = set()
        complete_tasks = set()

        def task(i):
            pending_tasks.add(i)
            if i >= 1:
                self.assertNotIn(0, pending_tasks, "Task 0 was pending during task %d" % i)
            time.sleep(1)
            complete_tasks.add(i)
            pending_tasks.remove(i)

        def tasks():
            for i in range(8):
                yield task, [i], {}

        #
        # |---0---|
        #           |---1---|
        #           |---2---|
        #           |---3---|
        #
        # Assert that 0 doesn't overlap with 1, 2, or 3
        list(response_iterator(tasks(), get_futures_threadpool(8), do_first_task_sequentially=True))