def test_basic_iteration(self): def task(i, sleep_for=1): print("Task", i, "sleeping for", sleep_for) time.sleep(sleep_for) return i def tasks(): for i in range(8): yield task, [i], {"sleep_for": i / 8.0} for i, res in enumerate( response_iterator(tasks(), get_futures_threadpool(3))): self.assertEqual(i, res) for i, res in enumerate( response_iterator(tasks(), get_futures_threadpool(5))): self.assertEqual(i, res) for i, res in enumerate( response_iterator(tasks(), get_futures_threadpool(5), max_active_tasks=2)): self.assertEqual(i, res) for i, res in enumerate( response_iterator(tasks(), get_futures_threadpool(5), max_active_tasks=6)): self.assertEqual(i, res)
def test_basic_iteration(self): def task(i, sleep_for=1): print("Task", i, "sleeping for", sleep_for) time.sleep(sleep_for) return i def tasks(): for i in range(8): yield task, [i], {"sleep_for": i/8.0} for i, res in enumerate(response_iterator(tasks(), get_futures_threadpool(3))): self.assertEqual(i, res) for i, res in enumerate(response_iterator(tasks(), get_futures_threadpool(5))): self.assertEqual(i, res) for i, res in enumerate(response_iterator(tasks(), get_futures_threadpool(5), max_active_tasks=2)): self.assertEqual(i, res) for i, res in enumerate(response_iterator(tasks(), get_futures_threadpool(5), max_active_tasks=6)): self.assertEqual(i, res) def tasks2(): for i in range(8): yield task, [i], {"sleep_for": (8-i)/8.0} for i, res in enumerate(response_iterator(tasks2(), get_futures_threadpool(5), num_retries=2, retry_after=0.1)): self.assertEqual(i, res)
def test_first_chunk_sequentially(self): pending_tasks = set() complete_tasks = set() def task(i): pending_tasks.add(i) if i >= 1: self.assertNotIn(0, pending_tasks, "Task 0 was pending during task %d" % i) time.sleep(1) complete_tasks.add(i) pending_tasks.remove(i) def tasks(): for i in range(8): yield task, [i], {} # # |---0---| # |---1---| # |---2---| # |---3---| # # Assert that 0 doesn't overlap with 1, 2, or 3 list( response_iterator(tasks(), get_futures_threadpool(8), do_first_task_sequentially=True))
def test_first_chunk_sequentially(self): pending_tasks = set() complete_tasks = set() def task(i): pending_tasks.add(i) if i >= 1: self.assertNotIn(0, pending_tasks, "Task 0 was pending during task %d" % i) time.sleep(1) complete_tasks.add(i) pending_tasks.remove(i) def tasks(): for i in range(8): yield task, [i], {} # # |---0---| # |---1---| # |---2---| # |---3---| # # Assert that 0 doesn't overlap with 1, 2, or 3 list(response_iterator(tasks(), get_futures_threadpool(8), do_first_task_sequentially=True))