def test_basic_iteration(self):
def task(i, sleep_for=1):
print("Task", i, "sleeping for", sleep_for)
time.sleep(sleep_for)
return i
def tasks():
for i in range(8):
yield task, [i], {"sleep_for": i / 8.0}
for i, res in enumerate(
response_iterator(tasks(), get_futures_threadpool(3))):
self.assertEqual(i, res)
for i, res in enumerate(
response_iterator(tasks(), get_futures_threadpool(5))):
self.assertEqual(i, res)
for i, res in enumerate(
response_iterator(tasks(),
get_futures_threadpool(5),
max_active_tasks=2)):
self.assertEqual(i, res)
for i, res in enumerate(
response_iterator(tasks(),
get_futures_threadpool(5),
max_active_tasks=6)):
self.assertEqual(i, res)
def test_basic_iteration(self):
def task(i, sleep_for=1):
print("Task", i, "sleeping for", sleep_for)
time.sleep(sleep_for)
return i
def tasks():
for i in range(8):
yield task, [i], {"sleep_for": i/8.0}
for i, res in enumerate(response_iterator(tasks(), get_futures_threadpool(3))):
self.assertEqual(i, res)
for i, res in enumerate(response_iterator(tasks(), get_futures_threadpool(5))):
self.assertEqual(i, res)
for i, res in enumerate(response_iterator(tasks(), get_futures_threadpool(5), max_active_tasks=2)):
self.assertEqual(i, res)
for i, res in enumerate(response_iterator(tasks(), get_futures_threadpool(5), max_active_tasks=6)):
self.assertEqual(i, res)
def tasks2():
for i in range(8):
yield task, [i], {"sleep_for": (8-i)/8.0}
for i, res in enumerate(response_iterator(tasks2(), get_futures_threadpool(5), num_retries=2, retry_after=0.1)):
self.assertEqual(i, res)
def test_first_chunk_sequentially(self):
pending_tasks = set()
complete_tasks = set()
def task(i):
pending_tasks.add(i)
if i >= 1:
self.assertNotIn(0, pending_tasks,
"Task 0 was pending during task %d" % i)
time.sleep(1)
complete_tasks.add(i)
pending_tasks.remove(i)
def tasks():
for i in range(8):
yield task, [i], {}
#
# |---0---|
# |---1---|
# |---2---|
# |---3---|
#
# Assert that 0 doesn't overlap with 1, 2, or 3
list(
response_iterator(tasks(),
get_futures_threadpool(8),
do_first_task_sequentially=True))
def test_first_chunk_sequentially(self):
pending_tasks = set()
complete_tasks = set()
def task(i):
pending_tasks.add(i)
if i >= 1:
self.assertNotIn(0, pending_tasks, "Task 0 was pending during task %d" % i)
time.sleep(1)
complete_tasks.add(i)
pending_tasks.remove(i)
def tasks():
for i in range(8):
yield task, [i], {}
#
# |---0---|
# |---1---|
# |---2---|
# |---3---|
#
# Assert that 0 doesn't overlap with 1, 2, or 3
list(response_iterator(tasks(), get_futures_threadpool(8), do_first_task_sequentially=True))