def do_some_expensive_things(number):
"""
Perform one expensive computation cooperatively with any
other iterator passed into twisted's cooperate, then
use it's result to pass into the second computation.
:param number:
:return:
"""
result = yield batch_accumulate(1000, expensive(number))
total = reduce(add, result, 0)
log.msg("first for {}: {}".format(number, total))
result = yield batch_accumulate(1000, expensive(int(total/1e9)))
total = reduce(add, result, 0)
log.msg("second for {}: {}".format(number, total))
defer.returnValue(total)
def do_some_expensive_things(number):
"""
Perform one expensive computation cooperatively with any
other iterator passed into twisted's cooperate, then
use it's result to pass into the second computation.
:param number:
:return:
"""
result = yield batch_accumulate(1000, expensive(number))
total = reduce(add, result, 0)
log.msg("first for {}: {}".format(number, total))
result = yield batch_accumulate(1000, expensive(int(total / 1e9)))
total = reduce(add, result, 0)
log.msg("second for {}: {}".format(number, total))
defer.returnValue(total)
def run(self):
"""
Cooperatively iterator over two iterators consecutively and the
result of the final one is returned.
:return:
"""
# first deferred
self.count += 1
result = yield accumulate(i_get_tenth_11(range(110, 150)), self.cooperator)
log.msg("accumulated {} at {}".format(result, self.count))
result = yield batch_accumulate(2, range(self.count), self.cooperator)
log.msg("batch accumulated {} at {}".format(result, self.count))
defer.returnValue(result)
def test_multi_deux_batched(self):
"""
Ensure multiple inline callback functions will run cooperatively.
Ensure the result of gatherResults can be chained together
in order.
Ensure cooperatively run generators will complete
no matter the length.
Ensure the longest one will continue to iterate after the
others run out of iterations.
Ensure those called with batch_accumulate will
iterate over the generator in batches the size of max_size.
:return:
"""
called = []
def watcher(value):
"""
A pass through generator for i_get_tenth_11 that
captures the value in a list, which can show the
order of the generator iteration.
:param value:
:return:
"""
for item in i_get_tenth_11(value):
called.append(item)
yield item
def deux_watcher(value):
"""
A pass through generator for i_get_tenth_11 that
captures the value in a list, which can show the
order of the generator iteration.
:param value:
:return:
"""
for item in i_get_tenth_11(value):
called.append(item)
yield item
for item in i_get_tenth_11(value):
called.append(item)
yield item
result = yield defer.gatherResults([
accumulate(watcher(range(0, 15))),
accumulate(watcher(range(15, 100))),
batch_accumulate(3, deux_watcher(range(1098, 10200))),
accumulate(watcher(range(145, 189)))
])
final_result = list(chain.from_iterable(result))
self.assertEqual(final_result,
[10, 11, 25, 26, 1108, 1109, 1108, 1109, 155, 156])
self.assertEqual(set(called), set(final_result))
self.assertNotEqual(called, final_result)
# The iteration is shown to alternate between generators passed
# to cooperate, except the batched one does three at a time
# when it is it's turn.
self.assertEqual(called,
[10, 25, 1108, 1109, 1108, 155, 11, 26, 1109, 156])
