def batch_accumulate(max_batch_size, a_generator, cooperator=None):
"""
Start a Deferred whose callBack arg is a deque of the accumulation
of the values yielded from a_generator which is iterated over
in batches the size of max_batch_size.
It should be more efficient to iterate over the generator in
batches and still provide enough speed for non-blocking execution.
:param max_batch_size: The number of iterations of the generator
to consume at a time.
:param a_generator: An iterator which yields some not None values.
:return: A Deferred to which the next callback will be called with
the yielded contents of the generator function.
"""
if cooperator:
own_cooperate = cooperator.cooperate
else:
own_cooperate = cooperate
spigot = ValueBucket()
items = stream_tap((spigot,), a_generator)
d = own_cooperate(i_batch(max_batch_size, items)).whenDone()
d.addCallback(accumulation_handler, spigot)
return d
def batch_accumulate(max_batch_size, a_generator, cooperator=None):
"""
Start a Deferred whose callBack arg is a deque of the accumulation
of the values yielded from a_generator which is iterated over
in batches the size of max_batch_size.
It should be more efficient to iterate over the generator in
batches and still provide enough speed for non-blocking execution.
:param max_batch_size: The number of iterations of the generator
to consume at a time.
:param a_generator: An iterator which yields some not None values.
:return: A Deferred to which the next callback will be called with
the yielded contents of the generator function.
"""
if cooperator:
own_cooperate = cooperator.cooperate
else:
own_cooperate = cooperate
spigot = ValueBucket()
items = stream_tap((spigot, ), a_generator)
d = own_cooperate(i_batch(max_batch_size, items)).whenDone()
d.addCallback(accumulation_handler, spigot)
return d
def certain_kind_tap(data_items):
"""
As the stream of data items go by, get different
kinds of information from them, in this case,
the things that are fruit and metal, collecting
each kind with a different spigot.
stream_tap doesn't consume the data_items iterator
by itself, it's a generator and must be consumed
by something else. In this case, it's consuming
the items by casting the iterator to a tuple,
but doing it in batches.
Since each batch is not referenced by anything
the memory can be freed by the garbage collector,
so no matter the size of the data_items, only a little
memory is needed. The only things retained
are the results, which should just be a subset
of the items and in this case, the getter functions
only return a portion of each item it matches.
:param data_items: A sequence of unicode strings
"""
fruit_spigot = Bucket(get_fruit)
metal_spigot = Bucket(get_metal)
items = stream_tap((fruit_spigot, metal_spigot), data_items)
for batch in i_batch(100, items):
tuple(batch)
return fruit_spigot.contents(), metal_spigot.contents()
def accumulate(a_generator, cooperator=None):
"""
Start a Deferred whose callBack arg is a deque of the accumulation
of the values yielded from a_generator.
:param a_generator: An iterator which yields some not None values.
:return: A Deferred to which the next callback will be called with
the yielded contents of the generator function.
"""
if cooperator:
own_cooperate = cooperator.cooperate
else:
own_cooperate = cooperate
spigot = ValueBucket()
items = stream_tap((spigot,), a_generator)
d = own_cooperate(items).whenDone()
d.addCallback(accumulation_handler, spigot)
return d
def accumulate(a_generator, cooperator=None):
"""
Start a Deferred whose callBack arg is a deque of the accumulation
of the values yielded from a_generator.
:param a_generator: An iterator which yields some not None values.
:return: A Deferred to which the next callback will be called with
the yielded contents of the generator function.
"""
if cooperator:
own_cooperate = cooperator.cooperate
else:
own_cooperate = cooperate
spigot = ValueBucket()
items = stream_tap((spigot, ), a_generator)
d = own_cooperate(items).whenDone()
d.addCallback(accumulation_handler, spigot)
return d
