def batch_accumulate(max_batch_size, a_generator, cooperator=None): """ Start a Deferred whose callBack arg is a deque of the accumulation of the values yielded from a_generator which is iterated over in batches the size of max_batch_size. It should be more efficient to iterate over the generator in batches and still provide enough speed for non-blocking execution. :param max_batch_size: The number of iterations of the generator to consume at a time. :param a_generator: An iterator which yields some not None values. :return: A Deferred to which the next callback will be called with the yielded contents of the generator function. """ if cooperator: own_cooperate = cooperator.cooperate else: own_cooperate = cooperate spigot = ValueBucket() items = stream_tap((spigot,), a_generator) d = own_cooperate(i_batch(max_batch_size, items)).whenDone() d.addCallback(accumulation_handler, spigot) return d
def batch_accumulate(max_batch_size, a_generator, cooperator=None): """ Start a Deferred whose callBack arg is a deque of the accumulation of the values yielded from a_generator which is iterated over in batches the size of max_batch_size. It should be more efficient to iterate over the generator in batches and still provide enough speed for non-blocking execution. :param max_batch_size: The number of iterations of the generator to consume at a time. :param a_generator: An iterator which yields some not None values. :return: A Deferred to which the next callback will be called with the yielded contents of the generator function. """ if cooperator: own_cooperate = cooperator.cooperate else: own_cooperate = cooperate spigot = ValueBucket() items = stream_tap((spigot, ), a_generator) d = own_cooperate(i_batch(max_batch_size, items)).whenDone() d.addCallback(accumulation_handler, spigot) return d
def certain_kind_tap(data_items): """ As the stream of data items go by, get different kinds of information from them, in this case, the things that are fruit and metal, collecting each kind with a different spigot. stream_tap doesn't consume the data_items iterator by itself, it's a generator and must be consumed by something else. In this case, it's consuming the items by casting the iterator to a tuple, but doing it in batches. Since each batch is not referenced by anything the memory can be freed by the garbage collector, so no matter the size of the data_items, only a little memory is needed. The only things retained are the results, which should just be a subset of the items and in this case, the getter functions only return a portion of each item it matches. :param data_items: A sequence of unicode strings """ fruit_spigot = Bucket(get_fruit) metal_spigot = Bucket(get_metal) items = stream_tap((fruit_spigot, metal_spigot), data_items) for batch in i_batch(100, items): tuple(batch) return fruit_spigot.contents(), metal_spigot.contents()
def accumulate(a_generator, cooperator=None): """ Start a Deferred whose callBack arg is a deque of the accumulation of the values yielded from a_generator. :param a_generator: An iterator which yields some not None values. :return: A Deferred to which the next callback will be called with the yielded contents of the generator function. """ if cooperator: own_cooperate = cooperator.cooperate else: own_cooperate = cooperate spigot = ValueBucket() items = stream_tap((spigot,), a_generator) d = own_cooperate(items).whenDone() d.addCallback(accumulation_handler, spigot) return d
def accumulate(a_generator, cooperator=None): """ Start a Deferred whose callBack arg is a deque of the accumulation of the values yielded from a_generator. :param a_generator: An iterator which yields some not None values. :return: A Deferred to which the next callback will be called with the yielded contents of the generator function. """ if cooperator: own_cooperate = cooperator.cooperate else: own_cooperate = cooperate spigot = ValueBucket() items = stream_tap((spigot, ), a_generator) d = own_cooperate(items).whenDone() d.addCallback(accumulation_handler, spigot) return d