class DynamicForEachSource(object):
"""A source that for each given argument creates a new source that
will be iterated by this source.
For example, useful for directories where a CSVSource should be created
for each file.
The user must provide a function that when called with a single argument,
returns a new source to iterate. A DynamicForEachSource instance can be
given to several ProcessSource instances.
"""
def __init__(self, seq, callee):
"""Arguments:
- seq: A sequence with the elements for each of which a unique source
must be created. The elements are given (one by one) to callee.
- callee: A function f(e) that must accept elements as those in the seq
argument. The function should return a source which then will be
iterated by this source. The function is called once for every
element in seq.
"""
self.__queue = Queue() # a multiprocessing.Queue
if not callable(callee):
raise TypeError, 'callee must be callable'
self.__callee = callee
for e in seq:
# put them in a safe queue such that this object can be used from
# different fork'ed processes
self.__queue.put(e)
def __iter__(self):
while True:
try:
arg = self.__queue.get(False)
src = self.__callee(arg)
for row in src:
yield row
except Empty:
raise StopIteration
