class DynamicForEachSource(object): """A source that for each given argument creates a new source that will be iterated by this source. For example, useful for directories where a CSVSource should be created for each file. The user must provide a function that when called with a single argument, returns a new source to iterate. A DynamicForEachSource instance can be given to several ProcessSource instances. """ def __init__(self, seq, callee): """Arguments: - seq: A sequence with the elements for each of which a unique source must be created. The elements are given (one by one) to callee. - callee: A function f(e) that must accept elements as those in the seq argument. The function should return a source which then will be iterated by this source. The function is called once for every element in seq. """ self.__queue = Queue() # a multiprocessing.Queue if not callable(callee): raise TypeError, 'callee must be callable' self.__callee = callee for e in seq: # put them in a safe queue such that this object can be used from # different fork'ed processes self.__queue.put(e) def __iter__(self): while True: try: arg = self.__queue.get(False) src = self.__callee(arg) for row in src: yield row except Empty: raise StopIteration