def consume_stream(target, input_dir=None): """Read lines from the standard input or files in a directory. Behaves as a generator, should receive a .send() call to send a line to the target. """ file_names = get_file_names(input_dir) if input_dir else [] with contextlib.closing( fileinput.FileInput(file_names, openhook=fileinput.hook_compressed)) as lines: targets = [target] if target is not None else [] with consumers.closing(*targets): for line in lines: if not line.strip(): continue if isinstance(line, bytes): line = line.decode('utf-8') if target is not None: result = target.send(line) if result is not consumers.SendNext: yield result else: yield line
def consume_stream(target, input_dir=None): """Read lines from the standard input or files in a directory. Behaves as a generator, should receive a .send() call to send a line to the target. """ file_names = get_file_names(input_dir) if input_dir else [] with contextlib.closing(fileinput.FileInput(file_names, openhook=fileinput.hook_compressed)) as lines: targets = [target] if target is not None else [] with consumers.closing(*targets): for line in lines: if not line.strip(): continue if isinstance(line, bytes): line = line.decode('utf-8') if target is not None: result = target.send(line) if result is not consumers.SendNext: yield result else: yield line
def _conductor(merger, tagged_sources): result = None with closing(merger): # The first step: push each target once. for source in tagged_sources.values(): try: result = next(source) if result is None: yield except StopIteration: pass item, source_priority = result # The second step: push targets as merge requests. while True: yield item, source_priority while source_priority: tag = source_priority.pop(0) try: item, source_priority = next(tagged_sources[tag]) except StopIteration: pass else: break else: return
def SpecialTarget(target): try: with consumers.closing(target): while True: item = yield target.send(item) except ThrownException: pass
def from_simple_queue(target, queue): with consumers.closing(target): while True: item = queue.get() if item is StopIteration: break target.send(item)
def _merger(tagged_targets, provider=None): if provider is None: def provider(current): return current targets = set(tagged_targets.values()) with closing(*targets): items = [] heapq.heapify(items) applied_sources = set() closed_sources = set() all_sources = set(tagged_targets) try: # First step: get at least one item from each source while applied_sources != all_sources: try: tag, item = yield except TaggedGeneratorExit as e: applied_sources.add(e.tag) closed_sources.add(e.tag) else: heapq.heappush(items, (provider(item), tag, item)) applied_sources.add(tag) # Second step: push forward the smallest item to its target # and push back to conductor the priorities of the sources. while closed_sources != applied_sources: priorities = [i[1] for i in sorted(items)] _, tag, item = heapq.heappop(items) result = tagged_targets[tag].send(item) try: tag, item = yield result, priorities except TaggedGeneratorExit as e: closed_sources.add(e.tag) logger.debug("%s is closed", tag) else: heapq.heappush(items, (provider(item), tag, item)) finally: for _, tag, item in sorted(items): tagged_targets[tag].send(item)
def _merger(tagged_targets, provider=None): if provider is None: def provider(current): return current targets = set(tagged_targets.values()) with closing(*targets): items = [] heapq.heapify(items) applied_sources = set() closed_sources = set() all_sources = set(tagged_targets) try: # First step: get at least one item from each source while applied_sources != all_sources: try: tag, item = yield except TaggedGeneratorExit as e: applied_sources.add(e.tag) closed_sources.add(e.tag) else: heapq.heappush(items, (provider(item), tag, item)) applied_sources.add(tag) # Second step: push forward the smallest item to its target # and push back to conductor the priorities of the sources. while closed_sources != applied_sources: priorities = [i[1] for i in sorted(items)] _, tag, item = heapq.heappop(items) result = tagged_targets[tag].send(item) try: tag, item = yield result, priorities except TaggedGeneratorExit as e: closed_sources.add(e.tag) logger.debug('%s is closed', tag) else: heapq.heappush(items, (provider(item), tag, item)) finally: for _, tag, item in sorted(items): tagged_targets[tag].send(item)
def consume_stream(target, input_dir=None): """Read lines from the standard input or files in a directory. Behaves as a generator, should receive a .send() call to send a line to the target. """ file_names = get_file_names(input_dir) if input_dir else [] with contextlib.closing(fileinput.FileInput(file_names, openhook=fileinput.hook_compressed)) as lines: with consumers.closing(target): for line in lines: result = target.send(line) if result is not consumers.SendNext: yield result
def consume_stream(target, input_dir=None): """Read lines from the standard input or files in a directory. Behaves as a generator, should receive a .send() call to send a line to the target. """ file_names = get_file_names(input_dir) if input_dir else [] with contextlib.closing( fileinput.FileInput(file_names, openhook=fileinput.hook_compressed)) as lines: with consumers.closing(target): for line in lines: result = target.send(line) if result is not consumers.SendNext: yield result