def consume_stream(target, input_dir=None):
"""Read lines from the standard input or files in a directory.
Behaves as a generator, should receive a .send() call to send a
line to the target.
"""
file_names = get_file_names(input_dir) if input_dir else []
with contextlib.closing(
fileinput.FileInput(file_names,
openhook=fileinput.hook_compressed)) as lines:
targets = [target] if target is not None else []
with consumers.closing(*targets):
for line in lines:
if not line.strip():
continue
if isinstance(line, bytes):
line = line.decode('utf-8')
if target is not None:
result = target.send(line)
if result is not consumers.SendNext:
yield result
else:
yield line
def consume_stream(target, input_dir=None):
"""Read lines from the standard input or files in a directory.
Behaves as a generator, should receive a .send() call to send a
line to the target.
"""
file_names = get_file_names(input_dir) if input_dir else []
with contextlib.closing(fileinput.FileInput(file_names, openhook=fileinput.hook_compressed)) as lines:
targets = [target] if target is not None else []
with consumers.closing(*targets):
for line in lines:
if not line.strip():
continue
if isinstance(line, bytes):
line = line.decode('utf-8')
if target is not None:
result = target.send(line)
if result is not consumers.SendNext:
yield result
else:
yield line
def _conductor(merger, tagged_sources):
result = None
with closing(merger):
# The first step: push each target once.
for source in tagged_sources.values():
try:
result = next(source)
if result is None:
yield
except StopIteration:
pass
item, source_priority = result
# The second step: push targets as merge requests.
while True:
yield item, source_priority
while source_priority:
tag = source_priority.pop(0)
try:
item, source_priority = next(tagged_sources[tag])
except StopIteration:
pass
else:
break
else:
return
def _conductor(merger, tagged_sources):
result = None
with closing(merger):
# The first step: push each target once.
for source in tagged_sources.values():
try:
result = next(source)
if result is None:
yield
except StopIteration:
pass
item, source_priority = result
# The second step: push targets as merge requests.
while True:
yield item, source_priority
while source_priority:
tag = source_priority.pop(0)
try:
item, source_priority = next(tagged_sources[tag])
except StopIteration:
pass
else:
break
else:
return
def SpecialTarget(target):
try:
with consumers.closing(target):
while True:
item = yield
target.send(item)
except ThrownException:
pass
def SpecialTarget(target):
try:
with consumers.closing(target):
while True:
item = yield
target.send(item)
except ThrownException:
pass
def from_simple_queue(target, queue):
with consumers.closing(target):
while True:
item = queue.get()
if item is StopIteration:
break
target.send(item)
def from_simple_queue(target, queue):
with consumers.closing(target):
while True:
item = queue.get()
if item is StopIteration:
break
target.send(item)
def _merger(tagged_targets, provider=None):
if provider is None:
def provider(current):
return current
targets = set(tagged_targets.values())
with closing(*targets):
items = []
heapq.heapify(items)
applied_sources = set()
closed_sources = set()
all_sources = set(tagged_targets)
try:
# First step: get at least one item from each source
while applied_sources != all_sources:
try:
tag, item = yield
except TaggedGeneratorExit as e:
applied_sources.add(e.tag)
closed_sources.add(e.tag)
else:
heapq.heappush(items, (provider(item), tag, item))
applied_sources.add(tag)
# Second step: push forward the smallest item to its target
# and push back to conductor the priorities of the sources.
while closed_sources != applied_sources:
priorities = [i[1] for i in sorted(items)]
_, tag, item = heapq.heappop(items)
result = tagged_targets[tag].send(item)
try:
tag, item = yield result, priorities
except TaggedGeneratorExit as e:
closed_sources.add(e.tag)
logger.debug("%s is closed", tag)
else:
heapq.heappush(items, (provider(item), tag, item))
finally:
for _, tag, item in sorted(items):
tagged_targets[tag].send(item)
def _merger(tagged_targets, provider=None):
if provider is None:
def provider(current):
return current
targets = set(tagged_targets.values())
with closing(*targets):
items = []
heapq.heapify(items)
applied_sources = set()
closed_sources = set()
all_sources = set(tagged_targets)
try:
# First step: get at least one item from each source
while applied_sources != all_sources:
try:
tag, item = yield
except TaggedGeneratorExit as e:
applied_sources.add(e.tag)
closed_sources.add(e.tag)
else:
heapq.heappush(items, (provider(item), tag, item))
applied_sources.add(tag)
# Second step: push forward the smallest item to its target
# and push back to conductor the priorities of the sources.
while closed_sources != applied_sources:
priorities = [i[1] for i in sorted(items)]
_, tag, item = heapq.heappop(items)
result = tagged_targets[tag].send(item)
try:
tag, item = yield result, priorities
except TaggedGeneratorExit as e:
closed_sources.add(e.tag)
logger.debug('%s is closed', tag)
else:
heapq.heappush(items, (provider(item), tag, item))
finally:
for _, tag, item in sorted(items):
tagged_targets[tag].send(item)
def consume_stream(target, input_dir=None):
"""Read lines from the standard input or files in a directory.
Behaves as a generator, should receive a .send() call to send a
line to the target.
"""
file_names = get_file_names(input_dir) if input_dir else []
with contextlib.closing(fileinput.FileInput(file_names, openhook=fileinput.hook_compressed)) as lines:
with consumers.closing(target):
for line in lines:
result = target.send(line)
if result is not consumers.SendNext:
yield result
def consume_stream(target, input_dir=None):
"""Read lines from the standard input or files in a directory.
Behaves as a generator, should receive a .send() call to send a
line to the target.
"""
file_names = get_file_names(input_dir) if input_dir else []
with contextlib.closing(
fileinput.FileInput(file_names,
openhook=fileinput.hook_compressed)) as lines:
with consumers.closing(target):
for line in lines:
result = target.send(line)
if result is not consumers.SendNext:
yield result