def multiplex(sources):
in_q = queue.Queue()
consumers = []
for src in sources:
t = threading.Thread(target=sendto_queue, args=(src, in_q))
t.start()
consumers.append(genfrom_queue(in_q))
return gen_cat(consumers)
def multiplex(sources):
""" Generate items from multiple generators
"""
in_q = queue.Queue()
consumers = []
for src in sources:
thr = threading.Thread(target=sendto_queue, args=(src, in_q))
thr.start()
consumers.append(genfrom_queue(in_q))
return gen_cat(consumers)
# gengrep.py
#
# Grep a sequence of lines that match a re pattern
import re
def gen_grep(pat, lines):
patc = re.compile(pat)
for line in lines:
if patc.search(line):
yield line
# Example use
if __name__ == '__main__':
from genfind import gen_find
from genopen import gen_open
from gencat import gen_cat
lognames = gen_find("access-log*", "www")
logfiles = gen_open(lognames)
loglines = gen_cat(logfiles)
# Look for ply downloads (PLY is my own Python package)
plylines = gen_grep(r'ply-.*\.gz', loglines)
for line in plylines:
print(line, )
def lines_from_dir(filepat, dirname):
names = gen_find(filepat,dirname)
files = gen_open(names)
lines = gen_cat(files)
return lines
# Concatenate multiple generators into a single sequence
def gen_cat(sources):
for s in sources:
for item in s:
yield item
# Example use
if __name__ == '__main__':
from genfind import gen_find
from genopen import gen_open
lognames = gen_find("access-log*","www")
logfiles = gen_open(lognames)
loglines = gen_cat(logfiles)
for line in loglines:
print line,
# gengrep.py
#
# Grep a sequence of lines that match a re pattern
import re
def gen_grep(pat,lines):
patc = re.compile(pat)
for line in lines:
if patc.search(line): yield line
# Example use
def lines_from_dir(filepat, dirname):
names = Path(dirname).rglob(filepat)
files = gen_open(names)
lines = gen_cat(files)
return lines