# genopen.py
#
# Takes a sequence of filenames as input and yields a sequence of file
# objects that have been suitably open
import gzip, bz2
def gen_open(filenames):
for name in filenames:
if name.endswith(".gz"):
yield gzip.open(name)
elif name.endswith(".bz2"):
yield bz2.BZ2File(name)
else:
yield open(name)
# Example use
if __name__ == '__main__':
from genfind import gen_find
lognames = gen_find("access-log*", "www")
logfiles = gen_open(lognames)
for f in logfiles:
print(f)
def lines_from_dir(filepat, dirname):
names = gen_find(filepat,dirname)
files = gen_open(names)
lines = gen_cat(files)
return lines
# gencat.py
#
# Concatenate multiple generators into a single sequence
def gen_cat(sources):
for s in sources:
for item in s:
yield item
# Example use
if __name__ == '__main__':
from genfind import gen_find
from genopen import gen_open
lognames = gen_find("access-log*","www")
logfiles = gen_open(lognames)
loglines = gen_cat(logfiles)
for line in loglines:
print line,
import gzip, bz2
def gen_open(filenames):
for name in filenames:
if name.endswith(".gz"):
yield gzip.open(name)
elif name.endswith(".bz2"):
yield bz2.BZ2File(name)
else:
yield open(name)
# Example use
if __name__ == '__main__':
from genfind import gen_find
lognames = gen_find("access-log*","www")
logfiles = gen_open(lognames)
for f in logfiles:
print f
# gencat.py
#
# Concatenate multiple generators into a single sequence
def gen_cat(sources):
for s in sources:
for item in s:
yield item
# Example use