def main(input, output):
with open(output, 'wb') as outfile:
for fname in reader.smart_find_logs(input):
print "Processing", fname
with open(fname, 'rb') as f:
results = pipeline.inline(f, filt)
outfile.writelines(util.stringify(results))
def writefiles(inpath, outpath):
records = iter_records(inpath)
for i, lines in enumerate(chunk(records, write_blocksize)):
fname = os.path.join(outpath, 'block-%04d.log.gz'%i)
with gzip.open(fname, 'wb') as f:
f.writelines(stringify(lines))
def mapper(it):
return util.stringify(filt(it))