def main(input, output): with open(output, 'wb') as outfile: for fname in reader.smart_find_logs(input): print "Processing", fname with open(fname, 'rb') as f: results = pipeline.inline(f, filt) outfile.writelines(util.stringify(results))
def writefiles(inpath, outpath): records = iter_records(inpath) for i, lines in enumerate(chunk(records, write_blocksize)): fname = os.path.join(outpath, 'block-%04d.log.gz'%i) with gzip.open(fname, 'wb') as f: f.writelines(stringify(lines))
def mapper(it): return util.stringify(filt(it))