Exemplo n.º 1
0
def main(input, output):
    with open(output, 'wb') as outfile:
        for fname in reader.smart_find_logs(input):
            print "Processing", fname
            with open(fname, 'rb') as f:
                results = pipeline.inline(f, filt)
                outfile.writelines(util.stringify(results))
Exemplo n.º 2
0
def writefiles(inpath, outpath):
    records = iter_records(inpath)
    for i, lines in enumerate(chunk(records, write_blocksize)):
        fname = os.path.join(outpath, 'block-%04d.log.gz'%i)    
        with gzip.open(fname, 'wb') as f:
            f.writelines(stringify(lines))
Exemplo n.º 3
0
def mapper(it):
    return util.stringify(filt(it))