Python reader示例

示例#1

文件： pymtbl_wf_merge.py 项目： rep/pymtbl

def main(input_fnames, output_fname):
    merger = mtbl.merger(merge_func)
    writer = mtbl.writer(output_fname, compression=mtbl.COMPRESSION_SNAPPY)
    for fname in input_fnames:
        reader = mtbl.reader(fname)
        merger.add_reader(reader)
    merger.write(writer)

示例#2

文件： pymtbl_search_random_table.py 项目： rep/pymtbl

def main(fname, num_keys, num_iters):
    reader = mtbl.reader(fname)

    a = time.time()
    last = a
    num_found = 0
    count = 0
    while count < num_iters:
        key = '%010d' % random.randint(0, num_keys)
        if reader.has_key(key):
            val = reader[key]
            num_found += 1
        count += 1
        if (count % report_interval) == 0:
            b = time.time()
            last_secs = b - last
            last = b
            sys.stderr.write(
                '%s lookups, %s keys found in %s seconds, %s lookups/second\n'
                % (locale.format('%d', count, grouping=True),
                   locale.format('%d', num_found, grouping=True),
                   locale.format('%f', last_secs, grouping=True),
                   locale.format(
                       '%d', report_interval / last_secs, grouping=True)))
    b = time.time()
    total_secs = b - a
    sys.stderr.write(
        '%s total lookups, %s keys found in %s seconds, %s lookups/second\n' %
        (locale.format('%d', count, grouping=True),
         locale.format('%d', num_found, grouping=True),
         locale.format('%f', total_secs, grouping=True),
         locale.format('%d', count / total_secs, grouping=True)))

示例#3

文件： pymtbl_wf_merge.py 项目： edmonds/pymtbl

def main(input_fnames, output_fname):
    merger = mtbl.merger(merge_func)
    writer = mtbl.writer(output_fname, compression=mtbl.COMPRESSION_SNAPPY)
    for fname in input_fnames:
        reader = mtbl.reader(fname)
        merger.add_reader(reader)
    merger.write(writer)

示例#4

文件： pymtbl_merge_tables.py 项目： edmonds/pymtbl

def main(output_fname, input_fnames):
    merger = mtbl.merger(merge_func)
    writer = mtbl.writer(output_fname, compression=mtbl.COMPRESSION_SNAPPY)
    for fname in input_fnames:
        reader = mtbl.reader(fname)
        merger.add_reader(reader)
    for k, v in merger.iteritems():
        writer[k] = v
    writer.close()

示例#5

def main(output_fname, input_fnames):
    merger = mtbl.merger(merge_func)
    writer = mtbl.writer(output_fname, compression=mtbl.COMPRESSION_SNAPPY)
    for fname in input_fnames:
        reader = mtbl.reader(fname)
        merger.add_reader(reader)
    for k, v in merger.iteritems():
        writer[k] = v
    writer.close()

示例#6

文件： pymtbl_search_random_table.py 项目： edmonds/pymtbl

def main(fname, num_keys, num_iters):
    reader = mtbl.reader(fname)

    a = time.time()
    last = a
    num_found = 0
    count = 0
    while count < num_iters:
        key = "%010d" % random.randint(0, num_keys)
        if reader.has_key(key):
            val = reader[key]
            num_found += 1
        count += 1
        if (count % report_interval) == 0:
            b = time.time()
            last_secs = b - last
            last = b
            sys.stderr.write(
                "%s lookups, %s keys found in %s seconds, %s lookups/second\n"
                % (
                    locale.format("%d", count, grouping=True),
                    locale.format("%d", num_found, grouping=True),
                    locale.format("%f", last_secs, grouping=True),
                    locale.format("%d", report_interval / last_secs, grouping=True),
                )
            )
    b = time.time()
    total_secs = b - a
    sys.stderr.write(
        "%s total lookups, %s keys found in %s seconds, %s lookups/second\n"
        % (
            locale.format("%d", count, grouping=True),
            locale.format("%d", num_found, grouping=True),
            locale.format("%f", total_secs, grouping=True),
            locale.format("%d", count / total_secs, grouping=True),
        )
    )

示例#7

def main(mtbl_fname):
    reader = mtbl.reader(mtbl_fname)
    for k, v in reader.items():
        word = k
        count = mtbl.varint_decode(v)
        print '%s\t%s' % (count, word)

示例#8

文件： pymtbl_wf_dump.py 项目： edmonds/pymtbl

def main(mtbl_fname):
    reader = mtbl.reader(mtbl_fname)
    for k, v in reader.items():
        word = k
        count = mtbl.varint_decode(v)
        print '%s\t%s' % (count, word)