Exemple #1
0
        pass

def onSingleUpdate(ctr, src, trg):
    if op == "gen":
        print(VOC.str_rpr(trg[0]))
    elif op == "sppl":
        print(PPL(trg))

def onDecodingFinish(data, output):
    if op == "gen":
        for src_id, (inp, out) in sorted(output.items(), key=lambda x:x[0]):
            print(TRG.str_rpr(out))
    elif op == "cppl":
        UF.trace("Corpus PPL:", PPL(output))
        print(PPL(output))

tester = Tester(load_lm_gen_data, VOC, onDecodingStart, onBatchUpdate, onSingleUpdate, onDecodingFinish, batch=args.batch, out_vocab=VOC, options=decoding_options)
if op == "sppl" or op == "cppl":
    if not args.src:
        _, data = load_lm_data(sys.stdin, VOC)
    else:
        with open(args.src) as src_fp:
            _, data = load_lm_data(src_fp, VOC)
    data = list(batch_generator(data, (VOC, VOC), args.batch))
    tester.eval(data, model)
elif op == "gen":
    tester.test(args.src, model)
else:
    raise NotImplementedError("Undefined operation:", op)
    
Exemple #2
0
def onBatchUpdate(ctr, src, trg):
    # Decoding
    if args.verbose:
        print_result(ctr, trg, TRG, src, SRC, sys.stderr)

def onSingleUpdate(ctr, src, trg):
    align_fp = ao_fp if ao_fp is not None else sys.stderr
    if args.verbose:
        print_result(ctr, trg, TRG, src, SRC, sys.stderr)
    print(TRG.str_rpr(trg.y[0]))
    if trg.a is not None:
        AlignmentVisualizer.print(trg.a, ctr, src, trg.y, SRC, TRG, fp=align_fp)

def onDecodingFinish(data, output):
    align_fp = ao_fp if ao_fp is not None else sys.stderr
    for src_id, (inp, out) in sorted(output.items(), key=lambda x:x[0]):
        if type(out) == tuple:
            out, align = out
            AlignmentVisualizer.print([align], src_id, [inp], [out], SRC, TRG, fp=align_fp)
        print(TRG.str_rpr(out))

# Execute testing
tester = Tester(load_nmt_test_data, SRC, onDecodingStart, onBatchUpdate, onSingleUpdate, onDecodingFinish, options=decoding_options, batch=args.batch)
tester.test(args.src, model)

# Finishing up
if ao_fp is not None:
    ao_fp.close()