예제 #1
0
def main():
    if len(sys.argv) != 2:
        print('usage: python postagger-runner.py model-prefix < input > output', file=sys.stderr)
        return

    model_prefix = sys.argv[1]

    # load dictionary
    word_ids = Dictionary.load(model_prefix + '.wordid')
    pos_ids = Dictionary.load(model_prefix + '.posid')

    # load and test tagger
    tagger = POSTagger.load(model_prefix)
    
    # output queue
    qs = []
    wss = []

    # input iterator
    def iterate_words():
        for l in sys.stdin:
            ls = l.strip('\n').split(' ')
            wss.append(ls)
            for w in ls:
                yield word_ids[w]

    for w, p in tagger.iterate(iterate_words()):
        qs.append(pos_ids.get_name(p))
        if len(qs) >= len(wss[0]):
            print(' '.join('%s/%s' % wq for wq in zip(wss[0], qs)))
            sys.stdout.flush()
            qs = []
            wss.pop(0)
예제 #2
0
def main():
    if len(sys.argv) != 3:
        print('usage: python postagger-test.py', file=sys.stderr)
        print('                <str: test prefix>', file=sys.stderr)
        print('                <str: model prefix>', file=sys.stderr)
        return

    test_prefix = sys.argv[1]
    model_prefix = sys.argv[2]

    print('loading data ...', file=sys.stderr)

    # load test data
    test_words = [w.lower() for w in utils.read_data(test_prefix + '.words')]
    test_pos = utils.read_data(test_prefix + '.pos')

    # load dictionary
    word_ids = Dictionary.load(model_prefix + '.wordid')
    pos_ids = Dictionary.load(model_prefix + '.posid')

    # make word/POS IDs
    test_wids = [word_ids[w] for w in test_words]
    test_pids = [pos_ids[w] for w in test_pos]

    # load and test tagger
    tagger = POSTagger.load(model_prefix)
    tagger.test(test_wids, test_pids)