Exemplo n.º 1
0
def main():
    train = "../data/Verne.80jours.en"
    # train = "../data/simplewiki-20140903-pages-articles.200000first.100000last.txt"
    # train = "../data/wsj.words"

    order = 3
    n_iter = 100
    vocabulary = Vocabulary()
    logging.info('Reading training corpus')
    with open(train) as train:
        training_corpus = read_corpus(train, vocabulary)
        base = Uniform(len(vocabulary))
        model = PYPLM(order, base)
        logging.info('Training model of order %d', order)
        run_sampler(model, training_corpus, n_iter)