Exemplos de SequenceCorpus.build em Python

Exemplo n.º 1

Arquivo: language_model.py Projeto: rudaoshi/neural_machine

from neural_machine.tasks.language.common.corpus.sequence_corpus import SequenceCorpus
from neural_machine.tasks.language.common.data_reader.bucket_iter import *

import sys

import logging

if __name__ == '__main__':

    head = '%(asctime)-15s %(message)s'
    logging.basicConfig(level=logging.DEBUG, format=head)

    segmenter = SpaceSegmenter()
    corpus = SequenceCorpus()

    corpus.build(open(sys.argv[1], 'r'), segmenter)
    cell_num = corpus.cell_num()

    problem = LanguageModelProblem(corpus)

    batch_size = 32

    data_train = BucketIter(problem, batch_size)

    val_corpus = corpus.make(open(sys.argv[2], 'r'), segmenter)
    val_problem = LanguageModelProblem(val_corpus)
    data_val = BucketIter(val_problem, batch_size)

    arch_param = LanguageModelArchParam(num_hidden=200,
                                        num_embed=200,
                                        num_lstm_layer=2,

Exemplo n.º 2

Arquivo: language_model.py Projeto: rudaoshi/neuralmachines

from neural_machine.tasks.language.common.data_reader.bucket_iter import *

import sys

import logging

if __name__ == '__main__':

    head = '%(asctime)-15s %(message)s'
    logging.basicConfig(level=logging.DEBUG, format=head)


    segmenter = SpaceSegmenter()
    corpus = SequenceCorpus()

    corpus.build(open(sys.argv[1], 'r'), segmenter)
    cell_num = corpus.cell_num()

    problem = LanguageModelProblem(corpus)

    batch_size = 32

    data_train = BucketIter(problem, batch_size)

    val_corpus = corpus.make(open(sys.argv[2], 'r'), segmenter)
    val_problem = LanguageModelProblem(val_corpus)
    data_val = BucketIter(val_problem, batch_size)


    arch_param = LanguageModelArchParam(
        num_hidden= 200,