示例#1
0
def get_grams_cached(lines):
    grams_filename = FLAGS.PICKLE_PATH + '/true-char-ngrams.pkl'
    if os.path.exists(grams_filename):
        return model_and_data_serialization.load_picklized(grams_filename)
    else:
        grams = get_grams(lines)
        model_and_data_serialization.save_picklized(grams, grams_filename)
        return grams
示例#2
0
def get_true_char_ngram_lms(lines, tokenize):
    true_char_filename = FLAGS.PICKLE_PATH + '/true-char-ngrams-%s.pkl' % tokenize
    if os.path.exists(true_char_filename):
        true_char_ngram_lms = model_and_data_serialization.load_picklized(
            true_char_filename)
    else:
        true_char_ngram_lms = build_vailidation_ngram_models(lines)
        model_and_data_serialization.save_picklized(true_char_ngram_lms,
                                                    true_char_filename)
    return true_char_ngram_lms
示例#3
0
def get_gt_grams_cached(lines, dataset='training'):
    grams_filename = 'true-char-ngrams.pkl'
    if dataset == 'heldout':
        grams_filename = 'heldout_' + grams_filename
    grams_filename = FLAGS.PICKLE_PATH + '/' + grams_filename
    if os.path.exists(grams_filename):
        return model_and_data_serialization.load_picklized(grams_filename)
    else:
        grams = get_grams(lines)
        model_and_data_serialization.save_picklized(grams, grams_filename)
        return grams