def get_grams_cached(lines): grams_filename = FLAGS.PICKLE_PATH + '/true-char-ngrams.pkl' if os.path.exists(grams_filename): return model_and_data_serialization.load_picklized(grams_filename) else: grams = get_grams(lines) model_and_data_serialization.save_picklized(grams, grams_filename) return grams
def get_true_char_ngram_lms(lines, tokenize): true_char_filename = FLAGS.PICKLE_PATH + '/true-char-ngrams-%s.pkl' % tokenize if os.path.exists(true_char_filename): true_char_ngram_lms = model_and_data_serialization.load_picklized( true_char_filename) else: true_char_ngram_lms = build_vailidation_ngram_models(lines) model_and_data_serialization.save_picklized(true_char_ngram_lms, true_char_filename) return true_char_ngram_lms
def get_gt_grams_cached(lines, dataset='training'): grams_filename = 'true-char-ngrams.pkl' if dataset == 'heldout': grams_filename = 'heldout_' + grams_filename grams_filename = FLAGS.PICKLE_PATH + '/' + grams_filename if os.path.exists(grams_filename): return model_and_data_serialization.load_picklized(grams_filename) else: grams = get_grams(lines) model_and_data_serialization.save_picklized(grams, grams_filename) return grams