def get_grams_cached(lines):
grams_filename = FLAGS.PICKLE_PATH + '/true-char-ngrams.pkl'
if os.path.exists(grams_filename):
return model_and_data_serialization.load_picklized(grams_filename)
else:
grams = get_grams(lines)
model_and_data_serialization.save_picklized(grams, grams_filename)
return grams
def get_true_char_ngram_lms(lines, tokenize):
true_char_filename = FLAGS.PICKLE_PATH + '/true-char-ngrams-%s.pkl' % tokenize
if os.path.exists(true_char_filename):
true_char_ngram_lms = model_and_data_serialization.load_picklized(
true_char_filename)
else:
true_char_ngram_lms = build_vailidation_ngram_models(lines)
model_and_data_serialization.save_picklized(true_char_ngram_lms,
true_char_filename)
return true_char_ngram_lms
def get_gt_grams_cached(lines, dataset='training'):
grams_filename = 'true-char-ngrams.pkl'
if dataset == 'heldout':
grams_filename = 'heldout_' + grams_filename
grams_filename = FLAGS.PICKLE_PATH + '/' + grams_filename
if os.path.exists(grams_filename):
return model_and_data_serialization.load_picklized(grams_filename)
else:
grams = get_grams(lines)
model_and_data_serialization.save_picklized(grams, grams_filename)
return grams
