import sys
params = utils.load_param_file(sys.argv[1])
vocab_file = os.path.join(utils.get_dict_value(params, 'output_location'),
'vocab.pkl')
ckpt = os.path.join(utils.get_dict_value(params, 'output_location'),
utils.get_dict_value(params, 'model_name') + '.ckpt')
#e = Evaluator.load_graphdef('commaV10.graphdef')
e = Evaluator.load2(ckpt)
#e.dump_variable_sizes()
i = TextIndexer.from_file(vocab_file)
test_data = ClassifierData.get_data(params=params, type='valid')
model_results = []
timestr = str(int(time()))
f = open(
os.path.join(utils.get_dict_value(params, 'output_location'),
'heldout_%s.txt' % timestr), 'w')
f.write('Exec Time\tModel Score\tGround Truth\tSentence\n')
for batch_no in range(10):
print("WORKING ON BATCH %s" % batch_no)
batch = test_data.next_batch(batch_size=10000)
for sentence, ground_truth in zip(batch['sentence'], batch['y']):
_, indexed, _, _ = i.index_wordlist(sentence)
before_time = time()
r = e.eval({'sentence': [indexed]}, {'sm_decision'})
after_time = time()
indexer.add_token('<pad>')
if utils.get_dict_value(params, 'all_lowercase', False):
indexer.add_token('<s>')
else:
indexer.add_token('<s>')
indexer.add_token('unk')
os.makedirs(utils.get_dict_value(params, 'output_location'), exist_ok=True)
indexer.save_vocab_as_pkl(
os.path.join(utils.get_dict_value(params, 'output_location'), 'vocab.pkl'))
shutil.copyfile(
param_file,
os.path.join(utils.get_dict_value(params, 'output_location'), param_file))
params['vocab_size'] = indexer.vocab_size()
print("VOCAB SIZE: %s" % params['vocab_size'])
training_data = ClassifierData.get_data(params, type='train', indexer=indexer)
#if 'training_data_dir' in params:
# training_data = ClassifierData.get_training_data(base_dir=params['training_data_dir'], indexer=indexer, params=params)
#else:
# training_data = ClassifierData.get_monolingual_training(base_dir=params['monolingual_dir'],
# indexer=indexer,
# params=params)
def on_checkpoint_saved(trainer, params, save_path):
msg = 'saved checkpoint: ' + save_path
print(msg)
def train_iteration_done(trainer, epoch, index, iteration_count, loss_value,
training_done, run_results, params):