def load(self, model_dir): self._model_dir = model_dir self._paramsfile = os.path.join(self._model_dir, 'params.py') self._params = utils.load_param_file(self._paramsfile) self._num_before = utils.get_dict_value(self._params, "num_words_before") self._num_after = utils.get_dict_value(self._params, "num_words_after") ckpt = os.path.join(utils.get_dict_value(self._params,'output_location'), utils.get_dict_value(self._params, 'model_name') + '.ckpt') vocab_file = os.path.join(utils.get_dict_value(self._params, 'output_location'), 'vocab.pkl') self._e = Evaluator.load2(ckpt) self._i = TextIndexer.from_file(vocab_file) self._keywords = self._params['keywords'] self._id_to_word = self._params['id_to_keyword']
def load(self, model_dir): self._model_dir = model_dir self._paramsfile = os.path.join(self._model_dir, 'params.py') self._params = utils.load_param_file(self._paramsfile) self._num_before = utils.get_dict_value(self._params, "num_words_before") self._num_after = utils.get_dict_value(self._params, "num_words_after") ckpt = os.path.join(utils.get_dict_value(self._params,'output_location'), utils.get_dict_value(self._params, 'model_name') + '.ckpt') vocab_file = os.path.join(utils.get_dict_value(self._params, 'output_location'), 'vocab.pkl') self._e = Evaluator.load2(ckpt) self._i = TextIndexer.from_file(vocab_file) with open(os.path.join( utils.get_dict_value(self._params, 'output_location'), 'keywords.pkl'), 'rb') as f: keywords = pickle.load(f) self._params['keywords'] = keywords self._keywords = self._params['keywords'] self._keyword_map, self._keyword_list = gen_keywords(self._params)
def eval(params, save_accuracy_file=True, batch_size=5000, num_batches=20, topn=1, verbose=True): num_before = utils.get_dict_value(params, "num_words_before") num_after = utils.get_dict_value(params, "num_words_after") ckpt = os.path.join(utils.get_dict_value(params,'output_location'), utils.get_dict_value(params, 'model_name') + '.ckpt') accuracy_file = os.path.join(utils.get_dict_value(params,'output_location'), 'accuracy.txt') vocab_file = os.path.join(utils.get_dict_value(params, 'output_location'), 'vocab.pkl') keywords_file = os.path.join(utils.get_dict_value(params, 'output_location'), 'keywords.pkl') e = Evaluator.load2(ckpt) i = TextIndexer.from_file(vocab_file) #test_sentence = "<S> ___ quick brown fox jumped over the lazy dog" test_sentence = "<S> ___ is no way to know whether it will work" #test_sentence = "<S> ___ house is on fire" # test_sentence = "<S> ___ in your best interest to lie" # test_sentence = "<S> ___ yours and I cannot touch it" #test_sentence = "<S> I ate a ___ and an apple" #test_sentence = "<S> I have to take ___ life away" # test_sentence = "<S> ___ may and it is raining" #test_sentence = "<S> This will take ___ before it will actually work" #test_sentence = "<S> this is probably bigger ___ that" # test_sentence = "<S> ___ is no place like home" #test_sentence = "I have ___ of money" #test_sentence = "<S> I think I ___ have it" test_sentence = "<S> don 't forget to get orange , banana , and ___ ." # test_sentence = "<S> in the heat ___ the night" # test_sentence = "<S> in the river , ___ the boat" # test_sentence = "<S> nothing can be ___ from the truth" # test_sentence = "<S> the ___ knot will unwind" # test_sentence = "<S> if you keep playing, you will ___ ." test_sentence = "<s> I ate a ___ of oranges ." # test_sentence = "<s> I ate a ___ and oranges ." # test_sentence = "<s> I live in a ___ ." # test_sentence = "<s> I ate a ___ of oranges ." test_sentence = "<s> I ate a ___ and oranges ." test_sentence = "<s> I live in a ___ ." test_sentence = "<s> I have seen it on him , and can ___ to it ." test_sentence = "<s> the thieves ___ the library and got very little for their pains ." # input data with open('/mnt/work/NeuralRewriting/eval/small_eval_data.json') as f: data = json.load(f) with open(keywords_file, 'rb') as f: k = pickle.load(f) unk_list = [] for q in data: query_word = q['query_word'] orig_sent = q['orig_sent'] options = q['options'] orig_sent = orig_sent.replace(query_word, "___") orig_sent = "<s> " + orig_sent test_sentence = orig_sent.lower() split_sentence = list(split_sentence_for_eval(test_sentence.split(), ["___"], num_before, num_after)) # print(split_sentence[0][0]) _, sentence, _, _ = i.index_wordlist(split_sentence[0][0]) bef = time() r = e.eval({'sentence': [sentence]}, {'sm_decision'}) aft = time() sm = r[0][0] for o in options: synonym = o['synonym'] if synonym not in k: score = -1000 unk_list += [synonym] else: score = math.log(sm[k.index(synonym)]) o['clmtV1'] = score print(score) # save output with open('/mnt/work/NeuralRewriting/eval/small_eval_data_out.json','w') as f: json.dump(data,f)
from framework.evaluator import Evaluator import framework.utils.common as utils from framework.trainer import Trainer import framework.subgraph.losses as losses import logging import sys import os from framework.utils.data.text_indexer import TextIndexer import tensorflow as tf sentences = ['Jimenez reported from San Jose Costa Rica ; Associated Press', "We went to the store and I bought a banana", "This is the cat that walked over the mat .", "<pad> <pad> In this case I think the problem is"] e = Evaluator.load2("outputv0.ckpt") i = TextIndexer.from_file('vocab.pkl') for sentence in sentences: # for j in range(5): # sentence = '<pad> ' + sentence + " <pad>" _,indexed,_,_ = i.index_wordlist(sentence.split()) r = e.eval({'sentence': [indexed]}, {'sm_decision'}) print(sentence) print(r)
from urllib.parse import urlparse from http.server import BaseHTTPRequestHandler, HTTPServer run_server = True paramsfile = "params.py" data_base_dir = "" http_port = 8080 params = utils.load_param_file(paramsfile) vocab_file = os.path.join(utils.get_dict_value(params,'output_location'), 'vocab.pkl') ckpt = os.path.join(utils.get_dict_value(params,'output_location'), utils.get_dict_value(params, 'model_name') + '.ckpt') print(ckpt) e = Evaluator.load2(ckpt) i = TextIndexer.from_file(vocab_file) num_before = utils.get_dict_value(params, "num_words_before") num_after = utils.get_dict_value(params, "num_words_after") pad_tok = utils.get_dict_value(params, "pad_tok", '<pad>') def split_sentence_for_eval(sentence, keywords, num_before, num_after): result = data.gen_data(sentence, keywords, num_before=num_before, num_after=num_after, ignore_negative_data=True, add_redundant_keyword_data=False) return result def merge_sentence(sentence, num_before, num_after, mid_word): before = sentence[:num_before] after = sentence[num_after:] rlist = (before + [mid_word] + after)
def eval(params, save_accuracy_file=True, batch_size=5000, num_batches=20, topn=1, verbose=True): num_before = utils.get_dict_value(params, "num_words_before") num_after = utils.get_dict_value(params, "num_words_after") ckpt = os.path.join(utils.get_dict_value(params,'output_location'), utils.get_dict_value(params, 'model_name') + '.ckpt') accuracy_file = os.path.join(utils.get_dict_value(params,'output_location'), 'accuracy.txt') keywords = params['keywords'] rkeywords = params['id_to_keyword'] vocab_file = os.path.join(utils.get_dict_value(params, 'output_location'), 'vocab.pkl') e = Evaluator.load2(ckpt) i = TextIndexer.from_file(vocab_file) e.dump_variable_sizes() exit(0) test_sentence = "it is better to die happy then to live miserably" # test_sentence = "there going two make a turkey today" # test_sentence = "they 're going two make a turkey today" test_sentence = "to big to fail my ass !" # test_sentence = "two big is not bad" # test_sentence = "<S> two big fishes in the same bucket" # test_sentence = "<S> there are two fishes in the same bucket ." # test_sentence = "<S> there are too fishes in the same bucket ." #test_sentence = "<S> I had two fishes for dinner" test_sentence = "<S> its raining men . hallelujah !" # test_sentence = "it 's head is too big" #test_sentence = "if it 's not one than it 's true" #test_sentence = "i would except it , but i don 't think it 's true" print("HELLO") print(keywords) split_sentence = list(split_sentence_for_eval(test_sentence.split(), keywords, num_before, num_after)) print(split_sentence) for j in range(len(split_sentence)): print(split_sentence[j][0]) _, sentence, _, _ = i.index_wordlist(split_sentence[j][0]) bef = time() r = e.eval({'sentence': [sentence]}, {'sm_decision'}) aft = time() #print(r[0][0]) sm = r[0][0] am = np.argmax(sm) #print(am) k = rkeywords if am == 0: print("DO NOTHING") else: print(k[am - 1]) k = [''] + k sm, k = zip(*sorted(zip(sm, k), reverse=True)) # print(k) for q,(x,y) in enumerate(zip(sm,k)): if q > 10: break print("%0.4f %s" %(x,y)) print(test_sentence) print("EVAL TIME = %s"%(aft-bef))