def __init__(self, n_word_vocab, n_role_vocab, n_factors_emb, n_hidden, word_vocabulary, role_vocabulary, unk_word_id, unk_role_id, missing_word_id, using_dropout, dropout_rate, optimizer, loss, metrics): self.n_word_vocab = n_word_vocab self.n_role_vocab = n_role_vocab self.n_factors_emb = n_factors_emb self.n_hidden = n_hidden self.word_vocabulary = word_vocabulary self.role_vocabulary = role_vocabulary self.word_decoder = get_reverse_map(word_vocabulary) self.role_decoder = get_reverse_map(role_vocabulary) self.unk_role_id = unk_role_id self.unk_word_id = unk_word_id self.missing_word_id = missing_word_id self.using_dropout = using_dropout self.dropout_rate = dropout_rate self.optimizer = optimizer self.loss = loss
def get_top_predictions(inputs, target, model, raw_word_list, n=5): """ Returns the top predicted filler for a target role, given a set of input roles + fillers Keyword arguments: inputs -- A dictionary of inputs with the role as the key and the filler as the value. target -- A singleton dictionary containing the target role as the key and target filler as the value. model -- The loaded model with which to make predictions raw_word_list -- A dictionary of vocabulary n -- The number of top predictions that should be retrieved """ #print(inputs) raw_word_list.update(inputs) #print(raw_word_list) assert len(raw_word_list) == len(model.role_vocabulary) t_r = [ model.role_vocabulary.get(r, model.unk_role_id) for r in target.keys() ] t_w = [model.unk_word_id] input_roles_words = {} for r, w in raw_word_list.items(): input_roles_words[model.role_vocabulary[r]] = utils.input_word_index( model.word_vocabulary, w, model.unk_word_id, warn_unk=False) #print input_roles_words, t_r[0] input_roles_words.pop(t_r[0]) x_w_i = numpy.asarray([input_roles_words.values()], dtype=numpy.int64) x_r_i = numpy.asarray([input_roles_words.keys()], dtype=numpy.int64) y_w_i = numpy.asarray(t_w, dtype=numpy.int64) y_r_i = numpy.asarray(t_r, dtype=numpy.int64) predicted_word_indices = model.top_words(x_w_i, x_r_i, y_w_i, y_r_i, n) results = [] reverse_vocabulary = utils.get_reverse_map(model.word_vocabulary) for t_w_i in predicted_word_indices: t_w = model.word_vocabulary.get(t_w_i, model.unk_word_id) y_w_i = numpy.asarray([t_w_i], dtype=numpy.int64) p = model.p_words(x_w_i, x_r_i, y_w_i, y_r_i, batch_size=1, verbose=0)[0] n = numpy.round(p / 0.005) fb = numpy.floor(n) hb = n % 2 lemma = reverse_vocabulary[int(t_w_i)] #print u"{:<5} {:7.6f} {:<20} ".format(i+1, float(p), lemma) + u"\u2588" * int(fb) + u"\u258C" * int(hb) results.append((lemma, p)) return results
def punctuator(input_file, vocab_file, punct_vocab_file, output_file, get_post): # Convert text to ids. (NOTE: fake outputs) vocabulary = load_vocabulary(vocab_file) punctuations = get_punctuations(punct_vocab_file) punct_vocab_reverse_map = utils.get_reverse_map(punctuations) inputs, outputs, lens = inference_sentences_to_ids(input_file, vocabulary, punctuations) # Get predicts if get_post: posteriors = get_predicts(inputs, outputs, lens, get_post) write_posteriors(input_file, posteriors, punct_vocab_reverse_map, output_file) return else: predicts = get_predicts(inputs, outputs, lens) # Write punctuations write_punctuations(input_file, predicts, punct_vocab_reverse_map, output_file)
output_file.write("%s %s" % (punctuation[:1], word)) else: output_file.write(" %s %s" % (punctuation, word)) else: word = token if __name__ == "__main__": if len(sys.argv) > 3: model_name = sys.argv[1] net = utils.load_model(model_name) net.batch_size = 1 net.reset_state() punctuation_reverse_map = utils.get_reverse_map(net.out_vocabulary) write_readable_text = bool(int(sys.argv[2])) output_file_path = sys.argv[3] if output_file_path == "-": output_file_path = sys.stdout if len(sys.argv) > 4: with open(sys.argv[4], 'r') as unpunctuated_file: unpunctuated_text = " ".join(unpunctuated_file.readlines()) else: unpunctuated_text = " ".join(sys.stdin.readlines()) write_punctuations(net, unpunctuated_text, output_file_path, punctuation_reverse_map, write_readable_text)
def evaluate(model_name, experiment_name, batch_size): MODEL_NAME = experiment_name repr_file = os.path.join(MODEL_PATH, 'confusionM_' + MODEL_NAME) description = model_builder.load_description(MODEL_PATH, MODEL_NAME) net = model_builder.build_model(model_name, description) net.load(MODEL_PATH, MODEL_NAME, description) n_roles = len(net.role_vocabulary) print(net.role_vocabulary) #Added () to print (team1-change) print("unk_word_id", net.unk_word_id) print("missing_word_id", net.missing_word_id) net.model.summary() print(net.model.metrics_names) #Added () to print (team1-change) reverse_role_vocabulary = utils.get_reverse_map(net.role_vocabulary) test_sample_size = config.OCT_TEST_SIZE test_steps = test_sample_size / batch_size # # DEBUG # test_steps = 10 print('Testing...') #Added () to print (team1-change) test_start = time.process_time() #Changed from time.clock() (team1-change) # Always use generator in Keras if re.search('NAME_WHICH_YOU_NEED_OLD_BATCHER', experiment_name): test_gen = get_minibatch(DATA_PATH + "NN_test", net.unk_word_id, net.unk_role_id, net.missing_word_id, n_roles, random=False, batch_size=batch_size) else: test_gen = generator(DATA_PATH + "NN_test", model_name, net.unk_word_id, net.unk_role_id, net.missing_word_id, n_roles, random=False, batch_size=batch_size) # Test the model test_result = net.model.evaluate_generator(generator=test_gen, steps=test_steps, max_q_size=1, workers=1, pickle_safe=False) print('test_result', test_result) # Compute confusion matrix metrics_names = net.model.metrics_names result_dict = {(x, 0) for x in metrics_names} batch_n = 0 confusionM = np.zeros((n_roles, n_roles), dtype='int32') ppl_role_list = dict() ppl_role = dict() result_list = [] for ([i_w, i_r, t_w, t_r], _) in generator(DATA_PATH + "NN_test", model_name, net.unk_word_id, net.unk_role_id, net.missing_word_id, n_roles, random=False, batch_size=batch_size): result_role = net.predict_role(i_w, i_r, t_w, t_r, batch_size) result_word_likelihood = net.predict(i_w, i_r, t_w, t_r, batch_size)[0] neg_log_likelihoods = -np.log(result_word_likelihood) for i, row in enumerate(neg_log_likelihoods, start=0): target_word = t_w[i][0] target_role = t_r[i][0] neg_log_likelihood = row[target_word] ppl_role_list.setdefault(target_role, []).append(neg_log_likelihood) for i, true_r in enumerate(t_r, start=0): confusionM[true_r, result_role[i]] += 1 if true_r == result_role[i]: result_list.append(1) batch_n += 1 print(batch_n) #Added () to print (team1-change) if batch_n >= test_steps: break for k, v in ppl_role_list.items(): neg_log_likelihood_role = np.mean(np.array(v)) ppl_role[k] = np.exp(neg_log_likelihood_role) print("Confusion Matrix: ") #Added () to print (team1-change) print(" A0, A1, LOC, TMP, MNR, V, <UNKNOWN>" ) #Added () to print (team1-change) print(confusionM) #Added () to print (team1-change) np.savetxt('confusionM_' + experiment_name + '.csv', confusionM, delimiter=',') np.savetxt('result_list_' + experiment_name + '.csv', result_list, delimiter=',') stats(net, confusionM) print("Loss(neg_log_likelihood) by role: " ) #Added () to print (team1-change) for r in ppl_role.keys(): print(reverse_role_vocabulary[r], np.log(ppl_role[r])) print("PPL by role: ") #Added () to print (team1-change) for r in ppl_role.keys(): print(reverse_role_vocabulary[r], ppl_role[r]) with open(repr_file, 'w') as f_out: f_out.write('[') for i in range(n_roles): f_out.write('[') for j in range(n_roles): f_out.write(str(confusionM[i][j]) + ", ") f_out.write('] \n') f_out.write(']') test_end = time.process_time() #Changed from time.clock() (team1-change) print('test time: %f, sps: %f' % (test_end - test_start, test_steps * batch_size / (test_end - test_start))) #Added () to print (team1-change)
def evaluate(model_name, experiment_name, test_name, batch_size, VR_SP_SRL=True, bootstrapping=False, majority_baseline=False): MODEL_NAME = experiment_name # repr_file = os.path.join(MODEL_PATH, 'confusionM_' + MODEL_NAME) description = model_builder.load_description(MODEL_PATH, MODEL_NAME) net = model_builder.build_model(model_name, description) net.load(MODEL_PATH, MODEL_NAME, description) n_roles = len(net.role_vocabulary) reverse_word_vocabulary = utils.get_reverse_map(net.word_vocabulary) reverse_role_vocabulary = utils.get_reverse_map(net.role_vocabulary) # net.set_0_bias() print net.role_vocabulary print("unk_word_id", net.unk_word_id) print("missing_word_id", net.missing_word_id) net.model.summary() # print net.model.metrics_names test_sample_size = 0 with open(EVAL_PATH + test_name, 'r') as lines: for l in lines: test_sample_size += 1 print(test_sample_size) test_steps = test_sample_size / float(batch_size) # test_steps = test_sample_size # # DEBUG # test_steps = 10 print 'Testing ' + test_name + ' ...' print 'VR_SP_SRL: ' + str(VR_SP_SRL) test_start = time.clock() # if re.search('NNRF_1e8', experiment_name) or re.search('MTRF_dev', experiment_name): # test_gen = get_minibatch(DATA_PATH + "NN_test", net.unk_word_id, net.unk_role_id, net.missing_word_id, # n_roles, random=False, batch_size=batch_size) # else: # test_gen = generator(DATA_PATH + "NN_test", model_name, net.unk_word_id, net.unk_role_id, net.missing_word_id, # n_roles, random=False, batch_size=batch_size) # # Test the model # test_result = net.model.evaluate_generator( # generator = test_gen, # steps = test_steps, # max_q_size = 1, # workers = 1, # pickle_safe = False # ) # print ('test_result', test_result) # Compute confusion matrix metrics_names = net.model.metrics_names result_dict = {(x, 0) for x in metrics_names} batch_n = 0 confusionM = np.zeros((n_roles, n_roles), dtype='int32') ppl_role_list = dict() result_list = [] output_list = [] for ([i_w, i_r, t_w, t_r], _) in data_gen(EVAL_PATH + test_name, model_name, net, batch_size, VR_SP_SRL=VR_SP_SRL): # zeros = np.zeros(t_r.shape) result_role = net.predict_role(i_w, i_r, t_w, t_r, batch_size) # word_emb, avg_emb, event_emb = net.avg_emb.predict([i_w, i_r, t_w, t_r], batch_size) # print word_emb.shape, avg_emb.shape, event_emb.shape # assert np.multiply(word_emb[0][0], avg_emb[0])[0] == event_emb[0][0][0] # assert np.multiply(word_emb[0][0], avg_emb[0])[1] == event_emb[0][0][1] # test role prediction of MTRF_dev, result: role prediction is useless # print i_r # print t_r.reshape(-1) # print result_role # result_word_likelihood = net.predict(i_w, i_r, t_w, t_r, batch_size)[0] # neg_log_likelihoods = -np.log(result_word_likelihood) # for i, row in enumerate(neg_log_likelihoods, start=0): # target_word = t_w[i][0] # target_role = t_r[i][0] # neg_log_likelihood = row[target_word] # ppl_role_list.setdefault(target_role, []).append(neg_log_likelihood) # print i_w, i_r, t_w, t_r for i, true_r in enumerate(t_r, start=0): # if reverse_role_vocabulary.get(t_r[0][0], '<unknown>') == 'AM-LOC': # print ("input words", [reverse_word_vocabulary.get(w, '<unknown>') for w in i_w[0]]) # print ("input roles", [reverse_role_vocabulary.get(r, '<unknown>') for r in i_r[0]]) # print ("target word", [reverse_word_vocabulary.get(w, '<unknown>') for w in t_w[0]]) # print ("target role", [reverse_role_vocabulary.get(r, '<unknown>') for r in t_r[0]]) # print ("predicted role", [reverse_role_vocabulary.get(result_role[i], '<unknown>') for r in t_r[0]]) # print '' confusionM[true_r, result_role[i]] += 1 if true_r == result_role[i]: result_list.append(1) output_list.append((true_r, result_role[i])) batch_n += 1 if batch_n % 100 == 0: print(batch_n) if batch_n >= test_steps: break # ppl_role = dict() # for k, v in ppl_role_list.items(): # neg_log_likelihood_role = np.mean(np.array(v)) # ppl_role[k] = np.exp(neg_log_likelihood_role) # obtain ZeroR baseline print confusionM majority = 1 if majority_baseline == True: for i in range(7): confusionM[i][majority] = confusionM[i][:].sum() confusionM[i][majority - 1] = 0 confusionM[i][majority + 1:] = 0 print confusionM dir_P, dir_R, dir_F1, precision, recall, F1 = stats(net, confusionM) print "Dir: %.2f \t %.2f \t %.2f" % (dir_P, dir_R, dir_F1) # np.savetxt('confusionM_' + experiment_name + '.' + test_name.strip('.dat') + '.csv', confusionM, delimiter = ',') # np.savetxt('output_' + experiment_name + '.' + test_name.strip('.dat') + '.csv', output_list, delimiter = ',') # with open(repr_file, 'w') as f_out: # f_out.write('[') # for i in range(n_roles): # f_out.write('[') # for j in range(n_roles): # f_out.write(str(confusionM[i][j]) + ", ") # f_out.write('] \n') # f_out.write(']') # print "Loss(neg_log_likelihood) by role: " # for r in ppl_role.keys(): # print (reverse_role_vocabulary[r], np.log(ppl_role[r])) print("Result by role: ") for r in range(len(precision)): print('%s: \t %.2f \t %.2f \t %.2f' % (reverse_role_vocabulary[r], precision[r], recall[r], F1[r])) test_end = time.clock() print 'test time: %f, sps: %f' % (test_end - test_start, test_steps * batch_size / (test_end - test_start)) if bootstrapping: P_mean, P_std, R_mean, R_std, F1_mean, F1_std = bootstrap( experiment_name, test_name, net, n_roles, output_list=output_list) return P_mean, P_std, R_mean, R_std, F1_mean, F1_std
def query(model_name, experiment_name, inputs, target): MODEL_NAME = experiment_name description = model_builder.load_description(MODEL_PATH, MODEL_NAME) net = model_builder.build_model(model_name, description) net.load(MODEL_PATH, MODEL_NAME, description) # net.model.summary() # print net.model.get_layer(name="embedding_2").get_weights()[0] print net.role_vocabulary print("unk_word_id", net.unk_word_id) print("missing_word_id", net.missing_word_id) # net.set_0_bias() net.model.summary() propbank_map = { "subj" : "A0", "obj" : "A1", "ARG0" : "A0", "ARG1" : "A1", "ARG2" : "A2", } # tr_map = { # "A0": numpy.asarray([[net.role_vocabulary["A0"]]], dtype=numpy.int64), # "A1": numpy.asarray([[net.role_vocabulary["A1"]]], dtype=numpy.int64), # "A2": numpy.asarray([[net.role_vocabulary["<UNKNOWN>"]]], dtype=numpy.int64) # } # net.word_vocabulary["<NOTHING>"] = net.missing_word_id # net.role_vocabulary["<UNKNOWN>"] = net.unk_role_id reverse_vocabulary = utils.get_reverse_map(net.word_vocabulary) reverse_role_vocabulary = utils.get_reverse_map(net.role_vocabulary) print reverse_role_vocabulary raw_words = dict((reverse_role_vocabulary[r], reverse_vocabulary[net.missing_word_id]) for r in net.role_vocabulary.values()) # print raw_words raw_words.update(inputs) # print raw_words # print len(raw_words) assert len(raw_words) == len(net.role_vocabulary) # print repr(raw_words) # n = int(sys.argv[3]) t_r = [net.role_vocabulary.get(r, net.unk_role_id) for r in target.keys()] t_w = [net.word_vocabulary.get(w, net.unk_word_id) for w in target.values()] input_roles_words = {} for r, w in raw_words.items(): input_roles_words[net.role_vocabulary[r]] = utils.input_word_index(net.word_vocabulary, w, net.unk_word_id, warn_unk=True) print input_roles_words, t_r input_roles_words.pop(t_r[0]) # default_roles_words = dict((r, net.missing_word_id) for r in (net.role_vocabulary.values())) # default_roles_words.update(input_roles_words) # input_roles_words = default_roles_words x_w_i = numpy.asarray([input_roles_words.values()], dtype=numpy.int64) x_r_i = numpy.asarray([input_roles_words.keys()], dtype=numpy.int64) y_w_i = numpy.asarray(t_w, dtype=numpy.int64) y_r_i = numpy.asarray(t_r, dtype=numpy.int64) topN=20 predicted_word_indices = net.top_words(x_w_i, x_r_i, y_w_i, y_r_i, topN) # print predicted_word_indices # print len(predicted_word_indices) print(x_w_i, x_r_i, y_w_i, y_r_i) p_w = net.p_words(x_w_i, x_r_i, y_w_i, y_r_i, batch_size=1, verbose=0)[0] print ('p_t_w: ', p_w) resultlist = predicted_word_indices # print resultlist for i, t_w_i in enumerate(resultlist): t_w = net.word_vocabulary.get(t_w_i, net.unk_word_id) y_w_i = numpy.asarray([t_w_i], dtype=numpy.int64) p = net.p_words(x_w_i, x_r_i, y_w_i, y_r_i, batch_size=1, verbose=0)[0] n = numpy.round(p / 0.005) fb = numpy.floor(n) hb = n % 2 print u"{:<5} {:7.6f} {:<20} ".format(i+1, float(p), reverse_vocabulary[int(t_w_i)]) + u"\u2588" * int(fb) + u"\u258C" * int(hb)
if punctuation.strip() == "": sys.stdout.write("%s%s%s" % (punctuation, tagstring, word)) else: sys.stdout.write("%s %s%s" % (punctuation[:1], tagstring, word)) first_word = False else: if is_word(token): word = token else: tags.append(token) sys.stdout.write("\n") sys.stdout.flush() if __name__ == "__main__": assert len(sys.argv) > 1, "Give model path as first argument" model_path = sys.argv[1] net = utils.load_model(model_path) net.batch_size = 1 punctuation_reverse_map = utils.get_reverse_map(net.out_vocabulary) for line in iter(sys.stdin.readline, ""): net.reset_state() write_punctuations(net, punctuation_reverse_map, line)
def pd_themfit(model_name, experiment_name, df, predict_role='V', input_roles='all_available_args', function='filler_prob', n=5, debug=False): """ Adds a column to a pandas df with a role filler probability. For each row in the pandas df, calculates the probability that a particular role filler will fill a particular role, given a set of input roles and fillers (from that row). Keyword arguments: model_name -- The name of the model experiment_name -- The name of the model plus the name of the experiment, separated by '_' df -- The pandas dataframe. Must include columns for all propbank labels in predict_role and input_roles predict_role -- the target role (in propbank labels) for which the filler will be predicted (default: 'V') input_roles -- the set of roles (in propbank labels) that should be used as inputs (default: 'all_args') """ possible_roles = set( ['A0', 'A1', 'AM-LOC', 'AM-TMP', 'AM-MNR', '<UNKNOWN>', 'V']) try: assert predict_role in df.columns assert predict_role in possible_roles if input_roles != 'all_available_args': for r in input_roles: assert r in df.columns assert r in possible_roles except: print("NOT ALL ROLES ARE AVAILABLE AS DF COLUMNS") MODEL_NAME = experiment_name description = model_builder.load_description(MODEL_PATH, MODEL_NAME) net = model_builder.build_model(model_name, description) net.load(MODEL_PATH, MODEL_NAME, description) bias = net.set_0_bias() # net.model.summary() # print net.model.get_layer(name="embedding_2").get_weights()[0] # If no <UNKNOWN> role in the role vocabulary, add it. if net.role_vocabulary.get("<UNKNOWN>", -1) == -1: net.role_vocabulary["<UNKNOWN>"] = len(net.role_vocabulary) - 1 print("Role vocabulary", net.role_vocabulary) print("unk_word_id", net.unk_word_id) print("missing_word_id", net.missing_word_id) reverse_vocabulary = utils.get_reverse_map(net.word_vocabulary) reverse_role_vocabulary = utils.get_reverse_map(net.role_vocabulary) print("Reverse role vocabulary", reverse_role_vocabulary) raw_words = dict( (reverse_role_vocabulary[r], reverse_vocabulary[net.missing_word_id]) for r in net.role_vocabulary.values()) if input_roles == 'all_available_args': possible_roles.remove(predict_role) input_roles = possible_roles.intersection(set(df.columns)) all_roles = input_roles all_roles.add(predict_role) df = df.apply( lambda x: process_row(predict_role=predict_role, role_fillers={i: x[i] for i in all_roles}, model=net, raw_word_list=raw_words, function=function, n=n, debug=debug), axis=1) return df