def main(): parser = argparse.ArgumentParser() parser.add_argument('-task', required=True) parser.add_argument('-model', required=True) parser.add_argument('-eval_step', type=int, default=10) parser.add_argument('-epoch', type=int, default=400) parser.add_argument('-d_word_vec', type=int, default=300) parser.add_argument('-batch_size', type=int, default=100) parser.add_argument('-save_model', default=None) parser.add_argument('-save_mode', type=str, choices=['all', 'best'], default='best') parser.add_argument('-no_cuda', action='store_true') parser.add_argument('-lr', type=float, default=0.001) parser.add_argument('-n_bins', type=float, default=21) opt = parser.parse_args() opt.cuda = not opt.no_cuda opt.mu = kernal_mus(opt.n_bins) opt.sigma = kernel_sigmas(opt.n_bins) print opt # ========= Preparing DataLoader =========# if opt.task == "wikiqa": train_filename = "./data/wikiqa/wiki_train_pair.pkl" test_filename = "./data/wikiqa/wiki_test.pkl" dev_filename = "./data/wikiqa/wiki_dev.pkl" train_data = pickle.load(open(train_filename, 'r')) test_data = pickle.load(open(test_filename, 'r')) dev_data = pickle.load(open(dev_filename, 'r')) weights = np.load("./data/wikiqa/embed.txt") else: raise ("Not implement!") train_data = Dataloader(data = train_data, opt = opt, shuffle=True) test_data = DataloaderTest(data = test_data, opt = opt) dev_data = DataloaderTest(data = dev_data, opt = opt) if opt.model == "knrm": model = KNRM.knrm(opt, weights) else: raise ("No such model!") crit = nn.MarginRankingLoss(margin=1, size_average=True) if opt.cuda: model = model.cuda() crit = crit.cuda() optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr) train(model, opt, crit, optimizer, train_data, dev_data, test_data)
train = optimizer.minimize(loss) #estimator acc pred_number = tf.argmax(y_, 1) correct_pred = tf.equal(tf.argmax(y_, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) #init init = tf.global_variables_initializer() sess = tf.Session() sess.run(init) #data IO dl = Dataloader(epoch=4, total_num=50000) valid_x, valid_y = dl.get_valid_sample(num=5000) train_x, train_y = dl.get_train_sample() train_acc = [] test_acc = [] loss_set = [] while (dl.epoch > 0): #batch from dataloader batch_x, batch_y = dl.batch_train_sample(batch_size=batch_size) #run traning step if (dl.epoch > 0): sess.run(train, feed_dict={
class HMM(object): def __init__(self, config): self.dataLoader = Dataloader(config) self.file_pathtype = config.file_pathtype self.smoothing_d = config.smoothing_d def viterbi(self, states, word_states_prob, emissions_prob, transitions_prob, sentence): probability = [{}] path = {} # The first state for state in states: if (sentence[0] in emissions_prob[state]) and ( state in transitions_prob['START']): probability[0][state] = np.log( transitions_prob['START'][state]) + np.log( emissions_prob[state][sentence[0]]) elif (state in transitions_prob['START']): probability[0][state] = np.log( transitions_prob['START'][state]) + np.log( emissions_prob[state]['TBD']) else: probability[0][state] = np.log( transitions_prob['START']['TBD']) + np.log( emissions_prob[state]['TBD']) path[state] = [state] #other words in sentense for index in range(1, len(sentence)): update_path = {} probability.append({}) for state in states: trans_prob = {} for pri_state in states: if state in transitions_prob[pri_state] and ( sentence[index] in emissions_prob[state]): trans_prob[pri_state] = probability[ index - 1][pri_state] + np.log( transitions_prob[pri_state][state]) + np.log( emissions_prob[state][sentence[index]]) elif state in transitions_prob[pri_state]: trans_prob[pri_state] = probability[ index - 1][pri_state] + np.log( transitions_prob[pri_state][state]) + np.log( emissions_prob[state]['TBD']) elif (sentence[index] in emissions_prob[state]): trans_prob[pri_state] = probability[ index - 1][pri_state] + np.log( transitions_prob[pri_state]['TBD']) + np.log( emissions_prob[state][sentence[index]]) else: trans_prob[pri_state] = probability[ index - 1][pri_state] + np.log( transitions_prob[pri_state]['TBD']) + np.log( emissions_prob[state]['TBD']) # find the best path in state t-1 max_key = max(trans_prob, key=trans_prob.get) max_prob = trans_prob[max_key] probability[index][state] = max_prob update_path[state] = path[max_key] + [state] # update the path path = update_path probability.append({}) # consider for the last word in sentence for state in states: probability[len(sentence)][state] = probability[len( sentence) - 1][state] + np.log(transitions_prob[state]['END']) key = max(probability[-1], key=probability[-1].get) return path[key] def evalution(self): f = codecs.open(self.file_pathtype + '_test.txt', 'r', 'utf8') text = f.readlines() f.close() len_text = len(text) states, word_states_prob, emissions_prob, transitions_prob = self.dataLoader.Load( self.smoothing_d, mode='train', ) sentence = [] features = [] perdict = [] all_sentence = [] count = 0 for words in text: if words.strip(): word = words.strip().split() all_sentence.append(word[0]) sentence.append(word[0]) features.append(word[1]) count += 1 else: # process the sentence perdict += self.viterbi(states, word_states_prob, emissions_prob, transitions_prob, sentence) sentence = [] f = codecs.open(self.file_pathtype + '_result.txt', 'w', 'utf8') for i in range(len(perdict)): wri = all_sentence[i] + '\t' + features[i] + '\t' + perdict[ i] + '\n' f.write(wri) f.close() # Evaluation TP, FP, TN, FN, type_correct, sum = 0, 0, 0, 0, 0, 0 sumlen = len(perdict) for i in range(sumlen): if features[i] != 'O' and perdict[i] != 'O': TP += 1 if features[i] == perdict[i]: type_correct += 1 if features[i] != 'O' and perdict[i] == 'O': FN += 1 if features[i] == 'O' and perdict[i] != 'O': FP += 1 if features[i] == 'O' and perdict[i] == 'O': TN += 1 recall = TP / (TP + FN) precision = TP / (TP + FP) accuracy = (TP + TN) / sumlen F1 = 2 * precision * recall / (precision + recall) print('=====' + self.file_pathtype + ' labeling result=====') print("type_correct: ", round(type_correct / TP, 4)) print("accuracy: ", round(accuracy, 4)) print("precision: ", round(precision, 4)) print("recall: ", round(recall, 4)) print("F1: ", round(F1, 4))
def __init__(self, config): self.dataLoader = Dataloader(config) self.file_pathtype = config.file_pathtype self.smoothing_d = config.smoothing_d
class HMM(object): def __init__(self, config): self.dataLoader = Dataloader(config) self.file_pathtype = config.file_pathtype self.smoothing_d = config.smoothing_d def viterbi(self, states, word_states_prob, transitions_triprob, transitions_bigprob, emissions_prob, sentence): probability = [{}] path = {} # The first state for state in states: if (sentence[0] in emissions_prob[state]) and ( state in transitions_bigprob['START']): probability[0][state] = np.log( transitions_bigprob['START'][state]) + np.log( emissions_prob[state][sentence[0]]) elif (state in transitions_bigprob['START']): probability[0][state] = np.log( transitions_bigprob['START'][state]) + np.log( emissions_prob[state]['TBD']) else: probability[0][state] = np.log( transitions_bigprob['START']['TBD']) + np.log( emissions_prob[state]['TBD']) path[state] = [state] # The second state (use the Trigram and insert a string "START" as the start) index = 1 update_path = {} probability.append({}) for state in states: trans_prob = {} for pri_state in states: if state in transitions_bigprob[pri_state] and ( sentence[index] in emissions_prob[state]): trans_prob[pri_state] = probability[ index - 1][pri_state] + np.log( transitions_bigprob[pri_state][state]) + np.log( emissions_prob[state][sentence[index]]) elif state in transitions_bigprob[pri_state]: trans_prob[pri_state] = probability[ index - 1][pri_state] + np.log( transitions_bigprob[pri_state][state]) + np.log( emissions_prob[state]['TBD']) elif (sentence[index] in emissions_prob[state]): trans_prob[pri_state] = probability[ index - 1][pri_state] + np.log( transitions_bigprob[pri_state]['TBD']) + np.log( emissions_prob[state][sentence[index]]) else: trans_prob[pri_state] = probability[ index - 1][pri_state] + np.log( transitions_bigprob[pri_state]['TBD']) + np.log( emissions_prob[state]['TBD']) max_key = max(trans_prob, key=trans_prob.get) max_prob = trans_prob[max_key] probability[index][state] = max_prob update_path[state] = path[max_key] + [state] path = update_path # Other word in sentence for index in range(2, len(sentence)): update_path = {} probability.append({}) for state in states: trans_prob = {} for pri_state in states: # The index for trigram transition model pri2_state = ' '.join(path[pri_state][-2:]) if pri2_state in transitions_triprob: if state in transitions_triprob[pri2_state] and ( sentence[index] in emissions_prob[state]): trans_prob[pri_state] = probability[ index - 1][pri_state] + np.log( transitions_triprob[pri2_state] [state]) + np.log( emissions_prob[state][sentence[index]]) elif state in transitions_triprob[pri2_state]: trans_prob[pri_state] = probability[ index - 1][pri_state] + np.log( transitions_triprob[pri2_state][state] ) + np.log(emissions_prob[state]['TBD']) elif (sentence[index] in emissions_prob[state]): trans_prob[pri_state] = probability[ index - 1][pri_state] + np.log( transitions_triprob[pri2_state] ['TBD']) + np.log( emissions_prob[state][sentence[index]]) else: trans_prob[pri_state] = probability[ index - 1][pri_state] + np.log( transitions_triprob[pri2_state]['TBD'] ) + np.log(emissions_prob[state]['TBD']) else: if (sentence[index] in emissions_prob[state]): trans_prob[pri_state] = probability[ index - 1][pri_state] - 10 + np.log( emissions_prob[state][sentence[index]]) else: trans_prob[pri_state] = probability[ index - 1][pri_state] - 10 + np.log( emissions_prob[state]['TBD']) # Find the best path max_key = max(trans_prob, key=trans_prob.get) max_prob = trans_prob[max_key] probability[index][state] = max_prob update_path[state] = path[max_key] + [state] # Update the path path = update_path # The last word probability.append({}) for state in states: pri2_state = ' '.join(path[state][-2:]) if pri2_state in transitions_triprob: if ('END' in transitions_triprob[pri2_state]): probability[len(sentence)][state] = probability[ len(sentence) - 1][state] + np.log( transitions_triprob[pri2_state]['END']) else: probability[len(sentence)][state] = probability[ len(sentence) - 1][state] + np.log( transitions_triprob[pri2_state]['TBD']) else: probability[len(sentence) - 1][state] -= 10 key = max(probability[-1], key=probability[-1].get) # Return the predict feature of the sentence return path[key] def evalution(self): f = codecs.open(self.file_pathtype + '_test.txt', 'r', 'utf8') text = f.readlines() f.close() # Initalize the probability states, word_states_prob, transitions_triprob, transitions_bigprob, emissions_prob = self.dataLoader.LoadforTrigram( self.smoothing_d, mode='train', ) sentence = [] features = [] perdict = [] all_sentence = [] count = 0 for words in text: if words.strip(): word = words.strip().split() all_sentence.append(word[0]) sentence.append(word[0]) features.append(word[1]) count += 1 else: perdict += self.viterbi(states, word_states_prob, transitions_triprob, transitions_bigprob, emissions_prob, sentence) sentence = [] # Read the data f = codecs.open(self.file_pathtype + '_Trigram_result.txt', 'w', 'utf8') for i in range(len(perdict)): wri = all_sentence[i] + '\t' + features[i] + '\t' + perdict[ i] + '\n' f.write(wri) f.close() TP, FP, TN, FN, type_correct, sum = 0, 0, 0, 0, 0, 0 sumlen = len(perdict) for i in range(sumlen): if features[i] != 'O' and perdict[i] != 'O': TP += 1 if features[i] == perdict[i]: type_correct += 1 if features[i] != 'O' and perdict[i] == 'O': FN += 1 if features[i] == 'O' and perdict[i] != 'O': FP += 1 if features[i] == 'O' and perdict[i] == 'O': TN += 1 recall = TP / (TP + FN) precision = TP / (TP + FP) accuracy = (TP + TN) / sumlen F1 = 2 * precision * recall / (precision + recall) print('=====' + self.file_pathtype + ' labeling result=====') print("type_correct: ", round(type_correct / TP, 4)) print("accuracy: ", round(accuracy, 4)) print("precision: ", round(precision, 4)) print("recall: ", round(recall, 4)) print("F1: ", round(F1, 4))
init = tf.global_variables_initializer() sess = tf.Session() sess.run(init) # paths = '..\\dataset\\training.h5' indces = np.array(down_sampling(paths)) #transform to one dim one_indces = indces.reshape(-1) one_indces = list(one_indces) one_indces.sort() #data IO dl = Dataloader(confidence=list(one_indces), epoch=5, total_num=len(one_indces)) valid_x, valid_y = dl.get_valid_sample(num=1000) train_acc = [] test_acc = [] loss_set = [] while (dl.epoch > 0): #batch from dataloader batch_x, batch_y = dl.batch_train_sample(batch_size=batch_size) #run traning step if (dl.epoch > 0): sess.run(train, feed_dict={
def train(self, num_epochs=10): data = Dataloader(batchsize=self.batch_size) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) train_X, _ = data.getTrainData() num_tr_iter = int(train_X.shape[0] / self.batch_size) global_step = 0 Loss = [] Acc = [] for epoch in range(num_epochs): print("-------------------------------") print("Training epoch: {}".format(epoch + 1)) data.i = 0 acc = np.zeros(num_tr_iter) loss = np.zeros(num_tr_iter) for iteration in range(num_tr_iter): global_step += 1 x_batch, y_batch = data.nextBatch() # Run optimization op (backprop) feed_dict_batch = { self.Model.inputs: x_batch, self.Model.labels: y_batch, } sess.run(self.Model.train_op, feed_dict=feed_dict_batch) if iteration % 3 == 0: # Calculate and display the batch loss and accuracy loss_batch, acc_batch = sess.run( [self.Model.loss, self.Model.accuracy], feed_dict=feed_dict_batch, ) print( "iter {0:3d}:\t Loss={1:.2f},\tTraining Accuracy={2:.01%}" .format(iteration, loss_batch, acc_batch)) acc[iteration] = acc_batch loss[iteration] = loss_batch x_valid, y_valid = data.getValiData() val_loss, val_acc = sess.run( [self.Model.loss, self.Model.accuracy], feed_dict={ self.Model.inputs: x_valid, self.Model.labels: y_valid }, ) print( "Epoch: {0}, validation loss: {1:.2f}, validation accuracy: {2:.01%}" .format(epoch + 1, val_loss, val_acc)) acc, loss = np.mean(acc), np.mean(loss) Loss.append(loss) Acc.append(acc) x_test, y_test = data.getTestData() feed_dict_test = { self.Model.inputs: x_test, self.Model.labels: y_test } test_loss, test_acc = sess.run( [self.Model.loss, self.Model.accuracy], feed_dict=feed_dict_test) print("--------------------") print("Test loss: {0:.2f}, test accuracy: {1:.01%}".format( test_loss, test_acc)) return Loss, Acc
import numpy as np import tensorflow as tf from DataLoader import Dataloader data = Dataloader(batchsize=32) train_X, train_Y = data.getTrainData() test_X, test_Y = data.getTestData() valid_X, valid_Y = data.getValiData() label_num = train_Y.shape[0] * train_Y.shape[1] + test_Y.shape[ 0] * test_Y.shape[1] + valid_Y.shape[0] * valid_Y.shape[1] label_1_num = np.sum(train_Y) + np.sum(test_Y) + np.sum(valid_Y) pass
# -*- coding: utf-8 -*- """ Created on Fri Nov 30 14:04:36 2018 @author: freeze """ from DataLoader import Dataloader dl = Dataloader(total_num=320) train_x, train_y = dl.get_train_sample() while (dl.epoch > 0): batch_x, batch_y = dl.batch_train_sample() print("epoch %d , random_list %d " % (dl.epoch, len(dl.random_list)))