def main():
parser = argparse.ArgumentParser()
parser.add_argument('-task', required=True)
parser.add_argument('-model', required=True)
parser.add_argument('-eval_step', type=int, default=10)
parser.add_argument('-epoch', type=int, default=400)
parser.add_argument('-d_word_vec', type=int, default=300)
parser.add_argument('-batch_size', type=int, default=100)
parser.add_argument('-save_model', default=None)
parser.add_argument('-save_mode', type=str, choices=['all', 'best'], default='best')
parser.add_argument('-no_cuda', action='store_true')
parser.add_argument('-lr', type=float, default=0.001)
parser.add_argument('-n_bins', type=float, default=21)
opt = parser.parse_args()
opt.cuda = not opt.no_cuda
opt.mu = kernal_mus(opt.n_bins)
opt.sigma = kernel_sigmas(opt.n_bins)
print opt
# ========= Preparing DataLoader =========#
if opt.task == "wikiqa":
train_filename = "./data/wikiqa/wiki_train_pair.pkl"
test_filename = "./data/wikiqa/wiki_test.pkl"
dev_filename = "./data/wikiqa/wiki_dev.pkl"
train_data = pickle.load(open(train_filename, 'r'))
test_data = pickle.load(open(test_filename, 'r'))
dev_data = pickle.load(open(dev_filename, 'r'))
weights = np.load("./data/wikiqa/embed.txt")
else:
raise ("Not implement!")
train_data = Dataloader(data = train_data, opt = opt, shuffle=True)
test_data = DataloaderTest(data = test_data, opt = opt)
dev_data = DataloaderTest(data = dev_data, opt = opt)
if opt.model == "knrm":
model = KNRM.knrm(opt, weights)
else:
raise ("No such model!")
crit = nn.MarginRankingLoss(margin=1, size_average=True)
if opt.cuda:
model = model.cuda()
crit = crit.cuda()
optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr)
train(model, opt, crit, optimizer, train_data, dev_data, test_data)
train = optimizer.minimize(loss)
#estimator acc
pred_number = tf.argmax(y_, 1)
correct_pred = tf.equal(tf.argmax(y_, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
#init
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
#data IO
dl = Dataloader(epoch=4, total_num=50000)
valid_x, valid_y = dl.get_valid_sample(num=5000)
train_x, train_y = dl.get_train_sample()
train_acc = []
test_acc = []
loss_set = []
while (dl.epoch > 0):
#batch from dataloader
batch_x, batch_y = dl.batch_train_sample(batch_size=batch_size)
#run traning step
if (dl.epoch > 0):
sess.run(train,
feed_dict={
class HMM(object):
def __init__(self, config):
self.dataLoader = Dataloader(config)
self.file_pathtype = config.file_pathtype
self.smoothing_d = config.smoothing_d
def viterbi(self, states, word_states_prob, emissions_prob,
transitions_prob, sentence):
probability = [{}]
path = {}
# The first state
for state in states:
if (sentence[0] in emissions_prob[state]) and (
state in transitions_prob['START']):
probability[0][state] = np.log(
transitions_prob['START'][state]) + np.log(
emissions_prob[state][sentence[0]])
elif (state in transitions_prob['START']):
probability[0][state] = np.log(
transitions_prob['START'][state]) + np.log(
emissions_prob[state]['TBD'])
else:
probability[0][state] = np.log(
transitions_prob['START']['TBD']) + np.log(
emissions_prob[state]['TBD'])
path[state] = [state]
#other words in sentense
for index in range(1, len(sentence)):
update_path = {}
probability.append({})
for state in states:
trans_prob = {}
for pri_state in states:
if state in transitions_prob[pri_state] and (
sentence[index] in emissions_prob[state]):
trans_prob[pri_state] = probability[
index - 1][pri_state] + np.log(
transitions_prob[pri_state][state]) + np.log(
emissions_prob[state][sentence[index]])
elif state in transitions_prob[pri_state]:
trans_prob[pri_state] = probability[
index - 1][pri_state] + np.log(
transitions_prob[pri_state][state]) + np.log(
emissions_prob[state]['TBD'])
elif (sentence[index] in emissions_prob[state]):
trans_prob[pri_state] = probability[
index - 1][pri_state] + np.log(
transitions_prob[pri_state]['TBD']) + np.log(
emissions_prob[state][sentence[index]])
else:
trans_prob[pri_state] = probability[
index - 1][pri_state] + np.log(
transitions_prob[pri_state]['TBD']) + np.log(
emissions_prob[state]['TBD'])
# find the best path in state t-1
max_key = max(trans_prob, key=trans_prob.get)
max_prob = trans_prob[max_key]
probability[index][state] = max_prob
update_path[state] = path[max_key] + [state]
# update the path
path = update_path
probability.append({})
# consider for the last word in sentence
for state in states:
probability[len(sentence)][state] = probability[len(
sentence) - 1][state] + np.log(transitions_prob[state]['END'])
key = max(probability[-1], key=probability[-1].get)
return path[key]
def evalution(self):
f = codecs.open(self.file_pathtype + '_test.txt', 'r', 'utf8')
text = f.readlines()
f.close()
len_text = len(text)
states, word_states_prob, emissions_prob, transitions_prob = self.dataLoader.Load(
self.smoothing_d,
mode='train',
)
sentence = []
features = []
perdict = []
all_sentence = []
count = 0
for words in text:
if words.strip():
word = words.strip().split()
all_sentence.append(word[0])
sentence.append(word[0])
features.append(word[1])
count += 1
else:
# process the sentence
perdict += self.viterbi(states, word_states_prob,
emissions_prob, transitions_prob,
sentence)
sentence = []
f = codecs.open(self.file_pathtype + '_result.txt', 'w', 'utf8')
for i in range(len(perdict)):
wri = all_sentence[i] + '\t' + features[i] + '\t' + perdict[
i] + '\n'
f.write(wri)
f.close()
# Evaluation
TP, FP, TN, FN, type_correct, sum = 0, 0, 0, 0, 0, 0
sumlen = len(perdict)
for i in range(sumlen):
if features[i] != 'O' and perdict[i] != 'O':
TP += 1
if features[i] == perdict[i]:
type_correct += 1
if features[i] != 'O' and perdict[i] == 'O':
FN += 1
if features[i] == 'O' and perdict[i] != 'O':
FP += 1
if features[i] == 'O' and perdict[i] == 'O':
TN += 1
recall = TP / (TP + FN)
precision = TP / (TP + FP)
accuracy = (TP + TN) / sumlen
F1 = 2 * precision * recall / (precision + recall)
print('=====' + self.file_pathtype + ' labeling result=====')
print("type_correct: ", round(type_correct / TP, 4))
print("accuracy: ", round(accuracy, 4))
print("precision: ", round(precision, 4))
print("recall: ", round(recall, 4))
print("F1: ", round(F1, 4))
def __init__(self, config):
self.dataLoader = Dataloader(config)
self.file_pathtype = config.file_pathtype
self.smoothing_d = config.smoothing_d
class HMM(object):
def __init__(self, config):
self.dataLoader = Dataloader(config)
self.file_pathtype = config.file_pathtype
self.smoothing_d = config.smoothing_d
def viterbi(self, states, word_states_prob, transitions_triprob,
transitions_bigprob, emissions_prob, sentence):
probability = [{}]
path = {}
# The first state
for state in states:
if (sentence[0] in emissions_prob[state]) and (
state in transitions_bigprob['START']):
probability[0][state] = np.log(
transitions_bigprob['START'][state]) + np.log(
emissions_prob[state][sentence[0]])
elif (state in transitions_bigprob['START']):
probability[0][state] = np.log(
transitions_bigprob['START'][state]) + np.log(
emissions_prob[state]['TBD'])
else:
probability[0][state] = np.log(
transitions_bigprob['START']['TBD']) + np.log(
emissions_prob[state]['TBD'])
path[state] = [state]
# The second state (use the Trigram and insert a string "START" as the start)
index = 1
update_path = {}
probability.append({})
for state in states:
trans_prob = {}
for pri_state in states:
if state in transitions_bigprob[pri_state] and (
sentence[index] in emissions_prob[state]):
trans_prob[pri_state] = probability[
index - 1][pri_state] + np.log(
transitions_bigprob[pri_state][state]) + np.log(
emissions_prob[state][sentence[index]])
elif state in transitions_bigprob[pri_state]:
trans_prob[pri_state] = probability[
index - 1][pri_state] + np.log(
transitions_bigprob[pri_state][state]) + np.log(
emissions_prob[state]['TBD'])
elif (sentence[index] in emissions_prob[state]):
trans_prob[pri_state] = probability[
index - 1][pri_state] + np.log(
transitions_bigprob[pri_state]['TBD']) + np.log(
emissions_prob[state][sentence[index]])
else:
trans_prob[pri_state] = probability[
index - 1][pri_state] + np.log(
transitions_bigprob[pri_state]['TBD']) + np.log(
emissions_prob[state]['TBD'])
max_key = max(trans_prob, key=trans_prob.get)
max_prob = trans_prob[max_key]
probability[index][state] = max_prob
update_path[state] = path[max_key] + [state]
path = update_path
# Other word in sentence
for index in range(2, len(sentence)):
update_path = {}
probability.append({})
for state in states:
trans_prob = {}
for pri_state in states:
# The index for trigram transition model
pri2_state = ' '.join(path[pri_state][-2:])
if pri2_state in transitions_triprob:
if state in transitions_triprob[pri2_state] and (
sentence[index] in emissions_prob[state]):
trans_prob[pri_state] = probability[
index - 1][pri_state] + np.log(
transitions_triprob[pri2_state]
[state]) + np.log(
emissions_prob[state][sentence[index]])
elif state in transitions_triprob[pri2_state]:
trans_prob[pri_state] = probability[
index - 1][pri_state] + np.log(
transitions_triprob[pri2_state][state]
) + np.log(emissions_prob[state]['TBD'])
elif (sentence[index] in emissions_prob[state]):
trans_prob[pri_state] = probability[
index - 1][pri_state] + np.log(
transitions_triprob[pri2_state]
['TBD']) + np.log(
emissions_prob[state][sentence[index]])
else:
trans_prob[pri_state] = probability[
index - 1][pri_state] + np.log(
transitions_triprob[pri2_state]['TBD']
) + np.log(emissions_prob[state]['TBD'])
else:
if (sentence[index] in emissions_prob[state]):
trans_prob[pri_state] = probability[
index - 1][pri_state] - 10 + np.log(
emissions_prob[state][sentence[index]])
else:
trans_prob[pri_state] = probability[
index - 1][pri_state] - 10 + np.log(
emissions_prob[state]['TBD'])
# Find the best path
max_key = max(trans_prob, key=trans_prob.get)
max_prob = trans_prob[max_key]
probability[index][state] = max_prob
update_path[state] = path[max_key] + [state]
# Update the path
path = update_path
# The last word
probability.append({})
for state in states:
pri2_state = ' '.join(path[state][-2:])
if pri2_state in transitions_triprob:
if ('END' in transitions_triprob[pri2_state]):
probability[len(sentence)][state] = probability[
len(sentence) - 1][state] + np.log(
transitions_triprob[pri2_state]['END'])
else:
probability[len(sentence)][state] = probability[
len(sentence) - 1][state] + np.log(
transitions_triprob[pri2_state]['TBD'])
else:
probability[len(sentence) - 1][state] -= 10
key = max(probability[-1], key=probability[-1].get)
# Return the predict feature of the sentence
return path[key]
def evalution(self):
f = codecs.open(self.file_pathtype + '_test.txt', 'r', 'utf8')
text = f.readlines()
f.close()
# Initalize the probability
states, word_states_prob, transitions_triprob, transitions_bigprob, emissions_prob = self.dataLoader.LoadforTrigram(
self.smoothing_d,
mode='train',
)
sentence = []
features = []
perdict = []
all_sentence = []
count = 0
for words in text:
if words.strip():
word = words.strip().split()
all_sentence.append(word[0])
sentence.append(word[0])
features.append(word[1])
count += 1
else:
perdict += self.viterbi(states, word_states_prob,
transitions_triprob,
transitions_bigprob, emissions_prob,
sentence)
sentence = []
# Read the data
f = codecs.open(self.file_pathtype + '_Trigram_result.txt', 'w',
'utf8')
for i in range(len(perdict)):
wri = all_sentence[i] + '\t' + features[i] + '\t' + perdict[
i] + '\n'
f.write(wri)
f.close()
TP, FP, TN, FN, type_correct, sum = 0, 0, 0, 0, 0, 0
sumlen = len(perdict)
for i in range(sumlen):
if features[i] != 'O' and perdict[i] != 'O':
TP += 1
if features[i] == perdict[i]:
type_correct += 1
if features[i] != 'O' and perdict[i] == 'O':
FN += 1
if features[i] == 'O' and perdict[i] != 'O':
FP += 1
if features[i] == 'O' and perdict[i] == 'O':
TN += 1
recall = TP / (TP + FN)
precision = TP / (TP + FP)
accuracy = (TP + TN) / sumlen
F1 = 2 * precision * recall / (precision + recall)
print('=====' + self.file_pathtype + ' labeling result=====')
print("type_correct: ", round(type_correct / TP, 4))
print("accuracy: ", round(accuracy, 4))
print("precision: ", round(precision, 4))
print("recall: ", round(recall, 4))
print("F1: ", round(F1, 4))
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
#
paths = '..\\dataset\\training.h5'
indces = np.array(down_sampling(paths)) #transform to one dim
one_indces = indces.reshape(-1)
one_indces = list(one_indces)
one_indces.sort()
#data IO
dl = Dataloader(confidence=list(one_indces),
epoch=5,
total_num=len(one_indces))
valid_x, valid_y = dl.get_valid_sample(num=1000)
train_acc = []
test_acc = []
loss_set = []
while (dl.epoch > 0):
#batch from dataloader
batch_x, batch_y = dl.batch_train_sample(batch_size=batch_size)
#run traning step
if (dl.epoch > 0):
sess.run(train,
feed_dict={
def train(self, num_epochs=10):
data = Dataloader(batchsize=self.batch_size)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
train_X, _ = data.getTrainData()
num_tr_iter = int(train_X.shape[0] / self.batch_size)
global_step = 0
Loss = []
Acc = []
for epoch in range(num_epochs):
print("-------------------------------")
print("Training epoch: {}".format(epoch + 1))
data.i = 0
acc = np.zeros(num_tr_iter)
loss = np.zeros(num_tr_iter)
for iteration in range(num_tr_iter):
global_step += 1
x_batch, y_batch = data.nextBatch()
# Run optimization op (backprop)
feed_dict_batch = {
self.Model.inputs: x_batch,
self.Model.labels: y_batch,
}
sess.run(self.Model.train_op, feed_dict=feed_dict_batch)
if iteration % 3 == 0:
# Calculate and display the batch loss and accuracy
loss_batch, acc_batch = sess.run(
[self.Model.loss, self.Model.accuracy],
feed_dict=feed_dict_batch,
)
print(
"iter {0:3d}:\t Loss={1:.2f},\tTraining Accuracy={2:.01%}"
.format(iteration, loss_batch, acc_batch))
acc[iteration] = acc_batch
loss[iteration] = loss_batch
x_valid, y_valid = data.getValiData()
val_loss, val_acc = sess.run(
[self.Model.loss, self.Model.accuracy],
feed_dict={
self.Model.inputs: x_valid,
self.Model.labels: y_valid
},
)
print(
"Epoch: {0}, validation loss: {1:.2f}, validation accuracy: {2:.01%}"
.format(epoch + 1, val_loss, val_acc))
acc, loss = np.mean(acc), np.mean(loss)
Loss.append(loss)
Acc.append(acc)
x_test, y_test = data.getTestData()
feed_dict_test = {
self.Model.inputs: x_test,
self.Model.labels: y_test
}
test_loss, test_acc = sess.run(
[self.Model.loss, self.Model.accuracy],
feed_dict=feed_dict_test)
print("--------------------")
print("Test loss: {0:.2f}, test accuracy: {1:.01%}".format(
test_loss, test_acc))
return Loss, Acc
import numpy as np
import tensorflow as tf
from DataLoader import Dataloader
data = Dataloader(batchsize=32)
train_X, train_Y = data.getTrainData()
test_X, test_Y = data.getTestData()
valid_X, valid_Y = data.getValiData()
label_num = train_Y.shape[0] * train_Y.shape[1] + test_Y.shape[
0] * test_Y.shape[1] + valid_Y.shape[0] * valid_Y.shape[1]
label_1_num = np.sum(train_Y) + np.sum(test_Y) + np.sum(valid_Y)
pass
# -*- coding: utf-8 -*-
"""
Created on Fri Nov 30 14:04:36 2018
@author: freeze
"""
from DataLoader import Dataloader
dl = Dataloader(total_num=320)
train_x, train_y = dl.get_train_sample()
while (dl.epoch > 0):
batch_x, batch_y = dl.batch_train_sample()
print("epoch %d , random_list %d " % (dl.epoch, len(dl.random_list)))