def test_precision(self):
self.assertAlmostEqual(
0.25,
evaluate.precision(0, [[0], [1], [2], [3]])
)
self.assertAlmostEqual(0, evaluate.precision(0, [[1], [2], [3]]))
self.assertAlmostEqual(0, evaluate.precision(0, []))
self.assertAlmostEqual(1, evaluate.precision(0, [[0], [0], [0]]))
def eval_cb(classifier, batch, task=1):
'''Callback-function, to evaluate performance of classifier.'''
iteration = batch if task==1 else (task-1)*iters_per_task + batch
# evaluate the solver on multiple tasks (and log to visdom)
if iteration % log == 0:
evaluate.precision(classifier, test_datasets, task, iteration,
classes_per_task=classes_per_task, scenario=scenario, test_size=test_size,
visdom=visdom, summary_graph=summary_graph, with_exemplars=with_exemplars)
def test_model(Xt, yt, model_path=MODEL_PATH):
# Load model and dictionaries
print("Loading model params...")
params = load_params('%s/best_model.npz' % model_path)
print("Loading dictionaries...")
with open('%s/dict.pkl' % model_path, 'rb') as f:
chardict = pkl.load(f)
with open('%s/label_dict.pkl' % model_path, 'rb') as f:
labeldict = pkl.load(f)
n_char = len(chardict.keys()) + 1
n_classes = len(labeldict.keys())
print "#classes:", n_classes
print labeldict
print("Building network...")
# Tweet variables
tweet = T.itensor3()
targets = T.imatrix()
# masks
t_mask = T.fmatrix()
# network for prediction
predictions = classify(tweet, t_mask, params, n_classes, n_char)
# Theano function
print("Compiling theano functions...")
predict = theano.function([tweet, t_mask], predictions)
# Test
print("Testing...")
preds = []
targs = []
# iterator over batches
xr, y = list(BatchTweets(Xt, yt, labeldict, batch_size=N_BATCH))[0]
print xr, y
x, x_m = prepare_data(xr, chardict, n_chars=n_char)
vp = predict(x, x_m)
ranks = np.argsort(vp)[:, ::-1]
for idx, item in enumerate(xr):
preds.append(ranks[idx, :])
targs.append(y[idx])
print[ranks[0] for ranks in preds]
# compute precision @1
validation_cost = precision(np.asarray(preds), targs, 1)
print validation_cost
def train_model(Xt, yt, Xv, yv, save_path=MODEL_PATH,
num_epochs=NUM_EPOCHS, lr=LEARNING_RATE, mu=MOMENTUM,
reg=REGULARIZATION, sch=SCHEDULE):
'''
Xt, yt X,y arrays with training data
Xv, yv X,y arrays with validation data split
save_path path to store the trained model
'''
global T1
print("Initializing model...")
# Build dictionaries from training data
chardict, charcount = build_dictionary(Xt)
n_char = len(chardict.keys()) + 1
print n_char, "unique characters"
# print chardict
save_dictionary(chardict, charcount, '%s/dict.pkl' % save_path)
# params
params = init_params(n_chars=n_char)
labeldict, labelcount = build_label_dictionary(yt)
save_dictionary(labeldict, labelcount, '%s/label_dict.pkl' % save_path)
n_classes = len(labeldict.keys())
print "#classes:", n_classes
# classification params
params['W_cl'] = theano.shared(np.random.normal(loc=0., scale=SCALE, size=(LDIM, n_classes)).astype('float32'), name='W_cl')
params['b_cl'] = theano.shared(np.zeros((n_classes)).astype('float32'), name='b_cl')
# iterators
train_iter = BatchTweets(Xt, yt, labeldict, batch_size=N_BATCH)
val_iter = BatchTweets(Xv, yv, labeldict, batch_size=N_BATCH)
print("Building...")
# Tweet variables
tweet = T.itensor3()
targets = T.ivector()
# masks
t_mask = T.fmatrix()
# network for prediction
predictions, net, emb = classify(tweet, t_mask, params, n_classes, n_char)
# batch loss
loss = lasagne.objectives.categorical_crossentropy(predictions, targets)
cost = T.mean(loss) + reg * lasagne.regularization.regularize_network_params(net, lasagne.regularization.l2)
cost_only = T.mean(loss)
reg_only = reg * lasagne.regularization.regularize_network_params(net, lasagne.regularization.l2)
# updates
print("Computing updates...")
updates = lasagne.updates.nesterov_momentum(cost, lasagne.layers.get_all_params(net), lr, momentum=mu)
# Theano functions
print("Compiling theano functions...")
inps = [tweet, t_mask, targets]
predict = theano.function([tweet, t_mask], predictions)
cost_val = theano.function(inps, [cost_only, emb])
train = theano.function(inps, cost, updates=updates)
reg_val = theano.function([], reg_only)
# Training
print("Training...")
uidx = 0
maxp = 0.
start = time.time()
valcosts = []
for epoch in range(num_epochs):
n_samples = 0
train_cost = 0.
print("Epoch {}".format(epoch))
# learning schedule
if len(valcosts) > 1 and sch:
change = (valcosts[-1] - valcosts[-2]) / abs(valcosts[-2])
if change < T1:
lr = schedule(lr)
updates = lasagne.updates.nesterov_momentum(cost, lasagne.layers.get_all_params(net), lr, momentum=mu)
train = theano.function(inps, cost, updates=updates)
T1 = T1 / 2
# stopping criterion: runs for minimum 7 epochs
if len(valcosts) > 6:
deltas = []
for i in range(5):
deltas.append((valcosts[-i-1] - valcosts[-i-2]) / abs(valcosts[-i-2]))
if sum(deltas) / len(deltas) < T2:
break
ud_start = time.time()
for xr, y in train_iter:
n_samples +=len(xr)
uidx += 1
x, x_m = prepare_data(xr, chardict, n_chars=n_char)
if x is None:
print("Minibatch with zero samples under maxlength.")
uidx -= 1
continue
curr_cost = train(x, x_m, y)
train_cost += curr_cost * len(xr)
ud = time.time() - ud_start
if np.isnan(curr_cost) or np.isinf(curr_cost):
print("Nan detected.")
return
if np.mod(uidx, DISPF) == 0:
print("Epoch {} Update {} Cost {} Time {}".format(epoch, uidx, curr_cost, ud))
if np.mod(uidx,SAVEF) == 0:
print("Saving...")
saveparams = OrderedDict()
for kk, vv in params.iteritems():
saveparams[kk] = vv.get_value()
np.savez('%s/model.npz' % save_path, **saveparams)
print("Done.")
print("Testing on Validation set...")
preds = []
targs = []
for xr,y in val_iter:
x, x_m = prepare_data(xr, chardict, n_chars=n_char)
if x is None:
print("Validation: Minibatch with zero samples under maxlength.")
continue
vp = predict(x, x_m)
ranks = np.argsort(vp)[:,::-1]
for idx, item in enumerate(xr):
preds.append(ranks[idx,:])
targs.append(y[idx])
# compute precision @1
validation_cost = precision(np.asarray(preds), targs, 1)
regularization_cost = reg_val()
if validation_cost > maxp:
maxp = validation_cost
saveparams = OrderedDict()
for kk,vv in params.iteritems():
saveparams[kk] = vv.get_value()
np.savez('%s/best_model.npz' % (save_path), **saveparams)
print("Epoch {} Training Cost {} Validation Precision {} Regularization Cost {} Max Precision {}".format(epoch, train_cost/n_samples, validation_cost, regularization_cost, maxp))
print("Seen {} samples.".format(n_samples))
valcosts.append(validation_cost)
print("Saving...")
saveparams = OrderedDict()
for kk, vv in params.iteritems():
saveparams[kk] = vv.get_value()
np.savez('%s/model_%d.npz' % (save_path,epoch),**saveparams)
print("Done.")
# stopping criterion: reached maximum precision on the validation set
if validation_cost == 1.0:
break
print("Finish. Total training time = {}".format(time.time()-start))
plt.scatter(x16, y16, c='purple')
plt.scatter(x17, y17, c='navy')
plt.scatter(x18, y18, c='violet')
plt.scatter(x19, y19, c='purple')
plt.scatter(x20, y20, c='coral')
plt.show()
#result2 is the predicted class of every sample
#label is the autual class of samples
label = []
result2 = []
for i in range(len(result[1])):
label.append(int(result[1][i].category))
for j in range(len(clf.labels_)):
result2.append(clf.labels_[j])
import evaluate as A
purity = A.purity(result2, label)
NMI = A.NMI(result2, label)
TP, TN, FP, FN = A.contingency_table(result2, label)
rand_index = A.rand_index(result2, label)
precision = A.precision(result2, label)
recall = A.recall(result2, label)
F_measure = A.F_measure(result2, label)
print("Purity:" + str(purity))
print("Precision:" + str(precision))
print("Recall:" + str(recall))
print("F_measue:" + str(F_measure))
def main(train_path,val_path,save_path,wordvec=None,num_epochs=NUM_EPOCHS):
global T1
# save settings
shutil.copyfile('src/settings_word.py','%s/settings_word.txt'%save_path)
print("Preparing Data...")
# Training data
Xt = []
yt = []
with io.open(train_path,'r',encoding='utf-8') as f:
for line in f:
(yc, Xc) = line.rstrip('\n').split('\t')
Xt.append(Xc)
yt.append(yc)
# Validation data
Xv = []
yv = []
with io.open(val_path,'r',encoding='utf-8') as f:
for line in f:
(yc, Xc) = line.rstrip('\n').split('\t')
Xv.append(Xc)
yv.append(yc.split(','))
print("Preparing Model...")
if not RELOAD_MODEL:
# Build dictionaries from training data
tokendict, tokencount = batch.build_dictionary(Xt)
n_token = min(len(tokendict.keys()) + 1, N_WORD)
batch.save_dictionary(tokendict,tokencount,'%s/dict.pkl' % save_path)
# params
if wordvec is None:
params = init_params(n_chars=n_token)
else:
params = init_params_add_wordvec(tokendict, n_chars=n_token, vector_bin=wordvec)
labeldict, labelcount = batch.build_label_dictionary(yt)
batch.save_dictionary(labeldict, labelcount, '%s/label_dict.pkl' % save_path)
n_classes = min(len(labeldict.keys()) + 1, MAX_CLASSES)
# classification params
params['W_cl'] = theano.shared(np.random.normal(loc=0., scale=SCALE, size=(2*WDIM,n_classes)).astype('float32'), name='W_cl')
params['b_cl'] = theano.shared(np.zeros((n_classes)).astype('float32'), name='b_cl')
else:
print("Loading model params...")
params = load_params_shared('%s/best_model.npz' % save_path)
print("Loading dictionaries...")
with open('%s/dict.pkl' % save_path, 'rb') as f:
tokendict = pkl.load(f)
with open('%s/label_dict.pkl' % save_path, 'rb') as f:
labeldict = pkl.load(f)
n_token = min(len(tokendict.keys()) + 1, N_WORD)
n_classes = min(len(labeldict.keys()) + 1, MAX_CLASSES)
# iterators
train_iter = batch.BatchTweets(Xt, yt, labeldict, batch_size=N_BATCH, max_classes=MAX_CLASSES)
val_iter = batch.BatchTweets(Xv, yv, labeldict, batch_size=N_BATCH, max_classes=MAX_CLASSES, test=True)
print("Building network...")
# Tweet variables
tweet = T.itensor3()
targets = T.ivector()
# masks
t_mask = T.fmatrix()
# network for prediction
predictions, net, emb = classify(tweet, t_mask, params, n_classes, n_token)
# batch loss
loss = lasagne.objectives.categorical_crossentropy(predictions, targets)
cost = T.mean(loss) + REGULARIZATION*lasagne.regularization.regularize_network_params(net, lasagne.regularization.l2)
cost_only = T.mean(loss)
reg_only = REGULARIZATION*lasagne.regularization.regularize_network_params(net, lasagne.regularization.l2)
# params and updates
print("Computing updates...")
global SCHEDULE
lr = LEARNING_RATE
if ADADELTA:
SCHEDULE = False
updates = lasagne.updates.adadelta(cost, lasagne.layers.get_all_params(net), learning_rate=lr)
else:
mu = MOMENTUM
updates = lasagne.updates.nesterov_momentum(cost, lasagne.layers.get_all_params(net), lr, momentum=mu)
# Theano function
print("Compiling theano functions...")
inps = [tweet,t_mask,targets]
predict = theano.function([tweet,t_mask],predictions)
encode = theano.function([tweet,t_mask],emb)
cost_val = theano.function(inps,[cost_only,emb])
train = theano.function(inps,cost,updates=updates)
reg_val = theano.function([],reg_only)
# Training
print("Training...")
uidx = 0
maxp = 0.
start = time.time()
valcosts = []
try:
for epoch in range(num_epochs):
n_samples = 0
train_cost = 0.
print("Epoch {}".format(epoch))
# learning schedule
if len(valcosts) > 1 and SCHEDULE:
change = (valcosts[-1]-valcosts[-2])/abs(valcosts[-2])
if change < T1:
lr, mu = schedule(lr, mu)
updates = lasagne.updates.nesterov_momentum(cost, lasagne.layers.get_all_params(net), lr, momentum=mu)
train = theano.function(inps,cost,updates=updates)
T1 = T1/2
# stopping criterion
if len(valcosts) > 6:
deltas = []
for i in range(5):
deltas.append((valcosts[-i-1]-valcosts[-i-2])/abs(valcosts[-i-2]))
if sum(deltas)/len(deltas) < T2:
break
ud_start = time.time()
for xr,y in train_iter:
n_samples +=len(xr)
uidx += 1
x, x_m = batch.prepare_data(xr, tokendict, n_tokens=n_token)
if x is None:
print("Minibatch with zero samples under maxlength.")
uidx -= 1
continue
curr_cost = train(x,x_m,y)
train_cost += curr_cost*len(xr)
ud = time.time() - ud_start
if np.isnan(curr_cost) or np.isinf(curr_cost):
print("Nan detected.")
return
if np.mod(uidx, DISPF) == 0:
print("Epoch {} Update {} Cost {} Time {}".format(epoch,uidx,curr_cost,ud))
sys.stdout.flush()
if np.mod(uidx,SAVEF) == 0:
print("Saving...")
saveparams = OrderedDict()
for kk,vv in params.iteritems():
saveparams[kk] = vv.get_value()
np.savez('%s/model.npz' % save_path,**saveparams)
print("Done.")
print("Testing on Validation set...")
preds = []
targs = []
for xr,y in val_iter:
x, x_m = batch.prepare_data(xr, tokendict, n_tokens=n_token)
if x is None:
print("Validation: Minibatch with zero samples under maxlength.")
continue
vp = predict(x,x_m)
ranks = np.argsort(vp)[:,::-1]
for idx,item in enumerate(xr):
preds.append(ranks[idx,:])
targs.append(y[idx])
validation_cost = precision(np.asarray(preds),targs,1)
regularization_cost = reg_val()
if validation_cost > maxp:
maxp = validation_cost
saveparams = OrderedDict()
for kk,vv in params.iteritems():
saveparams[kk] = vv.get_value()
np.savez('%s/best_model.npz' % (save_path),**saveparams)
print("Epoch {} Training Cost {} Validation Precision {} Regularization Cost {} Max Precision {}".format(epoch, train_cost/n_samples, validation_cost, regularization_cost, maxp))
print("Seen {} samples.".format(n_samples))
valcosts.append(validation_cost)
print("Saving...")
saveparams = OrderedDict()
for kk,vv in params.iteritems():
saveparams[kk] = vv.get_value()
np.savez('%s/model_%d.npz' % (save_path,epoch),**saveparams)
print("Done.")
except KeyboardInterrupt:
pass
print("Total training time = {}".format(time.time()-start))
def main(unused_argv):
train_dataset, validate_dataset, test_dataset = input.input(
shuffle_files=False)
#Text information
info = tf.constant([
"Batch size = %s" % f.FLAGS.batch_size,
"Epochs = %s" % f.FLAGS.num_epochs,
"Learning rate = %s" % f.FLAGS.learning_rate,
"Batch normalization = No",
"Window size = %s" % f.FLAGS.window_size, "Shuffle Files = No",
"CNN model = %s" % f.FLAGS.cnn_model, "Shuffle Samples = YES"
])
with tf.name_scope('input'):
x = tf.placeholder(tf.float32, [
None, input.SAMPLE_DEPTH, input.SAMPLE_HEIGHT, input.SAMPLE_WIDTH
])
y_ = tf.placeholder(tf.float32, [None, 2])
dropout_rate = tf.placeholder(tf.float32)
is_training = tf.placeholder(tf.bool)
with tf.name_scope('logits'):
if f.FLAGS.cnn_model == "lenet5":
logits = lenet5.model_fn(sample_input=x,
is_training=is_training,
summaries=summaries)
with tf.name_scope('loss'):
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_,
logits=logits)
mean_cross_entropy_loss = tf.reduce_mean(cross_entropy)
loss_summ = tf.summary.scalar('Mean_cross_entropy_loss',
mean_cross_entropy_loss)
summaries['train'].append(loss_summ)
#summaries['validate'].append(loss_summ)
with tf.name_scope('adam_optimizer'):
optimizer = tf.train.AdamOptimizer(
f.FLAGS.learning_rate).minimize(mean_cross_entropy_loss)
with tf.name_scope('accuracy'):
preds = tf.argmax(logits, 1)
correct_preds = tf.argmax(y_, 1)
equal = tf.equal(preds, correct_preds)
training_accuracy_op = tf.reduce_mean(tf.cast(equal, tf.float32))
summaries['train'].append(
tf.summary.scalar('Training_Accuracy', training_accuracy_op))
with tf.name_scope('Evaluation_Metrics'):
tp_op = evaluate.tp(logits=logits, labels=y_)
fp_op = evaluate.fp(logits=logits, labels=y_)
tn_op = evaluate.tn(logits=logits, labels=y_)
fn_op = evaluate.fn(logits=logits, labels=y_)
tp_sum = tf.placeholder(tf.float32)
tn_sum = tf.placeholder(tf.float32)
fp_sum = tf.placeholder(tf.float32)
fn_sum = tf.placeholder(tf.float32)
precision_op = evaluate.precision(tp=tp_sum,
fp=fp_sum,
tn=tn_sum,
fn=fn_sum)
accuracy_op = evaluate.accuracy(tp=tp_sum,
fp=fp_sum,
tn=tn_sum,
fn=fn_sum)
recall_op = evaluate.recall(tp=tp_sum, fp=fp_sum, tn=tn_sum, fn=fn_sum)
fscore_op = evaluate.fscore(tp=tp_sum, fp=fp_sum, tn=tn_sum, fn=fn_sum)
precision_summ = tf.summary.scalar('Precision', precision_op)
accuracy_summ = tf.summary.scalar('Accuracy', accuracy_op)
recall_summ = tf.summary.scalar('Recall', recall_op)
fscore_summ = tf.summary.scalar('Fscore', fscore_op)
summaries['validate'].append(accuracy_summ)
summaries['validate'].append(precision_summ)
summaries['validate'].append(recall_summ)
summaries['validate'].append(fscore_summ)
summaries['test'].append(accuracy_summ)
summaries['test'].append(precision_summ)
summaries['test'].append(recall_summ)
summaries['test'].append(fscore_summ)
print("Saving graph to %s" % f.FLAGS.log_dir)
train_writer = tf.summary.FileWriter(f.FLAGS.log_dir + "/train")
validate_writer = tf.summary.FileWriter(f.FLAGS.log_dir + "/validate")
test_writer = tf.summary.FileWriter(f.FLAGS.log_dir + "/test")
train_writer.add_graph(tf.get_default_graph())
train_summaries = tf.summary.merge(summaries['train'])
validate_summaries = tf.summary.merge(summaries['validate'])
test_summaries = tf.summary.merge(summaries['test'])
with tf.Session() as sess:
train_writer.add_summary(sess.run(tf.summary.text("Information",
info)))
train_iter = train_dataset.make_initializable_iterator()
train_next_elem = train_iter.get_next()
sess.run(tf.global_variables_initializer())
global_step = 0
display_freq = 10
validate_freq = 50
test_freq = 50
for epoch in range(1, f.FLAGS.num_epochs + 1):
sess.run(train_iter.initializer)
step_time = 0.0
fetch_time = 0.0
while True:
try:
a = time.time()
global_step += 1
sample, label = sess.run(train_next_elem)
fetch_time += time.time() - a
#print (sample.shape, label.shape)
#print (label)
#for s in sample[0][0]:
# print (s)
a = time.time()
_, summ = sess.run([optimizer, train_summaries],
feed_dict={
x: sample,
y_: label,
dropout_rate: 0.5,
is_training: True
})
train_writer.add_summary(summ, global_step)
step_time += time.time() - a
except tf.errors.OutOfRangeError:
break
if global_step % display_freq == 0:
batch_loss, batch_accuracy = sess.run(
[mean_cross_entropy_loss, training_accuracy_op],
feed_dict={
x: sample,
y_: label,
dropout_rate: 1.0,
is_training: False
})
print(
"Epoch {:3}\t Step {:5}:\t Loss={:.3f}, \tTraining Accuracy={:.5f} \tStep Time {:4.2f}m, Fetch Time {:4.2f}m"
.format(epoch, global_step, batch_loss, batch_accuracy,
step_time / 60, fetch_time / 60))
step_time = 0.0
fetch_time = 0.0
#Validate and test after each epoch
val_it = validate_dataset.make_one_shot_iterator()
val_next_elem = val_it.get_next()
tot_tp, tot_tn, tot_fp, tot_fn = 0, 0, 0, 0
while True:
try:
sample, label = sess.run(val_next_elem)
tp, fp, tn, fn = sess.run(
[tp_op, fp_op, tn_op, fn_op],
feed_dict={
x: sample,
y_: label,
dropout_rate: 1.0,
is_training: False
})
except tf.errors.OutOfRangeError:
break
tot_tp += tp
tot_fp += fp
tot_fn += fn
tot_tn += tn
precision, recall, accuracy, fscore, summ = sess.run([
precision_op, recall_op, accuracy_op, fscore_op,
validate_summaries
],
feed_dict={
tp_sum:
tot_tp,
tn_sum:
tot_tn,
fp_sum:
tot_fp,
fn_sum:
tot_fn
})
validate_writer.add_summary(summ, global_step)
print("Epoch %d, Step %d" % (epoch, global_step))
print("=" * 10, "Validating Results", "=" * 10)
print("TP: %g\nTN: %g\nFP: %g\nFN: %g" %
(tot_tp, tot_tn, tot_fp, tot_fn))
print(
"\tPrecision: %g\n\tRecall: %g\n\tF1_score: %g\n\tAccuracy: %g"
% (precision, recall, fscore, accuracy))
test_it = test_dataset.make_one_shot_iterator()
test_next_elem = test_it.get_next()
tot_tp, tot_tn, tot_fp, tot_tn = 0, 0, 0, 0
while True:
try:
sample, label = sess.run(test_next_elem)
tp, fp, tn, fn = sess.run(
[tp_op, fp_op, tn_op, fn_op],
feed_dict={
x: sample,
y_: label,
dropout_rate: 1.0,
is_training: False
})
except tf.errors.OutOfRangeError:
break
tot_tp += tp
tot_fp += fp
tot_fn += fn
tot_tn += tn
precision, recall, accuracy, fscore, summ = sess.run([
precision_op, recall_op, accuracy_op, fscore_op, test_summaries
],
feed_dict={
tp_sum:
tot_tp,
tn_sum:
tot_tn,
fp_sum:
tot_fp,
fn_sum:
tot_fn
})
test_writer.add_summary(summ, global_step)
print("=" * 10, "Testing Results", "=" * 10)
print("TP: %g\nTN: %g\nFP: %g\nFN: %g" %
(tot_tp, tot_tn, tot_fp, tot_fn))
print(
"\tPrecision: %g\n\tRecall: %g\n\tF1_score: %g\n\tAccuracy: %g"
% (precision, recall, fscore, accuracy))
print("=" * 10, "===============", "=" * 10)
def main(train_path, val_path, save_path, num_epochs=50):
global T1
print('NUM EPOCHS %i' % num_epochs)
# save settings
shutil.copyfile('settings_char.py', '%s/settings_char.txt' % save_path)
print("Preparing Data 123")
# Training data
Xt = []
yt = []
with io.open(train_path, 'r', encoding='utf-8') as f:
for line in f:
(yc, Xc) = line.rstrip('\n').split('\t')
Xt.append(Xc[:MAX_LENGTH])
yt.append(yc)
# Validation data
Xv = []
yv = []
with io.open(val_path, 'r', encoding='utf-8') as f:
for line in f:
(yc, Xc) = line.rstrip('\n').split('\t')
Xv.append(Xc[:MAX_LENGTH])
yv.append(yc.split(','))
print("Building Model...")
if not RELOAD_MODEL:
# Build dictionaries from training data
chardict, charcount = batch.build_dictionary(Xt)
n_char = len(chardict.keys()) + 1
batch.save_dictionary(chardict, charcount, '%s/dict.pkl' % save_path)
# params
params = init_params(n_chars=n_char)
labeldict, labelcount = batch.build_label_dictionary(yt)
batch.save_dictionary(labeldict, labelcount,
'%s/label_dict.pkl' % save_path)
n_classes = min(len(labeldict.keys()) + 1, MAX_CLASSES)
# classification params
params['W_cl'] = theano.shared(np.random.normal(
loc=0., scale=SCALE, size=(WDIM, n_classes)).astype('float32'),
name='W_cl')
params['b_cl'] = theano.shared(np.zeros((n_classes)).astype('float32'),
name='b_cl')
else:
print("Loading model params from base path: %s ..." % save_path)
params = load_params_shared('%s/model.npz' % save_path)
print("Loading dictionaries...")
with open('%s/dict.pkl' % save_path, 'rb') as f:
chardict = pkl.load(f)
with open('%s/label_dict.pkl' % save_path, 'rb') as f:
labeldict = pkl.load(f)
n_char = len(chardict.keys()) + 1
n_classes = min(len(labeldict.keys()) + 1, MAX_CLASSES)
# iterators
train_iter = batch.BatchTweets(Xt,
yt,
labeldict,
batch_size=N_BATCH,
max_classes=MAX_CLASSES)
val_iter = batch.BatchTweets(Xv,
yv,
labeldict,
batch_size=N_BATCH,
max_classes=MAX_CLASSES,
test=True)
print("Building network...")
# Tweet variables
tweet = T.itensor3()
targets = T.ivector()
# masks
t_mask = T.fmatrix()
# network for prediction
predictions, net, emb = classify(tweet, t_mask, params, n_classes, n_char)
# batch loss
loss = lasagne.objectives.categorical_crossentropy(predictions, targets)
cost = T.mean(
loss
) + REGULARIZATION * lasagne.regularization.regularize_network_params(
net, lasagne.regularization.l2)
cost_only = T.mean(loss)
reg_only = REGULARIZATION * lasagne.regularization.regularize_network_params(
net, lasagne.regularization.l2)
# params and updates
print("Computing updates...")
lr = LEARNING_RATE
mu = MOMENTUM
updates = lasagne.updates.nesterov_momentum(
cost, lasagne.layers.get_all_params(net), lr, momentum=mu)
# Theano function
print("Compiling theano functions...")
inps = [tweet, t_mask, targets]
predict = theano.function([tweet, t_mask], predictions)
cost_val = theano.function(inps, [cost_only, emb])
train = theano.function(inps, cost, updates=updates)
reg_val = theano.function([], reg_only)
# Training
print("Training...")
uidx = 0
maxp = 0.
start = time.time()
valcosts = []
try:
for epoch in range(num_epochs):
n_samples = 0
train_cost = 0.
print("Epoch {}".format(epoch))
# learning schedule
if len(valcosts) > 1 and SCHEDULE:
change = (valcosts[-1] - valcosts[-2]) / abs(valcosts[-2])
if change < T1:
lr, mu = schedule(lr, mu)
updates = lasagne.updates.nesterov_momentum(
cost,
lasagne.layers.get_all_params(net),
lr,
momentum=mu)
train = theano.function(inps, cost, updates=updates)
T1 = T1 / 2
# stopping criterion
if len(valcosts) > 6:
deltas = []
for i in range(5):
deltas.append((valcosts[-i - 1] - valcosts[-i - 2]) /
abs(valcosts[-i - 2]))
if sum(deltas) / len(deltas) < T2:
break
ud_start = time.time()
for xr, y in train_iter:
n_samples += len(xr)
uidx += 1
x, x_m = batch.prepare_data(xr, chardict, n_chars=n_char)
if x is None:
print("Minibatch with zero samples under maxlength.")
uidx -= 1
continue
curr_cost = train(x, x_m, y)
train_cost += curr_cost * len(xr)
ud = time.time() - ud_start
if np.isnan(curr_cost) or np.isinf(curr_cost):
print("Nan detected.")
return
if np.mod(uidx, DISPF) == 0:
print("Epoch {} Update {} Cost {} Time {}".format(
epoch, uidx, curr_cost, ud))
if np.mod(uidx, SAVEF) == 0:
print("Saving...")
saveparams = OrderedDict()
for kk, vv in params.iteritems():
saveparams[kk] = vv.get_value()
np.savez('%s/model.npz' % save_path, **saveparams)
print("Done.")
print("Testing on Validation set...")
preds = []
targs = []
for xr, y in val_iter:
x, x_m = batch.prepare_data(xr, chardict, n_chars=n_char)
if x is None:
print(
"Validation: Minibatch with zero samples under maxlength."
)
continue
vp = predict(x, x_m)
ranks = np.argsort(vp)[:, ::-1]
for idx, item in enumerate(xr):
preds.append(ranks[idx, :])
targs.append(y[idx])
validation_cost = precision(np.asarray(preds), targs, 1)
regularization_cost = reg_val()
if validation_cost > maxp:
maxp = validation_cost
saveparams = OrderedDict()
for kk, vv in params.iteritems():
saveparams[kk] = vv.get_value()
np.savez('%s/best_model.npz' % (save_path), **saveparams)
print(
"Epoch {} Training Cost {} Validation Precision {} Regularization Cost {} Max Precision {}"
.format(epoch, train_cost / n_samples, validation_cost,
regularization_cost, maxp))
print("Seen {} samples.".format(n_samples))
valcosts.append(validation_cost)
print("Saving...")
saveparams = OrderedDict()
for kk, vv in params.iteritems():
saveparams[kk] = vv.get_value()
np.savez('%s/model_%d.npz' % (save_path, epoch), **saveparams)
print("Done.")
except KeyboardInterrupt:
pass
print("Total training time = {}".format(time.time() - start))
alreadyAlign)
alreadyAlign = aligner.insertAlign(cekSequences, i, alreadyAlign)
alreadyAlign = aligner.insertAlign(cekneighbor, i, alreadyAlign)
alreadyAlign = aligner.insertAlign(cekstop, i, alreadyAlign)
alreadyAlign = aligner.insertAlign(cekSimilarex, i, alreadyAlign)
# # alreadyAlign = aligner.insertAlign(cekDep, i, alreadyAlign)
# Ubah hasil dari yang pasangan angka menjadi hasil kata dan indeks
resultMSR = tools.transformResult(alreadyAlign[i], sentenceLemma2)
# Memanggil fungsi dari class Evaluate untuk menampilkan hasil skor
jumlahBenar = evaluate.jumlahBenar(resultMSR, sentenceLemma2,
listAnnotation[i])
jumlahGoldAnnotation = evaluate.jumlahGoldAnnotation(listAnnotation[i])
jumlahDariSistem = evaluate.jumlahDariSistem(resultMSR)
precision = evaluate.precision(jumlahBenar, jumlahDariSistem)
recall = evaluate.recall(jumlahBenar, jumlahGoldAnnotation)
F1 = evaluate.F1Measure(precision, recall)
# Hitung rata-rata precision, recall dan F1
precisionAverage.append(precision)
recallAverage.append(recall)
F1Average.append(F1)
print("========================================")
print("Sentence Lemma 1 :", sentenceLemma1)
print("Sentence Lemma 2 :", sentenceLemma2)
print("Gold Annotation :",
evaluate.printAnnotation(sentenceLemma2, goldAnnotation[i]))
file_output.write("========================================" + '\n')
file_output.write("Sentence Lemma 1 : " + str(sentenceLemma1) + '\n')
