def test_precision(self): self.assertAlmostEqual( 0.25, evaluate.precision(0, [[0], [1], [2], [3]]) ) self.assertAlmostEqual(0, evaluate.precision(0, [[1], [2], [3]])) self.assertAlmostEqual(0, evaluate.precision(0, [])) self.assertAlmostEqual(1, evaluate.precision(0, [[0], [0], [0]]))
def eval_cb(classifier, batch, task=1): '''Callback-function, to evaluate performance of classifier.''' iteration = batch if task==1 else (task-1)*iters_per_task + batch # evaluate the solver on multiple tasks (and log to visdom) if iteration % log == 0: evaluate.precision(classifier, test_datasets, task, iteration, classes_per_task=classes_per_task, scenario=scenario, test_size=test_size, visdom=visdom, summary_graph=summary_graph, with_exemplars=with_exemplars)
def test_model(Xt, yt, model_path=MODEL_PATH): # Load model and dictionaries print("Loading model params...") params = load_params('%s/best_model.npz' % model_path) print("Loading dictionaries...") with open('%s/dict.pkl' % model_path, 'rb') as f: chardict = pkl.load(f) with open('%s/label_dict.pkl' % model_path, 'rb') as f: labeldict = pkl.load(f) n_char = len(chardict.keys()) + 1 n_classes = len(labeldict.keys()) print "#classes:", n_classes print labeldict print("Building network...") # Tweet variables tweet = T.itensor3() targets = T.imatrix() # masks t_mask = T.fmatrix() # network for prediction predictions = classify(tweet, t_mask, params, n_classes, n_char) # Theano function print("Compiling theano functions...") predict = theano.function([tweet, t_mask], predictions) # Test print("Testing...") preds = [] targs = [] # iterator over batches xr, y = list(BatchTweets(Xt, yt, labeldict, batch_size=N_BATCH))[0] print xr, y x, x_m = prepare_data(xr, chardict, n_chars=n_char) vp = predict(x, x_m) ranks = np.argsort(vp)[:, ::-1] for idx, item in enumerate(xr): preds.append(ranks[idx, :]) targs.append(y[idx]) print[ranks[0] for ranks in preds] # compute precision @1 validation_cost = precision(np.asarray(preds), targs, 1) print validation_cost
def train_model(Xt, yt, Xv, yv, save_path=MODEL_PATH, num_epochs=NUM_EPOCHS, lr=LEARNING_RATE, mu=MOMENTUM, reg=REGULARIZATION, sch=SCHEDULE): ''' Xt, yt X,y arrays with training data Xv, yv X,y arrays with validation data split save_path path to store the trained model ''' global T1 print("Initializing model...") # Build dictionaries from training data chardict, charcount = build_dictionary(Xt) n_char = len(chardict.keys()) + 1 print n_char, "unique characters" # print chardict save_dictionary(chardict, charcount, '%s/dict.pkl' % save_path) # params params = init_params(n_chars=n_char) labeldict, labelcount = build_label_dictionary(yt) save_dictionary(labeldict, labelcount, '%s/label_dict.pkl' % save_path) n_classes = len(labeldict.keys()) print "#classes:", n_classes # classification params params['W_cl'] = theano.shared(np.random.normal(loc=0., scale=SCALE, size=(LDIM, n_classes)).astype('float32'), name='W_cl') params['b_cl'] = theano.shared(np.zeros((n_classes)).astype('float32'), name='b_cl') # iterators train_iter = BatchTweets(Xt, yt, labeldict, batch_size=N_BATCH) val_iter = BatchTweets(Xv, yv, labeldict, batch_size=N_BATCH) print("Building...") # Tweet variables tweet = T.itensor3() targets = T.ivector() # masks t_mask = T.fmatrix() # network for prediction predictions, net, emb = classify(tweet, t_mask, params, n_classes, n_char) # batch loss loss = lasagne.objectives.categorical_crossentropy(predictions, targets) cost = T.mean(loss) + reg * lasagne.regularization.regularize_network_params(net, lasagne.regularization.l2) cost_only = T.mean(loss) reg_only = reg * lasagne.regularization.regularize_network_params(net, lasagne.regularization.l2) # updates print("Computing updates...") updates = lasagne.updates.nesterov_momentum(cost, lasagne.layers.get_all_params(net), lr, momentum=mu) # Theano functions print("Compiling theano functions...") inps = [tweet, t_mask, targets] predict = theano.function([tweet, t_mask], predictions) cost_val = theano.function(inps, [cost_only, emb]) train = theano.function(inps, cost, updates=updates) reg_val = theano.function([], reg_only) # Training print("Training...") uidx = 0 maxp = 0. start = time.time() valcosts = [] for epoch in range(num_epochs): n_samples = 0 train_cost = 0. print("Epoch {}".format(epoch)) # learning schedule if len(valcosts) > 1 and sch: change = (valcosts[-1] - valcosts[-2]) / abs(valcosts[-2]) if change < T1: lr = schedule(lr) updates = lasagne.updates.nesterov_momentum(cost, lasagne.layers.get_all_params(net), lr, momentum=mu) train = theano.function(inps, cost, updates=updates) T1 = T1 / 2 # stopping criterion: runs for minimum 7 epochs if len(valcosts) > 6: deltas = [] for i in range(5): deltas.append((valcosts[-i-1] - valcosts[-i-2]) / abs(valcosts[-i-2])) if sum(deltas) / len(deltas) < T2: break ud_start = time.time() for xr, y in train_iter: n_samples +=len(xr) uidx += 1 x, x_m = prepare_data(xr, chardict, n_chars=n_char) if x is None: print("Minibatch with zero samples under maxlength.") uidx -= 1 continue curr_cost = train(x, x_m, y) train_cost += curr_cost * len(xr) ud = time.time() - ud_start if np.isnan(curr_cost) or np.isinf(curr_cost): print("Nan detected.") return if np.mod(uidx, DISPF) == 0: print("Epoch {} Update {} Cost {} Time {}".format(epoch, uidx, curr_cost, ud)) if np.mod(uidx,SAVEF) == 0: print("Saving...") saveparams = OrderedDict() for kk, vv in params.iteritems(): saveparams[kk] = vv.get_value() np.savez('%s/model.npz' % save_path, **saveparams) print("Done.") print("Testing on Validation set...") preds = [] targs = [] for xr,y in val_iter: x, x_m = prepare_data(xr, chardict, n_chars=n_char) if x is None: print("Validation: Minibatch with zero samples under maxlength.") continue vp = predict(x, x_m) ranks = np.argsort(vp)[:,::-1] for idx, item in enumerate(xr): preds.append(ranks[idx,:]) targs.append(y[idx]) # compute precision @1 validation_cost = precision(np.asarray(preds), targs, 1) regularization_cost = reg_val() if validation_cost > maxp: maxp = validation_cost saveparams = OrderedDict() for kk,vv in params.iteritems(): saveparams[kk] = vv.get_value() np.savez('%s/best_model.npz' % (save_path), **saveparams) print("Epoch {} Training Cost {} Validation Precision {} Regularization Cost {} Max Precision {}".format(epoch, train_cost/n_samples, validation_cost, regularization_cost, maxp)) print("Seen {} samples.".format(n_samples)) valcosts.append(validation_cost) print("Saving...") saveparams = OrderedDict() for kk, vv in params.iteritems(): saveparams[kk] = vv.get_value() np.savez('%s/model_%d.npz' % (save_path,epoch),**saveparams) print("Done.") # stopping criterion: reached maximum precision on the validation set if validation_cost == 1.0: break print("Finish. Total training time = {}".format(time.time()-start))
plt.scatter(x16, y16, c='purple') plt.scatter(x17, y17, c='navy') plt.scatter(x18, y18, c='violet') plt.scatter(x19, y19, c='purple') plt.scatter(x20, y20, c='coral') plt.show() #result2 is the predicted class of every sample #label is the autual class of samples label = [] result2 = [] for i in range(len(result[1])): label.append(int(result[1][i].category)) for j in range(len(clf.labels_)): result2.append(clf.labels_[j]) import evaluate as A purity = A.purity(result2, label) NMI = A.NMI(result2, label) TP, TN, FP, FN = A.contingency_table(result2, label) rand_index = A.rand_index(result2, label) precision = A.precision(result2, label) recall = A.recall(result2, label) F_measure = A.F_measure(result2, label) print("Purity:" + str(purity)) print("Precision:" + str(precision)) print("Recall:" + str(recall)) print("F_measue:" + str(F_measure))
def main(train_path,val_path,save_path,wordvec=None,num_epochs=NUM_EPOCHS): global T1 # save settings shutil.copyfile('src/settings_word.py','%s/settings_word.txt'%save_path) print("Preparing Data...") # Training data Xt = [] yt = [] with io.open(train_path,'r',encoding='utf-8') as f: for line in f: (yc, Xc) = line.rstrip('\n').split('\t') Xt.append(Xc) yt.append(yc) # Validation data Xv = [] yv = [] with io.open(val_path,'r',encoding='utf-8') as f: for line in f: (yc, Xc) = line.rstrip('\n').split('\t') Xv.append(Xc) yv.append(yc.split(',')) print("Preparing Model...") if not RELOAD_MODEL: # Build dictionaries from training data tokendict, tokencount = batch.build_dictionary(Xt) n_token = min(len(tokendict.keys()) + 1, N_WORD) batch.save_dictionary(tokendict,tokencount,'%s/dict.pkl' % save_path) # params if wordvec is None: params = init_params(n_chars=n_token) else: params = init_params_add_wordvec(tokendict, n_chars=n_token, vector_bin=wordvec) labeldict, labelcount = batch.build_label_dictionary(yt) batch.save_dictionary(labeldict, labelcount, '%s/label_dict.pkl' % save_path) n_classes = min(len(labeldict.keys()) + 1, MAX_CLASSES) # classification params params['W_cl'] = theano.shared(np.random.normal(loc=0., scale=SCALE, size=(2*WDIM,n_classes)).astype('float32'), name='W_cl') params['b_cl'] = theano.shared(np.zeros((n_classes)).astype('float32'), name='b_cl') else: print("Loading model params...") params = load_params_shared('%s/best_model.npz' % save_path) print("Loading dictionaries...") with open('%s/dict.pkl' % save_path, 'rb') as f: tokendict = pkl.load(f) with open('%s/label_dict.pkl' % save_path, 'rb') as f: labeldict = pkl.load(f) n_token = min(len(tokendict.keys()) + 1, N_WORD) n_classes = min(len(labeldict.keys()) + 1, MAX_CLASSES) # iterators train_iter = batch.BatchTweets(Xt, yt, labeldict, batch_size=N_BATCH, max_classes=MAX_CLASSES) val_iter = batch.BatchTweets(Xv, yv, labeldict, batch_size=N_BATCH, max_classes=MAX_CLASSES, test=True) print("Building network...") # Tweet variables tweet = T.itensor3() targets = T.ivector() # masks t_mask = T.fmatrix() # network for prediction predictions, net, emb = classify(tweet, t_mask, params, n_classes, n_token) # batch loss loss = lasagne.objectives.categorical_crossentropy(predictions, targets) cost = T.mean(loss) + REGULARIZATION*lasagne.regularization.regularize_network_params(net, lasagne.regularization.l2) cost_only = T.mean(loss) reg_only = REGULARIZATION*lasagne.regularization.regularize_network_params(net, lasagne.regularization.l2) # params and updates print("Computing updates...") global SCHEDULE lr = LEARNING_RATE if ADADELTA: SCHEDULE = False updates = lasagne.updates.adadelta(cost, lasagne.layers.get_all_params(net), learning_rate=lr) else: mu = MOMENTUM updates = lasagne.updates.nesterov_momentum(cost, lasagne.layers.get_all_params(net), lr, momentum=mu) # Theano function print("Compiling theano functions...") inps = [tweet,t_mask,targets] predict = theano.function([tweet,t_mask],predictions) encode = theano.function([tweet,t_mask],emb) cost_val = theano.function(inps,[cost_only,emb]) train = theano.function(inps,cost,updates=updates) reg_val = theano.function([],reg_only) # Training print("Training...") uidx = 0 maxp = 0. start = time.time() valcosts = [] try: for epoch in range(num_epochs): n_samples = 0 train_cost = 0. print("Epoch {}".format(epoch)) # learning schedule if len(valcosts) > 1 and SCHEDULE: change = (valcosts[-1]-valcosts[-2])/abs(valcosts[-2]) if change < T1: lr, mu = schedule(lr, mu) updates = lasagne.updates.nesterov_momentum(cost, lasagne.layers.get_all_params(net), lr, momentum=mu) train = theano.function(inps,cost,updates=updates) T1 = T1/2 # stopping criterion if len(valcosts) > 6: deltas = [] for i in range(5): deltas.append((valcosts[-i-1]-valcosts[-i-2])/abs(valcosts[-i-2])) if sum(deltas)/len(deltas) < T2: break ud_start = time.time() for xr,y in train_iter: n_samples +=len(xr) uidx += 1 x, x_m = batch.prepare_data(xr, tokendict, n_tokens=n_token) if x is None: print("Minibatch with zero samples under maxlength.") uidx -= 1 continue curr_cost = train(x,x_m,y) train_cost += curr_cost*len(xr) ud = time.time() - ud_start if np.isnan(curr_cost) or np.isinf(curr_cost): print("Nan detected.") return if np.mod(uidx, DISPF) == 0: print("Epoch {} Update {} Cost {} Time {}".format(epoch,uidx,curr_cost,ud)) sys.stdout.flush() if np.mod(uidx,SAVEF) == 0: print("Saving...") saveparams = OrderedDict() for kk,vv in params.iteritems(): saveparams[kk] = vv.get_value() np.savez('%s/model.npz' % save_path,**saveparams) print("Done.") print("Testing on Validation set...") preds = [] targs = [] for xr,y in val_iter: x, x_m = batch.prepare_data(xr, tokendict, n_tokens=n_token) if x is None: print("Validation: Minibatch with zero samples under maxlength.") continue vp = predict(x,x_m) ranks = np.argsort(vp)[:,::-1] for idx,item in enumerate(xr): preds.append(ranks[idx,:]) targs.append(y[idx]) validation_cost = precision(np.asarray(preds),targs,1) regularization_cost = reg_val() if validation_cost > maxp: maxp = validation_cost saveparams = OrderedDict() for kk,vv in params.iteritems(): saveparams[kk] = vv.get_value() np.savez('%s/best_model.npz' % (save_path),**saveparams) print("Epoch {} Training Cost {} Validation Precision {} Regularization Cost {} Max Precision {}".format(epoch, train_cost/n_samples, validation_cost, regularization_cost, maxp)) print("Seen {} samples.".format(n_samples)) valcosts.append(validation_cost) print("Saving...") saveparams = OrderedDict() for kk,vv in params.iteritems(): saveparams[kk] = vv.get_value() np.savez('%s/model_%d.npz' % (save_path,epoch),**saveparams) print("Done.") except KeyboardInterrupt: pass print("Total training time = {}".format(time.time()-start))
def main(unused_argv): train_dataset, validate_dataset, test_dataset = input.input( shuffle_files=False) #Text information info = tf.constant([ "Batch size = %s" % f.FLAGS.batch_size, "Epochs = %s" % f.FLAGS.num_epochs, "Learning rate = %s" % f.FLAGS.learning_rate, "Batch normalization = No", "Window size = %s" % f.FLAGS.window_size, "Shuffle Files = No", "CNN model = %s" % f.FLAGS.cnn_model, "Shuffle Samples = YES" ]) with tf.name_scope('input'): x = tf.placeholder(tf.float32, [ None, input.SAMPLE_DEPTH, input.SAMPLE_HEIGHT, input.SAMPLE_WIDTH ]) y_ = tf.placeholder(tf.float32, [None, 2]) dropout_rate = tf.placeholder(tf.float32) is_training = tf.placeholder(tf.bool) with tf.name_scope('logits'): if f.FLAGS.cnn_model == "lenet5": logits = lenet5.model_fn(sample_input=x, is_training=is_training, summaries=summaries) with tf.name_scope('loss'): cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=logits) mean_cross_entropy_loss = tf.reduce_mean(cross_entropy) loss_summ = tf.summary.scalar('Mean_cross_entropy_loss', mean_cross_entropy_loss) summaries['train'].append(loss_summ) #summaries['validate'].append(loss_summ) with tf.name_scope('adam_optimizer'): optimizer = tf.train.AdamOptimizer( f.FLAGS.learning_rate).minimize(mean_cross_entropy_loss) with tf.name_scope('accuracy'): preds = tf.argmax(logits, 1) correct_preds = tf.argmax(y_, 1) equal = tf.equal(preds, correct_preds) training_accuracy_op = tf.reduce_mean(tf.cast(equal, tf.float32)) summaries['train'].append( tf.summary.scalar('Training_Accuracy', training_accuracy_op)) with tf.name_scope('Evaluation_Metrics'): tp_op = evaluate.tp(logits=logits, labels=y_) fp_op = evaluate.fp(logits=logits, labels=y_) tn_op = evaluate.tn(logits=logits, labels=y_) fn_op = evaluate.fn(logits=logits, labels=y_) tp_sum = tf.placeholder(tf.float32) tn_sum = tf.placeholder(tf.float32) fp_sum = tf.placeholder(tf.float32) fn_sum = tf.placeholder(tf.float32) precision_op = evaluate.precision(tp=tp_sum, fp=fp_sum, tn=tn_sum, fn=fn_sum) accuracy_op = evaluate.accuracy(tp=tp_sum, fp=fp_sum, tn=tn_sum, fn=fn_sum) recall_op = evaluate.recall(tp=tp_sum, fp=fp_sum, tn=tn_sum, fn=fn_sum) fscore_op = evaluate.fscore(tp=tp_sum, fp=fp_sum, tn=tn_sum, fn=fn_sum) precision_summ = tf.summary.scalar('Precision', precision_op) accuracy_summ = tf.summary.scalar('Accuracy', accuracy_op) recall_summ = tf.summary.scalar('Recall', recall_op) fscore_summ = tf.summary.scalar('Fscore', fscore_op) summaries['validate'].append(accuracy_summ) summaries['validate'].append(precision_summ) summaries['validate'].append(recall_summ) summaries['validate'].append(fscore_summ) summaries['test'].append(accuracy_summ) summaries['test'].append(precision_summ) summaries['test'].append(recall_summ) summaries['test'].append(fscore_summ) print("Saving graph to %s" % f.FLAGS.log_dir) train_writer = tf.summary.FileWriter(f.FLAGS.log_dir + "/train") validate_writer = tf.summary.FileWriter(f.FLAGS.log_dir + "/validate") test_writer = tf.summary.FileWriter(f.FLAGS.log_dir + "/test") train_writer.add_graph(tf.get_default_graph()) train_summaries = tf.summary.merge(summaries['train']) validate_summaries = tf.summary.merge(summaries['validate']) test_summaries = tf.summary.merge(summaries['test']) with tf.Session() as sess: train_writer.add_summary(sess.run(tf.summary.text("Information", info))) train_iter = train_dataset.make_initializable_iterator() train_next_elem = train_iter.get_next() sess.run(tf.global_variables_initializer()) global_step = 0 display_freq = 10 validate_freq = 50 test_freq = 50 for epoch in range(1, f.FLAGS.num_epochs + 1): sess.run(train_iter.initializer) step_time = 0.0 fetch_time = 0.0 while True: try: a = time.time() global_step += 1 sample, label = sess.run(train_next_elem) fetch_time += time.time() - a #print (sample.shape, label.shape) #print (label) #for s in sample[0][0]: # print (s) a = time.time() _, summ = sess.run([optimizer, train_summaries], feed_dict={ x: sample, y_: label, dropout_rate: 0.5, is_training: True }) train_writer.add_summary(summ, global_step) step_time += time.time() - a except tf.errors.OutOfRangeError: break if global_step % display_freq == 0: batch_loss, batch_accuracy = sess.run( [mean_cross_entropy_loss, training_accuracy_op], feed_dict={ x: sample, y_: label, dropout_rate: 1.0, is_training: False }) print( "Epoch {:3}\t Step {:5}:\t Loss={:.3f}, \tTraining Accuracy={:.5f} \tStep Time {:4.2f}m, Fetch Time {:4.2f}m" .format(epoch, global_step, batch_loss, batch_accuracy, step_time / 60, fetch_time / 60)) step_time = 0.0 fetch_time = 0.0 #Validate and test after each epoch val_it = validate_dataset.make_one_shot_iterator() val_next_elem = val_it.get_next() tot_tp, tot_tn, tot_fp, tot_fn = 0, 0, 0, 0 while True: try: sample, label = sess.run(val_next_elem) tp, fp, tn, fn = sess.run( [tp_op, fp_op, tn_op, fn_op], feed_dict={ x: sample, y_: label, dropout_rate: 1.0, is_training: False }) except tf.errors.OutOfRangeError: break tot_tp += tp tot_fp += fp tot_fn += fn tot_tn += tn precision, recall, accuracy, fscore, summ = sess.run([ precision_op, recall_op, accuracy_op, fscore_op, validate_summaries ], feed_dict={ tp_sum: tot_tp, tn_sum: tot_tn, fp_sum: tot_fp, fn_sum: tot_fn }) validate_writer.add_summary(summ, global_step) print("Epoch %d, Step %d" % (epoch, global_step)) print("=" * 10, "Validating Results", "=" * 10) print("TP: %g\nTN: %g\nFP: %g\nFN: %g" % (tot_tp, tot_tn, tot_fp, tot_fn)) print( "\tPrecision: %g\n\tRecall: %g\n\tF1_score: %g\n\tAccuracy: %g" % (precision, recall, fscore, accuracy)) test_it = test_dataset.make_one_shot_iterator() test_next_elem = test_it.get_next() tot_tp, tot_tn, tot_fp, tot_tn = 0, 0, 0, 0 while True: try: sample, label = sess.run(test_next_elem) tp, fp, tn, fn = sess.run( [tp_op, fp_op, tn_op, fn_op], feed_dict={ x: sample, y_: label, dropout_rate: 1.0, is_training: False }) except tf.errors.OutOfRangeError: break tot_tp += tp tot_fp += fp tot_fn += fn tot_tn += tn precision, recall, accuracy, fscore, summ = sess.run([ precision_op, recall_op, accuracy_op, fscore_op, test_summaries ], feed_dict={ tp_sum: tot_tp, tn_sum: tot_tn, fp_sum: tot_fp, fn_sum: tot_fn }) test_writer.add_summary(summ, global_step) print("=" * 10, "Testing Results", "=" * 10) print("TP: %g\nTN: %g\nFP: %g\nFN: %g" % (tot_tp, tot_tn, tot_fp, tot_fn)) print( "\tPrecision: %g\n\tRecall: %g\n\tF1_score: %g\n\tAccuracy: %g" % (precision, recall, fscore, accuracy)) print("=" * 10, "===============", "=" * 10)
def main(train_path, val_path, save_path, num_epochs=50): global T1 print('NUM EPOCHS %i' % num_epochs) # save settings shutil.copyfile('settings_char.py', '%s/settings_char.txt' % save_path) print("Preparing Data 123") # Training data Xt = [] yt = [] with io.open(train_path, 'r', encoding='utf-8') as f: for line in f: (yc, Xc) = line.rstrip('\n').split('\t') Xt.append(Xc[:MAX_LENGTH]) yt.append(yc) # Validation data Xv = [] yv = [] with io.open(val_path, 'r', encoding='utf-8') as f: for line in f: (yc, Xc) = line.rstrip('\n').split('\t') Xv.append(Xc[:MAX_LENGTH]) yv.append(yc.split(',')) print("Building Model...") if not RELOAD_MODEL: # Build dictionaries from training data chardict, charcount = batch.build_dictionary(Xt) n_char = len(chardict.keys()) + 1 batch.save_dictionary(chardict, charcount, '%s/dict.pkl' % save_path) # params params = init_params(n_chars=n_char) labeldict, labelcount = batch.build_label_dictionary(yt) batch.save_dictionary(labeldict, labelcount, '%s/label_dict.pkl' % save_path) n_classes = min(len(labeldict.keys()) + 1, MAX_CLASSES) # classification params params['W_cl'] = theano.shared(np.random.normal( loc=0., scale=SCALE, size=(WDIM, n_classes)).astype('float32'), name='W_cl') params['b_cl'] = theano.shared(np.zeros((n_classes)).astype('float32'), name='b_cl') else: print("Loading model params from base path: %s ..." % save_path) params = load_params_shared('%s/model.npz' % save_path) print("Loading dictionaries...") with open('%s/dict.pkl' % save_path, 'rb') as f: chardict = pkl.load(f) with open('%s/label_dict.pkl' % save_path, 'rb') as f: labeldict = pkl.load(f) n_char = len(chardict.keys()) + 1 n_classes = min(len(labeldict.keys()) + 1, MAX_CLASSES) # iterators train_iter = batch.BatchTweets(Xt, yt, labeldict, batch_size=N_BATCH, max_classes=MAX_CLASSES) val_iter = batch.BatchTweets(Xv, yv, labeldict, batch_size=N_BATCH, max_classes=MAX_CLASSES, test=True) print("Building network...") # Tweet variables tweet = T.itensor3() targets = T.ivector() # masks t_mask = T.fmatrix() # network for prediction predictions, net, emb = classify(tweet, t_mask, params, n_classes, n_char) # batch loss loss = lasagne.objectives.categorical_crossentropy(predictions, targets) cost = T.mean( loss ) + REGULARIZATION * lasagne.regularization.regularize_network_params( net, lasagne.regularization.l2) cost_only = T.mean(loss) reg_only = REGULARIZATION * lasagne.regularization.regularize_network_params( net, lasagne.regularization.l2) # params and updates print("Computing updates...") lr = LEARNING_RATE mu = MOMENTUM updates = lasagne.updates.nesterov_momentum( cost, lasagne.layers.get_all_params(net), lr, momentum=mu) # Theano function print("Compiling theano functions...") inps = [tweet, t_mask, targets] predict = theano.function([tweet, t_mask], predictions) cost_val = theano.function(inps, [cost_only, emb]) train = theano.function(inps, cost, updates=updates) reg_val = theano.function([], reg_only) # Training print("Training...") uidx = 0 maxp = 0. start = time.time() valcosts = [] try: for epoch in range(num_epochs): n_samples = 0 train_cost = 0. print("Epoch {}".format(epoch)) # learning schedule if len(valcosts) > 1 and SCHEDULE: change = (valcosts[-1] - valcosts[-2]) / abs(valcosts[-2]) if change < T1: lr, mu = schedule(lr, mu) updates = lasagne.updates.nesterov_momentum( cost, lasagne.layers.get_all_params(net), lr, momentum=mu) train = theano.function(inps, cost, updates=updates) T1 = T1 / 2 # stopping criterion if len(valcosts) > 6: deltas = [] for i in range(5): deltas.append((valcosts[-i - 1] - valcosts[-i - 2]) / abs(valcosts[-i - 2])) if sum(deltas) / len(deltas) < T2: break ud_start = time.time() for xr, y in train_iter: n_samples += len(xr) uidx += 1 x, x_m = batch.prepare_data(xr, chardict, n_chars=n_char) if x is None: print("Minibatch with zero samples under maxlength.") uidx -= 1 continue curr_cost = train(x, x_m, y) train_cost += curr_cost * len(xr) ud = time.time() - ud_start if np.isnan(curr_cost) or np.isinf(curr_cost): print("Nan detected.") return if np.mod(uidx, DISPF) == 0: print("Epoch {} Update {} Cost {} Time {}".format( epoch, uidx, curr_cost, ud)) if np.mod(uidx, SAVEF) == 0: print("Saving...") saveparams = OrderedDict() for kk, vv in params.iteritems(): saveparams[kk] = vv.get_value() np.savez('%s/model.npz' % save_path, **saveparams) print("Done.") print("Testing on Validation set...") preds = [] targs = [] for xr, y in val_iter: x, x_m = batch.prepare_data(xr, chardict, n_chars=n_char) if x is None: print( "Validation: Minibatch with zero samples under maxlength." ) continue vp = predict(x, x_m) ranks = np.argsort(vp)[:, ::-1] for idx, item in enumerate(xr): preds.append(ranks[idx, :]) targs.append(y[idx]) validation_cost = precision(np.asarray(preds), targs, 1) regularization_cost = reg_val() if validation_cost > maxp: maxp = validation_cost saveparams = OrderedDict() for kk, vv in params.iteritems(): saveparams[kk] = vv.get_value() np.savez('%s/best_model.npz' % (save_path), **saveparams) print( "Epoch {} Training Cost {} Validation Precision {} Regularization Cost {} Max Precision {}" .format(epoch, train_cost / n_samples, validation_cost, regularization_cost, maxp)) print("Seen {} samples.".format(n_samples)) valcosts.append(validation_cost) print("Saving...") saveparams = OrderedDict() for kk, vv in params.iteritems(): saveparams[kk] = vv.get_value() np.savez('%s/model_%d.npz' % (save_path, epoch), **saveparams) print("Done.") except KeyboardInterrupt: pass print("Total training time = {}".format(time.time() - start))
alreadyAlign) alreadyAlign = aligner.insertAlign(cekSequences, i, alreadyAlign) alreadyAlign = aligner.insertAlign(cekneighbor, i, alreadyAlign) alreadyAlign = aligner.insertAlign(cekstop, i, alreadyAlign) alreadyAlign = aligner.insertAlign(cekSimilarex, i, alreadyAlign) # # alreadyAlign = aligner.insertAlign(cekDep, i, alreadyAlign) # Ubah hasil dari yang pasangan angka menjadi hasil kata dan indeks resultMSR = tools.transformResult(alreadyAlign[i], sentenceLemma2) # Memanggil fungsi dari class Evaluate untuk menampilkan hasil skor jumlahBenar = evaluate.jumlahBenar(resultMSR, sentenceLemma2, listAnnotation[i]) jumlahGoldAnnotation = evaluate.jumlahGoldAnnotation(listAnnotation[i]) jumlahDariSistem = evaluate.jumlahDariSistem(resultMSR) precision = evaluate.precision(jumlahBenar, jumlahDariSistem) recall = evaluate.recall(jumlahBenar, jumlahGoldAnnotation) F1 = evaluate.F1Measure(precision, recall) # Hitung rata-rata precision, recall dan F1 precisionAverage.append(precision) recallAverage.append(recall) F1Average.append(F1) print("========================================") print("Sentence Lemma 1 :", sentenceLemma1) print("Sentence Lemma 2 :", sentenceLemma2) print("Gold Annotation :", evaluate.printAnnotation(sentenceLemma2, goldAnnotation[i])) file_output.write("========================================" + '\n') file_output.write("Sentence Lemma 1 : " + str(sentenceLemma1) + '\n')