def evaluate(dataset, classifier, groundTruth): '''evaluate the classifier on the given dataset, using only the points that are in the groundTruth. It then returns the confusion matrix. - dataset is the dataset from which to get the points - classifier is a function which given a point returns its class - groundTruth is a map from the points to classify to their respective class ''' total = len(groundTruth) done = 0 confusion = ConfusionMatrix() for (pointId, expected) in groundTruth.items(): try: found = classifier(dataset.point(pointId)) confusion.add(expected, found, pointId) except Exception, e: print 'WARNING: Could not classify point', pointId, 'because', str( e) done += 1 print '\r[%d%%]' % int(done * 100 / total),
def matrix(self): """Creates a Contingenz Matrix using ConfusionMatrix of NLTK""" cm = ConfusionMatrix(self.reference_tags, self.test_tags) # first reference, then test! #f = codecs.open("C:\\Users\\"+self.user+"\\Downloads\\continenzmatrix.txt", "w", "utf-8") f = codecs.open("Results\\contingenzmatrix.txt","w","utf-8") f.write(cm.pp()) f.close() ###print contingenzliste #g = codecs.open("C:\\Users\\"+self.user+"\\Downloads\\contingenzliste.txt", "w", "utf-8") #g.writelines(self.contingenzliste) #g.close() values_not_null = cm.get_values_not_null() return values_not_null
def __init__(self, num_classes, normalized=False, ignore_index=None): super(IoU, self).__init__() self.conf_metric = ConfusionMatrix(num_classes, normalized) if ignore_index is None: self.ignore_index = None elif isinstance(ignore_index, int): self.ignore_index = (ignore_index, ) else: try: self.ignore_index = tuple(ignore_index) except TypeError: raise ValueError("'ignore_index' must be an int or iterable")
def evaluate(classifier, dataset, groundTruth, confusion=None, verbose=True): """Evaluate the classifier on the given dataset and returns the confusion matrix. Uses only the points that are in the groundTruth parameter for the evaluation. Parameters ---------- classifier : a function which given a point returns its class dataset : the dataset from which to get the points groundTruth : a map from the points to classify to their respective class """ progress = TextProgress(len(groundTruth)) done = 0 confusion = confusion or ConfusionMatrix() for pointId, expected in groundTruth.items(): try: found = classifier(dataset.point(pointId)) confusion.add(expected, found, pointId) except Exception as e: log.warning('Could not classify point "%s" because %s' % (pointId, str(e))) raise done += 1 if verbose: progress.update(done) return confusion
def trial(trial_number, metric, K, N, distribution, xi): X = point.PointCloud.generate_with_classes(N, K, distribution, [True, False]) Y = point.PointCloud.generate(N, M, distribution) for y in Y: closest_point = y.nearest_point(X, metric) y.clazz = closest_point.clazz perturbed_Y = deepcopy(Y).perturb_points(xi) for y in perturbed_Y: closest_point = y.nearest_point(X, metric) y.clazz = closest_point.clazz confusion_matrix = ConfusionMatrix.generate_from_point_clouds(Y, perturbed_Y) result = [trial_number, metric, K, N, xi.sigma] + confusion_matrix.row() print(result) return result
def evaluate(dataset, classifier, groundTruth): '''evaluate the classifier on the given dataset, using only the points that are in the groundTruth. It then returns the confusion matrix. - dataset is the dataset from which to get the points - classifier is a function which given a point returns its class - groundTruth is a map from the points to classify to their respective class ''' total = len(groundTruth) done = 0 confusion = ConfusionMatrix() for (pointId, expected) in groundTruth.items(): try: found = classifier(dataset.point(pointId)) confusion.add(expected, found, pointId) except Exception, e: print 'WARNING: Could not classify point', pointId, 'because', str(e) done += 1 print '\r[%d%%]' % int(done*100 / total),
metric = random.choice(metrics) return N, K, distribution, xi, metric def trial(trial_number, metric, K, N, distribution, xi): X = point.PointCloud.generate_with_classes(N, K, distribution, [True, False]) Y = point.PointCloud.generate(N, M, distribution) for y in Y: closest_point = y.nearest_point(X, metric) y.clazz = closest_point.clazz perturbed_Y = deepcopy(Y).perturb_points(xi) for y in perturbed_Y: closest_point = y.nearest_point(X, metric) y.clazz = closest_point.clazz confusion_matrix = ConfusionMatrix.generate_from_point_clouds(Y, perturbed_Y) result = [trial_number, metric, K, N, xi.sigma] + confusion_matrix.row() print(result) return result results = [] for trial_number in range(trials): N, K, distribution, xi, metric = generate_params(N_Range, K_Range, metrics, distributions, sigma_range) results.append(trial(trial_number, metric, K, N, distribution, xi)) tablemaker.make_table('experiment1-2(1)', ['Trial', 'Metric', 'K', 'N', 'sigma'] + ConfusionMatrix.headers(), results)
all_mon = train_out[3:] grd_mon = train_out[:len(all_grads)] upd_mon = train_out[len(all_grads):] for pm, gm, um in zip(trainable_params, grd_mon, upd_mon): if '.b' not in pm.name: pad = (40-len(pm.name))*" " print "%s \t %.5e \t %.5e \t %.5e" % ( pm.name + pad, np.linalg.norm(gm), np.linalg.norm(um), np.linalg.norm(pm.get_value()) ) cost_train_lst += [cost_train] conf_train = ConfusionMatrix(num_classes) for i in range(x_train.shape[0] // 1000): probs_train, _ = f_eval(x_train[i*1000:(i+1)*1000]) preds_train_flat = probs_train.reshape((-1, num_classes)).argmax(-1) conf_train.batch_add( y_train[i*1000:(i+1)*1000].flatten(), preds_train_flat ) if last_decay > args.decayinterval and epoch > args.nodecay: last_decay = 0 old_lr = sh_lr.get_value(sh_lr) new_lr = old_lr / args.decayfac sh_lr.set_value(lasagne.utils.floatX(new_lr)) print "Decay lr from %f to %f" % (float(old_lr), float(new_lr)) else:
targetspace[tv[0]], tv[1]) logger("Done training files.", monitor) if outputmodel: # output character patterns to be able to generate new tweetvectors for separate testing on trained data stringspace.saveelementspace(charactervectorspacefilename) # output model here with info about the category of each model item with open(categorymodelfilename, "wb") as outfile: pickle.dump(targetspace, outfile) logger( "Testing targetspace with " + str(len(targetspace)) + " categories, " + str(testvectorantal) + " test items and " + str(trainvectorantal) + " training cases. ", monitor) confusion = ConfusionMatrix() averagerankofauthorhit = 0 logger( "Average linkage: " + str(averagelinkage) + " pool depth " + str(itempooldepth), monitor) for authorindex in testvectors: logger( str(authorindex) + "\t" + str(facittable[authornametable[authorindex]]) + "===============", debug) targetscore = {} for target in targets: targetscore[target] = 0 for testfile in testvectors[authorindex]: if averagelinkage: # take all test sentences and sum their scores for target in targets:
print("Validation shape: {}".format(X_val.shape)) print("Training shape: {}".format(X_tr.shape)) eps = [] best_val_acc = 0 print "Start training\n" for epoch in range(num_epochs): # Calculate epoch time start_time = time.time() # Full pass training set train_err = 0 train_batches = 0 confusion_train = ConfusionMatrix(n_class) # Generate minibatches and train on each one of them for batch in iterate_minibatches(X_tr, y_tr, mask_tr, batch_size, shuffle=True): inputs, targets, in_masks = batch tr_err, predict = train_fn(inputs, targets, in_masks) train_err += tr_err train_batches += 1 preds = np.argmax(predict, axis=-1) confusion_train.batch_add(targets, preds) train_loss = train_err / train_batches train_accuracy = confusion_train.accuracy() cf_train = confusion_train.ret_mat()
def main(): parser = argparse.ArgumentParser(description='Memlearn Chainer ver') parser.add_argument('--batchsize', '-b', type=int, default=100, help='Number of images in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=20, help='Number of sweeps over the dataset to train') parser.add_argument('--frequency', '-f', type=int, default=-1, help='Frequency of taking a snapshot') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') parser.add_argument('--train', '-i', nargs='*', help='Train data (all except test by default)') parser.add_argument('--test', '-t', nargs='*', help='Test data (you have to specify at least one') parser.add_argument('--sliceb', '-s', type=int, default=1, help='Time slice block') parser.add_argument('--list_dataset', action='store_true') parser.add_argument('--dataset', default="/data/data_filtered.pickle") args = parser.parse_args() if args.list_dataset: with open(args.dataset, 'rb') as fp: dataset = pickle.load(fp) print('\n'.join(['{} {}'.format(n, l) for n, l in dataset])) raise SystemExit if not args.test: raise SystemExit("you have to specify at least one test, see --list_dataset for available datasets") print('GPU: {}'.format(args.gpu)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('') # Set up a neural network to train # Classifier reports softmax cross entropy loss and accuracy at every # iteration, which will be used by the PrintReport extension below. model = L.Classifier(VGG(2)) #model = L.Classifier(SVM()) if args.gpu >= 0: # Make a specified GPU current chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() # Copy the model to the GPU # Setup an optimizer optimizer = chainer.optimizers.Adam() optimizer.setup(model) # Load the MEMORY dataset train, test = prep_windowed_datasets(args.dataset, args.test, args.train, slice_merge=args.sliceb) train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) # Set up a trainer updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) # Evaluate the model with the test dataset for each epoch #trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu)) trainer.extend(ConfusionMatrix(test_iter, model, device=args.gpu)) # Dump a computational graph from 'loss' variable at the first iteration # The "main" refers to the target link of the "main" optimizer. trainer.extend(extensions.dump_graph('main/loss')) # Take a snapshot for each specified epoch frequency = args.epoch if args.frequency == -1 else max(1, args.frequency) trainer.extend(extensions.snapshot(), trigger=(frequency, 'epoch')) # Write a log of evaluation statistics for each epoch trainer.extend(extensions.LogReport()) # Save two plot images to the result dir if extensions.PlotReport.available(): trainer.extend( extensions.PlotReport(['main/loss', 'validation/main/loss'], 'epoch', file_name='loss.png')) trainer.extend( extensions.PlotReport( ['main/accuracy', 'validation/main/accuracy'], 'epoch', file_name='accuracy.png')) # Print selected entries of the log to stdout # Here "main" refers to the target link of the "main" optimizer again, and # "validation" refers to the default name of the Evaluator extension. # Entries other than 'epoch' are reported by the Classifier link, called by # either the updater or the evaluator. trainer.extend(extensions.PrintReport( ['epoch', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'elapsed_time'])) # Print a progress bar to stdout trainer.extend(extensions.ProgressBar()) if args.resume: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) # Run the training trainer.run()
#print("The batch loss is : {} for itteration number : {}".format(res[1],i)) #embed()######new test #pred_network= sess.run(fetches=[y], feed_dict={x_pl: x_batch,is_training: False}) #pred=np.asarray(pred_network) #pred=pred[0][:,1] #fpr, tpr, thresholds = metrics.roc_curve(target_batch, pred, pos_label=0) #aucScore=metrics.auc(fpr, tpr) #aucList.append(aucScore) #print("The auc score is {} ".format(aucScore)) #accuracyList.append(acc) batch_loss = res[1] #this will do the complete backprob pass cur_loss += batch_loss loss += [cur_loss / batch_size] confusion_valid = ConfusionMatrix(num_classes) confusion_train = ConfusionMatrix(num_classes) print("Training done") trainLoss = [] trainAcc = [] trainAuc = [] for i in range(num_batches_train): # The model is evaluated index = range(i * batch_size, (i + 1) * batch_size) x_batch = x_train[index] targets_batch = targets_train[index] fetches_train = [cross_entropy] feed_dict_train = { x_pl: x_batch, y_: myOneHot(targets_batch), is_training: False
weight = Weighting(new_cleaned_data, new_terms) tfidf = weight.get_tf_idf_weighting() idf = weight.get_idf() nb = NBMultinomial() nb.fit(new_cleaned_data, new_terms, data_train[i]["target"], stopwords, idf, tfidf) for j in range(len(data_test[i]["tweet"])): print("Test ke " + str(j)) prediction = nb.predict(data_test[i]["tweet"][j], data_test[i]["target"][j]) y_test.append(data_test[i]["target"][j]) y_pred.append(prediction) cm = ConfusionMatrix() accuracy, accuracy_each_class, precision_each_class, recall_each_class, fmeasure_each_class = cm.score( y_test, y_pred) acc_neg.append(accuracy_each_class[0]) acc_net.append(accuracy_each_class[1]) acc_pos.append(accuracy_each_class[2]) prec_neg.append(precision_each_class[0]) prec_net.append(precision_each_class[1]) prec_pos.append(precision_each_class[2]) recall_neg.append(recall_each_class[0]) recall_net.append(recall_each_class[1]) recall_pos.append(recall_each_class[2]) fmeasure_neg.append(fmeasure_each_class[0]) fmeasure_net.append(fmeasure_each_class[1]) fmeasure_pos.append(fmeasure_each_class[2])
num_epochs = 25 train_acc = [] valid_acc = [] cur_loss = 0 loss = [] valid_loss = [] threshold, upper, lower = 0.5, 1, 0 Train = DP.get_paths("/home/xvt131/Functions/Adhish_copy/Training-Rand") Test = DP.get_paths("/home/xvt131/Functions/Adhish_copy/Validating-Rand") import gc for epoch in range(num_epochs): cur_loss = 0 val_loss = 0 confusion_valid = ConfusionMatrix(2) confusion_train = ConfusionMatrix(2) for im in Train: XY, XZ, YZ, Y_train = DP.Patch_triplanar_para(im, PS) num_samples_train = Y_train.shape[0] num_batches_train = num_samples_train // batch_size for i in range(num_batches_train): idx = range(i * batch_size, (i + 1) * batch_size) xy_batch = XY[idx] xz_batch = XZ[idx] yz_batch = YZ[idx] target_batch = np.float32(Y_train[idx].reshape(batch_size, 1)) batch_loss = f_train(xy_batch, xz_batch, yz_batch, target_batch) #this will do the backprop pass cur_loss += batch_loss[0] / batch_size
ffn = FeedforwardNetwork() ffn.add(LinearLayer(num_inputs, num_hidden_units)) ffn.add(ReluActivationLayer()) ffn.add(LinearLayer(num_hidden_units, num_classes)) ffn.add(SoftmaxActivationLayer()) losslayer = CrossEntropyLoss() print "Network" print ffn print "Loss", losslayer print "" acc = [] for epoch in range(num_epochs): confusion = ConfusionMatrix(num_classes) for i in range(num_batches): idx = range(i*batch_size, (i+1)*batch_size) x_batch = x_train[idx] target_batch = targets_train[idx] y_probs = ffn.forward(x_batch) loss = losslayer.fprop(y_probs, target_batch) delta = losslayer.bprop(y_probs, target_batch) ffn.backward(delta) ffn.update(learning_rate) confusion.batch_add(target_batch.argmax(-1), y_probs.argmax(-1)) curr_acc = confusion.accuracy() print "Epoch %i : Loss %f Train acc %f" % (epoch, loss, curr_acc) acc += [curr_acc]
print("\nEvaluation:") accuracy, predictions = dev_step(x_test, y_test, writer=dev_summary_writer) if accuracy > max_acc: max_acc = accuracy path = saver.save(sess, checkpoint_prefix, global_step=current_step) print("Saved model checkpoint to {}\n".format(path)) else: ckpt = tf.train.get_checkpoint_state(checkpoint_dir) saver.restore(sess, ckpt.model_checkpoint_path) accuracy, predictions = dev_step( x_test, y_test) # dev_step(x_dev, y_dev, writer=dev_summary_writer) outputs = np.argmax(predictions, axis=-1) print(outputs.shape) confusion_test = ConfusionMatrix(num_classes) y_test = np.argmax(y_test, axis=-1) print(y_test.shape) confusion_test.batch_add(y_test, outputs) test_accuracy = confusion_test.accuracy() cf_test = confusion_test.ret_mat() print("FINAL TEST RESULTS") print(confusion_test) print(" test accuracy:\t\t{:.2f} %".format(test_accuracy * 100)) print(" test Gorodkin:\t\t{:.2f}".format(gorodkin(cf_test))) print(" test IC:\t\t{:.2f}".format(IC(cf_test)))
def runbatchtest(fraction, n: int = 100): logger("{} {} {}".format(n, fraction, ticker), monitor) keylist = list(vectorrepositoryall.keys())[:n] random.shuffle(keylist) split = int(len(keylist) * fraction) train = keylist[:split] test = keylist[split:] logger("{} train vs {} test".format(len(train), len(test)), monitor) ones = [] nils = [] dummymaxconfusionmatrix = ConfusionMatrix() dummyrandomconfusionmatrix = ConfusionMatrix() centroidconfusionmatrix = ConfusionMatrix() poolconfusionmatrix = ConfusionMatrix() for trainitem in test: if illness[trainitem] == "1": ones.append(vectorrepositoryall[trainitem]) else: nils.append(vectorrepositoryall[trainitem]) onecentroid = sparsevectors.centroid(ones) nilcentroid = sparsevectors.centroid(nils) if len(nils) > len(ones): dummymaxguess = "0" else: dummymaxguess = "1" # factor = len(ones) / len(nils) # no, bad idea, go for fifty-fifty factor = 1 / 2 for testitem in test: dummymaxconfusionmatrix.addconfusion(illness[testitem], dummymaxguess) if random.random() > factor: dummyrandomguess = "0" else: dummyrandomguess = "1" dummyrandomconfusionmatrix.addconfusion(illness[testitem], dummyrandomguess) probe = vectorrepositoryall[testitem] resultc = "0" i1 = sparsevectors.sparsecosine(probe, onecentroid) n1 = sparsevectors.sparsecosine(probe, nilcentroid) if i1 > n1: resultc = "1" centroidconfusionmatrix.addconfusion(illness[testitem], resultc) probeneighbours = {} for targetitem in train: probeneighbours[targetitem] = sparsevectors.sparsecosine( probe, vectorrepositoryall[targetitem]) sortedfriends = sorted(probeneighbours, key=lambda hh: probeneighbours[hh], reverse=True)[:pooldepth] illity = 0 result = "0" for friend in sortedfriends: if illness[friend] == "1": illity += 1 if illity > pooldepth * factor: result = "1" nullity = pooldepth - illity poolconfusionmatrix.addconfusion(illness[testitem], result) print("{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format( testitem, illness[testitem], resultc, i1, n1, result, illity, nullity, pooldepth)) print("RANDOM ----------------") dummyrandomconfusionmatrix.evaluate() print("MAX ----------------") dummymaxconfusionmatrix.evaluate() print("CENTROID ----------------") centroidconfusionmatrix.evaluate() print("NEIGHBOURS --------------") poolconfusionmatrix.evaluate()
def run_kfold(topic, no_folds, results_type, conf_matrix): """ Runs kfold test using credentials in ../Credentials.py """ # get credentials, import + export folders import Credentials active_adoption = Credentials.active_adoption instance_creds = Credentials.ctx[active_adoption] workspace_id = Credentials.workspace_id[active_adoption][topic] workspace_thresh = Credentials.calculate_workspace_thresh(topic) conversation_version = Credentials.conversation_version # import + export folders import config import time data_folder = config.data_dir export_folder = config.output_folder timestr = time.strftime("%Y%m%d-%H%M") output_loc_results = os.path.join( export_folder, "{}_kfold_results_raw_{}.csv".format(topic, timestr)) output_loc_metrics = os.path.join( export_folder, "{}_kfold_results_metrics_{}.csv".format(topic, timestr)) output_loc_confmat = os.path.join( export_folder, "{}_kfold_confmat_{}.png".format(topic, timestr)) # authenticate if 'apikey' in instance_creds: logger.debug("Authenticating (apikey)") kf = kfoldtest(n_folds=no_folds, apikey=instance_creds['apikey'], url=instance_creds['url'], threshold=workspace_thresh, version=conversation_version) elif 'password' in instance_creds: logger.debug("Authenticating (username/password)") kf = kfoldtest(n_folds=no_folds, username=instance_creds['username'], password=instance_creds['password'], url=instance_creds['url'], threshold=workspace_thresh, version=conversation_version) # get train df from watson + check there are sufficient workspaces to run the test train_df = kf.intent_df_from_watson(workspace_id) kf.check_sufficient_workspaces() # create folds in WA if above is true folds = kf.create_folds(method='kfold') kf.create_kfold_WA(folds) available_flag = False while available_flag == False: logger.info("Checking workspaces..") available_flag = kf.check_workspaces_status() time.sleep(20) # run kfold test try: results = kf.run_kfold_test(folds) if (results_type == 'raw') or (results_type == 'all'): results.to_csv(output_loc_results) classification_report = kf.create_classification_report(results) if (results_type == 'metrics') or (results_type == 'all'): metrics = Metrics(workspace_thresh) metric_df = metrics.get_all_metrics_CV( results, fold_col='fold', detailed_results=False) metric_df.to_csv(output_loc_metrics) # TODO: confusion matrix if conf_matrix: from confusionmatrix import ConfusionMatrix cfn = ConfusionMatrix(workspace_thresh=workspace_thresh) cfn.create(results, fig_path=output_loc_confmat) logger.info("Confusion matrix saved to {}".format( output_loc_confmat)) finally: # regardless of what happens above, delete the temporary workspaces before exiting kf.delete_kfold_workspaces()
for item in testers: neighbours[item] = {} for otheritem in targetspace.items(): if testitemspace.name[item] == targetspace.name[otheritem]: continue neighbours[item][otheritem] = sparsevectors.sparsecosine( testitemspace.indexspace[item], targetspace.indexspace[otheritem]) logger("Done calculating neighbours", monitor) logger("Pool depth " + str(itempooldepth), monitor) if averagelinkage: logger("Averagelinkage", monitor) if votelinkage: logger("Votelinkage", monitor) confusion = ConfusionMatrix() primeconfusion = ConfusionMatrix() targetscore = {} for item in testers: sortedneighbours = sorted(neighbours[item], key=lambda hh: neighbours[item][hh], reverse=True)[:itempooldepth] primeconfusion.addconfusion(facittable[testitemspace.name[item]], targetspace.category[sortedneighbours[0]]) for target in categories: targetscore[target] = 0 if averagelinkage: # take all test neighbours and sum their scores for neighbour in sortedneighbours: targetscore[targetspace. category[neighbour]] += neighbours[item][neighbour] elif votelinkage:
num_epochs = 175 train_acc= [] valid_acc = [] cur_loss = 0 loss = [] valid_loss = [] Train = DP.get_paths("/home/xvt131/Biomediq/Validating-Rand") Test = DP.get_paths("/home/xvt131/Biomediq/Validating-Rand") import gc for epoch in range(num_epochs): cur_loss = 0 val_loss = 0 confusion_valid = ConfusionMatrix(2) confusion_train = ConfusionMatrix(2) for im in Train: XY, XZ, YZ, Y_train = DP.Patch_triplanar_para(im, PS) num_samples_train = Y_train.shape[0] num_batches_train = num_samples_train // batch_size for i in range(num_batches_train): idx = range(i*batch_size, (i+1)*batch_size) xy_batch = XY[idx] xz_batch = XZ[idx] yz_batch = YZ[idx] target_batch = Y_train[idx] batch_loss = f_train(xy_batch, xz_batch, yz_batch ,target_batch) #this will do the backprop pass
all_mon = train_out[3:] grd_mon = train_out[:len(all_grads)] upd_mon = train_out[len(all_grads):] for pm, gm, um in zip(trainable_params, grd_mon, upd_mon): if '.b' not in pm.name: pad = (40-len(pm.name))*" " print("%s \t %.5e \t %.5e \t %.5e" % ( pm.name + pad, np.linalg.norm(gm), np.linalg.norm(um), np.linalg.norm(pm.get_value()) )) cost_train_lst += [cost_train] conf_train = ConfusionMatrix(num_classes) for i in range(x_train.shape[0] // 1000): probs_train, _ = f_eval(x_train[i*1000:(i+1)*1000]) preds_train_flat = probs_train.reshape((-1, num_classes)).argmax(-1) conf_train.batch_add( y_train[i*1000:(i+1)*1000].flatten(), preds_train_flat ) if last_decay > args.decayinterval and epoch > args.nodecay: last_decay = 0 old_lr = sh_lr.get_value(sh_lr) new_lr = old_lr / args.decayfac sh_lr.set_value(lasagne.utils.floatX(new_lr)) print("Decay lr from %f to %f" % (float(old_lr), float(new_lr))) else:
print("\n") for item in items: print(str(item), itemspace.name[item]) result = {} prunedresult = {} for c in categories: result[c] = {} prunedresult[c] = {} logger("Pool depth " + str(itempooldepth), monitor) if averagelinkage: logger("Averagelinkage", monitor) if votelinkage: logger("Votelinkage", monitor) confusion = ConfusionMatrix() prunedconfusion = ConfusionMatrix() targetscore = {} prunedtargetscore = {} for item in testvectors: sortedneighbours = sorted(neighbours[item], key=lambda hh: neighbours[item][hh], reverse=True)[:itempooldepth] if cleanup: prunedsortedneighbours = sorted( prunedneighbours[item], key=lambda hh: prunedneighbours[item][hh], reverse=True)[:itempooldepth] for target in categories: targetscore[target] = 0 prunedtargetscore[target] = 0
class IoU(metric.Metric): """Computes the intersection over union (IoU) per class and corresponding mean (mIoU). Intersection over union (IoU) is a common evaluation metric for semantic segmentation. The predictions are first accumulated in a confusion matrix and the IoU is computed from it as follows: IoU = true_positive / (true_positive + false_positive + false_negative). Keyword arguments: - num_classes (int): number of classes in the classification problem - normalized (boolean, optional): Determines whether or not the confusion matrix is normalized or not. Default: False. - ignore_index (int or iterable, optional): Index of the classes to ignore when computing the IoU. Can be an int, or any iterable of ints. """ def __init__(self, num_classes, normalized=False, ignore_index=None): super(IoU, self).__init__() self.conf_metric = ConfusionMatrix(num_classes, normalized) if ignore_index is None: self.ignore_index = None elif isinstance(ignore_index, int): self.ignore_index = (ignore_index, ) else: try: self.ignore_index = tuple(ignore_index) except TypeError: raise ValueError("'ignore_index' must be an int or iterable") def reset(self): self.conf_metric.reset() def add(self, predicted, target): """Adds the predicted and target pair to the IoU metric. Keyword arguments: - predicted (Tensor): Can be a (N, K, H, W) tensor of predicted scores obtained from the model for N examples and K classes, or (N, H, W) tensor of integer values between 0 and K-1. - target (Tensor): Can be a (N, K, H, W) tensor of target scores for N examples and K classes, or (N, H, W) tensor of integer values between 0 and K-1. """ # Dimensions check assert predicted.size(0) == target.size(0), \ 'number of targets and predicted outputs do not match' assert predicted.dim() == 3 or predicted.dim() == 4, \ "predictions must be of dimension (N, H, W) or (N, K, H, W)" assert target.dim() == 3 or target.dim() == 4, \ "targets must be of dimension (N, H, W) or (N, K, H, W)" # If the tensor is in categorical format convert it to integer format # When the tensor is 4-dimensional, for each of the 3-dimensions of # class for each pixel, the value it contains is confidence of that # class. We take the "index" (class label) with the maximum confidence # among the classes (i.e 3 dimensions) if predicted.dim() == 4: _, predicted = predicted.max(1) if target.dim() == 4: _, target = target.max(1) # .view(-1) converts tensor to single dimensional tensor self.conf_metric.add(predicted.view(-1), target.view(-1)) def value(self): """Computes the IoU and mean IoU. The mean computation ignores NaN elements of the IoU array. Returns: Tuple: (IoU, mIoU). The first output is the per class IoU, for K classes it's numpy.ndarray with K elements. The second output, is the mean IoU. """ conf_matrix = self.conf_metric.value() if self.ignore_index is not None: for index in self.ignore_index: conf_matrix[:, self.ignore_index] = 0 conf_matrix[self.ignore_index, :] = 0 true_positive = np.diag(conf_matrix) false_positive = np.sum(conf_matrix, 0) - true_positive false_negative = np.sum(conf_matrix, 1) - true_positive # Just in case we get a division by 0, ignore/hide the error with np.errstate(divide='ignore', invalid='ignore'): iou = true_positive / (true_positive + false_positive + false_negative) return iou, np.nanmean(iou)
def main(instance_folder, results_folder, remap_file=None, parameters={'-c': 1}, multilabel=None, pattern=None, silent=False): ''' Returns a list of lists of predicted labels (per fold). Contains the main pipeline: * Remaps the class labels if a mapping is specified. * If a specific label is specified in <multilabel>, assume we're dealing with a multilabel dataset and only consider the given label. * For each fold, trains and tests each fold based on the instances in <instance_folder>. By default the script looks for files with the following filenames: N_GRAM_TOKEN_?1.TotalSet.ngrams.txt or Use the <pattern> option to specify your own pattern (can be a regex). ''' assert not (multilabel and remap_file), '--multilabel and --remap cannot be used at the same time' instance_files = getInstances(instance_folder, pattern=pattern) labels = getLabelMap(instance_folder) ''' Get all class (re-)mappings. Example: original label original SVM remapped remapped SVM A 1 X 1 B 2 Y 2 C 3 Y 2 D 4 Y 2 The stylene_mapping maps from the original label to its SVM equivalent The intermediate mapping maps from "orginal SVM" to "remapped SVM" The remapping maps from the original label to the remapped labels The true_mapping maps from "remapped" to its SVM equivalent ''' true_mapping = {} stylene_mapping = getLabelMap(instance_folder) stylene_start_idx = min(stylene_mapping.values()) stylene_max_idx = max(stylene_mapping.values()) if remap_file: intermediate_mapping = {} remapping = getRemapping(remap_file) assert not False in [(l in remapping) for l in stylene_mapping], "Your label remapping file doesn't contain all classes found in the Stylene mapping!" for i, label in enumerate(sorted(set(remapping.values()))): # Set instances to be excluded. if label == 'EXCLUDE': true_mapping[label] = 0 else: if stylene_start_idx == 0: true_mapping[label] = i+1 else: true_mapping[label] = i for label, mapped in stylene_mapping.items(): intermediate_mapping[mapped] = true_mapping[remapping[label]] elif multilabel: remapping = {} stylene_mapping['other'] = stylene_max_idx + 1 assert multilabel in stylene_mapping, "The --multilabel label doesn't seem to be present in the Stylene mapping!" for label in stylene_mapping: if label == multilabel: remapping[label] = label else: remapping[label] = 'other' true_mapping[multilabel] = stylene_mapping[multilabel] true_mapping['other'] = stylene_max_idx + 1 intermediate_mapping = {} for label, mapped in stylene_mapping.items(): intermediate_mapping[mapped] = true_mapping[remapping[label]] multilabel = (multilabel, stylene_mapping[multilabel]) else: intermediate_mapping = None true_mapping = stylene_mapping # Invert for reverse lookup true_mapping = invertLabelMap(true_mapping) gold_labels = [] predicted_labels = [] predicted_values = defaultdict(list) global_cm = ConfusionMatrix() fold_accuracies = [] # Train and test each fold. for i, (train_file, test_file) in enumerate(instance_files): # If the dictionary file exists, use it to add more meta-information. base_name = test_file.split('.ngrams.txt')[0] if base_name == test_file: dict_file = os.path.splitext(test_file)[0] + '.filelist.txt' else: dict_file = base_name + '.dictionary.ngrams.txt' base_name = os.path.basename(base_name.split('.TotalSet')[0]) if not os.path.exists(dict_file): original_filenames = None else: original_filenames = [] with open(dict_file, 'r') as fin: while True: line = fin.readline() if not line.strip(): break original_filenames.append(line.strip().split()[1]) print 'Processing fold {0}/{1}'.format(i+1, len(instance_files)) # Train! fold_model, fold_distribution = train(train_file, i, intermediate_mapping, parameters, multilabel, results_folder) if not silent: print 'Training distribution:', dict([(true_mapping[label], dist) for label, dist in fold_distribution.items()]) # Classify! pred_labels, pred_values, label_order, test_labels = test(test_file, fold_model, i, intermediate_mapping, multilabel) # Write the predicted (non-remapped) labels to file. pred_values_dict = dict([(label, []) for label in label_order]) output_folder = os.path.join(results_folder, 'fold-{0:02d}'.format(i+1)) if multilabel: predictions_file = os.path.join(output_folder, os.path.basename(test_file) + '.{0}.predicted'.format(multilabel[0])) else: predictions_file = os.path.join(output_folder, os.path.basename(test_file) + '.predicted') with open(predictions_file, 'w') as fout: # Headers fout.write('filename predicted\t') for label in label_order: fout.write('{0}\t'.format(label)) fout.write('{0}\n'.format(dict([(label, true_mapping[label]) for label in label_order]))) # Predictions for j, (pred_label, pred_vals) in enumerate(zip(pred_labels, pred_values)): for k, value in enumerate(pred_vals): pred_values_dict[label_order[k]].append(value) if not original_filenames: filename = 'unknown' else: filename = original_filenames[j].split('.ProcessedFrequencies.xml')[0].split(base_name)[0].rstrip('.') fout.write('{0}\t{1}\t{2}\n'.format(filename, int(pred_label), ' '.join([str(val) for val in pred_vals]))) # Remap to the true labels pred_labels = [true_mapping[label] for label in pred_labels] predicted_labels.extend(pred_labels) g_labels = [true_mapping[label] for label in test_labels] gold_labels.extend(g_labels) local_cm = ConfusionMatrix() for (gold, pred) in zip(g_labels, pred_labels): local_cm.update(gold, pred) global_cm.update(gold, pred) # Update the predicted values pred_values = dict([(true_mapping[label], values) for label, values in pred_values_dict.items()]) assert i==0 or sorted(pred_values) == sorted(predicted_values) for label, values in pred_values.items(): predicted_values[label].extend(values) if not silent: print local_cm print 'Fold Accuracy: {0:.3f}'.format(local_cm.accuracy) fold_accuracies.append(local_cm.accuracy) avg_acc = sum(fold_accuracies)/len(fold_accuracies) stdev = (sum([(acc - avg_acc)**2 for acc in fold_accuracies])/len(fold_accuracies))**0.5 correct = sum([global_cm.TP(c) for c in global_cm.classes]) total = len(global_cm) if not silent: print print 'Global confusion matrix:' print global_cm print for label in global_cm.classes: print '{0} P\t{1:.3f}'.format(label, global_cm.precision(label)) print '{0} R\t{1:.3f}'.format(label, global_cm.recall(label)) print '{0} F\t{1:.3f}'.format(label, global_cm.fmeasure(label)) print print 'Accuracy: {0:.3f} +/- {1:.3f} ({2}/{3})'.format(global_cm.accuracy, stdev, correct, total) print 'Micro-avg F:', global_cm.microfmeasure() print 'Macro-avg F:', global_cm.macrofmeasure() return global_cm
start = time.time() y_test = [] y_pred = [] for j in range(len(data_test[i]["tweet"])): prediction = nb.predict(data_test[i]["tweet"][j], data_test[i]["target"][j]) y_test.append(data_test[i]["target"][j]) y_pred.append(prediction) print("nb pred") print(time.time() - start) start = time.time() cm = ConfusionMatrix() accuracy, precision, recall, fmeasure = cm.score(y_test, y_pred) print("Stopwords") print(stopwords) print("\nRemoved Stopwords") print(removed_words) print("\nAccuracy : {}".format(accuracy)) print("Precision : {}".format(precision)) print("Recall : {}".format(recall)) print("FMeasure : {}".format(fmeasure)) # df = pd.DataFrame({'X':x_array,'Y':y_array,'L':l_array,'K-Fold':kfold_per_combination,'Accuracy':list_acc,'Precision':list_prec,'Recall':list_recall,'F-Measure':list_fmeasure,'Fold Accuracy':fold_accuracy,'Fold Precision':fold_precision,'Fold Recall':fold_recall,'Fold F-Measure':fold_fmeasure}) # print(df) # df.to_excel(r'cobabarunih.xlsx', index = False, header=True)
num_epochs = 150 train_acc = [] valid_acc = [] cur_loss = 0 loss = [] valid_loss = [] Train = DP.get_paths("/home/xvt131/Functions/Adhish_copy/Training-Rand") Test = DP.get_paths("/home/xvt131/Functions/Adhish_copy/Validating-Rand") import gc for epoch in range(num_epochs): cur_loss = 0 val_loss = 0 confusion_valid = ConfusionMatrix(2) confusion_train = ConfusionMatrix(2) for im in Train: XY, XZ, YZ, Pos, Y_train = DP.Patch_triplanar_para(im, PS) num_samples_train = Y_train.shape[0] num_batches_train = num_samples_train // batch_size for i in range(num_batches_train): idx = range(i * batch_size, (i + 1) * batch_size) xy_batch = XY[idx] xz_batch = XZ[idx] yz_batch = YZ[idx] pos_batch = Pos[idx] target_batch = Y_train[idx] batch_loss = f_train(xy_batch, xz_batch, yz_batch,
y_test, writer=dev_summary_writer) if accuracy > max_acc: max_acc = accuracy path = saver.save(sess, checkpoint_prefix, global_step=current_step) print("Saved model checkpoint to {}\n".format(path)) else: ckpt = tf.train.get_checkpoint_state(checkpoint_dir) saver.restore(sess, ckpt.model_checkpoint_path) accuracy, predictions = dev_step(x_test, y_test) #dev_step(x_dev, y_dev, writer=dev_summary_writer) outputs = np.argmax(predictions, axis=-1) print(outputs.shape) confusion_test = ConfusionMatrix(num_classes) y_test = np.argmax(y_test, axis=-1) print(y_test.shape) confusion_test.batch_add(y_test, outputs) test_accuracy = confusion_test.accuracy() a, positive_predictive_value = confusion_test.positive_predictive_value() b, negative_predictive_value = confusion_test.negative_predictive_value() e, F1 = confusion_test.F1() f, MCC = confusion_test.matthews_correlation() cf_val = confusion_test.ret_mat() print("FINAL TEST RESULTS") print(confusion_test) print(cf_val)
def run_blindset(topic, results_type, conf_matrix, blindset_name): """ Runs blindset test using credentials in ../Credentials.py """ # get credentials, import + export folders import Credentials active_adoption = Credentials.active_adoption instance_creds = Credentials.ctx[active_adoption] print(instance_creds) print('print works') workspace_id = Credentials.workspace_id[active_adoption][topic] workspace_thresh = Credentials.calculate_workspace_thresh(topic) conversation_version = Credentials.conversation_version # import + export folders import config import time data_folder = config.data_dir export_folder = config.output_folder timestr = time.strftime("%Y%m%d-%H%M") blindset_name = blindset_name or topic + "_blindset.csv" output_loc_results = os.path.join( export_folder, "{}_results_raw_{}.csv".format(topic, timestr)) output_loc_metrics = os.path.join( export_folder, "{}_results_metrics_{}.csv".format(topic, timestr)) output_loc_confmat = os.path.join( export_folder, "{}_confmat_{}.png".format(topic, timestr)) # authenticate if 'apikey' in instance_creds: logger.debug("Authenticating (apikey)") bs = blindset(apikey=instance_creds['apikey'], url=instance_creds['url'], threshold=workspace_thresh, version=conversation_version) elif 'password' in instance_creds: logger.debug("Authenticating (username/password)") bs = blindset(username=instance_creds['username'], password=instance_creds['password'], url=instance_creds['url'], threshold=workspace_thresh, version=conversation_version) # run test blindset_df = bs.import_blindset(os.path.join(data_folder, blindset_name)) # TODO: check blindset df results = bs.run_blind_test(blindset_df, workspace_id) # exports + metrics if (results_type == 'raw') or (results_type == 'all'): cols_export = [ col for col in results.columns.values if col != 'intent_correct' ] results[cols_export].to_csv(output_loc_results, encoding='utf-8') logger.info("Raw results exported to {}".format(output_loc_results)) if (results_type == 'metrics') or (results_type == 'all'): met = Metrics(workspace_thresh) metric_df, _ = met.get_all_metrics(results, detailed_results=True) metric_df.to_csv(output_loc_metrics, encoding='utf-8') logger.info( "Metrics per intent exported to {}".format(output_loc_metrics)) # confusion matrix if conf_matrix: from confusionmatrix import ConfusionMatrix cfn = ConfusionMatrix(workspace_thresh=workspace_thresh) cfn.create(results, fig_path=output_loc_confmat) #bs.plot_confusion_matrix(results, output_loc_confmat) logger.info("Confusion matrix saved to {}".format(output_loc_confmat)) # print high-level metrics overall_metrics = bs.calculate_overall_metrics(results, av_method="weighted") logger.info("Overall metrics for the workspace (weighted):") logger.info(overall_metrics)