def evaluate(classifier, dataset, groundTruth, confusion=None, verbose=True): """Evaluate the classifier on the given dataset and returns the confusion matrix. Uses only the points that are in the groundTruth parameter for the evaluation. Parameters ---------- classifier : a function which given a point returns its class dataset : the dataset from which to get the points groundTruth : a map from the points to classify to their respective class """ progress = TextProgress(len(groundTruth)) done = 0 confusion = confusion or ConfusionMatrix() for pointId, expected in groundTruth.items(): try: found = classifier(dataset.point(pointId)) confusion.add(expected, found, pointId) except Exception as e: log.warning('Could not classify point "%s" because %s' % (pointId, str(e))) raise done += 1 if verbose: progress.update(done) return confusion
def evaluate(dataset, classifier, groundTruth): '''evaluate the classifier on the given dataset, using only the points that are in the groundTruth. It then returns the confusion matrix. - dataset is the dataset from which to get the points - classifier is a function which given a point returns its class - groundTruth is a map from the points to classify to their respective class ''' total = len(groundTruth) done = 0 confusion = ConfusionMatrix() for (pointId, expected) in groundTruth.items(): try: found = classifier(dataset.point(pointId)) confusion.add(expected, found, pointId) except Exception, e: print 'WARNING: Could not classify point', pointId, 'because', str( e) done += 1 print '\r[%d%%]' % int(done * 100 / total),
def __init__(self, num_classes, normalized=False, ignore_index=None): super(IoU, self).__init__() self.conf_metric = ConfusionMatrix(num_classes, normalized) if ignore_index is None: self.ignore_index = None elif isinstance(ignore_index, int): self.ignore_index = (ignore_index, ) else: try: self.ignore_index = tuple(ignore_index) except TypeError: raise ValueError("'ignore_index' must be an int or iterable")
start = time.time() y_test = [] y_pred = [] for j in range(len(data_test[i]["tweet"])): prediction = nb.predict(data_test[i]["tweet"][j], data_test[i]["target"][j]) y_test.append(data_test[i]["target"][j]) y_pred.append(prediction) print("nb pred") print(time.time() - start) start = time.time() cm = ConfusionMatrix() accuracy, precision, recall, fmeasure = cm.score(y_test, y_pred) print("Stopwords") print(stopwords) print("\nRemoved Stopwords") print(removed_words) print("\nAccuracy : {}".format(accuracy)) print("Precision : {}".format(precision)) print("Recall : {}".format(recall)) print("FMeasure : {}".format(fmeasure)) # df = pd.DataFrame({'X':x_array,'Y':y_array,'L':l_array,'K-Fold':kfold_per_combination,'Accuracy':list_acc,'Precision':list_prec,'Recall':list_recall,'F-Measure':list_fmeasure,'Fold Accuracy':fold_accuracy,'Fold Precision':fold_precision,'Fold Recall':fold_recall,'Fold F-Measure':fold_fmeasure}) # print(df) # df.to_excel(r'cobabarunih.xlsx', index = False, header=True)
all_mon = train_out[3:] grd_mon = train_out[:len(all_grads)] upd_mon = train_out[len(all_grads):] for pm, gm, um in zip(trainable_params, grd_mon, upd_mon): if '.b' not in pm.name: pad = (40-len(pm.name))*" " print("%s \t %.5e \t %.5e \t %.5e" % ( pm.name + pad, np.linalg.norm(gm), np.linalg.norm(um), np.linalg.norm(pm.get_value()) )) cost_train_lst += [cost_train] conf_train = ConfusionMatrix(num_classes) for i in range(x_train.shape[0] // 1000): probs_train, _ = f_eval(x_train[i*1000:(i+1)*1000]) preds_train_flat = probs_train.reshape((-1, num_classes)).argmax(-1) conf_train.batch_add( y_train[i*1000:(i+1)*1000].flatten(), preds_train_flat ) if last_decay > args.decayinterval and epoch > args.nodecay: last_decay = 0 old_lr = sh_lr.get_value(sh_lr) new_lr = old_lr / args.decayfac sh_lr.set_value(lasagne.utils.floatX(new_lr)) print("Decay lr from %f to %f" % (float(old_lr), float(new_lr))) else:
targetspace[tv[0]], tv[1]) logger("Done training files.", monitor) if outputmodel: # output character patterns to be able to generate new tweetvectors for separate testing on trained data stringspace.saveelementspace(charactervectorspacefilename) # output model here with info about the category of each model item with open(categorymodelfilename, "wb") as outfile: pickle.dump(targetspace, outfile) logger( "Testing targetspace with " + str(len(targetspace)) + " categories, " + str(testvectorantal) + " test items and " + str(trainvectorantal) + " training cases. ", monitor) confusion = ConfusionMatrix() averagerankofauthorhit = 0 logger( "Average linkage: " + str(averagelinkage) + " pool depth " + str(itempooldepth), monitor) for authorindex in testvectors: logger( str(authorindex) + "\t" + str(facittable[authornametable[authorindex]]) + "===============", debug) targetscore = {} for target in targets: targetscore[target] = 0 for testfile in testvectors[authorindex]: if averagelinkage: # take all test sentences and sum their scores for target in targets:
print("Validation shape: {}".format(X_val.shape)) print("Training shape: {}".format(X_tr.shape)) eps = [] best_val_acc = 0 print "Start training\n" for epoch in range(num_epochs): # Calculate epoch time start_time = time.time() # Full pass training set train_err = 0 train_batches = 0 confusion_train = ConfusionMatrix(n_class) # Generate minibatches and train on each one of them for batch in iterate_minibatches(X_tr, y_tr, mask_tr, batch_size, shuffle=True): inputs, targets, in_masks = batch tr_err, predict = train_fn(inputs, targets, in_masks) train_err += tr_err train_batches += 1 preds = np.argmax(predict, axis=-1) confusion_train.batch_add(targets, preds) train_loss = train_err / train_batches train_accuracy = confusion_train.accuracy() cf_train = confusion_train.ret_mat()
num_epochs = 25 train_acc= [] valid_acc = [] cur_loss = 0 loss = [] valid_loss = [] Train = DP.get_paths("/home/xvt131/Functions/Adhish_copy/Training-Rand") Test = DP.get_paths("/home/xvt131/Functions/Adhish_copy/Validating-Rand") for epoch in range(num_epochs): print epoch cur_loss = 0 val_loss = 0 confusion_valid = ConfusionMatrix(classes) confusion_train = ConfusionMatrix(classes) B = np.array([]) Pos = np.empty((0, 1,3)) for im in Train: Scan, Y_train,A, Post = DP.Sampling(im) B = np.int32(np.append(B, Y_train)) random = np.arange(len(B)) print len(Pos) Y_train = B[random] Pos = A[random] num_samples_train = Y_train.shape[0] num_batches_train = num_samples_train // batch_size for i in range(num_batches_train):
def run_blindset(topic, results_type, conf_matrix, blindset_name): """ Runs blindset test using credentials in ../Credentials.py """ # get credentials, import + export folders import Credentials active_adoption = Credentials.active_adoption instance_creds = Credentials.ctx[active_adoption] print(instance_creds) print('print works') workspace_id = Credentials.workspace_id[active_adoption][topic] workspace_thresh = Credentials.calculate_workspace_thresh(topic) conversation_version = Credentials.conversation_version # import + export folders import config import time data_folder = config.data_dir export_folder = config.output_folder timestr = time.strftime("%Y%m%d-%H%M") blindset_name = blindset_name or topic + "_blindset.csv" output_loc_results = os.path.join( export_folder, "{}_results_raw_{}.csv".format(topic, timestr)) output_loc_metrics = os.path.join( export_folder, "{}_results_metrics_{}.csv".format(topic, timestr)) output_loc_confmat = os.path.join( export_folder, "{}_confmat_{}.png".format(topic, timestr)) # authenticate if 'apikey' in instance_creds: logger.debug("Authenticating (apikey)") bs = blindset(apikey=instance_creds['apikey'], url=instance_creds['url'], threshold=workspace_thresh, version=conversation_version) elif 'password' in instance_creds: logger.debug("Authenticating (username/password)") bs = blindset(username=instance_creds['username'], password=instance_creds['password'], url=instance_creds['url'], threshold=workspace_thresh, version=conversation_version) # run test blindset_df = bs.import_blindset(os.path.join(data_folder, blindset_name)) # TODO: check blindset df results = bs.run_blind_test(blindset_df, workspace_id) # exports + metrics if (results_type == 'raw') or (results_type == 'all'): cols_export = [ col for col in results.columns.values if col != 'intent_correct' ] results[cols_export].to_csv(output_loc_results, encoding='utf-8') logger.info("Raw results exported to {}".format(output_loc_results)) if (results_type == 'metrics') or (results_type == 'all'): met = Metrics(workspace_thresh) metric_df, _ = met.get_all_metrics(results, detailed_results=True) metric_df.to_csv(output_loc_metrics, encoding='utf-8') logger.info( "Metrics per intent exported to {}".format(output_loc_metrics)) # confusion matrix if conf_matrix: from confusionmatrix import ConfusionMatrix cfn = ConfusionMatrix(workspace_thresh=workspace_thresh) cfn.create(results, fig_path=output_loc_confmat) #bs.plot_confusion_matrix(results, output_loc_confmat) logger.info("Confusion matrix saved to {}".format(output_loc_confmat)) # print high-level metrics overall_metrics = bs.calculate_overall_metrics(results, av_method="weighted") logger.info("Overall metrics for the workspace (weighted):") logger.info(overall_metrics)
def runbatchtest(fraction, n: int = 100): logger("{} {} {}".format(n, fraction, ticker), monitor) keylist = list(vectorrepositoryall.keys())[:n] random.shuffle(keylist) split = int(len(keylist) * fraction) train = keylist[:split] test = keylist[split:] logger("{} train vs {} test".format(len(train), len(test)), monitor) ones = [] nils = [] dummymaxconfusionmatrix = ConfusionMatrix() dummyrandomconfusionmatrix = ConfusionMatrix() centroidconfusionmatrix = ConfusionMatrix() poolconfusionmatrix = ConfusionMatrix() for trainitem in test: if illness[trainitem] == "1": ones.append(vectorrepositoryall[trainitem]) else: nils.append(vectorrepositoryall[trainitem]) onecentroid = sparsevectors.centroid(ones) nilcentroid = sparsevectors.centroid(nils) if len(nils) > len(ones): dummymaxguess = "0" else: dummymaxguess = "1" # factor = len(ones) / len(nils) # no, bad idea, go for fifty-fifty factor = 1 / 2 for testitem in test: dummymaxconfusionmatrix.addconfusion(illness[testitem], dummymaxguess) if random.random() > factor: dummyrandomguess = "0" else: dummyrandomguess = "1" dummyrandomconfusionmatrix.addconfusion(illness[testitem], dummyrandomguess) probe = vectorrepositoryall[testitem] resultc = "0" i1 = sparsevectors.sparsecosine(probe, onecentroid) n1 = sparsevectors.sparsecosine(probe, nilcentroid) if i1 > n1: resultc = "1" centroidconfusionmatrix.addconfusion(illness[testitem], resultc) probeneighbours = {} for targetitem in train: probeneighbours[targetitem] = sparsevectors.sparsecosine( probe, vectorrepositoryall[targetitem]) sortedfriends = sorted(probeneighbours, key=lambda hh: probeneighbours[hh], reverse=True)[:pooldepth] illity = 0 result = "0" for friend in sortedfriends: if illness[friend] == "1": illity += 1 if illity > pooldepth * factor: result = "1" nullity = pooldepth - illity poolconfusionmatrix.addconfusion(illness[testitem], result) print("{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format( testitem, illness[testitem], resultc, i1, n1, result, illity, nullity, pooldepth)) print("RANDOM ----------------") dummyrandomconfusionmatrix.evaluate() print("MAX ----------------") dummymaxconfusionmatrix.evaluate() print("CENTROID ----------------") centroidconfusionmatrix.evaluate() print("NEIGHBOURS --------------") poolconfusionmatrix.evaluate()
def run_kfold(topic, no_folds, results_type, conf_matrix): """ Runs kfold test using credentials in ../Credentials.py """ # get credentials, import + export folders import Credentials active_adoption = Credentials.active_adoption instance_creds = Credentials.ctx[active_adoption] workspace_id = Credentials.workspace_id[active_adoption][topic] workspace_thresh = Credentials.calculate_workspace_thresh(topic) conversation_version = Credentials.conversation_version # import + export folders import config import time data_folder = config.data_dir export_folder = config.output_folder timestr = time.strftime("%Y%m%d-%H%M") output_loc_results = os.path.join( export_folder, "{}_kfold_results_raw_{}.csv".format(topic, timestr)) output_loc_metrics = os.path.join( export_folder, "{}_kfold_results_metrics_{}.csv".format(topic, timestr)) output_loc_confmat = os.path.join( export_folder, "{}_kfold_confmat_{}.png".format(topic, timestr)) # authenticate if 'apikey' in instance_creds: logger.debug("Authenticating (apikey)") kf = kfoldtest(n_folds=no_folds, apikey=instance_creds['apikey'], url=instance_creds['url'], threshold=workspace_thresh, version=conversation_version) elif 'password' in instance_creds: logger.debug("Authenticating (username/password)") kf = kfoldtest(n_folds=no_folds, username=instance_creds['username'], password=instance_creds['password'], url=instance_creds['url'], threshold=workspace_thresh, version=conversation_version) # get train df from watson + check there are sufficient workspaces to run the test train_df = kf.intent_df_from_watson(workspace_id) kf.check_sufficient_workspaces() # create folds in WA if above is true folds = kf.create_folds(method='kfold') kf.create_kfold_WA(folds) available_flag = False while available_flag == False: logger.info("Checking workspaces..") available_flag = kf.check_workspaces_status() time.sleep(20) # run kfold test try: results = kf.run_kfold_test(folds) if (results_type == 'raw') or (results_type == 'all'): results.to_csv(output_loc_results) classification_report = kf.create_classification_report(results) if (results_type == 'metrics') or (results_type == 'all'): metrics = Metrics(workspace_thresh) metric_df = metrics.get_all_metrics_CV( results, fold_col='fold', detailed_results=False) metric_df.to_csv(output_loc_metrics) # TODO: confusion matrix if conf_matrix: from confusionmatrix import ConfusionMatrix cfn = ConfusionMatrix(workspace_thresh=workspace_thresh) cfn.create(results, fig_path=output_loc_confmat) logger.info("Confusion matrix saved to {}".format( output_loc_confmat)) finally: # regardless of what happens above, delete the temporary workspaces before exiting kf.delete_kfold_workspaces()
for item in testers: neighbours[item] = {} for otheritem in targetspace.items(): if testitemspace.name[item] == targetspace.name[otheritem]: continue neighbours[item][otheritem] = sparsevectors.sparsecosine( testitemspace.indexspace[item], targetspace.indexspace[otheritem]) logger("Done calculating neighbours", monitor) logger("Pool depth " + str(itempooldepth), monitor) if averagelinkage: logger("Averagelinkage", monitor) if votelinkage: logger("Votelinkage", monitor) confusion = ConfusionMatrix() primeconfusion = ConfusionMatrix() targetscore = {} for item in testers: sortedneighbours = sorted(neighbours[item], key=lambda hh: neighbours[item][hh], reverse=True)[:itempooldepth] primeconfusion.addconfusion(facittable[testitemspace.name[item]], targetspace.category[sortedneighbours[0]]) for target in categories: targetscore[target] = 0 if averagelinkage: # take all test neighbours and sum their scores for neighbour in sortedneighbours: targetscore[targetspace. category[neighbour]] += neighbours[item][neighbour] elif votelinkage:
print("\n") for item in items: print(str(item), itemspace.name[item]) result = {} prunedresult = {} for c in categories: result[c] = {} prunedresult[c] = {} logger("Pool depth " + str(itempooldepth), monitor) if averagelinkage: logger("Averagelinkage", monitor) if votelinkage: logger("Votelinkage", monitor) confusion = ConfusionMatrix() prunedconfusion = ConfusionMatrix() targetscore = {} prunedtargetscore = {} for item in testvectors: sortedneighbours = sorted(neighbours[item], key=lambda hh: neighbours[item][hh], reverse=True)[:itempooldepth] if cleanup: prunedsortedneighbours = sorted( prunedneighbours[item], key=lambda hh: prunedneighbours[item][hh], reverse=True)[:itempooldepth] for target in categories: targetscore[target] = 0 prunedtargetscore[target] = 0
num_epochs = 175 train_acc= [] valid_acc = [] cur_loss = 0 loss = [] valid_loss = [] Train = DP.get_paths("/home/xvt131/Biomediq/Validating-Rand") Test = DP.get_paths("/home/xvt131/Biomediq/Validating-Rand") import gc for epoch in range(num_epochs): cur_loss = 0 val_loss = 0 confusion_valid = ConfusionMatrix(2) confusion_train = ConfusionMatrix(2) for im in Train: XY, XZ, YZ, Y_train = DP.Patch_triplanar_para(im, PS) num_samples_train = Y_train.shape[0] num_batches_train = num_samples_train // batch_size for i in range(num_batches_train): idx = range(i*batch_size, (i+1)*batch_size) xy_batch = XY[idx] xz_batch = XZ[idx] yz_batch = YZ[idx] target_batch = Y_train[idx] batch_loss = f_train(xy_batch, xz_batch, yz_batch ,target_batch) #this will do the backprop pass
def main(): parser = argparse.ArgumentParser(description='Memlearn Chainer ver') parser.add_argument('--batchsize', '-b', type=int, default=100, help='Number of images in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=20, help='Number of sweeps over the dataset to train') parser.add_argument('--frequency', '-f', type=int, default=-1, help='Frequency of taking a snapshot') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') parser.add_argument('--train', '-i', nargs='*', help='Train data (all except test by default)') parser.add_argument('--test', '-t', nargs='*', help='Test data (you have to specify at least one') parser.add_argument('--sliceb', '-s', type=int, default=1, help='Time slice block') parser.add_argument('--list_dataset', action='store_true') parser.add_argument('--dataset', default="/data/data_filtered.pickle") args = parser.parse_args() if args.list_dataset: with open(args.dataset, 'rb') as fp: dataset = pickle.load(fp) print('\n'.join(['{} {}'.format(n, l) for n, l in dataset])) raise SystemExit if not args.test: raise SystemExit("you have to specify at least one test, see --list_dataset for available datasets") print('GPU: {}'.format(args.gpu)) print('# Minibatch-size: {}'.format(args.batchsize)) print('# epoch: {}'.format(args.epoch)) print('') # Set up a neural network to train # Classifier reports softmax cross entropy loss and accuracy at every # iteration, which will be used by the PrintReport extension below. model = L.Classifier(VGG(2)) #model = L.Classifier(SVM()) if args.gpu >= 0: # Make a specified GPU current chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() # Copy the model to the GPU # Setup an optimizer optimizer = chainer.optimizers.Adam() optimizer.setup(model) # Load the MEMORY dataset train, test = prep_windowed_datasets(args.dataset, args.test, args.train, slice_merge=args.sliceb) train_iter = chainer.iterators.SerialIterator(train, args.batchsize) test_iter = chainer.iterators.SerialIterator(test, args.batchsize, repeat=False, shuffle=False) # Set up a trainer updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) # Evaluate the model with the test dataset for each epoch #trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu)) trainer.extend(ConfusionMatrix(test_iter, model, device=args.gpu)) # Dump a computational graph from 'loss' variable at the first iteration # The "main" refers to the target link of the "main" optimizer. trainer.extend(extensions.dump_graph('main/loss')) # Take a snapshot for each specified epoch frequency = args.epoch if args.frequency == -1 else max(1, args.frequency) trainer.extend(extensions.snapshot(), trigger=(frequency, 'epoch')) # Write a log of evaluation statistics for each epoch trainer.extend(extensions.LogReport()) # Save two plot images to the result dir if extensions.PlotReport.available(): trainer.extend( extensions.PlotReport(['main/loss', 'validation/main/loss'], 'epoch', file_name='loss.png')) trainer.extend( extensions.PlotReport( ['main/accuracy', 'validation/main/accuracy'], 'epoch', file_name='accuracy.png')) # Print selected entries of the log to stdout # Here "main" refers to the target link of the "main" optimizer again, and # "validation" refers to the default name of the Evaluator extension. # Entries other than 'epoch' are reported by the Classifier link, called by # either the updater or the evaluator. trainer.extend(extensions.PrintReport( ['epoch', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'elapsed_time'])) # Print a progress bar to stdout trainer.extend(extensions.ProgressBar()) if args.resume: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) # Run the training trainer.run()
y_test, writer=dev_summary_writer) if accuracy > max_acc: max_acc = accuracy path = saver.save(sess, checkpoint_prefix, global_step=current_step) print("Saved model checkpoint to {}\n".format(path)) else: ckpt = tf.train.get_checkpoint_state(checkpoint_dir) saver.restore(sess, ckpt.model_checkpoint_path) accuracy, predictions = dev_step(x_test, y_test) #dev_step(x_dev, y_dev, writer=dev_summary_writer) outputs = np.argmax(predictions, axis=-1) print(outputs.shape) confusion_test = ConfusionMatrix(num_classes) y_test = np.argmax(y_test, axis=-1) print(y_test.shape) confusion_test.batch_add(y_test, outputs) test_accuracy = confusion_test.accuracy() a, positive_predictive_value = confusion_test.positive_predictive_value() b, negative_predictive_value = confusion_test.negative_predictive_value() e, F1 = confusion_test.F1() f, MCC = confusion_test.matthews_correlation() cf_val = confusion_test.ret_mat() print("FINAL TEST RESULTS") print(confusion_test) print(cf_val)
}) d_acc = sess.run(model.domain_acc, feed_dict={ model.input_x: combined_test_imgs, model.domain: combined_test_domain, model.dropout_keep_prob: 1, model.training: False }) print('\nDomain adaptation training') print('Source (mouse) accuracy:', source_acc) print('target (human) accuracy:', target_acc) print('Domain accuracy:', d_acc) outputs = np.argmax(predictions, axis=-1) print(outputs.shape) confusion_test = ConfusionMatrix(10) y_test = np.argmax(y_human_test, axis=-1) print(y_test.shape) confusion_test.batch_add(y_test, outputs) test_accuracy = confusion_test.accuracy() positive_predictive_value = confusion_test.positive_predictive_value( ) negative_predictive_value = confusion_test.negative_predictive_value( ) F1 = confusion_test.F1() MCC = confusion_test.matthews_correlation() cf_val = confusion_test.ret_mat() print("FINAL TEST RESULTS")