def evaluate(model, acoustic_iterator, linguistic_iterator, criterion): model.eval() epoch_loss = 0 conf_mat = np.zeros((4, 4)) assert len(acoustic_iterator) == len(linguistic_iterator) with torch.no_grad(): for acoustic_tuple, linguistic_tuple in zip(acoustic_iterator(), linguistic_iterator()): acoustic_batch = acoustic_tuple[0] acoustic_labels = acoustic_tuple[1] linguistic_batch = linguistic_tuple[0] linguistic_labels = linguistic_tuple[1] predictions = model(acoustic_batch, linguistic_batch).squeeze(1) loss = criterion(predictions.float(), acoustic_labels) epoch_loss += loss.item() conf_mat += confusion_matrix(predictions, acoustic_labels) acc = sum([conf_mat[i, i] for i in range(conf_mat.shape[0])]) / conf_mat.sum() acc_per_class = [ conf_mat[i, i] / conf_mat[i].sum() for i in range(conf_mat.shape[0]) ] weighted_acc = sum(acc_per_class) / len(acc_per_class) return epoch_loss / len(acoustic_iterator), acc, weighted_acc, conf_mat
def confusion_matrix_age_filter(age_min, age_max, test, predicted_OUTPUT_with_age): count = 0 for i, row in test.iterrows(): if (row['AGE'] < age_min or row['AGE'] > age_max ) and predicted_OUTPUT_with_age[count] == 1: predicted_OUTPUT_with_age[count] = 0 count += 1 cnf_matrix = metrics.confusion_matrix(test.OUTPUT, predicted_OUTPUT_with_age) class_names = [0, 1] fig, ax = cnf_plt.subplots() tick_marks = np.arange(len(class_names)) cnf_plt.xticks(tick_marks, class_names, fontsize=14) cnf_plt.yticks(tick_marks, class_names, fontsize=14) # create heatmap sns.heatmap(pd.DataFrame(cnf_matrix), annot=True, annot_kws={"size": 22}, cmap="YlGnBu", fmt='g') ax.xaxis.set_label_position("top") cnf_plt.tight_layout() cnf_plt.title('Confusion Matrix', y=1.1, fontsize=14) cnf_plt.ylabel('Actual label', fontsize=14) cnf_plt.xlabel('Predicted label', fontsize=14) conf_matrix_fig = cnf_plt.gcf() conf_matrix_fig.tight_layout() cnf_plt.draw() conf_matrix_fig.savefig('plots/conf_matrix_age_{}_to_{}.png'.format( age_min, age_max)) cnf_plt.clf()
def _evaluations(predictions, labels, num_classes, class_names): """Returns a group of operations to calculate accuracy and other evaluation metrics.""" ops = [ metrics.accuracy(predictions, labels, num_classes), metrics.confusion_matrix(predictions, labels, num_classes), metrics.class_metrics(predictions, labels, class_names) ] return tf.group(*ops)
def test(classifier,x_test,y_test): prediction = classifier.predict(x_test) print("Confusion Matrix for Decision tree Classofier Model given : ") print(confusion_matrix(y_test,prediction)) print("Classification Report for given Decision tree Classisifer : ") print(classification_report(y_test,prediction)) print("Accuracy Score : ",accuracy_score(y_test,prediction)) return
def test_metrics(): y_true = [0, 1, 1, 0, 1, 1] y_pred = [0, 0, 1, 0, 0, 1] # metrics + cm counts observed_metrics = precision_recall_f1_accuracy(y_true, y_pred, label=1) # Confusion Matrix - counts and normalized observed_normalized_cm = confusion_matrix(y_true, y_pred, label=1, normalize=True) observed_cm = confusion_matrix(y_true, y_pred, label=1, normalize=False) expected_metrics = { "precision": 1.00, "recall": 0.50, "f1_score": 0.67, "accuracy": 66.67, "TP": 2, "FN": 2, "FP": 0, "TN": 2, } expected_normalized_cm = [[1.0, 0.0], [0.5, 0.5]] expected_cm = [[2, 0], [2, 2]] assert observed_metrics == expected_metrics assert observed_cm == expected_cm assert observed_normalized_cm == expected_normalized_cm actual = ["B", "B", "C", "A", "B", "B", "C", "A", "A", "B", "D"] predicted = ["A", "B", "B", "A", "C", "B", "C", "C", "A", "C", "D"] print_normalized_confusion_matrix(actual, predicted) matrix, labels = confusion_matrix(actual, predicted, normalize=True) assert matrix == [ [0.67, 0.33, 0.0, 0.0], [0.0, 0.67, 0.33, 0.0], [0.25, 0.5, 0.25, 0.0], [0.0, 0.0, 0.0, 1.0], ] assert sorted(labels) == ["A", "B", "C", "D"] compute_all(actual, predicted)
def test_1(self): actual = [1, 1, 0, 1, 1, 1, 0, 0, 1, 1] predicted = [0, 1, 0, 1, 0, 1, 0, 1, 0, 0] tp, fn, fp, tn = metrics.confusion_matrix(actual, predicted) self.assertEqual(tp, 3) self.assertEqual(fn, 4) self.assertEqual(fp, 1) self.assertEqual(tn, 2) self.assertEqual(metrics.accuracy(actual, predicted), 0.5) self.assertEqual(metrics.precision(actual, predicted), 3/4) self.assertEqual(metrics.recall(actual, predicted), 3/7) self.assertEqual(metrics.f1(actual, predicted), 6/11)
def metrics_calculator(masks, preds, mode_average=True, additional=False): batch_size, masks, predictions = mtr.standardize_for_metrics(masks, preds) accuracy_score = mtr.accuracy(batch_size, masks, predictions, mode_average) if additional: roc_auc_score = mtr.roc_auc(batch_size, masks, predictions, mode_average) jaccard_score = mtr.jaccard(batch_size, masks, predictions, mode_average) sens_score, spec_score, prec_score, f1_score = mtr.confusion_matrix( batch_size, masks, predictions, mode_average) pr_auc_score = mtr.precision_recall_auc(batch_size, masks, predictions, mode_average) iou_score = mtr.fast_hist(predictions, masks, 2) return roc_auc_score, accuracy_score, jaccard_score, sens_score, spec_score, prec_score, f1_score, pr_auc_score, iou_score return accuracy_score
def train(model, acoustic_iterator, linguistic_iterator, optimizer, criterion, reg_ratio): model.train() epoch_loss = 0 conf_mat = np.zeros((4, 4)) assert len(acoustic_iterator) == len(linguistic_iterator) for acoustic_tuple, linguistic_tuple in zip(acoustic_iterator(), linguistic_iterator()): acoustic_batch = acoustic_tuple[0] acoustic_labels = acoustic_tuple[1] linguistic_batch = linguistic_tuple[0] linguistic_labels = linguistic_tuple[1] optimizer.zero_grad() predictions = model(acoustic_batch, linguistic_batch).squeeze(1) loss = criterion(predictions, acoustic_labels) reg_loss = 0 for param in model.parameters(): reg_loss += param.norm(2) total_loss = loss + reg_ratio * reg_loss total_loss.backward() optimizer.step() epoch_loss += loss.item() conf_mat += confusion_matrix(predictions, acoustic_labels) acc = sum([conf_mat[i, i] for i in range(conf_mat.shape[0])]) / conf_mat.sum() acc_per_class = [ conf_mat[i, i] / conf_mat[i].sum() for i in range(conf_mat.shape[0]) ] weighted_acc = sum(acc_per_class) / len(acc_per_class) return epoch_loss / len(acoustic_iterator), acc, weighted_acc, conf_mat
def main(): # check for gpu device print(tf.test.gpu_device_name()) # print tf and keras version print(tf.VERSION) print(tf.keras.__version__) data, labels = data_prep.read_images("data/train", IMAGE_DIMS, general=True) print("Train Set loaded") data_test, labels_test = data_prep.read_images("data/test", IMAGE_DIMS, general=True) print("Test Set loaded") lb = LabelBinarizer() X_train, y_train = data_prep.binarize(data, labels, lb) X_test, y_test = data_prep.binarize(data_test, labels_test, lb) print(X_train.shape) print(y_train.shape) model = VGGNet.vgg_net(N_CLASSES, IMAGE_DIMS) # Train the model start = time.time() history = VGGNet.fit(model, X_train, y_train, X_test, y_test, EPOCHS, BS) end = time.time() print("Training Time: " + timer(start, end)) # Metrics metrics.plot_evaluation(history) y_pred = model.predict(X_test) classification_report = metrics.classification_report(y_test, y_pred, lb) print("Classification report : \n", classification_report) confusion_matrix = metrics.confusion_matrix(y_test, y_pred) print("Confusion Matrix : \n", confusion_matrix) metrics.print_confusion_matrix(y_test, y_pred, lb)
def confusion_matrix_multiple_diseases(test, predicted_OUTPUT_multiple_diseases, number_of_diseases): count = 0 for i, row in test.iterrows(): if (len(row['DIAGNOSIS'].split(';')) ) >= number_of_diseases and predicted_OUTPUT_multiple_diseases[ count] == 1: predicted_OUTPUT_multiple_diseases[count] = 0 count += 1 cnf_matrix = metrics.confusion_matrix(test.OUTPUT, predicted_OUTPUT_multiple_diseases) class_names = [0, 1] fig, ax = cnf_plt.subplots() tick_marks = np.arange(len(class_names)) cnf_plt.xticks(tick_marks, class_names, fontsize=14) cnf_plt.yticks(tick_marks, class_names, fontsize=14) # create heatmap sns.heatmap(pd.DataFrame(cnf_matrix), annot=True, annot_kws={"size": 22}, cmap="YlGnBu", fmt='g') ax.xaxis.set_label_position("top") cnf_plt.tight_layout() cnf_plt.title('Confusion Matrix', y=1.1, fontsize=14) cnf_plt.ylabel('Actual label', fontsize=14) cnf_plt.xlabel('Predicted label', fontsize=14) conf_matrix_fig = cnf_plt.gcf() conf_matrix_fig.tight_layout() cnf_plt.draw() conf_matrix_fig.savefig( 'plots/conf_matrix_{}_diseases_dismissed.png'.format( number_of_diseases)) cnf_plt.clf()
train_X, test_X = data.iloc[train_index], data.iloc[test_index] train_y, test_y = y.iloc[train_index], y.iloc[test_index] clf = None clf = RandomForestClassifier(n_estimators=n_estimators, criterion=criterion, max_depth=max_depth, min_samples_split=min_samples_split, n_jobs=n_processes) clf.fit(train_X, train_y) results.append(metrics.predict_table(clf, test_X, test_y)) ids.extend(test_X.index.tolist()) if validation == 'holdout': aux = metrics.predict_table(clf, test_X, test_y) aux.to_csv(result_path + 'Predicciones/hold_' + str(count) + '.csv') print 'hold ' + str(count) + ' ' + str(metrics.weighted_f_score(metrics.confusion_matrix(aux))) count += 1 result = pd.concat(results) result['indice'] = ids result.set_index('indice') result.index.name = catalog + '_id' result = result.drop('indice', axis=1) output = open(result_path + 'Arboles/Arbol_' + percentage + '.pkl', 'wb+') pickle.dump(clf, output) output.close() result.to_csv(result_path + 'Predicciones/result_' + percentage + '.csv') print metrics.weighted_f_score(metrics.confusion_matrix(result))
index_filter = args.index_filter feature_filter = args.feature_filter if index_filter is not None: index_filter = pd.read_csv(index_filter, index_col=0).index paths = [sets_path + catalog + '_sampled_' + str(i) + '.csv' for i in xrange(n_samples)] if model == 'tree': partial_fit = partial(parallel.fit_tree, feature_filter=feature_filter, folds=folds, inverse=inverse, max_depth=max_depth, min_samples_split=min_samples_split, lc_filter=lc_filter) elif model == 'rf': partial_fit = partial(parallel.fit_rf, feature_filter=feature_filter, folds=folds, inverse=inverse, lc_filter=lc_filter) elif model == 'sktree': partial_fit = partial(parallel.fit_sktree, feature_filter=feature_filter, folds=folds, inverse=inverse, max_depth=max_depth, min_samples_split=min_samples_split, lc_filter=lc_filter) pool = Pool(processes=n_processes, maxtasksperchild=2) resultados = pool.map(partial_fit, paths) pool.close() pool.join() result = metrics.aggregate_predictions(resultados) result.to_csv(result_path) print metrics.weighted_f_score(metrics.confusion_matrix(result))
ntrain = 6000 nvalid = 1000 nclass = 10 Xtrain,Ytrain,Xvalid,Yvalid = load_mnist.load_mnist(ntrain,nvalid) # (2) Define model nfeature = Xtrain.shape[0] np.random.seed(10) lamb = 0.0 model = NeuralNetwork.NeuralNetwork(nfeature) model.add_layer(128,"relu",lamb) model.add_layer(nclass,"softmax",lamb) # (3) Compile model optimizer = Optimizer.Adam(0.02,0.9,0.999,1e-7) model.compile("crossentropy",optimizer) model.summary() # (4) Train model epochs = 40 time_start = time.time() history = model.fit(Xtrain,Ytrain,epochs,batch_size=ntrain,validation_data=(Xvalid,Yvalid)) time_end = time.time() print("Train time: {}".format(time_end - time_start)) # (5) Predictions and plotting # confusion matrix print("Metrics for Validation Dataset") Yvalid_pred = model.predict(Xvalid) metrics.confusion_matrix(Yvalid,Yvalid_pred,nclass) # plot loss, accuracy, and animation of results plot_results.plot_results_history(history,["loss","valid_loss"]) plot_results.plot_results_history(history,["accuracy","valid_accuracy"]) plot_results.plot_results_mnist_animation(Xvalid,Yvalid,Yvalid_pred,model.get_Afinal(),100) plt.show()
# %% metrics.worst_samples(imgs_valid, labels_valid, scores_predict, class_to_index, top=16, names_valid=None) # %% metrics.classification_report(labels_valid, label_predict, class_to_index) # %% metrics.confusion_matrix(labels_valid, label_predict, class_to_index) # %% metrics.ROC(labels_valid, scores_predict, class_to_index, section=[0, 5, 11]) #%% metrics.ROCCompare(labels_valids, scores_predicts, class_to_index, subtitles, section=[0, 5]) # %%
features, targets, fraction) btrainf = np.ceil(np.asarray(trainf) / abs(np.asarray(trainf))) btestf = np.ceil(np.asarray(testf) / abs(np.asarray(testf))) # btrainf = btrainf.tolist() # btestf = btestf.tolist() if model == "decision_tree": tree = DecisionTree(attribute_names) tree.fit(btrainf, traint) num_nodes, max_depth = tree.tree_attributes(tree.tree) t = tree.predict(btestf) decision = tree.predict(today) #print(decision) #print("Should you buy tomorrow with today's data? %d" % decision) cm = metrics.confusion_matrix(testt, t) a = metrics.accuracy(testt, t) p, r = metrics.precision_and_recall(testt, t) f = metrics.f1_measure(testt, t) print("Accuracy = %f\n" % a) print("Precision = %f, Recall = %f\n" % (p, r)) print("F1 measure = %f\n" % f) print("Tomorrow's Forecast: %f\n" % decision[-1]) elif model == "prior_probability": prob = PriorProbability() print(btrainf) prob.fit(btrainf, traint) t = prob.predict(btestf) decision = prob.predict(today) #raise ValueError(t)
def evaluate(self, experiment_path: Path, task: str = 'aurora_clean', model_resolution=0.02, time_resolution=0.02, threshold=(0.5, 0.1), **kwargs): EVALUATION_DATA = { 'aurora_clean': { 'data': 'data/evaluation/hdf5/aurora_clean.h5', 'label': 'data/evaluation/labels/aurora_clean_labels.tsv', }, 'aurora_noisy': { 'data': 'data/evaluation/hdf5/aurora_noisy.h5', 'label': 'data/evaluation/labels/aurora_noisy_labels.tsv' }, 'dihard_dev': { 'data': 'data/evaluation/hdf5/dihard_dev.h5', 'label': 'data/evaluation/labels/dihard_dev.csv' }, 'dihard_eval': { 'data': 'data/evaluation/hdf5/dihard_eval.h5', 'label': 'data/evaluation/labels/dihard_eval.csv' }, 'aurora_snr_20': { 'data': 'data/evaluation/hdf5/aurora_noisy_musan_snr_20.0.hdf5', 'label': 'data/evaluation/labels/musan_labels.tsv' }, 'aurora_snr_15': { 'data': 'data/evaluation/hdf5/aurora_noisy_musan_snr_15.0.hdf5', 'label': 'data/evaluation/labels/musan_labels.tsv' }, 'aurora_snr_10': { 'data': 'data/evaluation/hdf5/aurora_noisy_musan_snr_10.0.hdf5', 'label': 'data/evaluation/labels/musan_labels.tsv' }, 'aurora_snr_5': { 'data': 'data/evaluation/hdf5/aurora_noisy_musan_snr_5.0.hdf5', 'label': 'data/evaluation/labels/musan_labels.tsv' }, 'aurora_snr_0': { 'data': 'data/evaluation/hdf5/aurora_noisy_musan_snr_0.0.hdf5', 'label': 'data/evaluation/labels/musan_labels.tsv' }, 'aurora_snr_-5': { 'data': 'data/evaluation/hdf5/aurora_noisy_musan_snr_-5.0.hdf5', 'label': 'data/evaluation/labels/musan_labels.tsv' }, 'dcase18': { 'data': 'data/evaluation/hdf5/dcase18.h5', 'label': 'data/evaluation/labels/dcase18.tsv', }, } assert task in EVALUATION_DATA, f"--task {'|'.join(list(EVALUATION_DATA.keys()))}" experiment_path = Path(experiment_path) if experiment_path.is_file(): # Model is given model_path = experiment_path experiment_path = experiment_path.parent else: model_path = next(Path(experiment_path).glob("run_model*")) config = torch.load(next(Path(experiment_path).glob("run_config*")), map_location='cpu') logger = utils.getfile_outlogger(None) # Use previous config, but update data such as kwargs config_parameters = dict(config, **kwargs) # Default columns to search for in data model_parameters = torch.load( model_path, map_location=lambda storage, loc: storage) encoder = torch.load('labelencoders/vad.pth') data = EVALUATION_DATA[task]['data'] label_df = pd.read_csv(EVALUATION_DATA[task]['label'], sep='\s+') label_df['filename'] = label_df['filename'].apply( lambda x: Path(x).name) logger.info(f"Label_df shape is {label_df.shape}") dset = dataset.EvalH5Dataset(data, fnames=np.unique( label_df['filename'].values)) dataloader = torch.utils.data.DataLoader(dset, batch_size=1, num_workers=4, shuffle=False) model = getattr(models, config_parameters['model'])( inputdim=dataloader.dataset.datadim, outputdim=len(encoder.classes_), **config_parameters['model_args']) model.load_state_dict(model_parameters) model = model.to(DEVICE).eval() ## VAD preprocessing data vad_label_helper_df = label_df.copy() vad_label_helper_df['onset'] = np.ceil(vad_label_helper_df['onset'] / model_resolution).astype(int) vad_label_helper_df['offset'] = np.ceil(vad_label_helper_df['offset'] / model_resolution).astype(int) vad_label_helper_df = vad_label_helper_df.groupby(['filename']).agg({ 'onset': tuple, 'offset': tuple, 'event_label': tuple }).reset_index() logger.trace(model) output_dfs = [] speech_label_idx = np.where('Speech' == encoder.classes_)[0].squeeze() speech_frame_predictions, speech_frame_ground_truth, speech_frame_prob_predictions = [], [],[] # Using only binary thresholding without filter if len(threshold) == 1: postprocessing_method = utils.binarize else: postprocessing_method = utils.double_threshold with torch.no_grad(), tqdm(total=len(dataloader), leave=False, unit='clip') as pbar: for feature, filename in dataloader: feature = torch.as_tensor(feature).to(DEVICE) # PANNS output a dict instead of 2 values prediction_tag, prediction_time = model(feature) prediction_tag = prediction_tag.to('cpu') prediction_time = prediction_time.to('cpu') if prediction_time is not None: # Some models do not predict timestamps cur_filename = filename[0] thresholded_prediction = postprocessing_method( prediction_time, *threshold) ## VAD predictions speech_frame_prob_predictions.append( prediction_time[..., speech_label_idx].squeeze()) ### Thresholded speech predictions speech_prediction = thresholded_prediction[ ..., speech_label_idx].squeeze() speech_frame_predictions.append(speech_prediction) targets = vad_label_helper_df[ vad_label_helper_df['filename'] == cur_filename][[ 'onset', 'offset' ]].values[0] target_arr = np.zeros_like(speech_prediction) for start, end in zip(*targets): target_arr[start:end] = 1 speech_frame_ground_truth.append(target_arr) #### SED predictions labelled_predictions = utils.decode_with_timestamps( encoder, thresholded_prediction) pred_label_df = pd.DataFrame( labelled_predictions[0], columns=['event_label', 'onset', 'offset']) if not pred_label_df.empty: pred_label_df['filename'] = cur_filename pred_label_df['onset'] *= model_resolution pred_label_df['offset'] *= model_resolution pbar.set_postfix(labels=','.join( np.unique(pred_label_df['event_label'].values))) pbar.update() output_dfs.append(pred_label_df) full_prediction_df = pd.concat(output_dfs) prediction_df = full_prediction_df[full_prediction_df['event_label'] == 'Speech'] assert set(['onset', 'offset', 'filename', 'event_label' ]).issubset(prediction_df.columns), "Format is wrong" assert set(['onset', 'offset', 'filename', 'event_label' ]).issubset(label_df.columns), "Format is wrong" logger.info("Calculating VAD measures ... ") speech_frame_ground_truth = np.concatenate(speech_frame_ground_truth, axis=0) speech_frame_predictions = np.concatenate(speech_frame_predictions, axis=0) speech_frame_prob_predictions = np.concatenate( speech_frame_prob_predictions, axis=0) vad_results = [] tn, fp, fn, tp = metrics.confusion_matrix( speech_frame_ground_truth, speech_frame_predictions).ravel() fer = 100 * ((fp + fn) / len(speech_frame_ground_truth)) acc = 100 * ((tp + tn) / (len(speech_frame_ground_truth))) p_miss = 100 * (fn / (fn + tp)) p_fa = 100 * (fp / (fp + tn)) for i in [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 0.7,0.9]: mp_fa, mp_miss = metrics.obtain_error_rates( speech_frame_ground_truth, speech_frame_prob_predictions, i) tn, fp, fn, tp = metrics.confusion_matrix( speech_frame_ground_truth, speech_frame_prob_predictions > i).ravel() sub_fer = 100 * ((fp + fn) / len(speech_frame_ground_truth)) logger.info( f"PFa {100*mp_fa:.2f} Pmiss {100*mp_miss:.2f} FER {sub_fer:.2f} t: {i:.2f}" ) auc = metrics.roc(speech_frame_ground_truth, speech_frame_prob_predictions) * 100 for avgtype in ('micro', 'macro', 'binary'): precision, recall, f1, _ = metrics.precision_recall_fscore_support( speech_frame_ground_truth, speech_frame_predictions, average=avgtype) vad_results.append( (avgtype, 100 * precision, 100 * recall, 100 * f1)) logger.info("Calculating segment based metric .. ") # Change order just for better printing in file prediction_df = prediction_df[[ 'filename', 'onset', 'offset', 'event_label' ]] metric = metrics.segment_based_evaluation_df( label_df, prediction_df, time_resolution=time_resolution) logger.info("Calculating event based metric .. ") event_metric = metrics.event_based_evaluation_df( label_df, prediction_df) prediction_df.to_csv(experiment_path / f'speech_predictions_{task}.tsv', sep='\t', index=False) full_prediction_df.to_csv(experiment_path / f'predictions_{task}.tsv', sep='\t', index=False) with open(experiment_path / f'evaluation_{task}.txt', 'w') as fp: for k, v in config_parameters.items(): print(f"{k}:{v}", file=fp) print(metric, file=fp) print(event_metric, file=fp) for avgtype, precision, recall, f1 in vad_results: print( f"VAD {avgtype} F1: {f1:<10.3f} {precision:<10.3f} Recall: {recall:<10.3f}", file=fp) print(f"FER: {fer:.2f}", file=fp) print(f"AUC: {auc:.2f}", file=fp) print(f"Pfa: {p_fa:.2f}", file=fp) print(f"Pmiss: {p_miss:.2f}", file=fp) print(f"ACC: {acc:.2f}", file=fp) logger.info(f"Results are at {experiment_path}") for avgtype, precision, recall, f1 in vad_results: print( f"VAD {avgtype:<10} F1: {f1:<10.3f} Pre: {precision:<10.3f} Recall: {recall:<10.3f}" ) print(f"FER: {fer:.2f}") print(f"AUC: {auc:.2f}") print(f"Pfa: {p_fa:.2f}") print(f"Pmiss: {p_miss:.2f}") print(f"ACC: {acc:.2f}") print(event_metric) print(metric)
print("XvalidT.shape: {}".format(XvalidT.shape)) print("YvalidT.shape: {}".format(YvalidT.shape)) # (2) Define model nfeature = Xtrain.shape[0] model = tf.keras.models.Sequential([ tf.keras.layers.Dense(200,input_shape=(nfeature,),activation="tanh"), tf.keras.layers.Dense(50,activation="tanh"), tf.keras.layers.Dense(1,activation="sigmoid")]) # (3) Compile model optimizer = tf.keras.optimizers.Adam(lr=0.02,beta_1=0.9,beta_2=0.999,epsilon=1e-7) model.compile(optimizer=optimizer, loss="binary_crossentropy", metrics=["accuracy"]) model.summary() # (4) Train model epochs = 20 time_start = time.time() ntrain = XtrainT.shape[0] history = model.fit(XtrainT,YtrainT,epochs=epochs,batch_size=ntrain,validation_data=(XvalidT,YvalidT)) time_end = time.time() print("Train time: {}".format(time_end - time_start)) # (5) Predictions and plotting # confusion matrix Afinal = model.predict(XvalidT).T Yvalid_pred = np.round(Afinal) metrics.confusion_matrix(Yvalid,Yvalid_pred,2) f1score,precision,recall = metrics.f1score(Yvalid,Yvalid_pred) print("F1Score: {} - Precision: {} - Recall: {}".format(f1score,precision,recall)) text_results.text_results(Yvalid,Yvalid_pred,Xvalid_raw) # plot loss and accuracy plot_results.plot_results_history(history.history,["loss","val_loss"]) plot_results.plot_results_history(history.history,["accuracy","val_accuracy"]) plt.show()
def evaluate(self, actual: np.array, predicted: np.array) -> confusion_matrix: return confusion_matrix(actual, predicted, labels=[0,1])
train_X, test_X = data.iloc[train_index], data.iloc[test_index] train_y, test_y = y.iloc[train_index], y.iloc[test_index] clf = None clf = RandomForestClassifier(n_estimators=p, criterion='entropy', max_depth=14, min_samples_split=20, n_jobs=2) clf.fit(train_X, train_y) results.append(metrics.predict_table(clf, test_X, test_y)) result = pd.concat(results) matrix = metrics.confusion_matrix(result) clases = matrix.columns.tolist() precisions = [metrics.precision(matrix, c) for c in clases] recalls = [metrics.recall(matrix, c) for c in clases] f_scores = [metrics.f_score(matrix, c) for c in clases] w_score = metrics.weighted_f_score(matrix) # f = open(result_dir + str(max_depth) + ' ' + str(min_samples_split) + '.txt', 'w') f = open(result_dir + str(p) + '.txt', 'w') f.write('F_score by class') f.write('\n') f.write(str(f_scores)) f.write('\n')
def valid(model, validation_loader, loss, best_loss, pad_size, orig_dim, num_img): # set the model to be ready to test on the validation set model.eval() validation_acc = 0.0 validation_loss = 0.0 validation_IoU = 0.0 IoU_count = 0.0 # test the model print("Testing model...", flush=True) # run through the batches of validation images for validation_imgs, validation_labels in validation_loader: # use GPU for images if possible if torch.cuda.is_available(): validation_imgs = validation_imgs.cuda() validation_labels = validation_labels.cuda() # predict the classes (whether a pixel contains a building or not) outputs = model(validation_imgs) # compute the loss loss_output = loss(outputs, validation_labels.long()) # calculate the validation loss and find the prediction for each pixel of an image validation_loss += loss_output.item() * validation_imgs.size(0) _, prediction = torch.max(outputs.data, 1) # if the image and label were padded beforehand crop them back into their original # dimensions if pad_size != 0: prediction = crop(prediction.float(), orig_dim) validation_labels = crop(validation_labels, orig_dim) for img in range(validation_labels.shape[0]): # set up the confusion matrix and calculate the metrics # check that there is an object to compute the metrics with if torch.sum(validation_labels[img]) != 0: IoU_count += 1 _, _, IoU = confusion_matrix( np.asarray(prediction[img].cpu()), np.asarray(validation_labels[img].cpu()), orig_dim) validation_IoU += IoU # calculate the accuracy after each batch validation_acc += torch.sum( prediction.long() == validation_labels.long().data) # calculate the total accuracy, validation loss, and IoU validation_acc = validation_acc.item() / (float(num_img) * orig_dim * orig_dim) validation_loss = validation_loss / num_img # make sure there is no divide by 0 if IoU_count != 0: validation_IoU = validation_IoU / IoU_count print("Val acc: " + str(validation_acc), "Val loss: " + str(validation_loss), flush=True) print("Val IoU: " + str(validation_IoU), flush=True) # save the model's learned parameters with the lowest loss if best_loss > validation_loss: best_loss = validation_loss save_model(model) return best_loss
feature_filter = args.feature_filter train_data = pd.read_csv(training_set_path, index_col=0) train_X, train_y = utils.filter_data(train_data, feature_filter=feature_filter) test_data = pd.read_csv(test_set_path, index_col=0) test_X, test_y = utils.filter_data(test_data, feature_filter=feature_filter) clf = None clf = RandomForestClassifier(n_estimators=n_estimators, criterion=criterion, max_depth=max_depth, min_samples_split=min_samples_split, n_jobs=n_processes) clf.fit(train_X, train_y) result = metrics.predict_table(clf, test_X, test_y) result['indice'] = test_X.index.tolist() result.set_index('indice') result.index.name = catalog + '_id' result = result.drop('indice', axis=1) output = open(result_path + 'Arboles/Arbol_' + percentage + '.pkl', 'wb+') pickle.dump(clf, output) output.close() result.to_csv(result_path + 'Predicciones/result_' + percentage + '.csv') print metrics.weighted_f_score(metrics.confusion_matrix(result))
# bikin confusion matrix dan metric dari training ini: prediction = model.predict(X_test) label_pred = [] for i in range(prediction.shape[1]): label_pred.append(np.argmax(prediction[:, i])) y_test_label = [] for i in range(y_test.shape[0]): y_test_label.append(np.argmax(y_test[i, :])) metrics = Metrics(y_test_label, label_pred) print("ACCURACY SLURRR: ", metrics.all_accuracy()) print("PRECISION SLURRR: ", metrics.all_precision()) print("RECALL SLURRR: ", metrics.all_recall()) print("F1 SLURRR: ", metrics.all_f1_score()) print("CONFUSION MATRIXX: ") print(confusion_matrix(y_test_label, label_pred)) print("--------------------------------------") # 10-fold cross validation k_fold = KFold(n_splits=10) print(k_fold) scores = [] for train_index, test_index in k_fold.split(X): print("TRAIN DATA INDEX") print(train_index) print("TEST DATA INDEX") print(test_index) X_train, X_test = X[train_index], X[test_index] y_train, y_test = y[train_index], y[test_index]
results = [] skf = cross_validation.StratifiedKFold(y, n_folds=folds) for train_index, test_index in skf: train_X, test_X = data.iloc[train_index], data.iloc[test_index] train_y, test_y = y.iloc[train_index], y.iloc[test_index] clf = None clf = RandomForestClassifier(n_estimators=n_estimators, criterion=criterion, max_depth=max_depth, min_samples_split=min_samples_split, n_jobs=n_processes) clf.fit(train_X, train_y) results.append(metrics.predict_table(clf, test_X, test_y)) result = pd.concat(results) output = open(result_path + 'Arboles/Arbol.pkl', 'wb+') pickle.dump(clf, output) output.close() result.to_csv(result_path + 'Predicciones/result.csv') matrix = metrics.confusion_matrix(result) matrix.to_csv(result_path + 'Metricas/soft_matrix_.csv') clases = matrix.columns.tolist() f_score = [metrics.f_score(matrix, c) for c in clases] with open(result_path + 'Metricas/results.txt') as f: f.write(clases + '\n') f.write(str(f_score) + '\n')
def validate(self, dataloader, is_val_set=True, measure_entropy=True): self.net.eval() self.hook.flag_hook = True prefix = self.get_prefix(is_val_set) self.metrics_epoch = collections.defaultdict(utils.Meter) matrix = np.zeros((self.c_dim, self.c_dim), dtype=np.uint32) for mb, (x, y) in enumerate(dataloader): x, y = x.to(self.device), y.to(self.device) y_one_hot = utils.to_one_hot(y, self.c_dim) logits = self.net(x) losses = self.criterion(logits, y_one_hot) matrix = matrix + metrics.confusion_matrix( utils.tensor2array(utils.get_class_outputs(logits)), utils.tensor2array(y), self.c_dim) self.metrics_epoch['{}_loss'.format(prefix)].update( utils.tensor2array(losses), x.shape[0]) if self.cfg.num_log > 0 and self.cfg.plot and mb == 0: num_log = min(self.cfg.num_log, x.shape[0]) name = '{}_{}_{}_epoch{:0=3d}_minibatch{}' filepath = '{}/{}'.format( os.path.join(self.cfg.plot_dir, self.cfg.model_type, self.cfg.model_name), name) x_ = x[0:num_log] x_np, y_np = utils.tensor2array( x[0:num_log]), utils.tensor2array(y[0:num_log]) losses_np = utils.tensor2array(losses[0:num_log]) plotting.make_grid( x_, filepath.format(prefix, 'data', 'x', self.metrics['epochs'][-1], mb + 1)) utils.save_array( x_np, filepath.format(prefix, 'data', 'x', self.metrics['epochs'][-1], mb + 1)) utils.save_array( y_np, filepath.format(prefix, 'data', 'y', self.metrics['epochs'][-1], mb + 1)) utils.save_array( losses_np, filepath.format(prefix, 'data', 'losses', self.metrics['epochs'][-1], mb + 1)) for (k, layer_name) in enumerate(self.hook.layers): layer_np = utils.tensor2array( self.hook.layers[layer_name][0:num_log]) utils.save_array( layer_np, filepath.format(prefix, 'data', layer_name, self.metrics['epochs'][-1], mb + 1)) if measure_entropy: entropy = metrics.entropy(utils.logits_to_probs(logits)) self.metrics_epoch['{}_entropy'.format(prefix)].update( utils.tensor2array(entropy), x.shape[0]) if self.cfg.num_log > 0 and self.cfg.plot and mb == 0: entropy_np = utils.tensor2array(entropy[0:num_log]) utils.save_array( entropy_np, filepath.format(prefix, 'data', 'entropy', self.metrics['epochs'][-1], mb + 1)) if is_val_set: x_rand = torch.randn(size=x.shape).to(self.device) logits_rand = self.net(x_rand) entropy_rand = metrics.entropy( utils.logits_to_probs(logits_rand)) self.metrics_epoch['entropy_rand'].update( utils.tensor2array(entropy_rand), x.shape[0]) if self.cfg.num_log > 0 and self.cfg.plot and mb == 0: name = '{}_{}_{}_epoch{:0=3d}_minibatch{}' filepath = '{}/{}'.format( os.path.join(self.cfg.plot_dir, self.cfg.model_type, self.cfg.model_name), name) x_ = x_rand[0:num_log] x_np = utils.tensor2array(x_rand[0:num_log]) entropy_np = utils.tensor2array( entropy_rand[0:num_log]) plotting.make_grid( x_, filepath.format(prefix, 'noise', 'x', self.metrics['epochs'][-1], mb + 1)) utils.save_array( x_np, filepath.format(prefix, 'noise', 'x', self.metrics['epochs'][-1], mb + 1)) utils.save_array( entropy_np, filepath.format(prefix, 'noise', 'entropy', self.metrics['epochs'][-1], mb + 1)) for (k, layer_name) in enumerate(self.hook.layers): layer_np = utils.tensor2array( self.hook.layers[layer_name][0:num_log]) utils.save_array( layer_np, filepath.format(prefix, 'noise', layer_name, self.metrics['epochs'][-1], mb + 1)) # disable hook after first minibatch by default - this is done for computational/speed purposes self.hook.flag_hook = False self.summarize_metrics(matrix, prefix)