def evaluate(model, evalX, evalY, labelNames): stats = {0: 0, 1: 0, 2: 0, 3: 0} top5 = [] for i, image in enumerate(evalX): if i % 10 == 0: print("[INFO] Evaluated using {} images".format(i)) image = image.astype("float32") / 255.0 image = np.expand_dims(image, axis=0) preds = model.predict(image) top = np.argsort(-preds, axis=1) if evalY[i] == top[0][0]: stats[0] += 1 elif evalY[i] == top[0][1]: stats[1] += 1 elif evalY[i] == top[0][2]: stats[2] += 1 else: stats[3] += 1 top5.append( [top[0][0], top[0][1], top[0][2], top[0][3], top[0][4], evalY[i]]) numLabels = len(np.unique(evalY)) evalY = to_categorical(evalY, numLabels) evalX = np.array(evalX, dtype=np.float32) / 255.0 predictions = model.predict(evalX) report = classification_report(evalY.argmax(axis=1), predictions.argmax(axis=1), target_names=labelNames) confusion = math.confusion_matrix(evalY.argmax(axis=1), predictions.argmax(axis=1)) return stats, top5, report, confusion
def model_performance(model, x_test, y_test): test_score = model.evaluate(x_test, y_test) y_pred_prob = model.predict(x_test) y_pred = np.argmax(y_pred_prob, axis=1) y_test = np.argmax(y_test, axis=1) confusion_mtx = confusion_matrix(y_test, y_pred) return test_score, y_pred, confusion_mtx
def matriz_confusion(y_test, y_predecida): # le hacemos el argmax para desaher el to categorical matriz_conf = confusion_matrix(labels=argmax(y_test, axis=1), predictions=argmax(y_predecida, axis=1)).numpy() matriz_normalizada = np.around(matriz_conf.astype('float') / matriz_conf.sum(axis=1)[:, np.newaxis], decimals=2) clases = [i for i in range(NUM_CLASES)] dataframe_matriz = pd.DataFrame(matriz_normalizada, index=clases, columns=clases) figure = plt.figure(figsize=(8, 8)) sns.heatmap(dataframe_matriz, annot=True, cmap=plt.cm.Blues) plt.tight_layout() plt.ylabel('Valor real') plt.xlabel('Valor predecido') plt.show()
model.fit(fashion_mnist_train_images, fashion_mnist_train_marks, epochs=10) test_loss, test_acc = model.evaluate(fashion_mnist_test_images, fashion_mnist_test_marks, verbose=1) if test_acc > best_test_acc: best_test_acc = test_acc best_number = number number += 1 print("best test accuracy = " + str(best_test_acc)) predict = models[best_number].predict(fashion_mnist_test_images) p1 = [] for i in range(len(predict)): p1.append(argmax(predict[i])) print(math.confusion_matrix(fashion_mnist_test_marks, p1)) classes = {} for i in range(len(fashion_mnist_test_marks)): if not classes.__contains__(fashion_mnist_test_marks[i]): classes[fashion_mnist_test_marks[i]] = [] classes[fashion_mnist_test_marks[i]].append(i) # print(classes) i = 0 for k in range(10): for j in range(10): i += 1 m = 0 num = 0 for index in classes[k]: if predict[index][j] > m:
def launch(self) -> int: """Execute the :class:`ClassificationNeuralNetwork <neural_networks.classification_neural_network.ClassificationNeuralNetwork>` neural_networks.classification_neural_network.ClassificationNeuralNetwork object.""" # check input/output paths and parameters self.check_data_params(self.out_log, self.err_log) # Setup Biobb if self.check_restart(): return 0 self.stage_files() # load dataset fu.log( 'Getting dataset from %s' % self.io_dict["in"]["input_dataset_path"], self.out_log, self.global_log) if 'columns' in self.features: labels = getHeader(self.io_dict["in"]["input_dataset_path"]) skiprows = 1 else: labels = None skiprows = None data = pd.read_csv(self.io_dict["in"]["input_dataset_path"], header=None, sep="\s+|;|:|,|\t", engine="python", skiprows=skiprows, names=labels) targets_list = data[getTargetValue(self.target)].to_numpy() X = getFeatures(self.features, data, self.out_log, self.__class__.__name__) fu.log('Features: [%s]' % (getIndependentVarsList(self.features)), self.out_log, self.global_log) # target y = getTarget(self.target, data, self.out_log, self.__class__.__name__) fu.log('Target: %s' % (str(getTargetValue(self.target))), self.out_log, self.global_log) # weights if self.weight: w = getWeight(self.weight, data, self.out_log, self.__class__.__name__) # shuffle dataset fu.log('Shuffling dataset', self.out_log, self.global_log) shuffled_indices = np.arange(X.shape[0]) np.random.shuffle(shuffled_indices) np_X = X.to_numpy() shuffled_X = np_X[shuffled_indices] shuffled_y = targets_list[shuffled_indices] if self.weight: shuffled_w = w[shuffled_indices] # train / test split fu.log('Creating train and test sets', self.out_log, self.global_log) arrays_sets = (shuffled_X, shuffled_y) # if user provide weights if self.weight: arrays_sets = arrays_sets + (shuffled_w, ) X_train, X_test, y_train, y_test, w_train, w_test = train_test_split( *arrays_sets, test_size=self.test_size, random_state=self.random_state) else: X_train, X_test, y_train, y_test = train_test_split( *arrays_sets, test_size=self.test_size, random_state=self.random_state) # scale dataset if self.scale: fu.log('Scaling dataset', self.out_log, self.global_log) X_train = scale(X_train) # build model fu.log('Building model', self.out_log, self.global_log) model = self.build_model((X_train.shape[1], ), np.unique(y_train).size) # model summary stringlist = [] model.summary(print_fn=lambda x: stringlist.append(x)) model_summary = "\n".join(stringlist) fu.log('Model summary:\n\n%s\n' % model_summary, self.out_log, self.global_log) # get optimizer mod = __import__('tensorflow.keras.optimizers', fromlist=[self.optimizer]) opt_class = getattr(mod, self.optimizer) opt = opt_class(lr=self.learning_rate) # compile model model.compile(optimizer=opt, loss='sparse_categorical_crossentropy', metrics=['accuracy', 'mse']) # fitting fu.log('Training model', self.out_log, self.global_log) # set an early stopping mechanism # set patience=2, to be a bit tolerant against random validation loss increases early_stopping = EarlyStopping(patience=2) if self.weight: sample_weight = w_train class_weight = [] else: # TODO: class_weight not working since TF 2.4.1 update #fu.log('No weight provided, class_weight will be estimated from the target data', self.out_log, self.global_log) fu.log('No weight provided', self.out_log, self.global_log) sample_weight = None class_weight = [ ] #compute_class_weight('balanced', np.unique(y_train), y_train) print(class_weight) # fit the model mf = model.fit(X_train, y_train, class_weight=class_weight, sample_weight=sample_weight, batch_size=self.batch_size, epochs=self.max_epochs, callbacks=[early_stopping], validation_split=self.validation_size, verbose=1) fu.log('Total epochs performed: %s' % len(mf.history['loss']), self.out_log, self.global_log) train_metrics = pd.DataFrame() train_metrics['metric'] = [ 'Train loss', ' Train accuracy', 'Train MSE', 'Validation loss', 'Validation accuracy', 'Validation MSE' ] train_metrics['coefficient'] = [ mf.history['loss'][-1], mf.history['accuracy'][-1], mf.history['mse'][-1], mf.history['val_loss'][-1], mf.history['val_accuracy'][-1], mf.history['val_mse'][-1] ] fu.log( 'Training metrics\n\nTRAINING METRICS TABLE\n\n%s\n' % train_metrics, self.out_log, self.global_log) # confusion matrix train_predictions = model.predict(X_train) train_predictions = np.around(train_predictions, decimals=2) norm_pred = [] [ norm_pred.append(np.argmax(pred, axis=0)) for pred in train_predictions ] cnf_matrix_train = math.confusion_matrix(y_train, norm_pred).numpy() np.set_printoptions(precision=2) if self.normalize_cm: cnf_matrix_train = cnf_matrix_train.astype( 'float') / cnf_matrix_train.sum(axis=1)[:, np.newaxis] cm_type = 'NORMALIZED CONFUSION MATRIX' else: cm_type = 'CONFUSION MATRIX, WITHOUT NORMALIZATION' fu.log( 'Calculating confusion matrix for training dataset\n\n%s\n\n%s\n' % (cm_type, cnf_matrix_train), self.out_log, self.global_log) # testing if self.scale: X_test = scale(X_test) fu.log('Testing model', self.out_log, self.global_log) test_loss, test_accuracy, test_mse = model.evaluate(X_test, y_test) test_metrics = pd.DataFrame() test_metrics['metric'] = ['Test loss', ' Test accuracy', 'Test MSE'] test_metrics['coefficient'] = [test_loss, test_accuracy, test_mse] fu.log( 'Testing metrics\n\nTESTING METRICS TABLE\n\n%s\n' % test_metrics, self.out_log, self.global_log) # predict data from X_test test_predictions = model.predict(X_test) test_predictions = np.around(test_predictions, decimals=2) tpr = tuple(map(tuple, test_predictions)) test_table = pd.DataFrame() test_table['P' + np.array2string(np.unique(y_train))] = tpr test_table['target'] = y_test fu.log('TEST DATA\n\n%s\n' % test_table, self.out_log, self.global_log) # confusion matrix norm_pred = [] [ norm_pred.append(np.argmax(pred, axis=0)) for pred in test_predictions ] cnf_matrix_test = math.confusion_matrix(y_test, norm_pred).numpy() np.set_printoptions(precision=2) if self.normalize_cm: cnf_matrix_test = cnf_matrix_test.astype( 'float') / cnf_matrix_test.sum(axis=1)[:, np.newaxis] cm_type = 'NORMALIZED CONFUSION MATRIX' else: cm_type = 'CONFUSION MATRIX, WITHOUT NORMALIZATION' fu.log( 'Calculating confusion matrix for testing dataset\n\n%s\n\n%s\n' % (cm_type, cnf_matrix_test), self.out_log, self.global_log) # save test data if (self.io_dict["out"]["output_test_table_path"]): fu.log( 'Saving testing data to %s' % self.io_dict["out"]["output_test_table_path"], self.out_log, self.global_log) test_table.to_csv(self.io_dict["out"]["output_test_table_path"], index=False, header=True) # create test plot if (self.io_dict["out"]["output_plot_path"]): vs = np.unique(targets_list) vs.sort() if len(vs) > 2: plot = plotResultsClassMultCM(mf.history, cnf_matrix_train, cnf_matrix_test, self.normalize_cm, vs) fu.log( 'Saving confusion matrix plot to %s' % self.io_dict["out"]["output_plot_path"], self.out_log, self.global_log) else: plot = plotResultsClassBinCM(mf.history, train_predictions, test_predictions, y_train, y_test, cnf_matrix_train, cnf_matrix_test, self.normalize_cm, vs) fu.log( 'Saving binary classifier evaluator plot to %s' % self.io_dict["out"]["output_plot_path"], self.out_log, self.global_log) plot.savefig(self.io_dict["out"]["output_plot_path"], dpi=150) # save model and parameters vs = np.unique(targets_list) vs.sort() vars_obj = { 'features': self.features, 'target': self.target, 'scale': self.scale, 'vs': vs.tolist(), 'type': 'classification' } variables = json.dumps(vars_obj) fu.log('Saving model to %s' % self.io_dict["out"]["output_model_path"], self.out_log, self.global_log) with h5py.File(self.io_dict["out"]["output_model_path"], mode='w') as f: hdf5_format.save_model_to_hdf5(model, f) f.attrs['variables'] = variables return 0
def visualize_detections(images, ground_truths, detections, id2code, label_codes, label_names, out_dir='/tmp'): """Create visualizations. Consist of the original image, the confusion matrix, ground truth labels and the model predicted labels. :param images: original images :param ground_truths: ground truth labels :param detections: the model label predictions :param id2code: dictionary mapping label ids to their codes :param label_codes: list with label codes :param label_names: list with label names :param out_dir: directory where the output visualizations will be saved """ max_id = max(id2code.values()) name_range = range(len(label_names)) for i in range(0, np.shape(detections)[0]): fig = plt.figure(figsize=(17, 17)) # original image ax1 = fig.add_subplot(2, 2, 1) # TODO: expect also other data than S2 a = np.stack( (images[i][:, :, 3], images[i][:, :, 2], images[i][:, :, 1]), axis=2) ax1.imshow((255 / a.max() * a).astype(np.uint8)) ax1.title.set_text('Actual image') # ground truths ax3 = fig.add_subplot(2, 2, 3) ax3.set_title('Ground truth labels') gt_labels = ground_truths[i] gt_labels = onehot_decode(gt_labels, id2code) ax3.imshow(gt_labels * 4) # detections ax4 = fig.add_subplot(2, 2, 4) ax4.set_title('Predicted labels') pred_labels = onehot_decode(detections[i], id2code) ax4.imshow(pred_labels * 4) # confusion matrix ax2 = fig.add_subplot(2, 2, 2) ax2.set_title('Confusion matrix') conf_matrix = confusion_matrix(gt_labels[:, :, 0].flatten(), pred_labels[:, :, 0].flatten(), max_id + 1) # subset to existing classes conf_matrix = conf_matrix.numpy()[label_codes][:, label_codes] # normalize the confusion matrix row_sums = conf_matrix.sum(axis=1)[:, np.newaxis] # TODO: solve division by 0 cm_norm = np.around(conf_matrix.astype('float') / row_sums, decimals=2) # visualize ax2.imshow(cm_norm, cmap=plt.cm.Blues) y_labels = [ '{}\n{}'.format(label_names[j], row_sums[j]) for j in name_range ] plt.xticks(name_range, label_names) plt.yticks(name_range, y_labels) plt.xlabel('Predicted label') plt.ylabel('True label') # write percentage values (0.00 -- 1.00) into the confusion matrix threshold = cm_norm.max() / 2. # used to decide for the font colour for row in range(len(conf_matrix)): for col in range(len(conf_matrix)): if cm_norm[col, row] > threshold: colour = 'white' else: colour = 'black' # TODO: class names, not codes ax2.text(row, col, cm_norm[col, row], color=colour, horizontalalignment='center') plt.savefig(os.path.join(out_dir, str(i))) plt.close()
r = model.fit( # np.append(data_train, data_test, axis = 0), # np.append(Y_multi_train, Y_multi_test, axis =0), data_train, Y_multi_train, epochs=10, validation_data=(data_val, np.array(Y_multi_val),), batch_size=64, verbose=1, shuffle=True, callbacks=callbacks_list ) predicted_test = model.predict(data_test)>0.5 #predicting test data in trained model predicted_test = predicted_test*1 predicted_val = model.predict(data_val)>0.5 #predicting test data in trained model predicted_val = predicted_val*1 #Inspecting confusion matrix for val data set as well as test data set val_con = math.confusion_matrix(Y_multi_test.argmax(axis = 1), predicted_test.argmax(axis = 1)) test_con = math.confusion_matrix(Y_multi_val.argmax(axis = 1), predicted_val.argmax(axis = 1)) print(val_con) print(test_con) test = "all the days the stock went low and it did not improve even at the end of the day" test = clean_text_round1(test) # print(test) print(test) test_sen = np.argmax(prediction_(test, model, T)) print(test_sen) if test_sen == 0: print("Postive sentance ") elif test_sen == 1: print("negative sentance") else:
model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"]) history = model.fit(X[train], y[train], epochs=N_EPOCHS, batch_size=B_SIZE, shuffle=1, validation_split=0.2, verbose=0) fitness = model.evaluate(X[test], y[test]) accuracy_scores.append(fitness[1]) cm = confusion_matrix(y[test], tf.math.round(model.predict(X[test]))) cm_scores.append(cm.numpy()) y_pred = tf.math.round(model.predict(X[test])).numpy() mcc_scores.append(matthews_corrcoef(y[test], y_pred)) print("\n\nEnd training! ") accuracy_std_deviation = np.std(accuracy_scores) print("accuracy:") print(f"mean: {np.mean(accuracy_scores)}") print(f"standard deviation: {accuracy_std_deviation}") # compute confidence interval cfi = scipy.stats.t.interval(0.95, len(accuracy_scores) - 1, loc=np.mean(accuracy_scores),
print('Loading dataset...') X_train, y_train, X_val, y_val, X_test, y_test = loadAugmentedBinaryDatasetFromFiles( ) print('Loading model...') model = load_model(full_path) print('Evaluating model on validation dataset...') results = model.evaluate(X_val, y_val, batch_size=4, return_dict=True, verbose=0) predictions = model.predict(X_val, batch_size=4, verbose=0) predictions = [round(p[0]) for p in predictions] conf_matrix = confusion_matrix(y_val, predictions) # Define print results function def printResults(res): for metric in res: print(" {}: {}".format(metric, res[metric])) def printConfusionMatrix(cf): print(cf.numpy()) print('Validation Results:') printResults(results) printConfusionMatrix(conf_matrix)
def plotAndSave(filename, history, model, X_val, y_val): # Create the figure. f = plt.figure(figsize=(15, 10)) f.tight_layout() # Create subplots for each metric and confusion matrix. ax1 = f.add_subplot(231, xlabel='Epoch', ylabel='Accuracy', ylim=[0, 1]) ax2 = f.add_subplot(232, xlabel='Epoch', ylabel='MSE', ylim=[0, 1]) ax3 = f.add_subplot(233, xlabel='Epoch', ylabel='Precision', ylim=[0, 1]) ax4 = f.add_subplot(234, xlabel='Epoch', ylabel='Recall', ylim=[0, 1]) ax5 = f.add_subplot(235, xlabel='Epoch', ylabel='Loss') ax6 = f.add_subplot(236, xlabel='Predicted', ylabel='Actual') ax1.plot(history.history['accuracy'], label='accuracy') ax1.plot(history.history['val_accuracy'], label='val_accuracy') ax1.legend(loc='lower right') ax2.plot(history.history['mse'], label='mse') ax2.plot(history.history['val_mse'], label='val_mse') ax2.legend(loc='lower right') ax3.plot(history.history['precision'], label='precision') ax3.plot(history.history['val_precision'], label='val_precision') ax3.legend(loc='lower right') ax4.plot(history.history['recall'], label='recall') ax4.plot(history.history['val_recall'], label='val_recall') ax4.legend(loc='lower right') ax5.plot(history.history['loss'], label='loss') ax5.plot(history.history['val_loss'], label='val_loss') ax5.set_yscale('log') ax5.legend(loc='lower right') # Make validation predictions and construct confusion matrix. predictions = model.predict(X_val, batch_size=2) predictions = [round(p[0]) for p in predictions] # print("Predictions:", predictions) # print("Y_val:", y_val) cf = confusion_matrix(y_val, predictions) cf = np.array(cf) # Plot confusion matrix. im_id = ax6.imshow(cf, interpolation='nearest') ax6.set_xticks(np.arange(2)) ax6.set_yticks(np.arange(2)) ax6.set_xticklabels(['positive', 'negative']) ax6.set_yticklabels(['positive', 'negative']) ax6.set_title("Confusion Matrix") f.colorbar(im_id) plt.setp(ax6.get_yticklabels(), rotation=90, ha="center", va="baseline", rotation_mode="anchor") for i in range(2): for j in range(2): ax6.text(j, i, cf[i, j], ha="center", va="center", color="w") # Save the figure as an image. f.savefig(filename)
# train_dataset, epochs=EPOCHS, batch_size=BATCH_SIZE, validation_data=(X_val, y_val), # validation_data=val_dataset, callbacks=[earlyStopping, mcp_save, reduce_lr_loss, csv_logger], verbose=VERBOSITY_1) plotAndSave('./baseline_results.png', history, model, X_val, y_val) print('Evaluating results...') # Generate confusion matrix again, for evaluation and training dataset. cf_text = "" predictions = model.predict(X_val, batch_size=BATCH_SIZE) predictions = [round(p[0]) for p in predictions] cf = confusion_matrix(y_val, predictions) cf_text += "Validation\n" + str(cf.numpy())[1:-1].replace('\n ', '\n') + "\n" predictions = model.predict(X_train, batch_size=BATCH_SIZE) predictions = [round(p[0]) for p in predictions] cf = confusion_matrix(y_train, predictions) cf_text += "Training\n" + str(cf.numpy())[1:-1].replace('\n ', '\n') + "\n" # Save confusion matrices in text format into the file. file = open('vgg16_confusion_matrix.txt', 'w') file.write(cf_text) file.close() print(cf_text)
# Normalize the data x_train = x_train.astype('float32') / 255 x_test = x_test.astype('float32') / 255 y_train = keras.utils.to_categorical(y_train, num_classes) y_test = keras.utils.to_categorical(y_test, num_classes) # Define our neural network model model = Sequential() model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=(28,28,1))) model.add(Conv2D(64, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(128, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(num_classes, activation='softmax')) # Compile the model model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy']) # Fit the model according to the parameters model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test, y_test)) # Evaluate the result, and generate confusion matrix result = model.evaluate(x_test, y_test, verbose=0) print('Test loss:', result[0]) print('Test accuracy:', result[1] * 100) print('Confusion Matrix:', math.confusion_matrix(x_test, y_test))