def accuracy(y_true, y_pred): shape = tf.shape(y_pred) sequence_lengths = tf.ones(shape[0], dtype=tf.int32) * (shape[1]) viterbi_sequence, _ = tf.contrib.crf.crf_decode( y_pred, self.transitions, sequence_lengths) output = keras.backend.one_hot(viterbi_sequence, self.output_dim) return categorical_accuracy(y_true, output)
def _generic_accuracy(y_true, y_pred): if K.int_shape(y_pred)[1] == 1: return binary_accuracy(y_true, y_pred) if K.int_shape(y_true)[-1] == 1: return sparse_categorical_accuracy(y_true, y_pred) return categorical_accuracy(y_true, y_pred)
def flatten_accuracy_fn(y_true, y_pred): rebatched_pred = K.reshape(y_pred, (-1, K.int_shape(y_pred)[-1])) rebatched_true = K.reshape(y_true, (-1, K.int_shape(y_true)[-1])) # Remove any predictions from padded values. rebatched_pred = rebatched_pred * K.expand_dims( K.cast_to_floatx(K.sum(rebatched_true, -1) > 0)) return categorical_accuracy(rebatched_true, rebatched_pred)
def acc_of_all(y_true, y_pred): y_true = tf.split(y_true, 64, axis=-1) # a list of 64 * vectors y_pred = tf.split(y_pred, 64, axis=-1) num = 0 for (true, pred) in zip(y_true, y_pred): num += tf.reduce_mean(metrics.categorical_accuracy(true, pred)) return num / 64
def train_step(inputs, target): with tf.GradientTape() as tape: predictions = model(inputs, training=True) loss = loss_fn(target, predictions) + sum(model.losses) gradients = tape.gradient(loss, model.trainable_variables) optimizer.apply_gradients(zip(gradients, model.trainable_variables)) acc = tf.reduce_mean(categorical_accuracy(target, predictions)) return loss, acc
def acc_of_valid(y_true, y_pred): y_true = tf.split(y_true, 64, axis=-1) # a list of 64 * vectors y_pred = tf.split(y_pred, 64, axis=-1) num = 0 for idx, (true, pred) in enumerate(zip(y_true, y_pred)): if idx not in [0, 1, 2, 3, 4, 5, 11, 25, 32, 39, 53, 59, 60, 61, 62, 63]: num += tf.reduce_mean(metrics.categorical_accuracy(true, pred)) return num/48
def evaluate(): predictions = model([x, a], training=False) losses = [] accuracies = [] for mask in [mask_tr, mask_va, mask_te]: loss = loss_fn(y[mask], predictions[mask]) loss += sum(model.losses) losses.append(loss) acc = tf.reduce_mean(categorical_accuracy(y[mask], predictions[mask])) accuracies.append(acc) return losses, accuracies
def test(): predictions = model([X, fltr], training=False) losses = [] accuracies = [] for mask in [train_mask, val_mask, test_mask]: loss = loss_fn(y[mask], predictions[mask]) loss += sum(model.losses) losses.append(loss) acc = tf.reduce_mean(categorical_accuracy(y[mask], predictions[mask])) accuracies.append(acc) return losses, accuracies
def evaluate(loader): output = [] step = 0 while step < loader.steps_per_epoch: step += 1 inputs, target = loader.__next__() pred = model(inputs, training=False) outs = ( loss_fn(target, pred), tf.reduce_mean(categorical_accuracy(target, pred)), ) output.append(outs) return np.mean(output, 0)
def evaluate(loader): step = 0 results = [] for batch in loader: step += 1 inputs, target = batch predictions = model(inputs, training=False) loss = loss_fn(target, predictions) acc = tf.reduce_mean(categorical_accuracy(target, predictions)) results.append((loss, acc, len(target))) # Keep track of batch size if step == loader.steps_per_epoch: results = np.array(results) return np.average(results[:, :-1], 0, weights=results[:, -1])
def approx_accuracy_round(y_true, y_pred): nb_classes = len(Nuscene_dataset.correspondances_classes_index.keys()) # Extraction des informations des tenseurs y_pred_extracted = tf.slice(y_pred, [0, 0, 0], size=[dataset.batch_size, 1, nb_classes]) y_pred_extracted = tf.reshape(y_pred_extracted, [dataset.batch_size, nb_classes]) y_true_label = tf.slice(y_true, [0, 0, 0], size=[dataset.batch_size, 1, nb_classes]) y_true_label = tf.reshape(y_true_label, [dataset.batch_size, nb_classes]) return categorical_accuracy(y_true_label, fct_approx(y_pred_extracted))
def score(y_true, y_pred): y_t_rank = len(y_true.shape.as_list()) y_p_rank = len(y_pred.shape.as_list()) y_t_last_dim = y_true.shape.as_list()[-1] y_p_last_dim = y_pred.shape.as_list()[-1] is_binary = y_p_last_dim == 1 is_sparse_categorical = (y_t_rank < y_p_rank or y_t_last_dim == 1 and y_p_last_dim > 1) if isinstance(metric_function, six.string_types): if metric_function in ["accuracy", "acc"]: if is_binary: metric = binary_accuracy(y_true, y_pred) elif is_sparse_categorical: metric = sparse_categorical_accuracy(y_true, y_pred) else: metric = categorical_accuracy(y_true, y_pred) else: metric = categorical_accuracy(y_true, y_pred) else: metric = metric_function(y_true, y_pred) return K.cast(metric * (1.0 + delta), K.floatx())
def on_epoch_end(self, epoch, logs=None): target = self.validation[1] prediction = np.asarray(self.model.predict(self.validation[0])) f1score = self.f1_score(target, prediction) precision = self.precision(target, prediction) recall = self.recall(target, prediction) accuracy = np.mean(categorical_accuracy(target, prediction)) tf.summary.scalar("f1score", f1score, epoch) tf.summary.scalar("precision", precision, epoch) tf.summary.scalar("recall", recall, epoch) tf.summary.scalar("accuracy", accuracy, epoch) print( f"Metrics for epoch {epoch}:" f" -- val_f1score: {f1score} -- val_precision: {precision} -- val_recall: {recall} " f" -- val_accuracy: {accuracy}")
def evaluate(loader): output = [] step = 0 while step < loader.steps_per_epoch: step += 1 inputs, target = loader.__next__() pred = model(inputs, training=False) outs = ( loss_fn(target, pred), tf.reduce_mean(categorical_accuracy(target, pred)), len(target), # Keep track of batch size ) output.append(outs) if step == loader.steps_per_epoch: output = np.array(output) return np.average(output[:, :-1], 0, weights=output[:, -1])
def _update_metric(self, key: str, y_true: tf.Tensor, y_pred: tf.Tensor, loss: tuple, setup_changes: tf.Tensor, step=0): """ Updates the metrics. :param key: :param y_true: :param y_pred: :param loss: :param grads: """ loss_tt, loss_ms, loss_total = loss self._loss_tt.get(key).append(loss_tt) self._loss_ms.get(key).append(loss_ms) self._loss_total.get(key).append(loss_total) self._setup_changes.get(key).append(setup_changes) self._tp_obj.update_state(y_true, y_pred) self._tp.get(key).append(self._tp_obj.result()) self._tn_obj.update_state(y_true, y_pred) self._tn.get(key).append(self._tn_obj.result()) self._fp_obj.update_state(y_true, y_pred) self._fp.get(key).append(self._fp_obj.result()) self._fn_obj.update_state(y_true, y_pred) self._fn.get(key).append(self._fn_obj.result()) self._pre_obj.update_state(y_true, y_pred) self._pre.get(key).append(self._pre_obj.result()) self._rec_obj.update_state(y_true, y_pred) self._rec.get(key).append(self._rec_obj.result()) shape = tf.shape(y_true) y_true = tf.squeeze(tf.transpose(y_true, [0, 2, 1, 3])) y_pred = tf.squeeze(tf.transpose(y_pred, [0, 2, 1, 3])) y_pred = tf.reshape(y_pred, [shape[0], shape[2], -1]) y_true = tf.reshape(y_true, [shape[0], shape[2], -1]) self._acc.get(key).append(categorical_accuracy(y_true, y_pred))
def mu_accuracy(self, y_true, y_pred, **args): """ metric that outputs the accuracy when only considering the logits_mu. This accuracy should be the same that was obtained with the fake classifier in its best epoch. Parameters ---------- y_true: `np.array` the labels in one hot encoding format y_pred: `np.array` output of the model formed by the concatenation of the original prediction in the first num_classes positions, and a beta scalar in the last one Returns ------- the accuracy of the original predictions """ mu_probs = y_pred[:, :self.num_classes_] y_true_probs = y_true[:, :self.num_classes_] return categorical_accuracy(y_true_probs, mu_probs)
def categorical_error(y_true: tf.Tensor, y_pred: tf.Tensor) -> tf.Tensor: return 1.0 - categorical_accuracy(y_true, y_pred)
def categorical_accuracy(y_true, y_pred): return K.mean(metrics.categorical_accuracy(y_true, y_pred))
def accuracy(y_true, y_pred): return metrics.categorical_accuracy(y_true, y_pred) * correct_mean(y_true)
def trainRBM(data, learning_rate, k1, k2, epochs, batch_size, dims): # import data print("importing training data") if data == "fashion_mnist": fashion_mnist = tf.keras.datasets.fashion_mnist (x_train, y_train), (x_test, y_test) = fashion_mnist.load_data() elif data == "mnist": mnist = tf.keras.datasets.mnist (x_train, y_train), (x_test, y_test) = mnist.load_data() y_train = tf.keras.utils.to_categorical(y_train, 10) y_test = tf.keras.utils.to_categorical(y_test, 10) elif data == "faces": x_train = [resize(mpimg.imread(file),(28,28)) for file in glob.glob("data/faces/*")] x_train = np.asarray(x_train) # make images sparse for easier distinctions for img in x_train: img[img < np.mean(img)+0.5*np.std(img)] = 0 else: raise NameError("unknown data type: %s" % data) if data == "mnist" or data == "fashion_mnist": x_train = x_train/255.0 x_test = x_test/255.0 x_train = tf.cast(tf.reshape(x_train, shape=(x_train.shape[0], 784)), "float32") x_test = tf.cast(tf.reshape( x_test, shape=(x_test.shape[0], 784)), "float32") elif data == "faces": # auto conversion to probabilities in earlier step x_train = tf.cast(tf.reshape( x_train, shape=(x_train.shape[0], 784)), "float32") # train_path = tf.keras.utils.get_file( # "iris_training.csv", "https://storage.googleapis.com/download.tensorflow.org/data/iris_training.csv") # test_path = tf.keras.utils.get_file( # "iris_test.csv", "https://storage.googleapis.com/download.tensorflow.org/data/iris_test.csv") # CSV_COLUMN_NAMES = ['SepalLength', 'SepalWidth', # 'PetalLength', 'PetalWidth', 'Species'] # # SPECIES = ['Setosa', 'Versicolor', 'Virginica'] # train = pd.read_csv(train_path, names=CSV_COLUMN_NAMES, header=0) # test = pd.read_csv(test_path, names=CSV_COLUMN_NAMES, header=0) # y_train, y_test = train.pop('Species').values, test.pop('Species').values # x_train, x_test = train.values, test.values # y_train = tf.keras.utils.to_categorical(y_train, 3) # y_test = tf.keras.utils.to_categorical(y_test, 3) feature_dim = x_train.shape[-1] output_dim = y_train.shape[-1] # x_train = x_train.reshape(x_train.shape[0], x_train.shape[-1]) # create log directory # parse string input into integer list inputs = tf.keras.Input(feature_dim) outputs = OnlineBolzmannCell(50, online=True, return_hidden=True)(inputs) outputs = tf.keras.layers.Dropout(0.2)(outputs) # outputs = tf.keras.layers.Dense(100)(inputs) outputs = tf.keras.layers.Dense(output_dim, activation='sigmoid')(outputs) rbm = tf.keras.Model(inputs, outputs) rbm.compile(loss='categorical_crossentropy', optimizer="adam", metrics=["accuracy"]) print('Training...') rbm.fit(x_train, y_train, batch_size=batch_size, epochs=1, validation_split=0.1) for i in range(epochs): pred = rbm.predict(x_train) score = rbm.evaluate(x_train, y_train, verbose=0) print(f'Epoch {i+1}, loss={score[0]} Accuracy={score[1]}') score = rbm.evaluate(x_test, y_test, verbose=0) print("Test loss:", score[0]) print("Test accuracy:", score[1]) print(10*'#') score_train = classification_report( y_train.astype(np.bool), rbm.predict(x_train).astype(np.bool)) score_test = classification_report( y_test, rbm.predict(x_test).astype(np.bool)) acc_train = categorical_accuracy( y_train, rbm.predict(x_train)) acc_test = categorical_accuracy( y_test, rbm.predict(x_test)) print(f'Train score:\nAccuracy: {acc_train}\n{score_train}') print(f'Test score:\nAccuracy: {acc_test}\n{score_test}') print(10*'#') # plot_confusion_matrix(KerasClassifier(rbm, batch_size=batch_size), x_train, y_train>0.51) # plot_confusion_matrix(KerasClassifier( # rbm, batch_size=batch_size), x_test, y_test > 0.51) plt.show()
def acc(y_true, y_pred): y_true = y_true[:, :nb_classes] y_pred = y_pred[:, :nb_classes] return categorical_accuracy(y_true, y_pred)
def categorical_error(y_true: np.ndarray, y_pred: np.ndarray) -> float: return 1.0 - categorical_accuracy(y_true, y_pred) # type: ignore
with tf.GradientTape() as tape: pred = model(X, training=True) loss = gcnloss(model, y_train[train_mask, :], pred[train_mask]) # Gradient gradients = tape.gradient(loss, model.trainable_variables) # Update weights within our model optimizer.apply_gradients(zip(gradients, model.trainable_variables)) # Early stopping pred = model(X, training=False) loss_val = gcnloss(model, y_val[val_mask, :], pred[val_mask]) if loss_val > old_loss_val: count += 1 print("------------------------------------------------------") print("Loss ascending count: %d, epoch: %d, loss: %.4f" % (count, epoch, loss_val)) if count >= 10: break else: count = 0 if epoch % 5 == 0: print("Epoch: %d, loss: %.4f" % (epoch, loss_val)) old_loss_val = loss_val # Estimation on the test dataset print("------------------------------------------------------") pred = model(X, training=False) loss_test = gcnloss(model, y_test[test_mask, :], pred[test_mask]) print("Loss on test dataset: %.4f" % loss_test) accu = categorical_accuracy(y_test[test_mask, :], pred[test_mask]) print("Accuracy: %.4f" % (np.sum(accu) / len(accu)))
def __init__(self, model, intensity_range, regularization, input_shape, init_cost, steps, mini_batch, lr, num_classes, upsample_size=UPSAMPLE_SIZE, attack_succ_threshold=ATTACK_SUCC_THRESHOLD, patience=PATIENCE, cost_multiplier=COST_MULTIPLIER, reset_cost_to_zero=RESET_COST_TO_ZERO, mask_min=MASK_MIN, mask_max=MASK_MAX, color_min=COLOR_MIN, color_max=COLOR_MAX, img_color=IMG_COLOR, shuffle=SHUFFLE, batch_size=BATCH_SIZE, verbose=VERBOSE, return_logs=RETURN_LOGS, save_last=SAVE_LAST, epsilon=EPSILON, early_stop=EARLY_STOP, early_stop_threshold=EARLY_STOP_THRESHOLD, early_stop_patience=EARLY_STOP_PATIENCE, save_tmp=SAVE_TMP, tmp_dir=TMP_DIR, raw_input_flag=RAW_INPUT_FLAG): assert intensity_range in {'imagenet', 'inception', 'mnist', 'raw'} assert regularization in {None, 'l1', 'l2'} self.model = model self.intensity_range = intensity_range self.regularization = regularization self.input_shape = input_shape self.init_cost = init_cost self.steps = steps self.mini_batch = mini_batch self.lr = lr self.num_classes = num_classes self.upsample_size = upsample_size self.attack_succ_threshold = attack_succ_threshold self.patience = patience self.cost_multiplier_up = cost_multiplier self.cost_multiplier_down = cost_multiplier**1.5 self.reset_cost_to_zero = reset_cost_to_zero self.mask_min = mask_min self.mask_max = mask_max self.color_min = color_min self.color_max = color_max self.img_color = img_color self.shuffle = shuffle self.batch_size = batch_size self.verbose = verbose self.return_logs = return_logs self.save_last = save_last self.epsilon = epsilon self.early_stop = early_stop self.early_stop_threshold = early_stop_threshold self.early_stop_patience = early_stop_patience self.save_tmp = save_tmp self.tmp_dir = tmp_dir self.raw_input_flag = raw_input_flag mask_size = np.ceil( np.array(input_shape[0:2], dtype=float) / upsample_size) mask_size = mask_size.astype(int) self.mask_size = mask_size mask = np.zeros(self.mask_size) pattern = np.zeros(input_shape) mask = np.expand_dims(mask, axis=2) mask_tanh = np.zeros_like(mask) pattern_tanh = np.zeros_like(pattern) # prepare mask related tensors self.mask_tanh_tensor = K.variable(mask_tanh) mask_tensor_unrepeat = (K.tanh(self.mask_tanh_tensor) / (2 - self.epsilon) + 0.5) mask_tensor_unexpand = K.repeat_elements(mask_tensor_unrepeat, rep=self.img_color, axis=2) self.mask_tensor = K.expand_dims(mask_tensor_unexpand, axis=0) upsample_layer = UpSampling2D(size=(self.upsample_size, self.upsample_size)) mask_upsample_tensor_uncrop = upsample_layer(self.mask_tensor) uncrop_shape = K.int_shape(mask_upsample_tensor_uncrop)[1:] cropping_layer = Cropping2D( cropping=((0, uncrop_shape[0] - self.input_shape[0]), (0, uncrop_shape[1] - self.input_shape[1]))) self.mask_upsample_tensor = cropping_layer(mask_upsample_tensor_uncrop) reverse_mask_tensor = (K.ones_like(self.mask_upsample_tensor) - self.mask_upsample_tensor) def keras_preprocess(x_input, intensity_range): if intensity_range is 'raw': x_preprocess = x_input elif intensity_range is 'imagenet': # 'RGB'->'BGR' x_tmp = x_input[..., ::-1] # Zero-center by mean pixel mean = K.constant([[[103.939, 116.779, 123.68]]]) x_preprocess = x_tmp - mean elif intensity_range is 'inception': x_preprocess = (x_input / 255.0 - 0.5) * 2.0 elif intensity_range is 'mnist': x_preprocess = x_input / 255.0 else: raise Exception('unknown intensity_range %s' % intensity_range) return x_preprocess def keras_reverse_preprocess(x_input, intensity_range): if intensity_range is 'raw': x_reverse = x_input elif intensity_range is 'imagenet': # Zero-center by mean pixel mean = K.constant([[[103.939, 116.779, 123.68]]]) x_reverse = x_input + mean # 'BGR'->'RGB' x_reverse = x_reverse[..., ::-1] elif intensity_range is 'inception': x_reverse = (x_input / 2 + 0.5) * 255.0 elif intensity_range is 'mnist': x_reverse = x_input * 255.0 else: raise Exception('unknown intensity_range %s' % intensity_range) return x_reverse # prepare pattern related tensors self.pattern_tanh_tensor = K.variable(pattern_tanh) self.pattern_raw_tensor = ((K.tanh(self.pattern_tanh_tensor) / (2 - self.epsilon) + 0.5) * 255.0) # prepare input image related tensors # ignore clip operation here # assume input image is already clipped into valid color range input_tensor = K.placeholder(model.input_shape) if self.raw_input_flag: input_raw_tensor = input_tensor else: input_raw_tensor = keras_reverse_preprocess( input_tensor, self.intensity_range) # IMPORTANT: MASK OPERATION IN RAW DOMAIN X_adv_raw_tensor = ( reverse_mask_tensor * input_raw_tensor + self.mask_upsample_tensor * self.pattern_raw_tensor) X_adv_tensor = keras_preprocess(X_adv_raw_tensor, self.intensity_range) output_tensor = model(X_adv_tensor) y_true_tensor = K.placeholder(model.output_shape) self.loss_acc = categorical_accuracy(output_tensor, y_true_tensor) self.loss_ce = categorical_crossentropy(output_tensor, y_true_tensor) if self.regularization is None: self.loss_reg = K.constant(0) elif self.regularization is 'l1': self.loss_reg = (K.sum(K.abs(self.mask_upsample_tensor)) / self.img_color) elif self.regularization is 'l2': self.loss_reg = K.sqrt( K.sum(K.square(self.mask_upsample_tensor)) / self.img_color) cost = self.init_cost self.cost_tensor = K.variable(cost) self.loss = self.loss_ce + self.loss_reg * self.cost_tensor self.opt = Adam(lr=self.lr, beta_1=0.5, beta_2=0.9) self.updates = self.opt.get_updates( params=[self.pattern_tanh_tensor, self.mask_tanh_tensor], loss=self.loss) self.train = K.function( [input_tensor, y_true_tensor], [self.loss_ce, self.loss_reg, self.loss, self.loss_acc], updates=self.updates) pass
def accuracy(self, y_true, y_pred): logits = self.get_logits(y_true, y_pred) accuracy = categorical_accuracy(y_true, logits) return accuracy
def mu_accuracy(self, y_true, y_pred, **args): logits_phi = y_pred[:, :self.num_classes] labels_phi = y_true[:, :self.num_classes] return categorical_accuracy(labels_phi, logits_phi)
def accuracy(self, y_true, y_pred): y_true = y_true[:, :self.num_class] y_pred = y_pred[:, :self.num_class] return categorical_accuracy(y_true, y_pred)
def accuracy(y_true,y_pred): return categorical_accuracy(y_true,y_pred)
def acc(y_true, y_pred): num_classes = util.NUM_CLASS y_true = y_true[:, :num_classes] y_pred = y_pred[:, :num_classes] return categorical_accuracy(y_true, y_pred)
def acc(y_true, y_pred): y_true = y_true[:, :10] y_pred = y_pred[:, :10] return categorical_accuracy(y_true, y_pred)