def test_cce_one_hot():
y_a = K.variable(np.random.randint(0, 7, (5, 6)))
y_b = K.variable(np.random.random((5, 6, 7)))
objective_output = losses.sparse_categorical_crossentropy(y_a, y_b)
assert K.eval(objective_output).shape == (5, 6)
y_a = K.variable(np.random.randint(0, 7, (6,)))
y_b = K.variable(np.random.random((6, 7)))
assert K.eval(losses.sparse_categorical_crossentropy(y_a, y_b)).shape == (6,)
def test_cce_one_hot():
y_a = K.variable(np.random.randint(0, 7, (5, 6)))
y_b = K.variable(np.random.random((5, 6, 7)))
objective_output = sparse_categorical_crossentropy(y_a, y_b)
assert K.eval(objective_output).shape == (5, 6)
y_a = K.variable(np.random.randint(0, 7, (6, )))
y_b = K.variable(np.random.random((6, 7)))
assert K.eval(sparse_categorical_crossentropy(y_a, y_b)).shape == (6, )
def sample(self, model):
"""
Sort the loss values of the training samples.
Variables
"""
idx = np.random.choice(np.arange(0, x_train.shape[0]),
size=args.fwd_batch_size,
replace=False)
if args.dataset == "ptb":
res = model.predict_proba(x_train[idx])
print("res.shape", res.shape)
res = K.get_value(
sparse_categorical_crossentropy(
tf.convert_to_tensor(y_train[idx], np.float32),
tf.convert_to_tensor(res)))
print("res.shape", res.shape)
else:
res = model.predict_proba(x_train[idx])
print(y_train.shape)
print(res.shape)
res = K.get_value(
tf.nn.softmax_cross_entropy_with_logits(labels=y_train[idx],
logits=res))
res = res / np.sum(res)
return np.random.choice(idx,
size=args.batch_size,
replace=False,
p=res)
def crf_loss(self, y_true, y_pred):
"""General CRF loss function depanding on the learning mode.
# Arguments
y_true: tensor with true targets.
y_pred: tensor with predicted targets.
# Returns
If the CRF layer is being trained in the join mode, returns the negative
log-likelihood. Otherwise returns the categorical crossentropy implemented
by the underlying Keras backend.
# About Codes
This code is from Keras-Team/keras_contrib.losses.crf_losses,
change some details.
"""
crf, idx = y_pred._keras_history[:2]
if crf.learn_mode == "join":
return self.crf_nll(y_true, y_pred)
else:
if crf.sparse_target:
return sparse_categorical_crossentropy(y_true, y_pred)
else:
return categorical_crossentropy(y_true, y_pred)
def __triplet_loss(y_true, y_pred):
from keras.losses import sparse_categorical_crossentropy
triplet_contribution = triplet_semihard_loss(
K.argmax(y_true, axis=1), embeddings)
classification_contribution = sparse_categorical_crossentropy(
y_true, y_pred)
return triplet_contribution + classification_contribution
def custom_loss(y_true, y_pred):
epsilon = 0.001
main_loss = losses.sparse_categorical_crossentropy(y_true, y_pred)
pred_indices = K.argmax(y_pred, axis=-1)
pred_indices = K.cast(pred_indices, dtype='float32')
distance_penalty = K.constant(1.0, dtype='float32') / (K.abs(pred_indices - K.constant(50/ 2.0, dtype='float32')) + epsilon)
return main_loss + 5000 * distance_penalty
def test_sparse_categorical_crossentropy():
y_pred = K.variable(np.array([[0.3, 0.6, 0.1],
[0.1, 0.2, 0.7]]))
y_true = K.variable(np.array([1, 2]))
expected_loss = - (np.log(0.6) + np.log(0.7)) / 2
loss = K.eval(losses.sparse_categorical_crossentropy(y_true, y_pred))
assert np.isclose(expected_loss, np.mean(loss))
def test_sparse_categorical_crossentropy():
y_pred = K.variable(np.array([[0.3, 0.6, 0.1],
[0.1, 0.2, 0.7]]))
y_true = K.variable(np.array([1, 2]))
expected_loss = - (np.log(0.6) + np.log(0.7)) / 2
loss = K.eval(losses.sparse_categorical_crossentropy(y_true, y_pred))
assert np.isclose(expected_loss, np.mean(loss))
def loss_D(y_true, y_pred):
if self.n_disc_outputs == 1:
return c * KL.binary_crossentropy(y_pred, y_true)
# return c * KL.binary_crossentropy(1.001/(K.exp(-y_pred) + 1), y_true)
# return c * KL.binary_crossentropy(y_pred, y_true)
else:
return c * KL.sparse_categorical_crossentropy(y_pred, y_true)
def my_loss(y_true, y_pred): crossentropy = losses.sparse_categorical_crossentropy(y_true, y_pred, axis=-1) bool = tf.ones(shape=(1, 13, 13, 1), dtype=tf.float32) mask = tf.math.minimum(bool, y_true) mask = tf.squeeze(mask, axis=-1) final = mask * crossentropy loss = K.sum(final, axis=(1,2)) return loss
def evaluate(model,
dataset,
section,
adv=None,
validation_size=5000,
adv_iterations=40,
adv_restarts=1,
verbose=1):
if dataset == "CIFAR10":
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
elif dataset == "MNIST":
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = np.expand_dims(x_train, axis=-1)
x_test = np.expand_dims(x_test, axis=-1)
else:
raise ValueError("Unrecognised dataset")
# Leave aside a validation set
x_valid = x_train[-validation_size:]
y_valid = y_train[-validation_size:]
x_train = x_train[:-validation_size]
y_train = y_train[:-validation_size]
if section == "train":
x = x_train
y = y_train
elif section == "validation":
x = x_valid
y = y_valid
elif section == "test":
x = x_test
y = y_test
else:
raise ValueError("Invalid dataset section")
# Normalize data
x = x.astype("float32") / 255
if adv is None or adv == 0:
return model.evaluate(x, y, verbose=verbose)[1]
else:
# Keras gives no easy way of just getting the cross-entropy loss (without the
# regularisation/rs loss) and it's needed for PGD, so we need to create it again
model.xent_loss = sparse_categorical_crossentropy(
model.targets[0], model.outputs[0])
adv_generator = AdversarialExampleGenerator(model,
x,
y,
batch_size=64,
epsilon=adv,
k=adv_iterations,
a=adv / 10.0,
incremental=False)
return model.evaluate_generator(adv_generator,
workers=0,
verbose=verbose)[1]
def crf_loss(y_true, y_pred):
"""General CRF loss function, depending on the learning mode."""
crf, idx = y_pred._keras_history[:2]
if crf.learn_mode == 'join':
return crf_nll(y_true, y_pred)
else:
if crf.sparse_target:
return sparse_categorical_crossentropy(y_true, y_pred)
else:
return categorical_crossentropy(y_true, y_pred)
def scce(y_true, y_pred):
""" sparse categorical cross entropy for sparse labels
# Args:
y_true: (batch_size, height, width, 1).
y_pred: (batch_size, height, width, 1).
"""
y_true_f = tf.reshape(y_true, [-1])
y_pred_f = tf.reshape(y_pred, [-1])
return losses.sparse_categorical_crossentropy(y_true_f, y_pred_f)
def test_sparse_categorical_crossentropy_4d():
y_pred = K.variable(
np.array([[[[0.7, 0.1, 0.2], [0.0, 0.3, 0.7], [0.1, 0.1, 0.8]],
[[0.3, 0.7, 0.0], [0.3, 0.4, 0.3], [0.2, 0.5, 0.3]],
[[0.8, 0.1, 0.1], [1.0, 0.0, 0.0], [0.4, 0.3, 0.3]]]]))
y_true = K.variable(np.array([[[0, 1, 0], [2, 1, 0], [2, 2, 1]]]))
expected_loss = -(np.log(0.7) + np.log(0.3) + np.log(0.1) +
np.log(K.epsilon()) + np.log(0.4) + np.log(0.2) +
np.log(0.1) + np.log(K.epsilon()) + np.log(0.3)) / 9
loss = K.eval(losses.sparse_categorical_crossentropy(y_true, y_pred))
assert np.isclose(expected_loss, np.mean(loss))
def policy_gradient_loss(args):
y_true, y_pred = args
#adv = k.squeeze(advantage, axis=-1)
#y_true = k.stack([k.arange(k.shape(y_true)[0]), k.cast(k.squeeze(y_true, axis=-1), dtype='int32')], axis=-1)
y_true = k.squeeze(y_true, axis=-1)
#policy_loss = k.log(k.clip(tf.gather_nd(y_pred, y_true), 1e-12, 1.))
policy_loss = sparse_categorical_crossentropy(
y_true, y_pred) # self.compute_log_prob(y_true, y_pred)
return policy_loss
def create_model(observation_space, action_space, args):
assert isinstance(observation_space, gym.spaces.Box)
assert isinstance(action_space, gym.spaces.Box) \
or isinstance(action_space, gym.spaces.Discrete)
h = x = Input(shape=observation_space.shape)
for i in range(args.hidden_layers):
h = Dense(args.hidden_nodes, activation=args.activation_function)(h)
# baseline output
bh = x
for i in range(args.hidden_layers):
bh = Dense(args.hidden_nodes, activation=args.activation_function)(bh)
b = Dense(1)(bh)
if isinstance(action_space, gym.spaces.Discrete):
# produce logits for all actions
h = Dense(action_space.n)(h)
# sample action from logits
a = Lambda(lambda x: tf.multinomial(x, num_samples=1))(h)
# turn logits into probabilities
p = Activation('softmax')(h)
# model outputs sampled action and baseline
model = Model(x, [a, b])
# loss is between true values and probabilities
model.compile(optimizer=RMSprop(lr=args.learning_rate),
loss=[
lambda y_true, y_pred:
sparse_categorical_crossentropy(y_true, p), 'mse'
],
loss_weights=[1, args.baseline_weight])
else:
# number of actions
n = np.prod(action_space.shape)
# produce means and stddevs for Gaussian
mu = Dense(n)(h)
std = args.stddev
# sample action from Gaussian
a = Lambda(lambda mu: mu + std * K.random_normal(K.shape(mu)))(mu)
# model outputs sampled action
model = Model(x, [a, b])
# negative log likelihood of Gaussian
model.compile(optimizer=RMSprop(lr=args.learning_rate, clipnorm=1.),
loss=[lambda y_true, y_pred: mse(y_true, mu), 'mse'],
loss_weights=[1, args.baseline_weight])
model.summary()
return model
def test_sparse_categorical_crossentropy_4d():
y_pred = K.variable(np.array([[[[0.7, 0.1, 0.2],
[0.0, 0.3, 0.7],
[0.1, 0.1, 0.8]],
[[0.3, 0.7, 0.0],
[0.3, 0.4, 0.3],
[0.2, 0.5, 0.3]],
[[0.8, 0.1, 0.1],
[1.0, 0.0, 0.0],
[0.4, 0.3, 0.3]]]]))
y_true = K.variable(np.array([[[0, 1, 0],
[2, 1, 0],
[2, 2, 1]]]))
expected_loss = - (np.log(0.7) + np.log(0.3) + np.log(0.1) +
np.log(K.epsilon()) + np.log(0.4) + np.log(0.2) +
np.log(0.1) + np.log(K.epsilon()) + np.log(0.3)) / 9
loss = K.eval(losses.sparse_categorical_crossentropy(y_true, y_pred))
assert np.isclose(expected_loss, np.mean(loss))
def sparse_categorical_cross_entropy_pos_contrib(target_rank, pred_rank):
target_class_ids = tf.reshape(target_rank, (-1,))
# Only positive ROIs contribute to the loss. And only
# the right class_id of each ROI. Get their indices.
positive_ix = tf.where(target_class_ids > 0)[:, 0]
# Gather the ranks (predicted and true) that contribute to loss
y_true = tf.gather(target_rank, positive_ix, axis=1)
y_pred = tf.gather(pred_rank, positive_ix, axis=1)
loss = K.switch(tf.size(y_true) > 0,
losses.sparse_categorical_crossentropy(y_true=y_true, y_pred=y_pred),
tf.constant(0.0))
loss = K.mean(loss)
return loss
def create_model(observation_space, action_space, args):
assert isinstance(observation_space, gym.spaces.Box)
assert isinstance(action_space, gym.spaces.Box) \
or isinstance(action_space, gym.spaces.Discrete)
h = x = Input(shape=observation_space.shape)
for i in range(args.hidden_layers):
h = Dense(args.hidden_nodes, activation=args.activation_function)(h)
if isinstance(action_space, gym.spaces.Discrete):
# produce logits for all actions
h = Dense(action_space.n)(h)
# sample action from logits
a = Lambda(lambda x: tf.multinomial(x, num_samples=1))(h)
# turn logits into probabilities
p = Activation('softmax')(h)
# model outputs sampled action
model = Model(x, a)
# loss is between true values and probabilities
model.compile(optimizer=RMSprop(lr=args.learning_rate),
loss=lambda y_true, y_pred:
sparse_categorical_crossentropy(y_true, p))
else:
# number of actions
n = np.prod(action_space.shape)
# produce means and stddevs for Gaussian
mu = Dense(n)(h)
# sample action from Gaussian
gaussian = SampleGaussian(initial_std=args.stddev)
a = gaussian(mu)
global std
std = gaussian.std
# model outputs sampled action
model = Model(x, a)
# negative log likelihood of Gaussian
model.compile(optimizer=RMSprop(lr=args.learning_rate, clipnorm=1.),
loss=lambda y_true, y_pred: 0.5 * np.log(
2 * np.pi) + gaussian.logstd + 0.5 *
((y_true - mu) / gaussian.std)**2)
model.summary()
return model
def crf_loss(y_true, y_pred):
"""General CRF loss function depending on the learning mode.
# Arguments
y_true: tensor with true targets.
y_pred: tensor with predicted targets.
# Returns
If the CRF layer is being trained in the join mode, returns the negative
log-likelihood. Otherwise returns the categorical crossentropy implemented
by the underlying Keras backend.
# About GitHub
If you open an issue or a pull request about CRF, please
add `cc @lzfelix` to notify Luiz Felix.
"""
crf, idx = y_pred._keras_history[:2]
if crf.learn_mode == 'join':
return crf_nll(y_true, y_pred)
else:
if crf.sparse_target:
return sparse_categorical_crossentropy(y_true, y_pred)
else:
return categorical_crossentropy(y_true, y_pred)
def custom_loss(y_true, y_pred):
return sparse_categorical_crossentropy(K.reshape(y_true[:, 0], (-1, 1)),
y_pred) * y_true[:, 1]
def masked_loss(y_true, y_pred):
masked_true = K.not_equal(y_true, 0)
masked_true = K.cast(masked_true, tf.float32)
loss = sparse_categorical_crossentropy(y_true, y_pred)
masked_loss = masked_true * loss
return K.mean(masked_loss)
tmp.iloc[-1, ]
#> loss 0.067497
#> accuracy 0.973214
#> val_loss 0.143529
#> val_accuracy 0.921053
tmp = my_model.predict(X)
y_ = np.argmax(tmp, axis=-1)
(y_ == y).mean()
#> 0.96
### 11.2.1 交差エントロピー
-np.log([0.8, 0.7, 0.3, 0.8]).mean()
#> 0.5017337127232719
-np.log([0.7, 0.6, 0.2, 0.7]).mean()
#> 0.708403356019389
y = [2, 1, 0, 1]
y_1 = [[0.1, 0.1, 0.8], [0.1, 0.7, 0.2], [0.3, 0.4, 0.3], [0.1, 0.8, 0.1]]
y_2 = [[0.1, 0.2, 0.7], [0.2, 0.6, 0.2], [0.2, 0.5, 0.3], [0.2, 0.7, 0.1]]
[
losses.sparse_categorical_crossentropy(y_true=y,
y_pred=y_1).numpy().mean(),
losses.sparse_categorical_crossentropy(y_true=y,
y_pred=y_2).numpy().mean()
]
#> [0.5017337, 0.70840335]
def loss_with_var(y_true, y_pred):
main_loss = losses.sparse_categorical_crossentropy(y_true, y_pred)
pred_indices = K.argmax(y_pred, axis=-1)
pred_indices = K.cast(pred_indices, dtype='float32')
var_penalty = K.var(pred_indices)
return main_loss + config.distance_weight * var_penalty
min_lr=config.lr_reduce_min,
verbose=1)
early_stopping = EarlyStopping(monitor="val_loss",
patience=config.early_stop_patience,
verbose=1)
callbacks = [checkpoint, tensor_board]
if (config.lr_reduce):
callbacks.append(lr_reducer)
if (config.early_stop):
callbacks.append(early_stopping)
if config.adv_train:
# Keras gives no easy way of just getting the cross-entropy loss (without the
# regularisation/rs loss) and it's needed for PGD, so we need to create it again
model.xent_loss = sparse_categorical_crossentropy(model.targets[0],
model.outputs[0])
incremental = (1, config.epsilon_incremental
) if config.epsilon_incremental > 0 else False
train_generator = AdversarialExampleGenerator(model,
x_train,
y_train,
config.batch_size,
epsilon=config.epsilon,
k=config.pgd_iter_train,
a=0.03,
incremental=incremental)
valid_generator = AdversarialExampleGenerator(model,
x_valid,
y_valid,
config.batch_size,
print(f"compute CCE by python: {loss}")
loss = log_loss(y_true, y_pred)
print(f"compute CCE by sklearn: {loss}")
loss = K.sum(categorical_crossentropy(tf.constant(y_true),
tf.constant(y_pred))) / 3
loss = K.eval(loss)
print(f"compute CCE by Keras: {loss}")
# Sparse Categorical Cross Entropy
t = LabelEncoder()
y_pred = tf.constant([[0.1, 0.1, 0.8], [0.1, 0.4, 0.5], [0.5, 0.3, 0.2],
[0.6, 0.3, 0.1]])
y_true = t.fit_transform(['Rain', 'Rain', 'High Changes of Rain', 'No Rain'])
print("transformed label: ", y_true)
y_true = tf.constant(y_true)
loss = sparse_categorical_crossentropy(y_true, y_pred)
loss = K.eval(loss)
print(f'Value of Sparse Categorical Cross Entropy is ', loss)
# hinge loss
y_true = tf.constant([[0., 1.], [0., 0.]])
y_pred = tf.constant([[0.7, 0.3], [0.4, 0.6]])
loss = hinge(y_true, y_pred)
a = K.eval(loss)
print("hinge loss: ", a)
# Custom Loss Function: categorical_crossentropy_with_label_smoothing
def categorical_crossentropy_with_label_smoothing(y_true,
y_pred,
label_smoothing=0.1):
def show_saliency(args):
model, config = load_model(args.name)
input_type = util.get_input_type(config)
def custom_loss(y_true, y_pred):
epsilon = 0.001
main_loss = losses.sparse_categorical_crossentropy(y_true, y_pred)
pred_indices = K.argmax(y_pred, axis=-1)
pred_indices = K.cast(pred_indices, dtype='float32')
distance_penalty = K.constant(1.0, dtype='float32') / (K.abs(pred_indices - K.constant(config.n_classes / 2.0, dtype='float32')) + epsilon)
return main_loss + config.distance_weight * distance_penalty
keras.losses.custom_loss = custom_loss
numeric_data, text_data, prices = preprocessing.load_tabular_data()
additional_num_data = np.load('tabular_data/add_num_data.npy')
numeric_data = util.preprocess_numeric_data(numeric_data, additional_num_data)
bins = util.get_bins(prices, num=config.n_classes)
number = 1024
img_files = os.listdir('imgs/')
np.random.shuffle(img_files)
img_files = img_files[:number]
x, y = util.load_data_batch(img_files, numeric_data, text_data, bins, config.img_shape,
False, number, 'train')
sequences = np.asarray(config.tokenizer.texts_to_matrix(x[2]))
sequences = pad_sequences(sequences, maxlen=config.max_seq_len)
x[2] = sequences
if input_type == 'full':
x = [x[0], x[1], sequences]
indices = np.arange(x[0].shape[0])
elif input_type == 'img':
x = x[1]
indices = np.arange(x.shape[0])
elif input_type == 'num':
x = x[0]
indices = np.arange(x.shape[0])
elif input_type == 'rnn':
x = sequences
indices = np.arange(x.shape[0])
else:
print('error')
exit()
_ = model.predict(x)
print('Visualizing saliency...')
folder = 'models/' + args.name + '/'
np.random.shuffle(indices)
indices = indices[:64]
if input_type == 'full':
imgs = x[1][indices]
x = [x[0][indices], x[1][indices], x[2][indices]]
else:
x = x[indices]
imgs = x
y = y[indices]
label_tensor = K.constant(y)
fn = K.function(model.inputs,
K.gradients(losses.sparse_categorical_crossentropy(label_tensor, model.outputs[0]),
model.inputs))
grads = fn([x])
grads = grads[0]
saliency = np.absolute(grads).max(axis=-1)
merged_sal = np.concatenate([saliency[i] for i in range(5)], axis=0)
merged_real = np.concatenate([imgs[i] for i in range(5)], axis=0)
plt.imsave(folder + 'saliency.jpg', merged_sal, cmap=plt.cm.hot)
plt.imsave(folder + 'saliency_real.jpg', merged_real, cmap=plt.cm.hot)
def lossfunction(y_true, y_pred):
return sparse_categorical_crossentropy(y_true, y_pred)