def call(self, position):
inputDim = K.ndim(position)
positionShape = K.shape(position)
targetDim = positionShape[-1]
position = K.reshape(position, (-1, targetDim))
samples = K.shape(position)[0]
theta = THT.zeros((samples, 3, 3))
chw = self.toChw(position)
chw = K.reshape(chw, (samples, targetDim))
dx = -self.distortion + 2.0 * self.distortion * self.srng.uniform((samples,))
dy = -self.distortion + 2.0 * self.distortion * self.srng.uniform((samples,))
cX = chw[:, 0] + dx
cY = chw[:, 1] + dy
h = K.maximum(chw[:, 2] * (1.0 + self.context), self.minSide)
w = K.maximum(chw[:, 3] * (1.0 + self.context), self.minSide)
# Calculating the parameters of the transformation
tx = cX
ty = cY
sx = w / 2.0 # Scale x
sy = h / 2.0 # Scale y
# Setting transformation
theta = THT.set_subtensor(theta[:, 0, 0], sx)
theta = THT.set_subtensor(theta[:, 1, 1], sy)
theta = THT.set_subtensor(theta[:, 0, 2], tx)
theta = THT.set_subtensor(theta[:, 1, 2], ty)
theta = THT.set_subtensor(theta[:, 2, 2], 1.0)
thetaShape = K.concatenate([positionShape[:-1], K.shape(theta)[-2:]])
theta = THT.reshape(theta, thetaShape, ndim=inputDim + 1)
return theta
def margin_loss(y_true, y_pred):
"""
Margin loss for Eq.(4). When y_true[i, :] contains not just one `1`, this loss should work too. Not test it.
:param y_true: [None, n_classes]
:param y_pred: [None, num_capsule]
:return: a scalar loss value.
"""
L = y_true * K.square(K.maximum(0., 0.9 - y_pred)) + \
0.5 * (1 - y_true) * K.square(K.maximum(0., y_pred - 0.1))
return K.mean(K.sum(L, 1))
def contrastive_loss(y, d):
'''Contrastive loss from Hadsell-et-al.'06
http://yann.lecun.cohe distance between the center of a source class and all the target points of this class.m/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
'''
margin = 1
return K.mean(y * K.square(d) + (1 - y) * K.maximum(margin - d, 0))
def get_updates(self, params, constraints, loss):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
t = self.iterations + 1
lr_t = self.lr / (1. - K.pow(self.beta_1, t))
shapes = [K.get_variable_shape(p) for p in params]
# zero init of 1st moment
ms = [K.zeros(shape) for shape in shapes]
# zero init of exponentially weighted infinity norm
us = [K.zeros(shape) for shape in shapes]
self.weights = [self.iterations] + ms + us
for p, g, m, u in zip(params, grads, ms, us):
m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
u_t = K.maximum(self.beta_2 * u, K.abs(g))
p_t = p - self.get_param_learning_rate_t(p,t,lr_t) * m_t / (u_t + self.epsilon)
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(u, u_t))
new_p = p_t
# apply constraints
if p in constraints:
c = constraints[p]
new_p = c(new_p)
self.updates.append(K.update(p, new_p))
return self.updates
def contrastive_loss(y_true, y_pred):
'''Contrastive loss from Hadsell-et-al.'06
http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
from https://github.com/fchollet/keras/blob/master/examples/mnist_siamese_graph.py
'''
margin = 1
return K.mean(y_true * K.square(y_pred) + (1 - y_true) * K.square(K.maximum(margin - y_pred, 0)))
def triplet_loss(y_true, y_pred, alpha=0.2):
"""
Implementation of the triplet loss as defined by formula (3)
Arguments:
y_true -- true labels, required when you define a loss in Keras, you don't need it in this function.
y_pred -- python list containing three objects:
anchor -- the encodings for the anchor images, of shape (None, 128)
positive -- the encodings for the positive images, of shape (None, 128)
negative -- the encodings for the negative images, of shape (None, 128)
Returns:
loss -- real number, value of the loss
"""
anchor, positive, negative = y_pred[0], y_pred[1], y_pred[2]
# Step 1: Compute the (encoding) distance between the anchor and the positive, you will need to sum over axis=-1
pos_dist = K.sum(K.square(anchor - positive), axis=-1)
# Step 2: Compute the (encoding) distance between the anchor and the negative, you will need to sum over axis=-1
neg_dist = K.sum(K.square(anchor - negative), axis=-1)
# Step 3: subtract the two previous distances and add alpha.
basic_loss = pos_dist - neg_dist + alpha
# Step 4: Take the maximum of basic_loss and 0.0. Sum over the training examples.
loss = K.sum(K.maximum(basic_loss, 0))
return loss
def contrastive_loss(y_true, y_pred):
'''Contrastive loss from Hadsell-et-al.'06
http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
'''
margin = 1
return K.mean(y_true * K.square(y_pred) +
(1 - y_true) * K.square(K.maximum(margin - y_pred, 0)))
def chopra_loss(y_true, y_pred):
''' (1-Y)(2/Q)(Ew)^2 + (Y) 2 Q e^(-2.77/Q * Ew)
Needs to use functions of keras.backend.theano_backend = K '''
#Q = 500.
#return (1 - y_true) * 2 / Q * K.square(y_pred) + y_true * 2 * Q * K.exp(-2.77 / Q * y_pred)
margin = 1
loss = K.mean(y_true * K.square(y_pred) + (1 - y_true) * K.square(K.maximum(margin - y_pred, 0)))
return loss
def max_margin2(y_true, y_pred):
# assumes the samples are interleaved positive and corrupt (p, c, p, c, ...)
v = - y_pred * y_true + y_pred * (1.0 - y_true) # (-p, c, -p, c,...)
v = K.reshape(v, (2, 64)) # ([-p, c], [-p, c],...)
v = 1. + K.sum(v, axis=0) # (1 - p + c, 1- p + c,...)
v = K.reshape(v, (64,))
v = K.maximum(0., v) # (max(0, 1 - p + c), max(0, 1 - p + c), ...)
return K.sum(v)
def calculateGpu(self, gtPosition, predPosition):
pShape = K.shape(gtPosition)
inputDim = K.ndim(gtPosition)
gtPosition = K.reshape(gtPosition, (-1, pShape[-1]))
predPosition = K.reshape(predPosition, (-1, pShape[-1]))
left = K.maximum(predPosition[:, 0], gtPosition[:, 0])
top = K.maximum(predPosition[:, 1], gtPosition[:, 1])
right = K.minimum(predPosition[:, 2], gtPosition[:, 2])
bottom = K.minimum(predPosition[:, 3], gtPosition[:, 3])
intersect = (right - left) * ((right - left) > 0) * (bottom - top) * ((bottom - top) > 0)
label_area = K.abs(gtPosition[:, 2] - gtPosition[:, 0]) * K.abs(gtPosition[:, 3] - gtPosition[:, 1])
predict_area = K.abs(predPosition[:, 2] - predPosition[:, 0]) * K.abs(predPosition[:, 3] - predPosition[:, 1])
union = label_area + predict_area - intersect
iou = intersect / union
#iouShape = K.concatenate([pShape[:-1], (1, )])
iou = THT.reshape(iou, (pShape[0], pShape[1], 1), ndim=inputDim)
return iou
def triplet_loss(self, y_true, y_pred):
y_pred = K.sigmoid(y_pred)
p_plus = K.sum(y_true * y_pred, axis=1, keepdims=True)
p_gaps = y_pred - p_plus + self.margin
L = K.maximum(0, p_gaps)
# return T.max(L, axis=1)
return K.sum(L, axis=1)
def contrastive_loss(y, d):
'''Contrastive loss from Hadsell-et-al.'06
http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
We want y and d to be different.
Loss is 0 if y = 1 and d = 0
Loss is 1 if y=d=1 or y=d=0
'''
margin = param_dict['margin']
return K.mean(y * K.square(d) + (1 - y) * K.square(K.maximum(margin - d, 0)))
def fn(y_true, y_pred):
class_id_true = K.argmax(y_true, axis=-1)
class_id_preds = K.argmax(y_pred, axis=-1)
# Replace class_id_preds with class_id_true for recall here
accuracy_mask = K.cast(K.equal(class_id_true, interesting_class_id), 'int32')
# accuracy_mask = K.cast(K.equal(class_id_true, interesting_class_id), 'int32')
class_acc_tensor = K.cast(K.equal(class_id_true, class_id_preds), 'int32') * accuracy_mask
class_acc = K.sum(class_acc_tensor) / K.maximum(K.sum(accuracy_mask), 1)
return class_acc
def weighted_bce_loss(y_true, y_pred, weight):
# avoiding overflow
epsilon = 1e-7
y_pred = K.clip(y_pred, epsilon, 1. - epsilon)
logit_y_pred = K.log(y_pred / (1. - y_pred))
# https://www.tensorflow.org/api_docs/python/tf/nn/weighted_cross_entropy_with_logits
loss = (1. - y_true) * logit_y_pred + (1. + (weight - 1.) * y_true) * \
(K.log(1. + K.exp(-K.abs(logit_y_pred))) + K.maximum(-logit_y_pred, 0.))
return K.sum(loss) / K.sum(weight)
def dice(self, y_true, y_pred):
"""
compute dice for given Tensors
"""
if self.crop_indices is not None:
y_true = utils.batch_gather(y_true, self.crop_indices)
y_pred = utils.batch_gather(y_pred, self.crop_indices)
if self.input_type == 'prob':
# We assume that y_true is probabilistic, but just in case:
y_true /= K.sum(y_true, axis=-1, keepdims=True)
y_true = K.clip(y_true, K.epsilon(), 1)
# make sure pred is a probability
y_pred /= K.sum(y_pred, axis=-1, keepdims=True)
y_pred = K.clip(y_pred, K.epsilon(), 1)
# Prepare the volumes to operate on
# If we're doing 'hard' Dice, then we will prepare one-hot-based matrices of size
# [batch_size, nb_voxels, nb_labels], where for each voxel in each batch entry,
# the entries are either 0 or 1
if self.dice_type == 'hard':
# if given predicted probability, transform to "hard max""
if self.input_type == 'prob':
if self.approx_hard_max:
y_pred_op = _hard_max(y_pred, axis=-1)
y_true_op = _hard_max(y_true, axis=-1)
else:
y_pred_op = _label_to_one_hot(K.argmax(y_pred, axis=-1), self.nb_labels)
y_true_op = _label_to_one_hot(K.argmax(y_true, axis=-1), self.nb_labels)
# if given predicted label, transform to one hot notation
else:
assert self.input_type == 'max_label'
y_pred_op = _label_to_one_hot(y_pred, self.nb_labels)
y_true_op = _label_to_one_hot(y_true, self.nb_labels)
# If we're doing soft Dice, require prob output, and the data already is as we need it
# [batch_size, nb_voxels, nb_labels]
else:
assert self.input_type == 'prob', "cannot do soft dice with max_label input"
y_pred_op = y_pred
y_true_op = y_true
# compute dice for each entry in batch.
# dice will now be [batch_size, nb_labels]
sum_dim = 1
top = 2 * K.sum(y_true_op * y_pred_op, sum_dim)
bottom = K.sum(K.square(y_true_op), sum_dim) + K.sum(K.square(y_pred_op), sum_dim)
# make sure we have no 0s on the bottom. K.epsilon()
bottom = K.maximum(bottom, self.area_reg)
return top / bottom
def _hard_max(tens, axis):
"""
we can't use the argmax function in a loss, as it's not differentiable
We can use it in a metric, but not in a loss function
therefore, we replace the 'hard max' operation (i.e. argmax + onehot)
with this approximation
"""
tensmax = K.max(tens, axis=axis, keepdims=True)
eps_hot = K.maximum(tens - tensmax + K.epsilon(), 0)
one_hot = eps_hot / K.epsilon()
return one_hot
def linear_correlation_loss(y_true, y_pred):
mean_y_true = K.mean(y_true)
mean_y_pred = K.mean(y_pred)
std_y_true = K.std(y_true)+1e-6
std_y_pred = K.std(y_pred)+1e-6
nSamples = K.shape(y_true)[0]
firstTerm = (y_true - mean_y_true)/std_y_true
secondTerm = (y_pred - mean_y_pred)/std_y_pred
pearsonCorr = K.sum(firstTerm*secondTerm)/(nSamples-1)
maeLoss = K.abs(y_true-y_pred)
return maeLoss*(1-K.maximum(0.,pearsonCorr))
def margin_triplet_loss(X):
user_latent, item_latent = X.values()
positive_item_latent, negative_item_latent = item_latent.values()
# Hinge loss: max(0, user * negative_item_latent + 1 - user * positive_item_latent)
loss = K.maximum(1.0
+ K.sum(user_latent * negative_item_latent, axis=-1, keepdims=True)
- K.sum(user_latent * positive_item_latent, axis=-1, keepdims=True),
0.0)
return loss
def compile(self, optimizer, **kwargs):
similarities = self.get_similarities()
loss = merge(similarities,
mode=lambda x: K.maximum(1e-6, self.config['margin'] - x[0] + x[1]),
output_shape=lambda x: x[0])
self.training_model = Model(input=[self.question, self.answer_good, self.answer_bad], output=loss)
self.training_model.compile(loss=lambda y_true, y_pred: y_pred, optimizer=optimizer, **kwargs)
self.prediction_model = Model(input=[self.question, self.answer_good], output=similarities[0])
self.prediction_model.compile(loss=lambda y_true, y_pred: y_pred, optimizer=optimizer, **kwargs)
def call(self, x, mask=None):
if mask is None:
return super(GlobalAveragePooling1D, self).call(x)
mask = K.expand_dims(mask)
mask = K.tile(mask, [1, 1, K.shape(x)[2]])
mask = K.cast(mask, K.dtype(x))
safe_mask_sum = K.sum(mask, axis=1)
safe_mask_sum = K.maximum(safe_mask_sum, K.ones_like(safe_mask_sum))
return K.sum(mask * x, axis=1) / safe_mask_sum
def KLdivergence(P, Y):
alpha = low_dim - 1.
sum_Y = K.sum(K.square(Y), axis=1)
eps = K.variable(10e-15)
D = sum_Y + K.reshape(sum_Y, [-1, 1]) - 2 * K.dot(Y, K.transpose(Y))
Q = K.pow(1 + D / alpha, -(alpha + 1) / 2)
Q *= K.variable(1 - np.eye(batch_size))
Q /= K.sum(Q)
Q = K.maximum(Q, eps)
C = K.log((P + eps) / (Q + eps))
C = K.sum(P * C)
return C
def get_similarity(self):
''' Specify similarity in configuration under 'similarity' -> 'mode'
If a parameter is needed for the model, specify it in 'similarity'
Example configuration:
config = {
... other parameters ...
'similarity': {
'mode': 'gesd',
'gamma': 1,
'c': 1,
}
}
cosine: dot(a, b) / sqrt(dot(a, a) * dot(b, b))
polynomial: (gamma * dot(a, b) + c) ^ d
sigmoid: tanh(gamma * dot(a, b) + c)
rbf: exp(-gamma * l2_norm(a-b) ^ 2)
euclidean: 1 / (1 + l2_norm(a - b))
exponential: exp(-gamma * l2_norm(a - b))
gesd: euclidean * sigmoid
aesd: (euclidean + sigmoid) / 2
'''
params = self.params
similarity = params['mode']
dot = lambda a, b: K.batch_dot(a, b, axes=1)
l2_norm = lambda a, b: K.sqrt(K.sum(K.square(a - b), axis=1, keepdims=True))
if similarity == 'cosine':
return lambda x: dot(x[0], x[1]) / K.maximum(K.sqrt(dot(x[0], x[0]) * dot(x[1], x[1])), K.epsilon())
elif similarity == 'polynomial':
return lambda x: (params['gamma'] * dot(x[0], x[1]) + params['c']) ** params['d']
elif similarity == 'sigmoid':
return lambda x: K.tanh(params['gamma'] * dot(x[0], x[1]) + params['c'])
elif similarity == 'rbf':
return lambda x: K.exp(-1 * params['gamma'] * l2_norm(x[0], x[1]) ** 2)
elif similarity == 'euclidean':
return lambda x: 1 / (1 + l2_norm(x[0], x[1]))
elif similarity == 'exponential':
return lambda x: K.exp(-1 * params['gamma'] * l2_norm(x[0], x[1]))
elif similarity == 'gesd':
euclidean = lambda x: 1 / (1 + l2_norm(x[0], x[1]))
sigmoid = lambda x: 1 / (1 + K.exp(-1 * params['gamma'] * (dot(x[0], x[1]) + params['c'])))
return lambda x: euclidean(x) * sigmoid(x)
elif similarity == 'aesd':
euclidean = lambda x: 0.5 / (1 + l2_norm(x[0], x[1]))
sigmoid = lambda x: 0.5 / (1 + K.exp(-1 * params['gamma'] * (dot(x[0], x[1]) + params['c'])))
return lambda x: euclidean(x) + sigmoid(x)
else:
raise Exception('Invalid similarity: {}'.format(similarity))
def __init__(self, filename, labels, model_type, feature_params=None, model=None,
layers=1, layer_dim=100, activation="tanh", normalize=False,
init="glorot_normal", max_num_labels=100, batch_size=10,
minibatch_size=200, nb_epochs=5, dropout=0,
optimizer="adam", loss="categorical_crossentropy",
regularizer="l2", regularization=1e-8):
"""
Create a new untrained NN or copy the weights from an existing one
:param labels: a list of labels that can be updated later to add a new label
:param feature_params: dict of feature type name -> FeatureInformation
:param model: if given, copy the weights (from a trained model)
:param layers: number of hidden layers
:param layer_dim: size of hidden layer
:param activation: activation function at hidden layers
:param normalize: perform batch normalization after each layer?
:param init: initialization type for hidden layers
:param max_num_labels: since model size is fixed, set maximum output size
:param batch_size: fit model every this many items
:param minibatch_size: batch size for SGD
:param nb_epochs: number of epochs for SGD
:param dropout: dropout to apply to input layer
:param optimizer: algorithm to use for optimization
:param loss: objective function to use for optimization
:param regularizer: regularization type (None, l1, l2 or l1l2)
:param regularization: regularization parameter lambda
"""
super(NeuralNetwork, self).__init__(model_type=model_type, filename=filename,
labels=labels, model=model)
assert feature_params is not None or model is not None
if self.is_frozen:
self.model = model
else:
self.max_num_labels = max_num_labels
self._layers = layers
self._layer_dim = layer_dim
self._activation = (lambda x: x*x*x) if activation == "cube" else activation
self._normalize = normalize
self._init = init
self._num_labels = self.num_labels
self._minibatch_size = minibatch_size
self._nb_epochs = nb_epochs
self._dropout = dropout
self._optimizer = optimizer
self._loss = (lambda t, p: K.sum(K.maximum(0., 1.-p*t+p*(1.-t)))) if loss == "max_margin" else loss
self._regularizer = (lambda: None) if regularizer is None else \
(lambda: regularizers.l1l2(regularization, regularization)) if regularizer == "l1l2" else \
(lambda: regularizers.get(regularizer, {"l": regularization}))
self.feature_params = feature_params
self.model = None
self._batch_size = batch_size
self._item_index = 0
self._iteration = 0
def box_iou(b1, b2):
'''Return iou tensor
Parameters
----------
b1: tensor, shape=(i1,...,iN, 4), xywh
b2: tensor, shape=(j, 4), xywh
Returns
-------
iou: tensor, shape=(i1,...,iN, j)
'''
# Expand dim to apply broadcasting.
b1 = K.expand_dims(b1, -2)
b1_xy = b1[..., :2]
b1_wh = b1[..., 2:4]
b1_wh_half = b1_wh/2.
b1_mins = b1_xy - b1_wh_half
b1_maxes = b1_xy + b1_wh_half
# Expand dim to apply broadcasting.
b2 = K.expand_dims(b2, 0)
b2_xy = b2[..., :2]
b2_wh = b2[..., 2:4]
b2_wh_half = b2_wh/2.
b2_mins = b2_xy - b2_wh_half
b2_maxes = b2_xy + b2_wh_half
intersect_mins = K.maximum(b1_mins, b2_mins)
intersect_maxes = K.minimum(b1_maxes, b2_maxes)
intersect_wh = K.maximum(intersect_maxes - intersect_mins, 0.)
intersect_area = intersect_wh[..., 0] * intersect_wh[..., 1]
b1_area = b1_wh[..., 0] * b1_wh[..., 1]
b2_area = b2_wh[..., 0] * b2_wh[..., 1]
iou = intersect_area / (b1_area + b2_area - intersect_area)
return iou
def compile_rt(self, optimizer, **kwargs):
qa_model_rt = self.get_qa_model_rt()
good_output = qa_model_rt([self.subject, self.relation, self.object_good])
bad_output = qa_model_rt([self.subject_bad, self.relation, self.object_good])
loss = merge([good_output, bad_output],
mode=lambda x: K.maximum(1e-6, self.config['margin'] - x[0] + x[1]),
output_shape=lambda x: x[0])
self.training_model_rt = Model(input=[self.subject, self.subject_bad, self.relation, self.object_good], output=loss)
self.training_model_rt.compile(loss=lambda y_true, y_pred: y_pred + y_true - y_true, optimizer=optimizer, **kwargs)
self.prediction_model_rt = Model(input=[self.subject, self.relation, self.object_good], output=good_output)
self.prediction_model_rt.compile(loss='binary_crossentropy', optimizer=optimizer, **kwargs)
def compile(self, optimizer, **kwargs):
qa_model = self.get_qa_model()
good_output = qa_model([self.question, self.answer_good])
bad_output = qa_model([self.question, self.answer_bad])
loss = merge([good_output, bad_output],
mode=lambda x: K.maximum(1e-6, self.config['margin'] - x[0] + x[1]),
output_shape=lambda x: x[0])
self.training_model = Model(input=[self.question, self.answer_good, self.answer_bad], output=loss)
self.training_model.compile(loss=lambda y_true, y_pred: y_pred, optimizer=optimizer, **kwargs)
self.prediction_model = Model(input=[self.question, self.answer_good], output=good_output)
self.prediction_model.compile(loss='binary_crossentropy', optimizer=optimizer, **kwargs)
def tsne(P, activations):
# d = K.shape(activations)[1]
d = 2 # TODO: should set this automatically, but the above is very slow for some reason
n = 1000 # TODO: should set this automatically
v = d - 1.
eps = K.variable(10e-15) # needs to be at least 10e-8 to get anything after Q /= K.sum(Q)
sum_act = K.sum(K.square(activations), axis=1)
Q = K.reshape(sum_act, [-1, 1]) + -2 * K.dot(activations, K.transpose(activations))
Q = (sum_act + Q) / v
Q = K.pow(1 + Q, -(v + 1) / 2)
Q *= K.variable(1 - np.eye(n))
Q /= K.sum(Q)
Q = K.maximum(Q, eps)
C = K.log((P + eps) / (Q + eps))
C = K.sum(P * C)
return C
def call(self, inputs, mask=None):
if type(inputs) is not list or len(inputs) <= 1:
raise Exception('Merge must be called on a list of tensors '
'(at least 2). Got: ' + str(inputs))
# case: "mode" is a lambda or function.
if hasattr(self.mode, '__call__'):
# TODO: consider making it possible to
# pass custom arguments to lambda.
arguments = {}
return self.mode(inputs, **arguments)
if self.mode == 'sum' or self.mode == 'ave':
s = inputs[0]
for i in range(1, len(inputs)):
s += inputs[i]
if self.mode == 'ave':
s /= len(inputs)
return s
elif self.mode == 'concat':
return K.concatenate(inputs, axis=self.concat_axis)
elif self.mode == 'mul':
s = inputs[0]
for i in range(1, len(inputs)):
s *= inputs[i]
return s
elif self.mode == 'dot':
l1 = inputs[0]
l2 = inputs[1]
output = K.batch_dot(l1, l2, self.dot_axes)
return output
elif self.mode == 'cos':
l1 = inputs[0]
l2 = inputs[1]
denominator = K.sqrt(K.batch_dot(l1, l1, self.dot_axes) *
K.batch_dot(l2, l2, self.dot_axes))
denominator = K.maximum(denominator, K.epsilon())
output = K.batch_dot(l1, l2, self.dot_axes) / denominator
output = K.expand_dims(output, 1)
return output
else:
raise Exception('Unknown merge mode.')
def call(self, inputs, mask=None):
if mask is None:
mask = K.zeros_like(inputs)
mask = K.sum(mask, axis=-1)
mask = 1 + mask
else:
mask = K.cast(mask, K.dtype(inputs))
safe_n1 = K.sum(mask, axis=1) - 1
safe_n1 = K.maximum(safe_n1, K.ones_like(safe_n1))
safe_n1 = K.expand_dims(safe_n1)
r = tf.cumsum(mask, axis=1) - 1
r = self.start + (self.stop - self.start) * r / safe_n1
r = mask * r
r = K.expand_dims(r)
return r
def loss(y_true, y_pred):
global output_feature;
y_pred_positive=y_pred[:,:output_feature]
y_pred_label=y_pred[:,output_feature:2*output_feature]
y_pred_negative1=y_pred[:,2*output_feature:3*output_feature]
y_pred_negative2=y_pred[:,3*output_feature:4*output_feature]
y_pred_negative3=y_pred[:,4*output_feature:5*output_feature]
y_pred_negative4=y_pred[:,5*output_feature:6*output_feature]
y_pred_negative5=y_pred[:,6*output_feature:7*output_feature]
y_pred_negative6=y_pred[:,7*output_feature:8*output_feature]
l=K.min(y_true-y_true, axis=1)
return K.maximum(cosine_proximity(y_pred_positive, y_pred_negative1)-cosine_proximity(y_pred_positive, y_pred_label)+0.3, 0.)\
+K.maximum(cosine_proximity(y_pred_positive, y_pred_negative2)-cosine_proximity(y_pred_positive, y_pred_label)+0.3, 0.)\
+K.maximum(cosine_proximity(y_pred_positive, y_pred_negative3)-cosine_proximity(y_pred_positive, y_pred_label)+0.3, 0.)\
+K.maximum(cosine_proximity(y_pred_positive, y_pred_negative4)-cosine_proximity(y_pred_positive, y_pred_label)+0.3, 0.)\
+K.maximum(cosine_proximity(y_pred_positive, y_pred_negative5)-cosine_proximity(y_pred_positive, y_pred_label)+0.3, 0.)\
+K.maximum(cosine_proximity(y_pred_positive, y_pred_negative6)-cosine_proximity(y_pred_positive, y_pred_label)+0.3, 0.)\
+K.maximum(0.98-cosine_proximity(y_pred_positive, y_pred_label), 0.)+l
def euclidean_distance(vects):
x, y = vects
sum_square = K.sum(K.square(x - y), axis=1, keepdims=True)
return K.sqrt(K.maximum(sum_square, K.epsilon()))
def identity_loss2(y_true, y_pred):
return K.sum(K.maximum(0.0, 1.0 + y_pred[1] - y_pred[0]), axis=-1)
def triplet_loss(y_true, y_pred):
margin = K.constant(0.2)
return K.mean(
K.maximum(
K.constant(0),
K.square(y_pred[:, 0, 0]) - K.square(y_pred[:, 1, 0]) + margin))
def vae_kl_loss(y_true, y_pred):
kl_loss = -0.5 * K.sum(1 + vae_z_log_var - K.square(vae_z_mean) -
K.exp(vae_z_log_var),
axis=1)
kl_loss = K.maximum(kl_loss, KL_TOLERANCE * Z_DIM)
return kl_loss
def euclidean_distance(vects): x, y = vects # x = tf.Print(x,[tf.shape(x)]) return K.sqrt(K.maximum(K.sum(K.square(x - y), axis=1, keepdims=True), K.epsilon()))
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
completed_updates = K.cast(
K.tf.floordiv(self.iterations, self.accum_iters), K.floatx())
if self.initial_decay > 0:
lr = lr * (1. / (1. + self.decay * completed_updates))
t = completed_updates + 1
lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
(1. - K.pow(self.beta_1, t)))
# self.iterations incremented after processing a batch
# batch: 1 2 3 4 5 6 7 8 9
# self.iterations: 0 1 2 3 4 5 6 7 8
# update_switch = 1: x x (if accum_iters=4)
update_switch = K.equal((self.iterations + 1) % self.accum_iters, 0)
update_switch = K.cast(update_switch, K.floatx())
ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
else:
vhats = [K.zeros(1) for _ in params]
self.weights = [self.iterations] + ms + vs + vhats
for p, g, m, v, vhat, tg in zip(params, grads, ms, vs, vhats, gs):
sum_grad = tg + g
avg_grad = sum_grad / self.accum_iters_float
m_t = (self.beta_1 * m) + (1. - self.beta_1) * avg_grad
v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(avg_grad)
if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
p_t = p - lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)
self.updates.append(
K.update(vhat, (1 - update_switch) * vhat +
update_switch * vhat_t))
else:
p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)
self.updates.append(
K.update(m, (1 - update_switch) * m + update_switch * m_t))
self.updates.append(
K.update(v, (1 - update_switch) * v + update_switch * v_t))
self.updates.append(K.update(tg, (1 - update_switch) * sum_grad))
new_p = p_t
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(
K.update(p, (1 - update_switch) * p + update_switch * new_p))
return self.updates
def triplet_loss(y_true, y_pred):
margin = K.constant(4)
# return K.mean(y_pred[:,0,0] - y_pred[:,1,0])
return K.mean(
K.maximum(K.constant(0), y_pred[:, 0, 0] - y_pred[:, 1, 0] + margin))
def euclidean_distance(self,vects):
x, y = vects
return K.sqrt(K.maximum(K.sum(K.square(x - y), axis=1, keepdims=True), K.epsilon()))
def contrastive_loss(self,y_true, y_pred):
margin = 1
return K.mean(y_true * K.square(y_pred) +
(1 - y_true) * K.square(K.maximum(margin - y_pred, 0)))
def customized_loss(args):
y_true, y_pred = args
loss = K.mean(K.maximum(1. - y_true + y_pred, 0.), axis=-1)
print loss
return loss
def yolo_loss(args,
anchors,
num_classes,
rescore_confidence=False,
print_loss=False):
"""YOLO localization loss function.
Parameters
----------
yolo_output : tensor
Final convolutional layer features.
_boxes : tensor
Ground truth boxes tensor with shape [batch, num_true_boxes, 5]
containing box x_center, y_center, width, height, and class.
detectors_mask : array
0/1 mask for detector positions where there is a matching ground truth.
matching_true_boxes : array
Corresponding ground truth boxes for positive detector positions.
Already adjusted for conv height and width.
anchors : tensor
Anchor boxes for model.
num_classes : int
Number of object classes.
rescore_confidence : bool, default=False
If true then set confidence target to IOU of best predicted box with
the closest matching ground truth box.
print_loss : bool, default=False
If True then use a tf.Print() to print the loss components.
Returns
-------
mean_loss : float
mean localization loss across minibatch
"""
(yolo_output, true_boxes, detectors_mask, matching_true_boxes) = args
num_anchors = len(anchors)
object_scale = 5
no_object_scale = 1
class_scale = 1
coordinates_scale = 1
pred_xy, pred_wh, pred_confidence, pred_class_prob = yolo_head(
yolo_output, anchors, num_classes)
# Unadjusted box predictions for loss.
# TODO: Remove extra computation shared with yolo_head.
yolo_output_shape = K.shape(yolo_output)
feats = K.reshape(yolo_output, [
-1, yolo_output_shape[1], yolo_output_shape[2], num_anchors,
num_classes + 5
])
pred_boxes = K.concatenate(
(K.sigmoid(feats[..., 0:2]), feats[..., 2:4]), axis=-1)
# TODO: Adjust predictions by image width/height for non-square images?
# IOUs may be off due to different aspect ratio.
# Expand pred x,y,w,h to allow comparison with ground truth.
# batch, conv_height, conv_width, num_anchors, num_true_boxes, box_params
pred_xy = K.expand_dims(pred_xy, 4)
pred_wh = K.expand_dims(pred_wh, 4)
pred_wh_half = pred_wh / 2.
pred_mins = pred_xy - pred_wh_half
pred_maxes = pred_xy + pred_wh_half
true_boxes_shape = K.shape(true_boxes)
# batch, conv_height, conv_width, num_anchors, num_true_boxes, box_params
true_boxes = K.reshape(true_boxes, [
true_boxes_shape[0], 1, 1, 1, true_boxes_shape[1], true_boxes_shape[2]
])
true_xy = true_boxes[..., 0:2]
true_wh = true_boxes[..., 2:4]
# Find IOU of each predicted box with each ground truth box.
true_wh_half = true_wh / 2.
true_mins = true_xy - true_wh_half
true_maxes = true_xy + true_wh_half
intersect_mins = K.maximum(pred_mins, true_mins)
intersect_maxes = K.minimum(pred_maxes, true_maxes)
intersect_wh = K.maximum(intersect_maxes - intersect_mins, 0.)
intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]
pred_areas = pred_wh[..., 0] * pred_wh[..., 1]
true_areas = true_wh[..., 0] * true_wh[..., 1]
#sess = tf.Session()
print ("*********************")
print ("True Area")
#print (sess.run(true_areas))
tf.Print(true_areas, [true_areas], message = "This is True Areas: ")
b = tf.add(true_areas, true_areas).eval()
print ("********************")
union_areas = pred_areas + true_areas - intersect_areas
iou_scores = intersect_areas / union_areas
# Best IOUs for each location.
best_ious = K.max(iou_scores, axis=4) # Best IOU scores.
best_ious = K.expand_dims(best_ious)
# A detector has found an object if IOU > thresh for some true box.
object_detections = K.cast(best_ious > 0.6, K.dtype(best_ious))
# TODO: Darknet region training includes extra coordinate loss for early
# training steps to encourage predictions to match anchor priors.
# Determine confidence weights from object and no_object weights.
# NOTE: YOLO does not use binary cross-entropy here.
no_object_weights = (no_object_scale * (1 - object_detections) *
(1 - detectors_mask))
no_objects_loss = no_object_weights * K.square(-pred_confidence)
if rescore_confidence:
objects_loss = (object_scale * detectors_mask *
K.square(best_ious - pred_confidence))
else:
objects_loss = (object_scale * detectors_mask *
K.square(1 - pred_confidence))
confidence_loss = objects_loss + no_objects_loss
# Classification loss for matching detections.
# NOTE: YOLO does not use categorical cross-entropy loss here.
matching_classes = K.cast(matching_true_boxes[..., 4], 'int32')
matching_classes = K.one_hot(matching_classes, num_classes)
classification_loss = (class_scale * detectors_mask *
K.square(matching_classes - pred_class_prob))
# Coordinate loss for matching detection boxes.
matching_boxes = matching_true_boxes[..., 0:4]
coordinates_loss = (coordinates_scale * detectors_mask *
K.square(matching_boxes - pred_boxes))
print ("coordinate loss\n\n")
print (coordinates_loss)
confidence_loss_sum = K.sum(confidence_loss)
classification_loss_sum = K.sum(classification_loss)
coordinates_loss_sum = K.sum(coordinates_loss)
total_loss = 0.5 * (
confidence_loss_sum + classification_loss_sum + coordinates_loss_sum)
if print_loss:
total_loss = tf.Print(
total_loss, [
total_loss, confidence_loss_sum, classification_loss_sum,
coordinates_loss_sum
],
message='yolo_loss, conf_loss, class_loss, box_coord_loss:')
return total_loss
def selective_loss(y_true, y_pred):
loss = K.categorical_crossentropy(
K.repeat_elements(y_pred[:, -1:], self.num_classes, axis=1) *
y_true[:, :-1], y_pred[:, :-1]) + lamda * K.maximum(
-K.mean(y_pred[:, -1]) + c, 0)**2
return loss
def fcrn_loss_new1(y_true, y_pred):
print("————————————————————————————————————")
images = []
loss = K.square(y_pred - y_true)
final_loss_parts = []
for i in range(0, mini_batch_size):
c_true = y_true[i, 6, :, :].reshape(
(1, delta,
delta)) # The last feature map in the true vals is the 'c' matrix
c_pred = y_pred[i, 6, :, :].reshape((1, delta, delta))
u_true = y_true[i, 0, :, :].reshape((1, delta, delta))
u_pred = y_pred[i, 0, :, :].reshape((1, delta, delta))
v_true = y_true[i, 1, :, :].reshape((1, delta, delta))
v_pred = y_pred[i, 1, :, :].reshape((1, delta, delta))
w_true = y_true[i, 2, :, :].reshape((1, delta, delta))
w_pred = y_pred[i, 2, :, :].reshape((1, delta, delta))
h_true = y_true[i, 3, :, :].reshape((1, delta, delta))
h_pred = y_pred[i, 3, :, :].reshape((1, delta, delta))
cos_true = y_true[i, 4, :, :].reshape((1, delta, delta))
cos_pred = y_pred[i, 4, :, :].reshape((1, delta, delta))
sin_true = y_true[i, 5, :, :].reshape((1, delta, delta))
sin_pred = y_pred[i, 5, :, :].reshape((1, delta, delta))
# square rect(true、pred)
tl_square_true = [u_true - w_true / 2, v_true - h_true / 2]
tr_square_true = [u_true + w_true / 2, v_true - h_true / 2]
bl_square_true = [u_true - w_true / 2, v_true + h_true / 2]
br_square_true = [u_true + w_true / 2, v_true + h_true / 2]
tl_square_pred = [u_pred - w_pred / 2, v_pred - h_pred / 2]
tr_square_pred = [u_pred + w_pred / 2, v_pred - h_pred / 2]
bl_square_pred = [u_pred - w_pred / 2, v_pred + h_pred / 2]
br_square_pred = [u_pred + w_pred / 2, v_pred + h_pred / 2]
# Slant Rectangle (true、pred)
tl_slant_true = [
tl_square_true[0] * cos_true - tl_square_true[1] * sin_true,
tl_square_true[0] * sin_true + tl_square_true[1] * cos_true
]
tr_slant_true = [
tr_square_true[0] * cos_true - tr_square_true[1] * sin_true,
tr_square_true[0] * sin_true + tr_square_true[1] * cos_true
]
bl_slant_true = [
bl_square_true[0] * cos_true - bl_square_true[1] * sin_true,
bl_square_true[0] * sin_true + bl_square_true[1] * cos_true
]
br_slant_true = [
br_square_true[0] * cos_true - br_square_true[1] * sin_true,
br_square_true[0] * sin_true + br_square_true[1] * cos_true
]
tl_slant_pred = [
tl_square_pred[0] * cos_pred - tl_square_pred[1] * sin_pred,
tl_square_pred[0] * sin_pred + tl_square_pred[1] * cos_pred
]
tr_slant_pred = [
tr_square_pred[0] * cos_pred - tr_square_pred[1] * sin_pred,
tr_square_pred[0] * sin_pred + tr_square_pred[1] * cos_pred
]
bl_slant_pred = [
bl_square_pred[0] * cos_pred - bl_square_pred[1] * sin_pred,
bl_square_pred[0] * sin_pred + bl_square_pred[1] * cos_pred
]
br_slant_pred = [
br_square_pred[0] * cos_pred - br_square_pred[1] * sin_pred,
br_square_pred[0] * sin_pred + br_square_pred[1] * cos_pred
]
#get center point
temp = K.minimum(tl_slant_true[0], tr_slant_true[0])
temp = K.minimum(temp, bl_slant_true[0])
temp = K.minimum(temp, br_slant_true[0])
x_true_min = K.minimum(
K.minimum(K.minimum(tl_slant_true[0], tr_slant_true[0]),
bl_slant_true[0]), br_slant_true[0])
x_true_max = K.maximum(
K.maximum(K.maximum(tl_slant_true[0], tr_slant_true[0]),
bl_slant_true[0]), br_slant_true[0])
y_true_min = K.minimum(
K.minimum(K.minimum(tl_slant_true[1], tr_slant_true[1]),
bl_slant_true[1]), br_slant_true[1])
y_true_max = K.maximum(
K.maximum(K.maximum(tl_slant_true[1], tr_slant_true[1]),
bl_slant_true[1]), br_slant_true[1])
center_true = [(x_true_min + x_true_max) / 2,
(y_true_min + y_true_max) / 2]
x_pred_min = K.minimum(
K.minimum(K.minimum(tl_slant_pred[0], tr_slant_pred[0]),
bl_slant_pred[0]), br_slant_pred[0])
x_pred_max = K.maximum(
K.maximum(K.maximum(tl_slant_pred[0], tr_slant_pred[0]),
bl_slant_pred[0]), br_slant_pred[0])
y_pred_min = K.minimum(
K.minimum(K.minimum(tl_slant_pred[1], tr_slant_pred[1]),
bl_slant_pred[1]), br_slant_pred[1])
y_pred_max = K.maximum(
K.maximum(K.maximum(tl_slant_pred[1], tr_slant_pred[1]),
bl_slant_pred[1]), br_slant_pred[1])
center_pred = [(x_pred_min + x_pred_max) / 2,
(y_pred_min + y_pred_max) / 2]
dx_true = center_true[0] - u_true
dy_true = center_true[1] - v_true
dx_pred = center_pred[0] - u_pred
dy_pred = center_pred[1] - v_pred
# move
tl_true = [tl_slant_true[0] - dx_true, tl_slant_true[1] - dy_true]
tr_true = [tr_slant_true[0] - dx_true, tr_slant_true[1] - dy_true]
bl_true = [bl_slant_true[0] - dx_true, bl_slant_true[1] - dy_true]
br_true = [br_slant_true[0] - dx_true, br_slant_true[1] - dy_true]
tl_pred = [tl_slant_pred[0] - dx_pred, tl_slant_pred[1] - dy_pred]
tr_pred = [tr_slant_pred[0] - dx_pred, tr_slant_pred[1] - dy_pred]
bl_pred = [bl_slant_pred[0] - dx_pred, bl_slant_pred[1] - dy_pred]
br_pred = [br_slant_pred[0] - dx_pred, br_slant_pred[1] - dy_pred]
# Calculation loss (coord 、 confideres)
loss_coord = 5 * (
K.abs(tl_pred[0] - tl_true[0]) + K.abs(tl_pred[1] - tl_true[1]) +
K.abs(tr_pred[0] - tr_true[0]) + K.abs(tr_pred[1] - tr_true[1]) +
K.abs(bl_pred[0] - bl_true[0]) + K.abs(bl_pred[1] - bl_true[1]) +
K.abs(br_pred[0] - br_true[0]) + K.abs(br_pred[1] - br_true[1]))
loss_c = K.abs(c_pred - c_true)
loss_c = T.set_subtensor(loss_c[(c_true <= 0.0).nonzero()],
loss_c[(c_true <= 0.0).nonzero()] * 0.5)
loss = loss_coord + loss_c
final_loss_parts.append(loss)
images.append(K.concatenate(final_loss_parts))
return K.mean(K.concatenate(images).reshape(
(mini_batch_size, 7, delta, delta)),
axis=1)
'''
def __call__(self, y_true, y_pred):
tp = self.tp(y_true, y_pred)
fp = self.fp(y_true, y_pred)
div = K.maximum((tp + fp), self.capping)
return truediv(tp, div)
def contrastive_loss(y_true, y_pred):
margin = 1
sqaure_pred = K.square(y_pred)
margin_square = K.square(K.maximum(margin - y_pred, 0))
return K.mean(y_true * sqaure_pred + (1 - y_true) * margin_square)
def l2_normalize(x, axis):
norm = K.sqrt(K.sum(K.square(x), axis=axis, keepdims=True))
return K.maximum(x, K.epsilon()) / K.maximum(norm, K.epsilon())
def yolo_loss(args, anchors, num_classes, ignore_thresh=.5, print_loss=False):
"""Return yolo_loss tensor
Parameters
----------
args:
yolo_outputs: list of tensor, the output of yolo_body or tiny_yolo_body,
y_true: list of array, the output of preprocess_true_boxes
anchors: array, shape=(N, 2), wh
num_classes: integer
ignore_thresh: float, the iou threshold whether to ignore object confidence loss
print_loss: print the loss after evaluation
Returns
-------
loss: tensor, shape=(1,)
"""
num_layers = len(anchors) // 3 # default setting
yolo_outputs = args[:num_layers]
y_true = args[num_layers:]
anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]
] if num_layers == 3 else [[3, 4, 5], [1, 2, 3]]
input_shape = K.cast(
K.shape(yolo_outputs[0])[1:3] * 32, K.dtype(y_true[0]))
grid_shapes = [
K.cast(K.shape(yolo_outputs[l])[1:3], K.dtype(y_true[0]))
for l in range(num_layers)
]
loss = 0
m = K.shape(yolo_outputs[0])[0] # batch size, tensor
mf = K.cast(m, K.dtype(yolo_outputs[0]))
# Area of smallest and largest anchor boxes
area_0 = anchors[0][0] * anchors[0][1]
area_2 = anchors[6][0] * anchors[6][1]
output_weight_multipl = 2.0 / (np.log2(area_2) - np.log2(area_0))
area_0_log2 = np.log2(area_0)
# there are 3 layers in default, first one has the lowest resolution
for l in range(0, num_layers):
object_mask = y_true[l][..., 4:5]
true_class_probs = y_true[l][..., 5:]
grid, raw_pred, pred_xy, pred_wh = yolo_head(yolo_outputs[l],
anchors[anchor_mask[l]],
num_classes,
input_shape,
calc_loss=True)
pred_box = K.concatenate([pred_xy, pred_wh])
# Darknet raw box to calculate loss.
raw_true_xy = y_true[l][..., :2] * grid_shapes[l][::-1] - grid
raw_true_wh = K.log(y_true[l][..., 2:4] / anchors[anchor_mask[l]] *
input_shape[::-1])
raw_true_wh = K.switch(object_mask, raw_true_wh,
K.zeros_like(raw_true_wh)) # avoid log(0)=-inf
box_loss_scale = 2 - y_true[l][..., 2:3] * y_true[l][..., 3:4]
# Find ignore mask, iterate over each of batch.
ignore_mask = tf.TensorArray(K.dtype(y_true[0]),
size=1,
dynamic_size=True)
object_mask_bool = K.cast(object_mask, 'bool')
def loop_body(b, _ignore_mask):
true_box = tf.boolean_mask(y_true[l][b, ..., 0:4],
object_mask_bool[b, ..., 0])
iou = box_iou(pred_box[b], true_box)
best_iou = K.max(iou, axis=-1)
_ignore_mask = _ignore_mask.write(
b, K.cast(best_iou < ignore_thresh, K.dtype(true_box)))
return b + 1, _ignore_mask
_, ignore_mask = K.control_flow_ops.while_loop(lambda b, *_args: b < m,
loop_body,
[0, ignore_mask])
ignore_mask = ignore_mask.stack()
ignore_mask = K.expand_dims(ignore_mask, -1)
# Find the best output layer according to the size of an object
area_pred = raw_pred[..., 2] * raw_pred[..., 3]
area_pred = K.map_fn(lambda x: K.maximum(x, area_0), area_pred)
area_pred = K.map_fn(lambda x: K.minimum(x, area_2), area_pred)
output_weight = K.map_fn(lambda x: x - area_0_log2,
K.log(area_pred)) * output_weight_multipl
# It will be a float between 0 and 2
output_weight = K.abs(K.map_fn(lambda x: x - l, output_weight))
output_weight = K.expand_dims(output_weight, -1)
obj_scale = 5
xywh_scale = 0.5
output_weight_scale = 1
xy_loss = xywh_scale * object_mask * box_loss_scale * K.square(
raw_true_xy - raw_pred[..., 0:2])
wh_loss = xywh_scale * object_mask * box_loss_scale * 0.5 * K.square(
raw_true_wh - raw_pred[..., 2:4])
confidence_loss = obj_scale * object_mask * K.binary_crossentropy(
object_mask, raw_pred[..., 4:5], from_logits=True)
confidence_loss += (1 - object_mask) * K.binary_crossentropy(
object_mask, raw_pred[..., 4:5], from_logits=True) * ignore_mask
class_loss = object_mask * K.binary_crossentropy(
true_class_probs, raw_pred[..., 5:], from_logits=True)
output_loss = object_mask * output_weight_scale * output_weight * ignore_mask
xy_loss = K.sum(xy_loss) / mf
wh_loss = K.sum(wh_loss) / mf
confidence_loss = K.sum(confidence_loss) / mf
class_loss = K.sum(class_loss) / mf
output_loss = K.sum(output_loss) / mf
loss += xy_loss + wh_loss + confidence_loss + class_loss + output_loss
if print_loss:
loss = tf.Print(loss, [
loss, xy_loss, wh_loss, confidence_loss, class_loss,
output_loss,
K.sum(ignore_mask)
],
message=str(l) + '. loss: ')
return loss
def triplet_loss(y_true, y_pred):
margin = K.constant(1)
return K.mean(K.maximum(K.constant(0), y_pred + margin))
def call(self, x):
x_std = K.std(x, axis=(1, 2), keepdims=True)
x_std = K.maximum(x_std, self.min_std_val / 10. + self.min_std)
return x_std
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr * (1. / (1. + self.decay *
K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx()) + 1
if self.initial_total_steps > 0:
warmup_steps = self.total_steps * self.warmup_proportion
decay_steps = self.total_steps - warmup_steps
lr = K.switch(
t <= warmup_steps,
lr * (t / warmup_steps),
lr * (1.0 - K.minimum(t, decay_steps) / decay_steps),
)
ms = [
K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i))
for (i, p) in enumerate(params)
]
vs = [
K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i))
for (i, p) in enumerate(params)
]
if self.amsgrad:
vhats = [
K.zeros(K.int_shape(p),
dtype=K.dtype(p),
name='vhat_' + str(i)) for (i, p) in enumerate(params)
]
else:
vhats = [
K.zeros(1, name='vhat_' + str(i)) for i in range(len(params))
]
self.weights = [self.iterations] + ms + vs + vhats
beta_1_t = K.pow(self.beta_1, t)
beta_2_t = K.pow(self.beta_2, t)
sma_inf = 2.0 / (1.0 - self.beta_2) - 1.0
sma_t = sma_inf - 2.0 * t * beta_2_t / (1.0 - beta_2_t)
for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
m_corr_t = m_t / (1.0 - beta_1_t)
if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
v_corr_t = K.sqrt(vhat_t / (1.0 - beta_2_t) + self.epsilon)
self.updates.append(K.update(vhat, vhat_t))
else:
v_corr_t = K.sqrt(v_t / (1.0 - beta_2_t) + self.epsilon)
r_t = K.sqrt((sma_t - 4.0) / (sma_inf - 4.0) * (sma_t - 2.0) /
(sma_inf - 2.0) * sma_inf / sma_t)
p_t = K.switch(sma_t > 5, r_t * m_corr_t / v_corr_t, m_corr_t)
if self.initial_weight_decay > 0:
p_t += self.weight_decay * p
p_t = p - lr * p_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
return self.updates
def smape(true, predicted):
epsilon = 0.1
summ = k.maximum(k.abs(true) + k.abs(predicted) + epsilon, 0.5 + epsilon)
smape_result = k.abs(predicted - true) / summ * 2.0
return smape_result
coeff=layer_contributions[layer_name]
activation=layer_dict[layer_name].output
scaling=K.prod(K.cast(K.shape(activation),'float32'))
loss=loss+coeff*K.sum(K.square(activation[:,2:-2,2:-2,:])) / scaling
K.prod(K.cast(K.shape(activation),'float32'))
layer_dict['mixed3'].output
model.summary()
import keras
dream=model.input
grads=K.gradients(loss,dream)[0]
grads=grads/K.maximum(K.mean(K.abs(grads)),1e-7)
outputs=[loss,grads]
fetch_loss_and_grads=K.function([dream],outputs)
def eval_loss_and_grads(x):
outs=fetch_loss_and_grads([x])
loss_val=outs[0]
grad_val=outs[1]
return loss_val,grad_val
def gradient_ascent(x,iterations,step,max_loss=None):
for i in range(iterations):
loss_value,grad_value=eval_loss_and_grads(x)
if max_loss is not None and loss_value>max_loss :
break
def custom_loss(y_true, y_pred):
margin = 1
return K.mean(0.25 * y_true * K.square(1 - y_pred) +
(1 - y_true) * K.square(K.maximum(y_pred, 0)))
def margin_loss(y_true, y_pred):
L = y_true * K.square(K.maximum(0., 0.9 - y_pred)) + \
0.5 * (1 - y_true) * K.square(K.maximum(0., y_pred - 0.1))
return K.mean(K.sum(L, 1))
# Add the L2 norm of the features of a layer to the loss.
assert (layer_name in layer_dict.keys(),
'Layer ' + layer_name + ' not found in model.')
coeff = settings['features'][layer_name]
x = layer_dict[layer_name].output
# We avoid border artifacts by only involving non-border pixels in the loss.
scaling = K.prod(K.cast(K.shape(x), 'float32'))
if K.image_data_format() == 'channels_first':
loss += coeff * K.sum(K.square(x[:, :, 2:-2, 2:-2])) / scaling
else:
loss += coeff * K.sum(K.square(x[:, 2:-2, 2:-2, :])) / scaling
# Compute the gradients of the dream wrt the loss.
grads = K.gradients(loss, dream)[0]
# Normalize gradients.
grads /= K.maximum(K.mean(K.abs(grads)), K.epsilon())
# Set up function to retrieve the value of the loss and gradients given an input image.
outputs = [loss, grads]
fetch_loss_and_grads = K.function([dream], outputs)
def eval_loss_and_grads(x):
outs = fetch_loss_and_grads([x])
loss_value = outs[0]
grad_values = outs[1]
return loss_value, grad_values
# Helper funtion to resize
def resize_img(img, size):
def cosine_triplet_loss(X):
_alpha = float(alpha)
positive_sim, negative_sim = X
losses = K.maximum(0.0, negative_sim - positive_sim + _alpha)
return K.mean(losses)
def contrastive_loss1(y_true, y_pred):
margin = 1.0
square_pred = K.square(y_pred)
margin_square = K.square(K.maximum(margin - y_pred, 0))
return K.mean((1 - y_true) * square_pred + y_true * margin_square)
def contrastive_loss(y_true, y_pred):
'''Contrastive loss from Hadsell-et-al.'06
http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
'''
margin = 5
return K.mean(y_true * K.square(y_pred) + (1 - y_true) * K.square(K.maximum(margin - y_pred, 0)))
def retanh(x):
return K.maximum(0, K.tanh(x))
def loss_yolo(y_pred, y_true):
# pred_boxes = K.Reshape(y_pred[...,3:], (-1,7*7,B,5)) ** QUITAMOS B POR AHORA
pred_boxes = K.reshape(y_pred[..., 3:], (-1, 7 * 7, 5)) #245
true_boxes = K.reshape(y_true[..., 3:], (-1, 7 * 7, 5)) #245
pred_boxes.shape
true_boxes.shape
# probabilidad de que haya un objeto
y_pred_conf = pred_boxes[..., 4]
y_true_conf = true_boxes[..., 4]
y_pred_conf.shape
y_true_conf.shape
### xy_loss--------------------------------------
y_pred_xy = pred_boxes[..., 0:2]
y_true_xy = true_boxes[..., 0:2]
y_pred_xy.shape
y_true_xy.shape
xy_loss = 5 * (K.sum(
K.sum(K.square(y_true_xy - y_pred_xy), axis=-1) * y_true_conf,
axis=-1))
### wh_loss---------------------------------------
y_pred_wh = pred_boxes[..., 2:4]
y_true_wh = true_boxes[..., 2:4]
wh_loss = 5 * (K.sum(
K.sum(K.square(tf.math.sqrt(y_true_wh) - tf.math.sqrt(y_pred_wh)),
axis=-1) * y_true_conf,
axis=-1))
### class_loss----------------------------------
#y_pred_class = y_pred[...,0:3]
#y_true_class = y_true[...,0:3]
y_pred_class = K.reshape(y_pred[..., 0:3], (-1, 7 * 7, 3))
y_true_class = K.reshape(y_true[..., 0:3], (-1, 7 * 7, 3))
clss_loss = K.sum(K.sum(K.square(y_true_class - y_pred_class), axis=-1) *
y_true_conf,
axis=-1)
### Conf_loss--------------------------------------
#(***Creo que esto solo tiene sentido cuando tenemos mas de una prediccion por celda (B)!!!!!)
#Calculo de intersection over union (iou)
#Coordenadas (xy) superior izquierda e inferior derecha de las cajas predichas y reales
x1y1_pred = y_pred_xy - (y_pred_wh / 2)
x2y2_pred = y_pred_xy + (y_pred_wh / 2)
x1y1_true = y_true_xy - (y_true_wh / 2)
x2y2_true = y_true_xy + (y_true_wh / 2)
#Coordenadas superior izquierda e inferior derecha del cuadrado de interseccion
xi1 = K.maximum(x1y1_pred[..., 0], x1y1_true[..., 0])
yi1 = K.maximum(x1y1_pred[..., 1], x1y1_true[..., 1])
xi2 = K.minimum(x2y2_pred[..., 0], x2y2_true[..., 0])
yi2 = K.minimum(x2y2_pred[..., 1], x2y2_true[..., 1])
#Calculo de areas
inter_area = (xi2 - xi1) * (yi2 - yi1)
true_area = y_true_wh[..., 0] * y_true_wh[..., 1]
pred_area = y_pred_wh[..., 0] * y_pred_wh[..., 1]
union_area = pred_area + true_area - inter_area
iou = inter_area / union_area
# -> Calculo del Primer termino de conf_loss (penaliza predicciones incorrectas)
conf_loss1 = K.sum(K.square(y_true_conf * iou - y_pred_conf) * y_true_conf,
axis=-1)
# -> Calculo del Segundo termino de conf_loss (penaliza predicciones cuando no hay en realidad objeto)
'''
Creamos el tensor y_true_conf_op que es igual que y_true_conf pero intercambiando
ceros por unos. Asi tenemos en cuenta las celdas donde no hay objetos y podemos calcular
la funcion de perdida cuando y_pred_conf != 0 (debe ser cero en las celdas donde no
hay objetos)
'''
ones_tensor = tf.ones(tf.shape(y_true_conf), dtype='float64')
y_true_conf_op = ones_tensor - y_true_conf
conf_loss2 = 0.5 * (K.sum(
K.square(y_true_conf * iou - y_pred_conf) * y_true_conf_op, axis=-1))
### LOSS FUNCTION
loss = clss_loss + xy_loss + wh_loss + conf_loss1 + conf_loss2
return loss
