def get_updates(self, params, constraints, loss): grads = self.get_gradients(loss, params) self.updates = [K.update_add(self.iterations, 1)] t = self.iterations + 1 lr_t = self.lr * K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t)) shapes = [K.get_variable_shape(p) for p in params] ms = [K.zeros(shape) for shape in shapes] vs = [K.zeros(shape) for shape in shapes] self.weights = [self.iterations] + ms + vs for p, g, m, v in zip(params, grads, ms, vs): m_t = (self.beta_1 * m) + (1. - self.beta_1) * g v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g) p_t = p - self.get_param_learning_rate_t(p,t,lr_t) * m_t / (K.sqrt(v_t) + self.epsilon) self.updates.append(K.update(m, m_t)) self.updates.append(K.update(v, v_t)) new_p = p_t # apply constraints if p in constraints: c = constraints[p] new_p = c(new_p) self.updates.append(K.update(p, new_p)) return self.updates
def loglik_discrete(y, u, a, b, epsilon=1e-35): hazard0 = K.pow((y + epsilon) / a, b) hazard1 = K.pow((y + 1.0) / a, b) loglikelihoods = u * \ K.log(K.exp(hazard1 - hazard0) - 1.0) - hazard1 return loglikelihoods
def build_loss(self): # Infinity norm if np.isinf(self.p): value = K.max(self.img) else: value = K.pow(K.sum(K.pow(K.abs(self.img), self.p)), 1. / self.p) return normalize(self.img, value)
def func(y_true, y_pred): Y_true = K.reshape(y_true, (-1, ) + img_shape) Y_pred = K.reshape(y_pred, (-1, ) + img_shape) t1 = K.pow(K.abs(Y_true[:, :, :, 1:, :] - Y_true[:, :, :, :-1, :]) - K.abs(Y_pred[:, :, :, 1:, :] - Y_pred[:, :, :, :-1, :]), alpha) t2 = K.pow(K.abs(Y_true[:, :, :, :, :-1] - Y_true[:, :, :, :, 1:]) - K.abs(Y_pred[:, :, :, :, :-1] - Y_pred[:, :, :, :, 1:]), alpha) out = K.mean(K.batch_flatten(t1 + t2), -1) return out
def focal_loss_fixed(y_true, y_pred): if(K.backend()=="tensorflow"): import tensorflow as tf pt = tf.where(tf.equal(y_true, 1), y_pred, 1 - y_pred) return -K.mean(alpha * K.pow(1. - pt, gamma) * K.log(pt)) if(K.backend()=="theano"): import theano.tensor as T pt = T.where(T.eq(y_true, 1), y_pred, 1 - y_pred) return -K.mean(alpha * K.pow(1. - pt, gamma) * K.log(pt))
def call(self, x): # statistics computed along features dimension, on every spatial position of the input tensor A = K.mean(K.abs(x), axis = self.channel_axis) # mean absolute value M1 = K.mean(x, axis = self.channel_axis) # mean value M2 = K.mean(x**2, axis = self.channel_axis) # squared quadratic average V = M2 - M1**2 # variance: V[X] = E[X^2] - E[X]^2 eps = 0.001 #K.epsilon() norm = K.pow(V + eps, self.norm_dev/2) * K.pow(M2 + eps, self.norm_mag/2) * K.pow(A + eps, self.norm_abs) return x / norm[...,None]
def normals_metric(y_true, y_pred): y_true = K.variable(y_true) y_pred = K.variable(y_pred) y_true = K.expand_dims(y_true,0) filter_y = K.variable(np.array([[ 0., -0.5 , 0.], [0., 0., 0.], [0., 0.5, 0.]]).reshape(3, 3, 1, 1)) filter_x = K.variable(np.array([ [0, 0., 0.], [0.5, 0., -0.5], [0., 0., 0.]]).reshape(3, 3, 1, 1)) dzdx = K.conv2d(K.exp(y_true), filter_x, padding='same') dzdy = K.conv2d(K.exp(y_true), filter_y, padding='same') dzdx_ = dzdx * -1.0#K.constant(-1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(-1.0, shape=K.int_shape(dzdx)) dzdy_ = dzdy * -1.0#K.constant(-1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(-1.0, shape=K.int_shape(dzdy)) mag_norm = K.pow(dzdx,2) + K.pow(dzdy,2) + 1.0#K.constant(1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(1.0, shape=K.int_shape(dzdx)) mag_norm = K.sqrt(mag_norm) N3 = 1.0 / mag_norm #K.constant(1.0, shape=K.int_shape(dzdx)) / mag_norm N1 = dzdx_ / mag_norm N2 = dzdy_ / mag_norm normals = K.concatenate(tensors=[N1,N2,N3],axis=-1) dzdx_pred = K.conv2d(K.exp(y_pred), filter_x, padding='same') dzdy_pred = K.conv2d(K.exp(y_pred), filter_y, padding='same') mag_norm_pred = K.pow(dzdx_pred,2) + K.pow(dzdy_pred,2) + 1.0 mag_norm_pred = K.sqrt(mag_norm_pred) grad_x = K.concatenate(tensors=[1.0/ mag_norm_pred, 0.0/ mag_norm_pred, dzdx_pred/ mag_norm_pred],axis=-1) grad_y = K.concatenate(tensors=[0.0/ mag_norm_pred, 1.0/ mag_norm_pred, dzdy_pred/ mag_norm_pred],axis=-1) dot_term_x = K.mean(K.sum(normals[0,:,:,:] * grad_x[0,:,:,:], axis=-1, keepdims=True), axis=-1) dot_term_y = K.mean(K.sum(normals[0,:,:,:] * grad_y[0,:,:,:], axis=-1, keepdims=True), axis=-1) dot_term_x = K.abs(dot_term_x) dot_term_y = K.abs(dot_term_y) return K.eval(K.mean(dot_term_x)),K.eval(K.mean(dot_term_y))
def call(self, x, mask=None): input_shape = K.int_shape(x) reduction_axes = list(range(len(input_shape))) del reduction_axes[self.axis] broadcast_shape = [1] * len(input_shape) broadcast_shape[self.axis] = input_shape[self.axis] alpha_pos = K.reshape(self.alpha_pos, broadcast_shape) alpha_neg = K.reshape(self.alpha_neg, broadcast_shape) beta_pos = K.reshape(self.beta_pos, broadcast_shape) beta_neg = K.reshape(self.beta_neg, broadcast_shape) rho_pos = K.reshape(self.rho_pos, broadcast_shape) rho_neg = K.reshape(self.rho_neg, broadcast_shape) pos = alpha_pos * K.pow(K.relu(x + beta_pos) + K.epsilon(), rho_pos) neg = alpha_neg * K.pow(K.relu(-x + beta_neg) + K.epsilon(), rho_neg) return pos + neg
def total_variation_loss(x): assert K.ndim(x) == 4 a = K.square(x[:, :, 1:, :img_width - 1] - x[:, :, :img_height - 1, :img_width - 1]) b = K.square(x[:, :, :img_height - 1, 1:] - x[:, :, :img_width - 1, :img_height - 1]) #a = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, 1:, :img_height-1]) #b = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, :img_width-1, 1:]) return K.sum(K.pow(a + b, 1.25))
def build_loss(self): r"""Implements the N-dim version of function $$TV^{\beta}(x) = \sum_{whc} \left ( \left ( x(h, w+1, c) - x(h, w, c) \right )^{2} + \left ( x(h+1, w, c) - x(h, w, c) \right )^{2} \right )^{\frac{\beta}{2}}$$ to return total variation for all images in the batch. """ image_dims = K.ndim(self.img) - 2 # Constructing slice [1:] + [:-1] * (image_dims - 1) and [:-1] * (image_dims) start_slice = [slice(1, None, None)] + [slice(None, -1, None) for _ in range(image_dims - 1)] end_slice = [slice(None, -1, None) for _ in range(image_dims)] samples_channels_slice = [slice(None, None, None), slice(None, None, None)] # Compute pixel diffs by rolling slices to the right per image dim. tv = None for i in range(image_dims): ss = tuple(samples_channels_slice + start_slice) es = tuple(samples_channels_slice + end_slice) diff_square = K.square(self.img[utils.slicer[ss]] - self.img[utils.slicer[es]]) tv = diff_square if tv is None else tv + diff_square # Roll over to next image dim start_slice = np.roll(start_slice, 1).tolist() end_slice = np.roll(end_slice, 1).tolist() tv = K.sum(K.pow(tv, self.beta / 2.)) return normalize(self.img, tv)
def __call__(self, loss): x = self.layer.get_output(True) assert K.ndim(x) == 4 a = K.square(x[:, :, 1:, :-1] - x[:, :, :-1, :-1]) b = K.square(x[:, :, :-1, 1:] - x[:, :, :-1, :-1]) loss += self.weight * K.mean(K.sum(K.pow(a + b, 1.25), axis=(1,2,3))) return loss
def get_updates(self, params, constraints, loss): grads = self.get_gradients(loss, params) self.updates = [K.update_add(self.iterations, 1)] t = self.iterations + 1 lr_t = self.lr / (1. - K.pow(self.beta_1, t)) shapes = [K.get_variable_shape(p) for p in params] # zero init of 1st moment ms = [K.zeros(shape) for shape in shapes] # zero init of exponentially weighted infinity norm us = [K.zeros(shape) for shape in shapes] self.weights = [self.iterations] + ms + us for p, g, m, u in zip(params, grads, ms, us): m_t = (self.beta_1 * m) + (1. - self.beta_1) * g u_t = K.maximum(self.beta_2 * u, K.abs(g)) p_t = p - self.get_param_learning_rate_t(p,t,lr_t) * m_t / (u_t + self.epsilon) self.updates.append(K.update(m, m_t)) self.updates.append(K.update(u, u_t)) new_p = p_t # apply constraints if p in constraints: c = constraints[p] new_p = c(new_p) self.updates.append(K.update(p, new_p)) return self.updates
def loglik_continuous_conditional_correction(y, u, a, b, epsilon=1e-35): """Integrated conditional excess loss. Explanation TODO """ ya = (y + epsilon) / a loglikelihoods = y * \ (u * (K.log(b) + b * K.log(ya)) - (b / (b + 1.)) * K.pow(ya, b)) return loglikelihoods
def total_variation_loss(y_pred): if K.image_data_format() == 'channels_first': a = K.square(y_pred[:, :, :m - 1, :n - 1] - y_pred[:, :, 1:, :n - 1]) b = K.square(y_pred[:, :, :m - 1, :n - 1] - y_pred[:, :, :m - 1, 1:]) else: a = K.square(y_pred[:, :m - 1, :n - 1, :] - y_pred[:, 1:, :n - 1, :]) b = K.square(y_pred[:, :m - 1, :n - 1, :] - y_pred[:, :m - 1, 1:, :]) return K.sum(K.pow(a + b, 1.25))
def variation_loss(comb): if K.image_dim_ordering() == "th": dx = K.square(comb[:, :, :RESIZED_WH-1, :RESIZED_WH-1] - comb[:, :, 1:, :RESIZED_WH-1]) dy = K.square(comb[:, :, :RESIZED_WH-1, :RESIZED_WH-1] - comb[:, :, :RESIZED_WH-1, 1:]) else: dx = K.square(comb[:, :RESIZED_WH-1, :RESIZED_WH-1, :] - comb[:, 1:, :RESIZED_WH-1, :]) dy = K.square(comb[:, :RESIZED_WH-1, :RESIZED_WH-1, :] - comb[:, :RESIZED_WH-1, 1:, :]) return K.sum(K.pow(dx + dy, 1.25))
def total_variation_loss(x): assert 4 == K.ndim(x) if K.image_dim_ordering() == 'th': a = K.square(x[:, :, :img_nrows - 1, :img_ncols - 1] - x[:, :, 1:, :img_ncols - 1]) b = K.square(x[:, :, :img_nrows - 1, :img_ncols - 1] - x[:, :, :img_nrows - 1, 1:]) else: a = K.square(x[:, :img_nrows - 1, :img_ncols - 1, :] - x[:, 1:, :img_ncols - 1, :]) b = K.square(x[:, :img_nrows - 1, :img_ncols - 1, :] - x[:, :img_nrows - 1, 1:, :]) return K.sum(K.pow(a + b, 1.25))
def continuity_loss(x): assert K.ndim(x) == 4 if K.image_dim_ordering() == "th": a = K.square(x[:, :, : img_width - 1, : img_height - 1] - x[:, :, 1:, : img_height - 1]) b = K.square(x[:, :, : img_width - 1, : img_height - 1] - x[:, :, : img_width - 1, 1:]) else: a = K.square(x[:, : img_width - 1, : img_height - 1, :] - x[:, 1:, : img_height - 1, :]) b = K.square(x[:, : img_width - 1, : img_height - 1, :] - x[:, : img_width - 1, 1:, :]) return K.sum(K.pow(a + b, 1.25))
def total_variation_loss(x): assert K.ndim(x) == 4 if K.image_data_format() == 'channels_first': a = K.square(x[:, :, :img_nrows - 1, :img_ncols - 1] - x[:, :, 1:, :img_ncols - 1]) b = K.square(x[:, :, :img_nrows - 1, :img_ncols - 1] - x[:, :, :img_nrows - 1, 1:]) else: a = K.square(x[:, :img_nrows - 1, :img_ncols - 1, :] - x[:, 1:, :img_ncols - 1, :]) b = K.square(x[:, :img_nrows - 1, :img_ncols - 1, :] - x[:, :img_nrows - 1, 1:, :]) return K.sum(K.pow(a + b, 1.25))
def total_variation_loss(x): assert K.ndim(x) == 4 if K.image_dim_ordering() == 'th': a = K.square(x[:, :, :img_width - 1, :img_height - 1] - x[:, :, 1:, :img_height - 1]) b = K.square(x[:, :, :img_width - 1, :img_height - 1] - x[:, :, :img_width - 1, 1:]) else: a = K.square(x[:, :img_width - 1, :img_height - 1, :] - x[:, 1:, :img_height - 1, :]) b = K.square(x[:, :img_width - 1, :img_height - 1, :] - x[:, :img_width - 1, 1:, :]) return K.sum(K.pow(a + b, 1.25))
def __call__(self, x): assert K.ndim(x) == 4 if K.image_dim_ordering() == 'th': a = K.square(x[:, :, :self.img_width - 1, :self.img_height - 1] - x[:, :, 1:, :self.img_height - 1]) b = K.square(x[:, :, :self.img_width - 1, :self.img_height - 1] - x[:, :, :self.img_width - 1, 1:]) else: a = K.square(x[:, :self.img_width - 1, :self.img_height - 1, :] - x[:, 1:, :self.img_height - 1, :]) b = K.square(x[:, :self.img_width - 1, :self.img_height - 1, :] - x[:, :self.img_width - 1, 1:, :]) loss = self.weight * K.mean(K.sum(K.pow(a + b, 1.25))) return loss
def call(self, x, mask=None): input_shape = self.input_spec[0].shape reduction_axes = list(range(len(input_shape))) del reduction_axes[self.axis] broadcast_shape = [1] * len(input_shape) broadcast_shape[self.axis] = input_shape[self.axis] alpha = K.reshape(self.alpha, broadcast_shape) rho = K.reshape(self.rho, broadcast_shape) return alpha * K.pow(K.relu(x) + K.epsilon(), rho)
def total_variation_loss(self): """ Total variation loss, designed to keep the generated image locally coherent :return: total variation loss """ x = self.input_tensor assert K.ndim(x) == 4 a = K.square(x[:, :, :img_width - 1, :img_height - 1] - x[:, :, 1:, :img_height - 1]) b = K.square(x[:, :, :img_width - 1, :img_height - 1] - x[:, :, :img_width - 1, 1:]) return K.sum(K.pow(a + b, 1.25))
def KLdivergence(P, Y): alpha = low_dim - 1. sum_Y = K.sum(K.square(Y), axis=1) eps = K.variable(10e-15) D = sum_Y + K.reshape(sum_Y, [-1, 1]) - 2 * K.dot(Y, K.transpose(Y)) Q = K.pow(1 + D / alpha, -(alpha + 1) / 2) Q *= K.variable(1 - np.eye(batch_size)) Q /= K.sum(Q) Q = K.maximum(Q, eps) C = K.log((P + eps) / (Q + eps)) C = K.sum(P * C) return C
def call(self, inputs, mask=None): if K.backend() == 'theano': a = K.pattern_broadcast(self.a, self.a_param_broadcast) k = K.pattern_broadcast(self.k, self.k_param_broadcast) n = K.pattern_broadcast(self.n, self.n_param_broadcast) z = K.pattern_broadcast(self.z, self.z_param_broadcast) else: a = self.a k = self.k n = self.n z = self.z return a / (K.pow((k / (inputs + 1e-5)), n) + z + 1e-5)
def get_updates(self, params, constraints, loss): grads = self.get_gradients(loss, params) self.updates = [K.update_add(self.iterations, 1)] t = self.iterations + 1 # Due to the recommendations in [2], i.e. warming momentum schedule momentum_cache_t = self.beta_1 * (1. - 0.5 * (K.pow(0.96, t * self.schedule_decay))) momentum_cache_t_1 = self.beta_1 * (1. - 0.5 * (K.pow(0.96, (t + 1) * self.schedule_decay))) m_schedule_new = self.m_schedule * momentum_cache_t m_schedule_next = self.m_schedule * momentum_cache_t * momentum_cache_t_1 self.updates.append((self.m_schedule, m_schedule_new)) shapes = [K.get_variable_shape(p) for p in params] ms = [K.zeros(shape) for shape in shapes] vs = [K.zeros(shape) for shape in shapes] self.weights = [self.iterations] + ms + vs for p, g, m, v in zip(params, grads, ms, vs): # the following equations given in [1] g_prime = g / (1. - m_schedule_new) m_t = self.beta_1 * m + (1. - self.beta_1) * g m_t_prime = m_t / (1. - m_schedule_next) v_t = self.beta_2 * v + (1. - self.beta_2) * K.square(g) v_t_prime = v_t / (1. - K.pow(self.beta_2, t)) m_t_bar = (1. - momentum_cache_t) * g_prime + momentum_cache_t_1 * m_t_prime self.updates.append(K.update(m, m_t)) self.updates.append(K.update(v, v_t)) p_t = p - get_learing_rate(p, self.lr) * m_t_bar / (K.sqrt(v_t_prime) + self.epsilon) new_p = p_t # apply constraints if p in constraints: c = constraints[p] new_p = c(new_p) self.updates.append(K.update(p, new_p)) return self.updates
def call(self, x): # print 'LocalNormalization.kernel_size:', self.kernel_size def mean2d(y): y = K.pool2d(y, (self.kernel_size[0], 1), pool_mode = 'avg', padding = 'same') y = K.pool2d(y, (1, self.kernel_size[1]), pool_mode = 'avg', padding = 'same') return y # return K.pool2d(y, self.kernel_size, pool_mode = 'avg', padding = 'same') # (dy, dx) = self.kernel_size # top = dy/2 + 1 # if even `dy`, averaging window is shifted to the top # left = dx/2 + 1 # if even `dx`, averaging window is shifted to the left # # padding = ((top, dy-top), (left, dx-left)) # # z = K.spatial_2d_padding(y, padding) # `y` padded with zeros # s1 = K.cumsum(z, axis = -3) # cumulative sums along Y axis only # s = K.cumsum(s1, axis = -2) # cumulative sums along (Y,X) axes # # t = s[...,dy:,dx:,:] + s[...,:-dy,:-dx,:] - s[...,dy:,:-dx,:] - s[...,:-dy,dx:,:] # # # t[0,0] = s[dy,dx] + s[0,0] - ... = cumsum(y)[0,0] + cumsum(y)[dy,dx] - ... = z[0,0] + (z[0,0]+...+z[dy,dx]) - ... # # = area_sum(z, (1,1)...(dy,dx)) = area_sum(y, (0,0)...(dy-top,dx-left)) = area_sum(y, (0,0)...(dy-(dy/2+1), dx-(dx/2+1))) = # # return t / float(dx*dy) # mean of `x` and x^2 in local area around given pixel M = mean2d(x) M2 = mean2d(x**2) V = mean2d((x-M)**2) eps = 0.001 #K.epsilon() scale = K.exp(self.scale) / K.pow(M2 + eps, self.normal_mag/2) / K.pow(V + eps, self.normal_dev/2) #(V + eps) #K.exp(K.log(D + eps) * self.normal[None,None,:]) return (x - self.shift * M) * scale
def total_variation_loss(x, img_nrows, img_ncols): """ Total variational loss. Encourages spatial smoothness in the output image. """ H, W = img_nrows, img_ncols if K.image_dim_ordering() == 'th': a = K.square(x[:, :, :H-1, :W-1] - x[:, :, 1:, :W-1]) b = K.square(x[:, :, :H-1, :W-1] - x[:, :, :H-1, 1:]) else: a = K.square(x[:, :H-1, :W-1, :] - x[:, 1:, :W-1, :]) b = K.square(x[:, :H-1, :W-1, :] - x[:, :H-1, 1:, :]) return K.sum(K.pow(a + b, 1.25))
def call(self, x, mask=None): if (self.size == None) or (self.mode == 'sum'): self.size = int(x.shape[-1]) batch_size, seq_len = K.shape(x)[0], K.shape(x)[1] position_j = 1. / K.pow(10000., 2 * K.arange(self.size / 2, dtype='float32') / self.size) position_j = K.expand_dims(position_j, 0) position_i = K.cumsum(K.ones_like(x[:, :, 0]), 1) - 1 # K.arange不支持变长,只好用这种方法生成 position_i = K.expand_dims(position_i, 2) position_ij = K.dot(position_i, position_j) position_ij = K.concatenate([K.cos(position_ij), K.sin(position_ij)], 2) if self.mode == 'sum': return position_ij + x elif self.mode == 'concat': return K.concatenate([position_ij, x], 2)
def continuity_loss(x): # continuity loss util function assert K.ndim(x) == 4 if K.image_data_format() == 'channels_first': a = K.square(x[:, :, :img_height - 1, :img_width - 1] - x[:, :, 1:, :img_width - 1]) b = K.square(x[:, :, :img_height - 1, :img_width - 1] - x[:, :, :img_height - 1, 1:]) else: a = K.square(x[:, :img_height - 1, :img_width - 1, :] - x[:, 1:, :img_width - 1, :]) b = K.square(x[:, :img_height - 1, :img_width - 1, :] - x[:, :img_height - 1, 1:, :]) return K.sum(K.pow(a + b, 1.25))
def generalized_loss_function(y_true, y_pred, var_a=1.0, cnst=1.0/255.0): """ generalized function used to return a large variety of mathematical loss functions primary benefit is smooth, differentiable version of L1 loss Barron, J. A More General Robust Loss Function https://arxiv.org/pdf/1701.03077.pdf Parameters: a: penalty factor. larger number give larger weight to large deviations c: scale factor used to adjust to the input scale (i.e. inputs of mean 1e-4 or 256) Return: a loss value from the results of function(y_pred - y_true) Example: a=1.0, x>>c , c=1.0/255.0 will give a smoothly differentiable version of L1 / MAE loss a=1.999999 (lim as a->2), c=1.0/255.0 will give L2 / RMSE loss """ var_x = y_pred - y_true loss = (K.abs(2.0-var_a)/var_a) * (K.pow(K.pow(var_x/cnst, 2.0)/K.abs(2.0-var_a) + 1.0, (var_a/2.0)) - 1.0) return K.mean(loss, axis=-1) * cnst
def total_variation_loss(x, height, width): a = backend.square(x[:, :height - 1, :width - 1, :] - x[:, 1:, :width - 1, :]) b = backend.square(x[:, :height - 1, :width - 1, :] - x[:, :height - 1, 1:, :]) return backend.sum(backend.pow(a + b, 1.25))
def total_variation_loss(img): assert(K.ndim(img) == 4) a = K.square(img[:, :nrows - 1, :ncols - 1, :] - img[:, 1:, :ncols - 1, :]) b = K.square(img[:, :nrows - 1, :ncols - 1, :] - img[:, :nrows - 1, 1:, :]) return K.sum(K.pow(a + b, 1.25))
def mse(y_true, y_pred): return K.mean(K.pow(y_true - y_pred, 2))
def Loss_tv(x_true, x_hat): # total variation loss a = K.square(x_hat[:, :, :H0 - 1, :W0 - 1] - x_hat[:, :, 1:, :W0 - 1]) / (C0 * H0 * W0) b = K.square(x_hat[:, :, :H0 - 1, :W0 - 1] - x_hat[:, :, :H0 - 1, 1:]) / (C0 * H0 * W0) return K.sum(K.pow(a + b, 1.25))
def potential_loss(y_true, y_pred): # shapes if y_true and y_pred must be the same return K.mean((1 - y_true) * K.exp(-10 * (1 - y_true) * y_pred) / K.maximum(K.epsilon(), y_pred) + y_true * K.pow(y_pred, 3))
def get_updates(self, loss, params): grads = self.get_gradients(loss, params) self.updates = [K.update_add(self.iterations, 1)] lr = self.lr if self.initial_decay > 0: lr = lr * (1. / (1. + self.decay * K.cast(self.iterations, K.dtype(self.decay)))) t = K.cast(self.iterations, K.floatx()) + 1 lr_t = lr * K.sqrt(1. - K.pow(self.beta_2, t)) / ( 1. - K.pow(self.beta_1, t)) shapes = [K.get_variable_shape(p) for p in params] ms = [K.zeros(shape) for shape in shapes] vs = [K.zeros(shape) for shape in shapes] self.weights = [self.iterations] + ms + vs for p, g, m, v in zip(params, grads, ms, vs): # if a weight tensor (len > 1) use weight normalized parameterization # this is the only part changed w.r.t. keras.optimizers.Adam ps = K.get_variable_shape(p) if len(ps) > 1: # get weight normalization parameters V, V_norm, V_scaler, g_param, grad_g, grad_V = get_weightnorm_params_and_grads( p, g) # Adam containers for the 'g' parameter V_scaler_shape = K.get_variable_shape(V_scaler) m_g = K.zeros(V_scaler_shape) v_g = K.zeros(V_scaler_shape) # update g parameters m_g_t = (self.beta_1 * m_g) + (1. - self.beta_1) * grad_g v_g_t = (self.beta_2 * v_g) + (1. - self.beta_2) * K.square(grad_g) new_g_param = g_param - lr_t * m_g_t / (K.sqrt(v_g_t) + self.epsilon) self.updates.append(K.update(m_g, m_g_t)) self.updates.append(K.update(v_g, v_g_t)) # update V parameters m_t = (self.beta_1 * m) + (1. - self.beta_1) * grad_V v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(grad_V) new_V_param = V - lr_t * m_t / (K.sqrt(v_t) + self.epsilon) self.updates.append(K.update(m, m_t)) self.updates.append(K.update(v, v_t)) # Apply constraints. if getattr(p, 'constraint', None) is not None: new_V_param = p.constraint(new_V_param) # wn param updates --> W updates add_weightnorm_param_updates(self.updates, new_V_param, new_g_param, p, V_scaler) else: # do optimization normally m_t = (self.beta_1 * m) + (1. - self.beta_1) * g v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g) p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon) self.updates.append(K.update(m, m_t)) self.updates.append(K.update(v, v_t)) new_p = p_t # Apply constraints. if getattr(p, 'constraint', None) is not None: new_p = p.constraint(new_p) self.updates.append(K.update(p, new_p)) return self.updates
def ContrastiveLoss(self, y_pre, y_align): weight_arr = np.diag([1.0 for i in range(self.Constant.BATCH_SIZE)]) weight_tensor = K.variable(weight_arr) loss_contrastive = K.mean(K.dot(weight_tensor, K.pow(y_pre - y_align, 2))) return loss_contrastive
def call(self, x): basis = K.concatenate([K.pow(x, i) for i in xrange(self.order + 1)]) return K.dot(basis, self.kernel)
def call(self, x): # norm the input norm = K.sqrt(K.sum(K.pow(x,2), axis=[1,2], keepdims=True)) x = x / norm * self.scale_factor return x
def focal_tversky_loss(y_true, y_pred): pt_1 = tversky(y_true, y_pred) gamma = 0.75 return K.pow((1 - pt_1), gamma)
def total_loss(features, height, width): a = K.square(features[:, :height - 1, :width - 1, :] - features[:, 1:, :width - 1, :]) b = K.square(features[:, :height - 1, :width - 1, :] - features[:, :height - 1, 1:, :]) return K.sum(K.pow(a + b, 1.25))
def get_updates(self, loss, params): grads = self.get_gradients(loss, params) self.updates = [K.update_add(self.iterations, 1)] lr = self.lrr completed_updates = K.cast( tf.math.floordiv(self.iterations, self.accum_iters), K.floatx()) if self.initial_decay > 0: lr = lr * (1. / (1. + self.decay * completed_updates)) t = completed_updates + 1 lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t))) update_switch = K.equal((self.iterations + 1) % self.accum_iters, 0) update_switch = K.cast(update_switch, K.floatx()) ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] if self.amsgrad: vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] else: vhats = [K.zeros(1) for _ in params] self.weights = [self.iterations] + ms + vs + vhats for p, g, m, v, vhat, tg in zip(params, grads, ms, vs, vhats, gs): sum_grad = tg + g avg_grad = sum_grad / self.accum_iters_float m_t = (self.beta_1 * m) + (1. - self.beta_1) * avg_grad v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(avg_grad) if self.amsgrad: vhat_t = K.maximum(vhat, v_t) p_t = p - lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon) self.updates.append( K.update(vhat, (1 - update_switch) * vhat + update_switch * vhat_t)) else: p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon) self.updates.append( K.update(m, (1 - update_switch) * m + update_switch * m_t)) self.updates.append( K.update(v, (1 - update_switch) * v + update_switch * v_t)) self.updates.append(K.update(tg, (1 - update_switch) * sum_grad)) new_p = p_t # Apply constraints. if getattr(p, 'constraint', None) is not None: new_p = p.constraint(new_p) self.updates.append( K.update(p, (1 - update_switch) * p + update_switch * new_p)) return self.updates
# get the predicted probabilities for each class predictions = model.predict(image_batch) return predictions ''' #print(test_inference(filename)) # ------------------------------------- # distribution elements to Keras distribution_elements_row = tf.constant(np.array( [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), dtype='float32', name='marks') distribution_elements = K.expand_dims(distribution_elements_row, -1) distribution_elements_square = K.square(distribution_elements) distribution_elements_cube = K.pow(distribution_elements, 3) # compute squared difference of first moments def first_moment(y_true, y_pred): means_true = K.dot(y_true, distribution_elements) means_pred = K.dot(y_pred, distribution_elements) return K.sqrt(K.mean(K.square(means_true - means_pred))) # compute squared difference of second moments def second_moment(y_true, y_pred): means_true = K.dot(y_true, distribution_elements)
def g_k(x): pi = tf.convert_to_tensor(np.pi, dtype=tf.float32) return 1. / K.pow(2. * pi, int(x.get_shape()[-1]) / 2) * K.exp( -0.5 * K.square(x))
def rmse(y_true, y_pred): y_pred = K.clip(y_pred, 1.0, 5.0) return K.sqrt(K.mean(K.pow(y_true - y_pred, 2)))
def get_TV(new_gram_matrix): x_diff = K.square(new_gram_matrix[:, :WIDTH - 1, :HEIGHT - 1, :] - new_gram_matrix[:, 1:, :HEIGHT - 1, :]) y_diff = K.square(new_gram_matrix[:, :WIDTH - 1, :HEIGHT - 1, :] - new_gram_matrix[:, :WIDTH - 1, 1:, :]) return TV_WEIGHT * K.mean(K.sum(K.pow(x_diff + y_diff, 1.25)))
def __call__(self, x): regularization = 0. regularization += K.pow(K.sum(self.lp * K.pow(x,self.p)),1/self.p) return regularization
def mse_powers(y_true, y_pred): m = mse(y_true, y_pred) return { 'mse_squared': K.pow(m, 2), 'mse_cubed': K.pow(m, 3) }
def norm(x1, x2, axis=1, norm=1): return Keras.pow(Keras.sum(Keras.pow(Keras.abs(x1 - x2), norm), axis=axis), 1.0 / norm)
def focal_loss_fixed(y_true, y_pred): pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred)) pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.ones_like(y_pred)) return -K.sum(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) - K.sum( 1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0)
def total_variation_loss(x): a = K.square(x[:, :resized_width - 1, :resized_height - 1, :] - x[:, 1:, :resized_height - 1, :]) b = K.square(x[:, :resized_width - 1, :resized_height - 1, :] - x[:, :resized_width - 1, 1:, :]) return K.sum(K.pow(a + b, 1.25))
def total_variation_loss(x): a = backend.square(x[:, :IMAGE_HEIGHT - 1, :IMAGE_WIDTH - 1, :] - x[:, 1:, :IMAGE_WIDTH - 1, :]) b = backend.square(x[:, :IMAGE_HEIGHT - 1, :IMAGE_WIDTH - 1, :] - x[:, :IMAGE_HEIGHT - 1, 1:, :]) return backend.sum(backend.pow(a + b, TOTAL_VARIATION_LOSS_FACTOR))
def test_model_methods(): a = Input(shape=(3,), name='input_a') b = Input(shape=(3,), name='input_b') a_2 = Dense(4, name='dense_1')(a) dp = Dropout(0.5, name='dropout') b_2 = dp(b) model = Model([a, b], [a_2, b_2]) optimizer = 'rmsprop' loss = 'mse' loss_weights = [1., 0.5] model.compile(optimizer, loss, metrics=[], loss_weights=loss_weights, sample_weight_mode=None) input_a_np = np.random.random((10, 3)) input_b_np = np.random.random((10, 3)) output_a_np = np.random.random((10, 4)) output_b_np = np.random.random((10, 3)) # test train_on_batch out = model.train_on_batch([input_a_np, input_b_np], [output_a_np, output_b_np]) out = model.train_on_batch({'input_a': input_a_np, 'input_b': input_b_np}, [output_a_np, output_b_np]) out = model.train_on_batch({'input_a': input_a_np, 'input_b': input_b_np}, {'dense_1': output_a_np, 'dropout': output_b_np}) # test fit out = model.fit([input_a_np, input_b_np], [output_a_np, output_b_np], nb_epoch=1, batch_size=4) out = model.fit({'input_a': input_a_np, 'input_b': input_b_np}, [output_a_np, output_b_np], nb_epoch=1, batch_size=4) out = model.fit({'input_a': input_a_np, 'input_b': input_b_np}, {'dense_1': output_a_np, 'dropout': output_b_np}, nb_epoch=1, batch_size=4) # test validation_split out = model.fit([input_a_np, input_b_np], [output_a_np, output_b_np], nb_epoch=1, batch_size=4, validation_split=0.5) out = model.fit({'input_a': input_a_np, 'input_b': input_b_np}, [output_a_np, output_b_np], nb_epoch=1, batch_size=4, validation_split=0.5) out = model.fit({'input_a': input_a_np, 'input_b': input_b_np}, {'dense_1': output_a_np, 'dropout': output_b_np}, nb_epoch=1, batch_size=4, validation_split=0.5) # test validation data out = model.fit([input_a_np, input_b_np], [output_a_np, output_b_np], nb_epoch=1, batch_size=4, validation_data=([input_a_np, input_b_np], [output_a_np, output_b_np])) out = model.fit({'input_a': input_a_np, 'input_b': input_b_np}, [output_a_np, output_b_np], nb_epoch=1, batch_size=4, validation_split=0.5, validation_data=({'input_a': input_a_np, 'input_b': input_b_np}, [output_a_np, output_b_np])) out = model.fit({'input_a': input_a_np, 'input_b': input_b_np}, {'dense_1': output_a_np, 'dropout': output_b_np}, nb_epoch=1, batch_size=4, validation_split=0.5, validation_data=({'input_a': input_a_np, 'input_b': input_b_np}, {'dense_1': output_a_np, 'dropout': output_b_np})) # test_on_batch out = model.test_on_batch([input_a_np, input_b_np], [output_a_np, output_b_np]) out = model.test_on_batch({'input_a': input_a_np, 'input_b': input_b_np}, [output_a_np, output_b_np]) out = model.test_on_batch({'input_a': input_a_np, 'input_b': input_b_np}, {'dense_1': output_a_np, 'dropout': output_b_np}) # predict_on_batch out = model.predict_on_batch([input_a_np, input_b_np]) out = model.predict_on_batch({'input_a': input_a_np, 'input_b': input_b_np}) # predict, evaluate input_a_np = np.random.random((10, 3)) input_b_np = np.random.random((10, 3)) output_a_np = np.random.random((10, 4)) output_b_np = np.random.random((10, 3)) out = model.evaluate([input_a_np, input_b_np], [output_a_np, output_b_np], batch_size=4) out = model.predict([input_a_np, input_b_np], batch_size=4) # with sample_weight input_a_np = np.random.random((10, 3)) input_b_np = np.random.random((10, 3)) output_a_np = np.random.random((10, 4)) output_b_np = np.random.random((10, 3)) sample_weight = [None, np.random.random((10,))] out = model.train_on_batch([input_a_np, input_b_np], [output_a_np, output_b_np], sample_weight=sample_weight) out = model.test_on_batch([input_a_np, input_b_np], [output_a_np, output_b_np], sample_weight=sample_weight) # test accuracy metric model.compile(optimizer, loss, metrics=['acc'], sample_weight_mode=None) out = model.train_on_batch([input_a_np, input_b_np], [output_a_np, output_b_np]) assert len(out) == 5 out = model.test_on_batch([input_a_np, input_b_np], [output_a_np, output_b_np]) assert len(out) == 5 # this should also work model.compile(optimizer, loss, metrics={'dense_1': 'acc'}, sample_weight_mode=None) out = model.train_on_batch([input_a_np, input_b_np], [output_a_np, output_b_np]) assert len(out) == 4 out = model.test_on_batch([input_a_np, input_b_np], [output_a_np, output_b_np]) assert len(out) == 4 # and this as well model.compile(optimizer, loss, metrics={'dense_1': ['acc']}, sample_weight_mode=None) out = model.train_on_batch([input_a_np, input_b_np], [output_a_np, output_b_np]) assert len(out) == 4 out = model.test_on_batch([input_a_np, input_b_np], [output_a_np, output_b_np]) assert len(out) == 4 # test starting from non-zero initial epoch trained_epochs = [] def on_epoch_begin(epoch, logs): trained_epochs.append(epoch) tracker_cb = LambdaCallback(on_epoch_begin=on_epoch_begin) out = model.fit([input_a_np, input_b_np], [output_a_np, output_b_np], nb_epoch=5, batch_size=4, initial_epoch=2, callbacks=[tracker_cb]) assert trained_epochs == [2, 3, 4] # test starting from non-zero initial epoch for generator too trained_epochs = [] def gen_data(batch_sz): while True: yield ([np.random.random((batch_sz, 3)), np.random.random((batch_sz, 3))], [np.random.random((batch_sz, 4)), np.random.random((batch_sz, 3))]) out = model.fit_generator(gen_data(4), samples_per_epoch=10, nb_epoch=5, initial_epoch=2, callbacks=[tracker_cb]) assert trained_epochs == [2, 3, 4] # test with a custom metric function mse = lambda y_true, y_pred: K.mean(K.pow(y_true - y_pred, 2)) def mse_powers(y_true, y_pred): m = mse(y_true, y_pred) return { 'mse_squared': K.pow(m, 2), 'mse_cubed': K.pow(m, 3) } model.compile(optimizer, loss, metrics=[mse, mse_powers], sample_weight_mode=None) out = model.train_on_batch([input_a_np, input_b_np], [output_a_np, output_b_np]) out_len = 1 + 2 * 4 # total loss, per layer: loss + 3 metrics assert len(out) == out_len out = model.test_on_batch([input_a_np, input_b_np], [output_a_np, output_b_np]) assert len(out) == out_len input_a_np = np.random.random((10, 3)) input_b_np = np.random.random((10, 3)) output_a_np = np.random.random((10, 4)) output_b_np = np.random.random((10, 3)) out = model.fit([input_a_np, input_b_np], [output_a_np, output_b_np], batch_size=4, nb_epoch=1) out = model.evaluate([input_a_np, input_b_np], [output_a_np, output_b_np], batch_size=4) out = model.predict([input_a_np, input_b_np], batch_size=4)
def smooth_l1(y_true, y_pred): diff = K.abs(y_true - y_pred) less_part = K.cast(K.less(diff, 1), tf.float32) greater_part = 1.0 - less_part return less_part * K.pow(diff, 2) + greater_part * (K.abs(0.5 - diff))
def focal_tversky(self, y_true, y_pred): pt_1 = self.tversky_index(y_true, y_pred) gamma = 0.75 return K.pow((1 - pt_1), gamma)
def gelu(x): return 0.5 * x * (1.0 + K.tanh(0.797884561 * (x + 0.044715 * K.pow(x, 3))))
def loss_img(self, y, y_pred): return K.sum(K.pow(y - y_pred, 2))
def focal_loss(target, output, gamma=2): output /= K.sum(output, axis=-1, keepdims=True) eps = K.epsilon() output = K.clip(output, eps, 1. - eps) return -K.sum(K.pow(1. - output, gamma) * target * K.log(output), axis=-1)
def call(self, x, **kwargs): cdf = 0.5 * (1.0 + K.tanh( (math.sqrt(2 / math.pi) * (x + 0.044715 * K.pow(x, 3))))) return x * cdf
def k_func(x): return KK.pow(x, 3.0)