def get_updates(self, params, loss): grads = self.get_gradients(loss, params) shapes = [K.get_variable_shape(p) for p in params] alphas = [ K.variable(K.ones(shape) * self.init_alpha) for shape in shapes ] old_grads = [K.zeros(shape) for shape in shapes] self.weights = alphas + old_grads self.updates = [] for p, grad, old_grad, alpha in zip(params, grads, old_grads, alphas): grad = K.sign(grad) new_alpha = K.switch( K.greater(grad * old_grad, 0), K.minimum(alpha * self.scale_up, self.max_alpha), K.switch(K.less(grad * old_grad, 0), K.maximum(alpha * self.scale_down, self.min_alpha), alpha)) grad = K.switch(K.less(grad * old_grad, 0), K.zeros_like(grad), grad) new_p = p - grad * new_alpha # Apply constraints. if getattr(p, 'constraint', None) is not None: new_p = p.constraint(new_p) self.updates.append(K.update(p, new_p)) self.updates.append(K.update(alpha, new_alpha)) self.updates.append(K.update(old_grad, grad)) return self.updates
def get_updates(self, loss, params): grads = self.get_gradients(loss, params) self.updates = [K.update_add(self.iterations, 1)] lr = self.lr if self.initial_decay > 0: lr *= (1. / (1. + self.decay * self.iterations)) t = self.iterations + 1 lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t))) ms = [K.zeros(K.get_variable_shape(p), dtype=K.dtype(p)) for p in params] vs = [K.zeros(K.get_variable_shape(p), dtype=K.dtype(p)) for p in params] self.weights = [self.iterations] + ms + vs # Multiplier for weights [0,2,4,6,...] and bias [1,3,5,7,...] if len(params) != len(self.lr_multipliers) : raise Exception("Check Multipliers !") count_multipliers = 0 for p, g, m, v in zip(params, grads, ms, vs): # Multiplier for weights [0,2,4,6,...] and bias [1,3,5,7,...] if self.lr_multipliers is None: new_lr = lr_t else: new_lr = lr_t * self.lr_multipliers[count_multipliers] count_multipliers += 1 m_t = (self.beta_1 * m) + (1. - self.beta_1) * g v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g) p_t = p - new_lr * m_t / (K.sqrt(v_t) + self.epsilon) self.updates.append(K.update(m, m_t)) self.updates.append(K.update(v, v_t)) new_p = p_t # Apply constraints. if getattr(p, 'constraint', None) is not None: new_p = p.constraint(new_p) self.updates.append(K.update(p, new_p)) return self.updates
def add_weightnorm_param_updates(updates, new_V_param, new_g_param, W, V_scaler): ps = K.get_variable_shape(new_V_param) norm_axes = [i for i in range(len(ps) - 1)] # update W and V_scaler new_V_norm = tf.sqrt(tf.reduce_sum(tf.square(new_V_param), norm_axes)) new_V_scaler = new_g_param / new_V_norm new_W = tf.reshape(new_V_scaler, [1] * len(norm_axes) + [-1]) * new_V_param updates.append(K.update(W, new_W)) updates.append(K.update(V_scaler, new_V_scaler))
def get_updates(self, params, loss): grads = self.get_gradients(loss, params) self.updates = [K.update_add(self.iterations, 1)] lr = self.lr if self.inital_decay > 0: lr *= (1. / (1. + self.decay * self.iterations)) t = self.iterations + 1 lr_t = lr * K.sqrt(1. - K.pow(self.beta_2, t)) / ( 1. - K.pow(self.beta_1, t)) shapes = [K.get_variable_shape(p) for p in params] ms = [K.zeros(shape) for shape in shapes] vs = [K.zeros(shape) for shape in shapes] f = K.variable(0) d = K.variable(1) self.weights = [self.iterations] + ms + vs + [f, d] cond = K.greater(t, K.variable(1)) small_delta_t = K.switch(K.greater(loss, f), self.small_k + 1, 1. / (self.big_K + 1)) big_delta_t = K.switch(K.greater(loss, f), self.big_K + 1, 1. / (self.small_k + 1)) c_t = K.minimum(K.maximum(small_delta_t, loss / (f + self.epsilon)), big_delta_t) f_t = c_t * f r_t = K.abs(f_t - f) / (K.minimum(f_t, f)) d_t = self.beta_3 * d + (1 - self.beta_3) * r_t f_t = K.switch(cond, f_t, loss) d_t = K.switch(cond, d_t, K.variable(1.)) self.updates.append(K.update(f, f_t)) self.updates.append(K.update(d, d_t)) for p, g, m, v in zip(params, grads, ms, vs): m_t = (self.beta_1 * m) + (1. - self.beta_1) * g v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g) p_t = p - lr_t * m_t / (d_t * K.sqrt(v_t) + self.epsilon) self.updates.append(K.update(m, m_t)) self.updates.append(K.update(v, v_t)) new_p = p_t self.updates.append(K.update(p, new_p)) return self.updates
def chopraloss3(y_pred,y_true): #g_shape = tf.shape(y_pred)[1] #g_length = tf.constant(tf.round(tf.divide(g_shape,tf.constant(2)))) tf.Print(y_pred,[tf.shape(y_pred)],"shape = ") g_one, g_two = tf.split(y_pred,2,axis=0) #g_one = y_pred[:,0:g_length].eval() # g_two = y_pred[:,g_length+1:g_length*2].eval() #E_w = || G_w(X_1) - G_w(X_2) || ml = K.abs(g_one-g_two) #ml = manhattan_distance(y_pred,y_true) y = K.round(ml) l_g = 1 l_l = 1 q = upper_b thisshape = K.get_variable_shape(y_pred)#.shape ones = K.ones_like(g_one) #negy = (ones-y) part_one = ((ones-y)*(2.0/q)*K.square(ml)) part_two = (y*2*q*K.exp(-(2.77/q)*ml)) return part_one + part_two
def get_weightnorm_params_and_grads(p, g): ps = K.get_variable_shape(p) # construct weight scaler: V_scaler = g/||V|| V_scaler_shape = (ps[-1], ) # assumes we're using tensorflow! V_scaler = K.ones( V_scaler_shape) # init to ones, so effective parameters don't change # get V parameters = ||V||/g * W norm_axes = [i for i in range(len(ps) - 1)] V = p / tf.reshape(V_scaler, [1] * len(norm_axes) + [-1]) # split V_scaler into ||V|| and g parameters V_norm = tf.sqrt(tf.reduce_sum(tf.square(V), norm_axes)) g_param = V_scaler * V_norm # get grad in V,g parameters grad_g = tf.reduce_sum(g * V, norm_axes) / V_norm grad_V = tf.reshape(V_scaler, [1] * len(norm_axes) + [-1]) * \ (g - tf.reshape(grad_g / V_norm, [1] * len(norm_axes) + [-1]) * V) return V, V_norm, V_scaler, g_param, grad_g, grad_V
def perceptual_loss(y_true, y_pred): ''' Loss function to calculate 2D perceptual loss Parameters ---------- y_ture : float 4D true image numpy array (batches, xres, yres, channels) y_pred : float 4D test image numpy array (batches, xres, yres, channels) Returns ------- float RMSE between extracted perceptual features @author: Akshay Chaudhari <*****@*****.**> Copyright Subtle Medical (https://www.subtlemedical.com) Created on 2018/04/20 ''' n_batches, xres, yres, n_channels = K.get_variable_shape(y_true) vgg = VGG16(include_top=False, weights='imagenet', input_shape=(xres, yres, 3)) loss_model = models.Model(inputs=vgg.input, outputs=vgg.get_layer('block3_conv3').output) loss_model.trainable = False # Convert to a 3D image and then calculate the RMS Loss y_true_rgb = tf.image.grayscale_to_rgb(y_true / K.max(y_true), name=None) y_pred_rgb = tf.image.grayscale_to_rgb(y_pred / K.max(y_true), name=None) loss = K.mean(K.square(loss_model(y_true_rgb) - loss_model(y_pred_rgb))) return loss
def get_updates(self, loss, params): grads = self.get_gradients(loss, params) self.updates = [K.update_add(self.iterations, 1)] lr = self.lr if self.initial_decay > 0: lr *= (1. / (1. + self.decay * K.cast(self.iterations, K.floatx()))) t = K.cast(self.iterations + 1, K.floatx()) lr_t = lr * K.sqrt(1. - K.pow(self.beta_2, t)) / ( 1. - K.pow(self.beta_1, t)) shapes = [K.get_variable_shape(p) for p in params] ms = [K.zeros(shape) for shape in shapes] vs = [K.zeros(shape) for shape in shapes] self.weights = [self.iterations] + ms + vs for p, g, m, v in zip(params, grads, ms, vs): # if a weight tensor (len > 1) use weight normalized parameterization # this is the only part changed w.r.t. keras.optimizers.Adam ps = K.get_variable_shape(p) if len(ps) > 1: # get weight normalization parameters V, V_norm, V_scaler, g_param, grad_g, grad_V = get_weightnorm_params_and_grads( p, g) # Adam containers for the 'g' parameter V_scaler_shape = K.get_variable_shape(V_scaler) m_g = K.zeros(V_scaler_shape) v_g = K.zeros(V_scaler_shape) # update g parameters m_g_t = (self.beta_1 * m_g) + (1. - self.beta_1) * grad_g v_g_t = (self.beta_2 * v_g) + (1. - self.beta_2) * K.square(grad_g) new_g_param = g_param - lr_t * m_g_t / (K.sqrt(v_g_t) + self.epsilon) self.updates.append(K.update(m_g, m_g_t)) self.updates.append(K.update(v_g, v_g_t)) # update V parameters m_t = (self.beta_1 * m) + (1. - self.beta_1) * grad_V v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(grad_V) new_V_param = V - lr_t * m_t / (K.sqrt(v_t) + self.epsilon) self.updates.append(K.update(m, m_t)) self.updates.append(K.update(v, v_t)) # if there are constraints we apply them to V, not W if getattr(p, 'constraint', None) is not None: new_V_param = p.constraint(new_V_param) # wn param updates --> W updates add_weightnorm_param_updates(self.updates, new_V_param, new_g_param, p, V_scaler) else: # do optimization normally m_t = (self.beta_1 * m) + (1. - self.beta_1) * g v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g) p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon) self.updates.append(K.update(m, m_t)) self.updates.append(K.update(v, v_t)) new_p = p_t # apply constraints if getattr(p, 'constraint', None) is not None: new_p = p.constraint(new_p) self.updates.append(K.update(p, new_p)) return self.updates
def get_updates(self, loss, params): grads = self.get_gradients(loss, params) self.updates = [] lr = self.lr if self.initial_decay > 0: lr *= (1. / (1. + self.decay * K.cast(self.iterations, K.floatx()))) self.updates.append(K.update_add(self.iterations, 1)) # momentum shapes = [K.get_variable_shape(p) for p in params] moments = [K.zeros(shape) for shape in shapes] self.weights = [self.iterations] + moments for p, g, m in zip(params, grads, moments): # if a weight tensor (len > 1) use weight # normalized parameterization ps = K.get_variable_shape(p) if len(ps) > 1: # get weight normalization parameters V, V_norm, V_scaler, g_param, grad_g, grad_V = \ get_weightnorm_params_and_grads(p, g) # momentum container for the 'g' parameter V_scaler_shape = K.get_variable_shape(V_scaler) m_g = K.zeros(V_scaler_shape) # update g parameters v_g = self.momentum * m_g - lr * grad_g # velocity self.updates.append(K.update(m_g, v_g)) if self.nesterov: new_g_param = g_param + self.momentum * v_g - lr * grad_g else: new_g_param = g_param + v_g # update V parameters v_v = self.momentum * m - lr * grad_V # velocity self.updates.append(K.update(m, v_v)) if self.nesterov: new_V_param = V + self.momentum * v_v - lr * grad_V else: new_V_param = V + v_v # if there are constraints we apply them to V, not W if getattr(p, 'constraint', None) is not None: new_V_param = p.constraint(new_V_param) # wn param updates --> W updates add_weightnorm_param_updates(self.updates, new_V_param, new_g_param, p, V_scaler) else: # normal SGD with momentum v = self.momentum * m - lr * g # velocity self.updates.append(K.update(m, v)) if self.nesterov: new_p = p + self.momentum * v - lr * g else: new_p = p + v # apply constraints if getattr(p, 'constraint', None) is not None: new_p = p.constraint(new_p) self.updates.append(K.update(p, new_p)) return self.updates