def less(f, other): """Element-wise comparison applied to the `Functional` objects. # Arguments f: Functional object. other: A python number or a tensor or a functional object. # Returns A Functional. """ validate_functional(f) inputs = f.inputs.copy() if is_functional(other): inputs += to_list(other.inputs) lmbd = [ Lambda(lambda x: K.cast_to_floatx(K.less(x[0], x[1])), name=graph_unique_name("less")) for X in f.outputs ] else: _warn_for_ndarray(other) lmbd = [ Lambda(lambda x: K.cast_to_floatx(K.less(x, other)), name=graph_unique_name("less")) for X in f.outputs ] Functional = f.get_class() res = Functional(inputs=unique_tensors(inputs), outputs=_apply_operation(lmbd, f, other), layers=lmbd) return res
def call(self, x): if self.mode == MODE_VISIBLE_BERNOULLI: return K.cast( K.less( K.random_uniform(shape=(self.hps['batch_size'], x.shape[1])) #? , K.sigmoid(K.dot(x, self.rbm_weight) + self.hidden_bias))) elif self.mode == MODE_VISIBLE_GAUSSIAN: return K.cast( K.less( K.random_uniform(shape=(self.hps['batch_size'], x.shape[1])), K.relu(K.dot(x, self.rbm_weight) + self.hidden_bias))) #?
def smooth_l1(y_true, y_pred, sigma=3.0, axis=None): """Compute the smooth L1 loss of y_pred w.r.t. y_true. Args: y_true: Tensor from the generator of shape (B, N, 5). The last value for each box is the state of the anchor (ignore, negative, positive). y_pred: Tensor from the network of shape (B, N, 4). sigma: The point where the loss changes from L2 to L1. Returns: The smooth L1 loss of y_pred w.r.t. y_true. """ if axis is None: axis = 1 if K.image_data_format( ) == 'channels_first' else K.ndim(y_pred) - 1 sigma_squared = sigma**2 # compute smooth L1 loss # f(x) = 0.5 * (sigma * x)^2 if |x| < 1 / sigma / sigma # |x| - 0.5 / sigma / sigma otherwise regression_diff = K.abs(y_true - y_pred) # |y - f(x)| regression_loss = tf.where(K.less(regression_diff, 1.0 / sigma_squared), 0.5 * sigma_squared * K.pow(regression_diff, 2), regression_diff - 0.5 / sigma_squared) return K.sum(regression_loss, axis=axis)
def _kernel_constraint(self, kernel): """Radially constraints a kernel with shape (height, width, channels).""" padding = K.constant([[1, 1], [1, 1]], dtype='int32') kernel_shape = K.shape(kernel)[0] start = K.cast(kernel_shape / 2, 'int32') kernel_new = K.switch( K.cast(math_ops.floormod(kernel_shape, 2), 'bool'), lambda: kernel[start - 1:start, start - 1:start], lambda: kernel[start - 1:start, start - 1:start] + K.zeros( # pylint: disable=g-long-lambda (2, 2), dtype=kernel.dtype)) index = K.switch(K.cast(math_ops.floormod(kernel_shape, 2), 'bool'), lambda: K.constant(0, dtype='int32'), lambda: K.constant(1, dtype='int32')) while_condition = lambda index, *args: K.less(index, start) def body_fn(i, array): return i + 1, array_ops.pad(array, padding, constant_values=kernel[start + i, start + i]) _, kernel_new = control_flow_ops.while_loop( while_condition, body_fn, [index, kernel_new], shape_invariants=[ index.get_shape(), tensor_shape.TensorShape([None, None]) ]) return kernel_new
def weighted_bce_dice_loss(y_true, y_pred): y_true = K.cast(y_true, 'float32') y_pred = K.cast(y_pred, 'float32') # if we want to get same size of output, kernel size must be odd number if K.int_shape(y_pred)[1] == 128: kernel_size = 11 elif K.int_shape(y_pred)[1] == 256: kernel_size = 21 elif K.int_shape(y_pred)[1] == 512: kernel_size = 21 elif K.int_shape(y_pred)[1] == 1024: kernel_size = 41 else: raise ValueError('Unexpected image size') averaged_mask = K.pool2d(y_true, pool_size=(kernel_size, kernel_size), strides=(1, 1), padding='same', pool_mode='avg') border = K.cast(K.greater(averaged_mask, 0.005), 'float32') * K.cast( K.less(averaged_mask, 0.995), 'float32') weight = K.ones_like(averaged_mask) w0 = K.sum(weight) weight += border * 2 w1 = K.sum(weight) weight *= (w0 / w1) loss = weighted_bce_loss(y_true, y_pred, weight) + ( 1 - weighted_dice_coeff(y_true, y_pred, weight)) return loss
def smooth_l1_loss(y_true, y_pred): """Implements Smooth-L1 loss. y_true and y_pred are typically: [N, 4], but could be any shape. """ diff = K.abs(y_true - y_pred) less_than_one = K.cast(K.less(diff, 1.0), "float32") loss = (less_than_one * 0.5 * diff**2) + (1 - less_than_one) * (diff - 0.5) return loss
def __init__(self, lr_decay_schedule=None, **kwargs): super(ConditionalAdamOptimizer, self).__init__(**kwargs) self.lr_decay_schedule = lr_decay_schedule if lr_decay_schedule.startswith('dropatc'): drop_at = int(lr_decay_schedule.replace('dropatc', '')) drop_at_generator = drop_at * 1000 self.lr_decay_schedule_generator = lambda iter: tf.where( K.less(iter, drop_at_generator), 1., 0.1) else: self.lr_decay_schedule_generator = lambda iter: 1.
def custom_loss(y_true, y_pred, loss_weights = loss_weights): # Verified zero_index = K.zeros_like(y_true[:, 0]) ones_index = K.ones_like(y_true[:, 0]) # Classifier labels = y_true[:, 0] class_preds = y_pred[:, 0] bi_crossentropy_loss = -labels * K.log(class_preds) - (1 - labels) * K.log(1 - class_preds) classify_valid_index = tf.where(K.less(y_true[:, 0], 0), zero_index, ones_index) classify_keep_num = K.cast(tf.cast(tf.reduce_sum(classify_valid_index), tf.float32) * SAMPLE_KEEP_RATIO, dtype = tf.int32) # For classification problem, only pick 70% of the valid samples. classify_loss_sum = bi_crossentropy_loss * tf.cast(classify_valid_index, bi_crossentropy_loss.dtype) classify_loss_sum_filtered, _ = tf.nn.top_k(classify_loss_sum, k = classify_keep_num) classify_loss = tf.where(K.equal(classify_keep_num, 0), tf.constant(0, dtype = tf.float32), K.mean(classify_loss_sum_filtered)) # Bounding box regressor rois = y_true[:, 1: 5] roi_preds = y_pred[:, 1: 5] roi_raw_mean_square_error = K.sum(K.square(rois - roi_preds), axis = 1) # mse # roi_raw_smooth_l1_loss = K.mean(tf.where(K.abs(rois - roi_preds) < 1, 0.5 * K.square(rois - roi_preds), K.abs(rois - roi_preds) - 0.5)) # L1 Smooth Loss roi_valid_index = tf.where(K.equal(K.abs(y_true[:, 0]), 1), ones_index, zero_index) roi_keep_num = K.cast(tf.reduce_sum(roi_valid_index), dtype = tf.int32) roi_valid_mean_square_error = roi_raw_mean_square_error * tf.cast(roi_valid_index, roi_raw_mean_square_error.dtype) roi_filtered_mean_square_error, _ = tf.nn.top_k(roi_valid_mean_square_error, k = roi_keep_num) roi_loss = tf.where(K.equal(roi_keep_num, 0), tf.constant(0, dtype = tf.float32), K.mean(roi_filtered_mean_square_error)) # roi_valid_smooth_l1_loss = roi_raw_smooth_l1_loss * roi_valid_index # roi_filtered_smooth_l1_loss, _ = tf.nn.top_k(roi_valid_smooth_l1_loss, k = roi_keep_num) # roi_loss = K.mean(roi_filtered_smooth_l1_loss) # Landmark regressor pts = y_true[:, 5: 17] pt_preds = y_pred[:, 5: 17] pts_raw_mean_square_error = K.sum(K.square(pts - pt_preds), axis = 1) # mse # pts_raw_smooth_l1_loss = K.mean(tf.where(K.abs(pts - pt_preds) < 1, 0.5 * K.square(pts - pt_preds), K.abs(pts - pt_preds) - 0.5)) # L1 Smooth Loss pts_valid_index = tf.where(K.equal(y_true[:, 0], -2), ones_index, zero_index) pts_keep_num = K.cast(tf.reduce_sum(pts_valid_index), dtype = tf.int32) pts_valid_mean_square_error = pts_raw_mean_square_error * tf.cast(pts_valid_index, tf.float32) pts_filtered_mean_square_error, _ = tf.nn.top_k(pts_valid_mean_square_error, k = pts_keep_num) pts_loss = tf.where(K.equal(pts_keep_num, 0), tf.constant(0, dtype = tf.float32), K.mean(pts_filtered_mean_square_error)) # pts_valid_smooth_l1_loss = pts_raw_smooth_l1_loss * pts_valid_index # pts_filtered_smooth_l1_loss, _ = tf.nn.top_k(pts_valid_smooth_l1_loss, k = pts_keep_num) # pts_loss = K.mean(pts_filtered_smooth_l1_loss) loss = classify_loss * loss_weights[0] + roi_loss * loss_weights[1] + pts_loss * loss_weights[2] return loss
def call(self, y): # Sanity Check if isinstance(y, list): raise ValueError('TSG layer has only 1 input') # y = tf_print(y, [y], message='{}: The unconstrained action is:'.format(y.name.split('/')[0]), summarize=-1) y = check_numerics(y, 'Problem with input y') # Calculate A.c Ac = tensordot(self.A_graph, self.c_graph, 1) # Calculate b - Ac bMinusAc = self.b_graph - Ac # Calculate y - c yMinusc = y - self.c_graph # Calculate A.(y - c) ADotyMinusc = K.sum((self.A_graph * expand_dims(yMinusc, -2)), axis=2) # Do elem-wise division intersection_points = bMinusAc / (ADotyMinusc + K.epsilon() ) # Do we need the K.epsilon()? # Enforce 0 <= intersection_points <= 1 because the point must lie between c and y greater_1 = K.greater(intersection_points, K.ones_like(intersection_points)) candidate_alpha = K.switch(greater_1, K.ones_like(intersection_points) + 1, intersection_points) less_0 = K.less(candidate_alpha, K.zeros_like(intersection_points)) candidate_alpha = K.switch(less_0, K.ones_like(intersection_points) + 1, candidate_alpha) # Find farthest intersection point from y to get projection point alpha = K.min(candidate_alpha, axis=-1, keepdims=True) # If it is an interior point, y itself is the projection point interior_point = K.greater(alpha, K.ones_like(alpha)) alpha = K.switch(interior_point, K.ones_like(alpha), alpha) # alpha = tf_print(alpha, [alpha], message="{}: The value of alpha is: ".format(alpha.name.split('/')[0])) # Return \alpha.y + (1 - \alpha).c z = alpha * y + ((1 - alpha) * self.c_graph) # z = tf_print(z, [z], message='{}: The constrained action is:'.format(z.name.split('/')[0]), summarize=-1) return z
def __call__(self, *args, **kwargs): gs = tf.train.get_global_step() if gs is None: # if not set - create a variable self.global_step = K.variable(tf.zeros(shape=(), dtype=tf.int64), dtype=tf.int64, name="lr_global_step") tf.train.global_step(K.get_session(), self.global_step) gs = K.update_add(self.global_step, 1) ###tf.train.get_global_step() else: self.global_step = gs assert (gs is not None) gstep = tf.cast(gs, dtype=tf.float32) lr_up = K.exp(self.step_accelerate_log * gstep) * self.min_lr lr_down = K.exp(self.step_deccelerate_log * (gstep - self.step_max_lr)) * self.max_lr lr = K.switch(K.less(gs, self.step_max_lr), lr_up, lr_down) if self.tensorboardimage and not self.added_scalar_to_tensorboard: self.tensorboardimage.add_scalar("learning_rate", lr) self.added_scalar_to_tensorboard = True # add once return lr
def build(self, input_shape): self.rbm_weight = self.add_weight( name='rbm_weight', shape=(input_shape[1], self.output_dim), initializer='uniform' # Which initializer is optimal? , trainable=True) self.hidden_bias = self.add_weight(name='rbm_hidden_bias', shape=(self.output_dim, ), initializer='uniform', trainable=True) self.visible_bias = K.variable(initializers.get('uniform')( (input_shape[1], )), dtype=K.floatx(), name='rbm_visible_bias') # Make symbolic computation objects. if self.mode == MODE_VISIBLE_BERNOULLI: # Transform visible units. self.input_visible = K.placeholder(shape=(None, input_shape[1]), name='input_visible') self.transform = K.cast( K.less( K.random_uniform(shape=(self.hps['batch_size'], input_shape[1])), K.sigmoid( K.dot(self.input_visible, self.rbm_weight) + self.hidden_bias))) self.transform_func = K.function([self.input_visible], [self.transform]) # Transform hidden units. self.input_hidden = K.placeholder(shape=(None, self.output_dim), name='input_hidden') self.inv_transform = K.cast( K.less( K.random_uniform(shape=(self.hps['batch_size'], input_shape[1])), K.sigmoid( K.dot(self.input_hidden, K.transpose(self.rbm_weight)) + self.visible_bias))) self.inv_transform_func = K.function([self.input_hidden], [self.inv_transform]) elif self.mode == MODE_VISIBLE_GAUSSIAN: # Transform visible units. self.input_visible = K.placeholder(shape=(None, input_shape[1]), name='input_visible') self.transform = K.cast( K.less( K.random_uniform(shape=(self.hps['batch_size'], input_shape[1])), K.relu( K.dot(self.input_visible, self.rbm_weight) + self.hidden_bias))) #? self.transform_func = K.function([self.input_visible], [self.transform]) # Transform hidden units. self.input_hidden = K.placeholder(shape=(None, self.output_dim), name='input_hidden') self.inv_transform = Ke.multivariate_normal_diag( loc=(K.dot(self.input_hidden, K.transpose(self.rbm_weight)) + self.visible_bias), scale_diag=np.ones(shape=(self.hps['batch_size'], input_shape[1]))).sample() self.inv_transform_func = K.function([self.input_hidden], [self.inv_transform]) else: # TODO pass # Calculate free energy. #? self.free_energy = -1 * (K.squeeze(K.dot(self.input_visible, K.expand_dims(self.visible_bias, axis=-1)), -1) +\ K.sum(K.log(1 + K.exp(K.dot(self.input_visible, self.rbm_weight) +\ self.hidden_bias)), axis=-1)) self.free_energy_func = K.function([self.input_visible], [self.free_energy]) super(RBM, self).build(input_shape)
def fit(self, V, verbose=1): """Train RBM with the data V. Parameters ---------- V : 2d numpy array Visible data (batch size x input_dim). verbose : integer Verbose mode (default, 1). """ num_step = V.shape[0] // self.hps['batch_size'] \ if V.shape[0] % self.hps['batch_size'] == 0 else V.shape[0] // self.hps['batch_size'] + 1 # Exception processing? for k in range(self.hps['epochs']): if verbose == 1: print(k + 1, '/', self.hps['epochs'], ' epochs', end='\r') if self.mode == MODE_VISIBLE_BERNOULLI: # Contrastive divergence. v_pos = self.input_visible h_pos = self.transform v_neg = K.cast(K.less( K.random_uniform(shape=(self.hps['batch_size'], V.shape[1])), K.sigmoid( K.dot(h_pos, K.transpose(self.rbm_weight)) + self.visible_bias)), dtype=np.float32) h_neg = K.sigmoid( K.dot(v_neg, self.rbm_weight) + self.hidden_bias) update = K.transpose(K.transpose(K.dot(K.transpose(v_pos), h_pos)) \ - K.dot(K.transpose(h_neg), v_neg)) self.rbm_weight_update_func = K.function( [self.input_visible], [K.update_add(self.rbm_weight, self.hps['lr'] * update)]) self.hidden_bias_update_func = K.function([self.input_visible] , [K.update_add(self.hidden_bias, self.hps['lr'] \ * (K.sum(h_pos, axis=0) - K.sum(h_neg, axis=0)))]) self.visible_bias_update_func = K.function([self.input_visible] , [K.update_add(self.visible_bias, self.hps['lr'] \ * (K.sum(v_pos, axis=0) - K.sum(v_neg, axis=0)))]) # Create the fist visible nodes sampling object. self.sample_first_visible = K.function([self.input_visible], [v_neg]) elif self.mode == MODE_VISIBLE_GAUSSIAN: # Contrastive divergence. v_pos = self.input_visible h_pos = self.transform v_neg = Ke.multivariate_normal_diag( loc=(K.dot(h_pos, K.transpose(self.rbm_weight)) + self.visible_bias), scale_diag=np.ones(shape=(self.hps['batch_size'], V.shape[1]))).sample() h_neg = K.sigmoid( K.dot(v_neg, self.rbm_weight) + self.hidden_bias) update = K.transpose(K.transpose(K.dot(K.transpose(v_pos), h_pos)) \ - K.dot(K.transpose(h_neg), v_neg)) self.rbm_weight_update_func = K.function( [self.input_visible], [K.update_add(self.rbm_weight, self.hps['lr'] * update)]) self.hidden_bias_update_func = K.function([self.input_visible] , [K.update_add(self.hidden_bias, self.hps['lr'] \ * (K.sum(h_pos, axis=0) - K.sum(h_neg, axis=0)))]) self.visible_bias_update_func = K.function([self.input_visible] , [K.update_add(self.visible_bias, self.hps['lr'] \ * (K.sum(v_pos, axis=0) - K.sum(v_neg, axis=0)))]) # Create the fist visible nodes sampling object. self.sample_first_visible = K.function([self.input_visible], [v_neg]) else: pass for i in range(num_step): if i == (num_step - 1): if self.mode == MODE_VISIBLE_BERNOULLI: # Contrastive divergence. v_pos = self.input_visible h_pos = self.transform v_neg = K.cast(K.less( K.random_uniform(shape=( V.shape[0] - int(i * self.hps['batch_size'], V.shape[1]))), K.sigmoid( K.dot(h_pos, K.transpose(self.rbm_weight)) + self.visible_bias)), dtype=np.float32) h_neg = K.sigmoid( K.dot(v_neg, self.rbm_weight) + self.hidden_bias) update = K.transpose(K.transpose(K.dot(K.transpose(v_pos), h_pos)) \ - K.dot(K.transpose(h_neg), v_neg)) self.rbm_weight_update_func = K.function( [self.input_visible], [ K.update_add(self.rbm_weight, self.hps['lr'] * update) ]) self.hidden_bias_update_func = K.function([self.input_visible] , [K.update_add(self.hidden_bias, self.hps['lr'] \ * (K.sum(h_pos, axis=0) - K.sum(h_neg, axis=0)))]) self.visible_bias_update_func = K.function([self.input_visible] , [K.update_add(self.visible_bias, self.hps['lr'] \ * (K.sum(v_pos, axis=0) - K.sum(v_neg, axis=0)))]) # Create the fist visible nodes sampling object. self.sample_first_visible = K.function( [self.input_visible], [v_neg]) elif self.mode == MODE_VISIBLE_GAUSSIAN: # Contrastive divergence. v_pos = self.input_visible h_pos = self.transform v_neg = Ke.multivariate_normal_diag( loc=(K.dot(h_pos, K.transpose(self.rbm_weight)) + self.visible_bias), scale_diag=np.ones(shape=( V.shape[0] - int(i * self.hps['batch_size'], V.shape[1]) ))).sample() h_neg = K.sigmoid( K.dot(v_neg, self.rbm_weight) + self.hidden_bias) update = K.transpose(K.transpose(K.dot(K.transpose(v_pos), h_pos)) \ - K.dot(K.transpose(h_neg), v_neg)) self.rbm_weight_update_func = K.function( [self.input_visible], [ K.update_add(self.rbm_weight, self.hps['lr'] * update) ]) self.hidden_bias_update_func = K.function([self.input_visible] , [K.update_add(self.hidden_bias, self.hps['lr'] \ * (K.sum(h_pos, axis=0) - K.sum(h_neg, axis=0)))]) self.visible_bias_update_func = K.function([self.input_visible] , [K.update_add(self.visible_bias, self.hps['lr'] \ * (K.sum(v_pos, axis=0) - K.sum(v_neg, axis=0)))]) # Create the fist visible nodes sampling object. self.sample_first_visible = K.function( [self.input_visible], [v_neg]) else: pass V_batch = [V[int(i * self.hps['batch_size']):V.shape[0]]] # Train. self.rbm_weight_update_func(V_batch) self.hidden_bias_update_func(V_batch) self.visible_bias_update_func(V_batch) else: V_batch = [ V[int(i * self.hps['batch_size']):int( (i + 1) * self.hps['batch_size'])] ] # Train. self.rbm_weight_update_func(V_batch) self.hidden_bias_update_func(V_batch) self.visible_bias_update_func(V_batch) # Calculate a training score by each step. # Free energy of the input visible nodes. fe = self.cal_free_energy(V_batch) # Free energy of the first sampled visible nodes. V_p_batch = self.sample_first_visible(V_batch) fe_p = self.cal_free_energy(V_p_batch) score = np.mean(np.abs(fe[0] - fe_p[0])) # Scale? print('\n{0:d}/{1:d}, score: {2:f}'.format( i + 1, num_step, score))