def call(self, inputs, **kwargs): x = self.conv1_1(inputs) x = self.norm1_1(x) x = self.conv1_2(x) x = self.norm1_2(x) x = self.maxpool1(x) skip = self.conv2_skip(x) x = self.conv2_1(x) x = self.norm2_1(x) x = self.conv2_2(x) x = self.norm2_2(x) x = K.relu(skip + x) x = self.maxpool2(x) skip = self.conv3_skip(x) x = self.conv3_1(x) x = self.norm3_1(x) x = self.conv3_2(x) x = self.norm3_2(x) x = K.relu(skip + x) x = self.maxpool3(x) skip = self.conv4_skip(x) x = self.conv4_1(x) x = self.norm4_1(x) x = self.conv4_2(x) x = self.norm4_2(x) x = K.relu(skip + x) x = self.maxpool4(x) return x
def call(self, inputs, mask=None): pos = K.relu(inputs) if K.backend() == 'theano': neg = (K.pattern_broadcast(self.alpha, self.param_broadcast) * (inputs - math_ops.abs(inputs)) * 0.5) else: neg = -self.alpha * K.relu(-inputs) return pos + neg
def call(self, inputs, mask=None): pos = K.relu(inputs) if K.backend() == 'theano': neg = ( K.pattern_broadcast(self.alpha, self.param_broadcast) * (inputs - math_ops.abs(inputs)) * 0.5) else: neg = -self.alpha * K.relu(-inputs) return pos + neg
def discriminative_instance_loss(y_true, y_pred, delta_v=0.5, delta_d=1.5, gamma=1e-3): """Discriminative loss between an output tensor and a target tensor. Args: y_true: A tensor of the same shape as y_pred. y_pred: A tensor of the vector embedding Returns: tensor: Output tensor. """ def temp_norm(ten, axis=None): if axis is None: axis = 1 if K.image_data_format( ) == 'channels_first' else K.ndim(ten) - 1 return K.sqrt(K.epsilon() + K.sum(K.square(ten), axis=axis)) rank = K.ndim(y_pred) channel_axis = 1 if K.image_data_format() == 'channels_first' else rank - 1 axes = [x for x in list(range(rank)) if x != channel_axis] # Compute variance loss cells_summed = tf.tensordot(y_true, y_pred, axes=[axes, axes]) n_pixels = K.cast(tf.count_nonzero(y_true, axis=axes), dtype=K.floatx()) + K.epsilon() n_pixels_expand = K.expand_dims(n_pixels, axis=1) + K.epsilon() mu = tf.divide(cells_summed, n_pixels_expand) delta_v = K.constant(delta_v, dtype=K.floatx()) mu_tensor = tf.tensordot(y_true, mu, axes=[[channel_axis], [0]]) L_var_1 = y_pred - mu_tensor L_var_2 = K.square(K.relu(temp_norm(L_var_1) - delta_v)) L_var_3 = tf.tensordot(L_var_2, y_true, axes=[axes, axes]) L_var_4 = tf.divide(L_var_3, n_pixels) L_var = K.mean(L_var_4) # Compute distance loss mu_a = K.expand_dims(mu, axis=0) mu_b = K.expand_dims(mu, axis=1) diff_matrix = tf.subtract(mu_b, mu_a) L_dist_1 = temp_norm(diff_matrix) L_dist_2 = K.square( K.relu(K.constant(2 * delta_d, dtype=K.floatx()) - L_dist_1)) diag = K.constant(0, dtype=K.floatx()) * tf.diag_part(L_dist_2) L_dist_3 = tf.matrix_set_diag(L_dist_2, diag) L_dist = K.mean(L_dist_3) # Compute regularization loss L_reg = gamma * temp_norm(mu) L = L_var + L_dist + K.mean(L_reg) return L
def call(self, inputs, **kwargs): inputs_real, inputs_imag = tf.math.real(inputs), tf.math.imag(inputs) real = K.relu(inputs_real, max_value=self.max_value_real, threshold=self.threshold_real, alpha=self.alpha_real) imag = K.relu(inputs_imag, max_value=self.max_value_imag, threshold=self.threshold_imag, alpha=self.alpha_imag) return tf.complex(real, imag)
def call(self, inputs): # alpha is used for leaky relu slope in activations instead of # negative_slope. return K.relu(inputs, alpha=self.negative_slope, max_value=self.max_value, threshold=self.threshold)
def call(self, inputs): # alpha is used for leaky relu slope in activations instead of # negative_slope. return K.relu(inputs, alpha=self.negative_slope, max_value=self.max_value, threshold=self.threshold)
def relu(x, alpha=0., max_value=None, threshold=0): """Rectified Linear Unit. The Rectified Linear Unit function is: x , if x>0 else 0 (for default parameter values) In the case that parameters are given values: `f(x) = max_value` for `x >= max_value`, `f(x) = x` for `threshold <= x < max_value`, `f(x) = alpha * (x - threshold)` otherwise. Example usage: ```python3 model.add(Convolution2D(20,kernel_size = 5, padding = 'same', input_shape = input_shape)) model.add(Activation("relu")) ``` Arguments: x: A tensor or variable. alpha: A scalar, slope of negative section (default=`0.`). max_value: float. Saturation threshold(upper limit) (default = `None`). threshold: float. Threshold value(lower limit) for thresholded activation. Returns: A tensor of the same shape as that of input x each element of which has undergone the ReLU activation. """ return K.relu(x, alpha=alpha, max_value=max_value, threshold=threshold)
def true_branch_renorm(): # We apply epsilon as part of the moving_stddev to mirror the training # code path. moving_stddev = _do_update(self.moving_stddev, math_ops.sqrt(new_variance + self.epsilon)) return self._assign_new_value( self.moving_variance, # Apply relu in case floating point rounding causes it to go # negative. K.relu(moving_stddev * moving_stddev - self.epsilon))
def get_updates(self, loss, params): grads = self.get_gradients(loss, params) self.updates = [K.update_add(self.iterations, 1)] lr = self.lr if self.initial_decay > 0: lr = lr * (1. / (1. + self.decay * K.cast(self.iterations, K.dtype(self.decay)))) t = K.cast(self.iterations, K.floatx()) + 1 beta_1_t = K.pow(self.beta_1, t) beta_2_t = K.pow(self.beta_2, t) rho = 2 / (1 - self.beta_2) - 1 rho_t = rho - 2 * t * beta_2_t / (1 - beta_2_t) r_t = K.sqrt( K.relu(rho_t - 4) * K.relu(rho_t - 2) * rho / ((rho - 4) * (rho - 2) * rho_t)) flag = K.cast(rho_t > 4, K.floatx()) ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] self.weights = [self.iterations] + ms + vs for p, g, m, v in zip(params, grads, ms, vs): m_t = (self.beta_1 * m) + (1. - self.beta_1) * g v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g) mhat_t = m_t / (1 - beta_1_t) vhat_t = K.sqrt(v_t / (1 - beta_2_t)) p_t = p - lr * mhat_t * (flag * r_t / (vhat_t + self.epsilon) + (1 - flag)) self.updates.append(K.update(m, m_t)) self.updates.append(K.update(v, v_t)) new_p = p_t # Apply constraints. if getattr(p, 'constraint', None) is not None: new_p = p.constraint(new_p) self.updates.append(K.update(p, new_p)) return self.updates
def call(self, x, mask=None): x = K.expand_dims(x, axis=2) x = tf.nn.separable_conv2d(x, self.depthwise_w, self.pointwise_w, strides=(1, 1, 1, 1), padding="SAME") x += self.bias x = K.relu(x) outputs = K.squeeze(x, axis=2) return outputs
def get_pairwise_model(self): if self.pairwise_model is None: if new_Para.param.pairwise and self.model is not None: # 如果使用pairwise型的目标函数 mashup_id_input = Input(shape=(1, ), dtype='int32', name='mashup_id_input') api_id_input = Input(shape=(1, ), dtype='int32', name='api_id_input') neg_api_id_input = Input(shape=(1, ), dtype='int32', name='neg_api_id_input') mashup_slt_apis_input = Input( shape=(new_Para.param.slt_item_num, ), dtype='int32', name='slt_api_ids_input') if self.old_new == 'new': pos_ratings = self.model( [mashup_id_input, api_id_input, mashup_slt_apis_input]) neg_ratings = self.model([ mashup_id_input, neg_api_id_input, mashup_slt_apis_input ]) # 再加一个负例api id elif self.old_new == 'old': pos_ratings = self.model([mashup_id_input, api_id_input]) neg_ratings = self.model( [mashup_id_input, neg_api_id_input]) loss = Lambda( lambda x: K.relu(new_Para.param.margin + x[0] - x[1]), name='sub_result')([neg_ratings, pos_ratings]) # 注意输入格式! if self.old_new == 'new': self.pairwise_model = Model(inputs=[ mashup_id_input, api_id_input, mashup_slt_apis_input, neg_api_id_input ], outputs=loss) elif self.old_new == 'old': self.pairwise_model = Model(inputs=[ mashup_id_input, api_id_input, neg_api_id_input ], outputs=loss) for layer in self.pairwise_model.layers: print(layer.name) # # 复用的是同一个对象! # print(self.pairwise_model.get_layer('predict_model'),id(self.pairwise_model.get_layer('predict_model'))) # print(self.model,id(self.model)) return self.pairwise_model
def relu(x, alpha=0., max_value=None): """Rectified Linear Unit. Arguments: x: Input tensor. alpha: Slope of the negative part. Defaults to zero. max_value: Maximum value for the output. Returns: The (leaky) rectified linear unit activation: `x` if `x > 0`, `alpha * x` if `x < 0`. If `max_value` is defined, the result is truncated to this value. """ return K.relu(x, alpha=alpha, max_value=max_value)
def call(self, x): if self.mode == MODE_VISIBLE_BERNOULLI: return K.cast( K.less( K.random_uniform(shape=(self.hps['batch_size'], x.shape[1])) #? , K.sigmoid(K.dot(x, self.rbm_weight) + self.hidden_bias))) elif self.mode == MODE_VISIBLE_GAUSSIAN: return K.cast( K.less( K.random_uniform(shape=(self.hps['batch_size'], x.shape[1])), K.relu(K.dot(x, self.rbm_weight) + self.hidden_bias))) #?
def variance_update(): """Update self.moving_variance with the most recent data point.""" if self.renorm: # We apply epsilon as part of the moving_stddev to mirror the training # code path. moving_stddev = self._assign_moving_average( self.moving_stddev, math_ops.sqrt(variance + self.epsilon), momentum, inputs_size) return self._assign_new_value( self.moving_variance, # Apply relu in case floating point rounding causes it to go # negative. K.relu(moving_stddev * moving_stddev - self.epsilon)) else: return self._assign_moving_average(self.moving_variance, variance, momentum, inputs_size)
def call(self, x): row = [] col = [] # 对特征进行两两组合 for r, c in combinations(x, 2): # [field * (field - 1)] / 2 row.append(r) col.append(c) p = K.concatenate( row, axis=1) # [batch_size, [field * (field - 1)] / 2, embedding_size] q = K.concatenate(col, axis=1) inner_product = p * q # 对应元素相乘 # 添加非线性, 进行激活 attention_tmp = K.relu( K.bias_add(K.dot(inner_product, self.attention_W), self.attention_b)) # [batch_size, [field * (field - 1)] / 2, embedding_size] * [embedding_size, attention_units] = > [batch_size, [field * (field - 1)] / 2, attention_units] # context 向量 attention_tmp_dot = K.dot( attention_tmp, self.projection_h) # [batch_size, [field * (field - 1)] / 2, 1] # 计算的是一个样本的sofmax, sum的是一个样本的所有特征 attention_weight = K.softmax( attention_tmp_dot, axis=1 ) # 等价于 K.exp(attention_tmp_dot) / K.sum(attention_tmp_dot, axis=1, keepdims=True) # [batch_size, [field * (field - 1)] / 2, 1] # 权重乘以内积 attention_output = K.sum(inner_product * attention_weight, axis=1) # [batch_size, embedding_size] # 经过dropout操作 attention_output = K.dropout( attention_output, self.dropout_rate) # [batch_size, embedding_size] # 等价于dense层 afm_out = K.dot(attention_output, self.projection_p) # [batch_size, 1] return afm_out
def call(self, inputs, mask=None): cos_m = math.cos(self.m) sin_m = math.sin(self.m) mm = sin_m * self.m threshold = math.cos(math.pi - self.m) # features X = inputs[0] # 1-D or one-hot label works as mask Y_mask = inputs[1] # If Y_mask is not in one-hot form, transfer it to one-hot form. if Y_mask.shape[-1] == 1: Y_mask = K.cast(Y_mask, tf.int32) Y_mask = K.reshape(K.one_hot(Y_mask, self.class_num), (-1, self.class_num)) X_normed = K.l2_normalize(X, axis=1) # L2 Normalized X W_normed = K.l2_normalize(self.W, axis=0) # L2 Normalized Weights # cos(theta + m) cos_theta = K.dot(X_normed, W_normed) # 矩阵乘法 cos_theta2 = K.square(cos_theta) sin_theta2 = 1. - cos_theta2 sin_theta = K.sqrt(sin_theta2 + K.epsilon()) cos_tm = self.s * ((cos_theta * cos_m) - (sin_theta * sin_m)) # This condition controls the theta + m should in range [0, pi] # 0 <= theta + m < = pi # -m <= theta <= pi - m cond_v = cos_theta - threshold cond = K.cast(K.relu(cond_v), dtype=tf.bool) keep_val = self.s * (cos_theta - mm) cos_tm_temp = tf.where(cond, cos_tm, keep_val) # mask by label # Y_mask =+ K.epsilon() # Why??? inv_mask = 1. - Y_mask s_cos_theta = self.s * cos_theta output = K.softmax((s_cos_theta * inv_mask) + (cos_tm_temp * Y_mask)) return output
def relu(x, alpha=0., max_value=None, threshold=0): """Rectified Linear Unit. With default values, it returns element-wise `max(x, 0)`. Otherwise, it follows: `f(x) = max_value` for `x >= max_value`, `f(x) = x` for `threshold <= x < max_value`, `f(x) = alpha * (x - threshold)` otherwise. Arguments: x: A tensor or variable. alpha: A scalar, slope of negative section (default=`0.`). max_value: float. Saturation threshold. threshold: float. Threshold value for thresholded activation. Returns: A tensor. """ return K.relu(x, alpha=alpha, max_value=max_value, threshold=threshold)
def relu(x, alpha=0., max_value=None, threshold=0): """Rectified Linear Unit. With default values, it returns element-wise `max(x, 0)`. Otherwise, it follows: `f(x) = max_value` for `x >= max_value`, `f(x) = x` for `threshold <= x < max_value`, `f(x) = alpha * (x - threshold)` otherwise. Arguments: x: A tensor or variable. alpha: A scalar, slope of negative section (default=`0.`). max_value: float. Saturation threshold. threshold: float. Threshold value for thresholded activation. Returns: A tensor. """ return K.relu(x, alpha=alpha, max_value=max_value, threshold=threshold)
def call(self, inputs, training=None, mask=None): identityInput = K.identity(inputs) stacks = K.array_ops.unstack(inputs, axis=1) ### [b,n] tempList = [] for oneTimeStepTensor in stacks: ### [b,1] bTensor = self.dense0(oneTimeStepTensor) bTensor = K.relu(bTensor) bTensor = self.dense1(bTensor) tempList.append(bTensor) ### [b,t,1] stackedTensor = K.array_ops.stack(tempList, axis=1) softMaxTensor = K.softmax(stackedTensor, axis=1) ###[b,t,1] weightEdTensor = K.math_ops.multiply(identityInput, softMaxTensor) unstack = K.array_ops.unstack(weightEdTensor, axis=1) temp1List = [] for oneTensor in unstack: temp1List.append(oneTensor) return K.math_ops.add_n(temp1List)
def relu(x, alpha=0., max_value=None, threshold=0): """Applies the rectified linear unit activation function. With default values, this returns the standard ReLU activation: `max(x, 0)`, the element-wise maximum of 0 and the input tensor. Modifying default parameters allows you to use non-zero thresholds, change the max value of the activation, and to use a non-zero multiple of the input for values below the threshold. For example: >>> foo = tf.constant([-10, -5, 0.0, 5, 10], dtype = tf.float32) >>> tf.keras.activations.relu(foo).numpy() array([ 0., 0., 0., 5., 10.], dtype=float32) >>> tf.keras.activations.relu(foo, alpha=0.5).numpy() array([-5. , -2.5, 0. , 5. , 10. ], dtype=float32) >>> tf.keras.activations.relu(foo, max_value=5).numpy() array([0., 0., 0., 5., 5.], dtype=float32) >>> tf.keras.activations.relu(foo, threshold=5).numpy() array([-0., -0., 0., 0., 10.], dtype=float32) Args: x: Input `tensor` or `variable`. alpha: A `float` that governs the slope for values lower than the threshold. max_value: A `float` that sets the saturation threshold (the largest value the function will return). threshold: A `float` giving the threshold value of the activation function below which values will be damped or set to zero. Returns: A `Tensor` representing the input tensor, transformed by the relu activation function. Tensor will be of the same shape and dtype of input `x`. """ return backend.relu(x, alpha=alpha, max_value=max_value, threshold=threshold)
def call(self, inputs, mask=None): return K.relu(inputs, max_value=self.alpha)
def call(self, inputs): return K.relu(inputs, alpha=self.alpha)
def call(self, inputs): return K.relu(inputs) + self.bias
def call(self, inputs, mask=None): pos = K.relu(inputs) neg = -self.alpha * K.relu(-inputs) return pos + neg
def relu(x, alpha=0., max_value=None): return K.relu(x, alpha=alpha, max_value=max_value)
def call(self, inputs): pos = K.relu(inputs) neg = -self.alpha * K.relu(-inputs) return pos + neg
def call(self, inputs): pos = backend.relu(inputs) neg = -self.alpha * backend.relu(-inputs) return pos + neg
def call(self, inputs): return K.relu(inputs, alpha=self.alpha)
def bounded_relu(x): return K.relu(x, max_value=1)
def relu6(x): return K.relu(x, max_value=6)
def relu6(x): return K.relu(x, max_value=6)
def antirectifier(x): x -= K.mean(x, axis=1, keepdims=True) x = K.l2_normalize(x, axis=1) pos = K.relu(x) neg = K.relu(-x) return K.concatenate([pos, neg], axis=1)
def relu6(x, alpha=0., max_value=None, threshold=0): return K.relu(x, alpha=alpha, max_value=6, threshold=threshold)
def call(self, inputs): neg = -self.alpha * K.relu(-inputs + self.threshold) pos = self.beta * K.relu(inputs - self.threshold) return pos + neg + self.bias
def relu(x, alpha=0., max_value=None): return K.relu(x, alpha=alpha, max_value=max_value)
def call(self, inputs): return backend.relu(inputs, alpha=self.alpha)