def build(self, input_shape): if not isinstance(input_shape, list) or len(input_shape) < 2: raise ValueError( 'A `Field-Wise Bi-Interaction` layer should be called ' 'on a list of at least 2 inputs') self.num_fields = len(input_shape) embedding_size = input_shape[0][-1] self.kernel_mf = self.add_weight( name='kernel_mf', shape=(int(self.num_fields * (self.num_fields - 1) / 2), 1), initializer=tf.keras.initializers.Ones(), regularizer=None, trainable=True) self.kernel_fm = self.add_weight( name='kernel_fm', shape=(self.num_fields, 1), initializer=tf.keras.initializers.Constant(value=0.5), regularizer=None, trainable=True) if self.use_bias: self.bias_mf = self.add_weight(name='bias_mf', shape=(embedding_size), initializer=Zeros()) self.bias_fm = self.add_weight(name='bias_fm', shape=(embedding_size), initializer=Zeros()) super(FieldWiseBiInteraction, self).build(input_shape) # Be sure to call this somewhere!
def init_from_pretrained_ssd(self, weights_path): self.ssd.load_weights(weights_path, by_name=True) for layer in self.ssd.layers: if layer in self.classification_headers or layer in self.regression_headers: if type(layer) == Sequential: for sub_layer in layer.layers: w = sub_layer.get_weights() if w: if len(w) == 2: # Layer has bias new_weights = [] new_weights.append(glorot_uniform()(w[0].shape)) new_weights.append(Zeros()(w[1].shape)) elif len(w) == 4: new_weights = [] new_weights.append(Ones()(w[0].shape)) new_weights.append(Zeros()(w[1].shape)) new_weights.append(Zeros()(w[2].shape)) new_weights.append(Ones()(w[3].shape)) else: new_weights = glorot_uniform()(w.shape) sub_layer.set_weights(new_weights) else: w = layer.get_weights() if len(w) > 1: # Layer has bias new_weights = [] new_weights.append(glorot_uniform()(w[0].shape)) new_weights.append(Zeros()(w[1].shape)) else: new_weights = glorot_uniform()(w.shape) layer.set_weights(new_weights)
def get_linear_logit(features, feature_columns, units=1, use_bias=False, seed=1024, prefix='linear', l2_reg=0): """ features: OrderedDict, {key: Input}形式 feature_columns: 特征。Input层转换后 """ linear_feature_columns = copy(feature_columns) for i in range(len(linear_feature_columns)): if isinstance(linear_feature_columns[i], SparseFeat): linear_feature_columns[i] = linear_feature_columns[i]._replace( embedding_dim=1, embeddings_initializer=Zeros()) if isinstance(linear_feature_columns[i], VarLenSparseFeat): linear_feature_columns[i] = linear_feature_columns[i]._replace( sparsefeat=linear_feature_columns[i].sparsefeat._replace( embedding_dim=1, embeddings_initializer=Zeros())) linear_emb_list = [ input_from_feature_columns(features, linear_feature_columns, l2_reg, seed, prefix=prefix + str(i))[0] for i in range(units) ] _, dense_input_list = input_from_feature_columns(features, linear_feature_columns, l2_reg, seed, prefix=prefix) linear_logit_list = [] for i in range(units): if len(linear_emb_list[i]) > 0 and len(dense_input_list) > 0: sparse_input = concat_func(linear_emb_list[i]) dense_input = concat_func(dense_input_list) linear_logit = Linear(l2_reg, mode=2, use_bias=use_bias, seed=seed)([sparse_input, dense_input]) elif len(linear_emb_list[i]) > 0: sparse_input = concat_func(linear_emb_list[i]) linear_logit = Linear(l2_reg, mode=0, use_bias=use_bias, seed=seed)(sparse_input) elif len(dense_input_list) > 0: dense_input = concat_func(dense_input_list) linear_logit = Linear(l2_reg, mode=1, use_bias=use_bias, seed=seed)(dense_input) else: # raise NotImplementedError return add_func([]) linear_logit_list.append(linear_logit) return concat_func(linear_logit_list)
def get_linear_logit(features, feature_columns, units=1, use_bias=False, seed=1024, prefix='linear', l2_reg=0): # units是线性logit的个数,可以在多个空间做线性logit(不同的权重),多个线性logit拼接,增加模型容量 linear_feature_columns = copy(feature_columns) for i in range(len(linear_feature_columns)): # 离散特征(单值,多值)的线性回归,这里的实现方式为长为1的embedding,等价于加权(而没有采用对输入one-hot的处理方式!!!) if isinstance(linear_feature_columns[i], SparseFeat): linear_feature_columns[i] = linear_feature_columns[i]._replace( embedding_dim=1, embeddings_initializer=Zeros()) if isinstance(linear_feature_columns[i], VarLenSparseFeat): linear_feature_columns[i] = linear_feature_columns[i]._replace( sparsefeat=linear_feature_columns[i].sparsefeat._replace( embedding_dim=1, embeddings_initializer=Zeros())) linear_emb_list = [ input_from_feature_columns(features, linear_feature_columns, l2_reg, seed, prefix=prefix + str(i))[0] for i in range(units) ] _, dense_input_list = input_from_feature_columns(features, linear_feature_columns, l2_reg, seed, prefix=prefix) linear_logit_list = [] for i in range(units): # 多个空间的logit if len(linear_emb_list[i]) > 0 and len(dense_input_list) > 0: sparse_input = concat_func(linear_emb_list[i]) dense_input = concat_func(dense_input_list) linear_logit = Linear(l2_reg, mode=2, use_bias=use_bias, seed=seed)([sparse_input, dense_input]) elif len(linear_emb_list[i]) > 0: sparse_input = concat_func(linear_emb_list[i]) linear_logit = Linear(l2_reg, mode=0, use_bias=use_bias, seed=seed)(sparse_input) elif len(dense_input_list) > 0: dense_input = concat_func(dense_input_list) linear_logit = Linear(l2_reg, mode=1, use_bias=use_bias, seed=seed)(dense_input) else: # raise NotImplementedError return add_func([]) linear_logit_list.append(linear_logit) return concat_func(linear_logit_list) # 拼接
def build(self, input_shape): self.bn = tf.keras.layers.BatchNormalization( axis=self.axis, epsilon=self.epsilon, center=False, scale=False) self.alphas = self.add_weight(shape=(input_shape[-1],), initializer=Zeros( ), dtype=tf.float32, name= 'dice_alpha') # name='alpha_'+self.name super(Dice, self).build(input_shape) # Be sure to call this somewhere! self.uses_learning_phase = True
def build(self, input_shape): self.alphas = self.add_weight(shape=(input_shape[-1], ), initializer=Zeros(), dtype=tf.float32, name=self.name + 'dice_alpha') # name='alpha_'+self.name super(Dice, self).build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape): # 这里 input_shape 是第一次运行 call() 时参数 inputs 的形状 if len(input_shape) != 3: raise ValueError( "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (len(input_shape))) embedding_size = int(input_shape[-1]) self.attention_W = self.add_weight( shape=(embedding_size, self.attention_factor), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg_w), name="attention_W") self.attention_b = self.add_weight(shape=(self.attention_factor, ), initializer=Zeros(), name="attention_b") self.projection_h = self.add_weight( shape=(self.attention_factor, 1), initializer=glorot_normal(seed=self.seed), name="projection_h") self.projection_p = self.add_weight( shape=(embedding_size, 1), initializer=glorot_normal(seed=self.seed), name="projection_p") self.dropout = tf.keras.layers.Dropout(self.dropout_rate, seed=self.seed) self.tensordot = tf.keras.layers.Lambda( lambda x: tf.tensordot(x[0], x[1], axes=(-1, 0))) # Be sure to call this somewhere! super(AFMLayer, self).build(input_shape)
def build(self, input_shape): input_size = input_shape[-1] hidden_units = [int(input_size)] + list(self.hidden_units) self.kernels = [ self.add_weight(name='kernel' + str(i), shape=(hidden_units[i], hidden_units[i + 1]), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) for i in range(len(self.hidden_units)) ] self.bias = [ self.add_weight(name='bias' + str(i), shape=(self.hidden_units[i], ), initializer=Zeros(), trainable=True) for i in range(len(self.hidden_units)) ] if self.use_bn: self.bn_layers = [ tf.keras.layers.BatchNormalization() for _ in range(len(self.hidden_units)) ] self.dropout_layers = [ tf.keras.layers.Dropout(self.dropout_rate, seed=self.seed + i) for i in range(len(self.hidden_units)) ] self.activation_layers = [tf.keras.layers.Activation(self.activation) \ for _ in range(len(self.hidden_units))] super(DNN, self).build(input_shape)
def build(self, input_shape): if self.use_bias: self.global_bias = self.add_weight( shape=(1,), initializer=Zeros(), name="global_bias") super(PredictionLayer, self).build(input_shape)
def build(self, input_shape): input_size = input_shape[-1] hidden_units = [int(input_size)] + list(self.hidden_units) # range(len(self.hidden_units)) self.kernels = [] self.bias = [] for i in range(len(self.hidden_units)): kernel = self.add_weight( name='kernel' + str(i), shape=(hidden_units[i], hidden_units[i + 1]), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True ) bias = self.add_weight( name='bias' + str(i), shape=(self.hidden_units[i],), initializer=Zeros(), trainable=True ) self.kernels.append(kernel) self.bias.append(bias) if self.use_bn: self.bn_layers = [tf.keras.layers.BatchNormalization()] * len(self.hidden_units) self.dropout_layers = [tf.keras.layers.Dropout()] * len(self.hidden_units) self.activation_layers = [activation_layer(self.activation)] * len(self.hidden_units) super(DNN, self).build(input_shape) # Be sure to call this somewhere! 传递信息?
def build(self, input_shape): if not isinstance(input_shape, list) or len(input_shape) != 2: raise ValueError('A `LocalActivationUnit` layer should be called ' 'on a list of 2 inputs') if len(input_shape[0]) != 3 or len(input_shape[1]) != 3: raise ValueError( "Unexpected inputs dimensions %d and %d, expect to be 3 dimensions" % (len(input_shape[0]), len(input_shape[1]))) if input_shape[0][-1] != input_shape[1][-1] or input_shape[0][1] != 1: raise ValueError( 'A `LocalActivationUnit` layer requires ' 'inputs of a two inputs with shape (None,1,embedding_size) and (None,T,embedding_size)' 'Got different shapes: %s,%s' % (input_shape)) size = 4 * \ int(input_shape[0][-1] ) if len(self.hidden_units) == 0 else self.hidden_units[-1] self.kernel = self.add_weight( shape=(size, 1), initializer=glorot_normal(seed=self.seed), name="kernel") self.bias = self.add_weight(shape=(1, ), initializer=Zeros(), name="bias") #self.dnn = DNN(self.hidden_units, self.activation, self.l2_reg, # self.dropout_rate, self.use_bn, seed=self.seed) super(LocalActivationUnit, self).build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape): X, A = input_shape embedding_size = int(X[-1]) self.weight = self.add_weight( name='weight', shape=[embedding_size, self.att_embedding_size * self.head_num], dtype=tf.float32, regularizer=l2(self.l2_reg), initializer=tf.keras.initializers.glorot_uniform()) self.att_self_weight = self.add_weight( name='att_self_weight', shape=[1, self.head_num, self.att_embedding_size], dtype=tf.float32, regularizer=l2(self.l2_reg), initializer=tf.keras.initializers.glorot_uniform()) self.att_neighs_weight = self.add_weight( name='att_neighs_weight', shape=[1, self.head_num, self.att_embedding_size], dtype=tf.float32, regularizer=l2(self.l2_reg), initializer=tf.keras.initializers.glorot_uniform()) if self.use_bias: self.bias_weight = self.add_weight( name='bias', shape=[1, self.head_num, self.att_embedding_size], dtype=tf.float32, initializer=Zeros()) self.in_dropout = Dropout(self.dropout_rate) self.feat_dropout = Dropout(self.dropout_rate, ) self.att_dropout = Dropout(self.dropout_rate, ) # Be sure to call this somewhere! super(GATLayer, self).build(input_shape)
def build(self, input_shapes): if self.feature_less: input_dim = int(input_shapes[0][-1]) else: assert len(input_shapes) == 2 features_shape = input_shapes[0] input_dim = int(features_shape[-1]) self.kernel = self.add_weight( shape=(input_dim, self.units), initializer=glorot_uniform(seed=self.seed), regularizer=l2(self.l2_reg), name='kernel', ) if self.use_bias: self.bias = self.add_weight( shape=(self.units, ), initializer=Zeros(), name='bias', ) self.dropout = Dropout(self.dropout_rate, seed=self.seed) self.built = True
def build(self, input_shape): # if len(self.hidden_units) == 0: # raise ValueError("hidden_units is empty") input_size = input_shape[-1] hidden_units = [int(input_size)] + list(self.hidden_units) self.kernels = [self.add_weight(name='kernel' + str(i), shape=(hidden_units[i], hidden_units[i + 1]), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) for i in range(len(self.hidden_units))] self.bias = [self.add_weight(name='bias' + str(i), shape=(self.hidden_units[i],), initializer=Zeros(), trainable=True) for i in range(len(self.hidden_units))] if self.use_bn: self.bn_layers = [tf.keras.layers.BatchNormalization() for _ in range(len(self.hidden_units))] self.dropout_layers = [tf.keras.layers.Dropout(self.dropout_rate, seed=self.seed + i) for i in range(len(self.hidden_units))] self.activation_layers = [activation_layer(self.activation) for _ in range(len(self.hidden_units))] if self.output_activation: self.activation_layers[-1] = activation_layer(self.output_activation) super(DNN, self).build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape): if self.use_bias: self.global_bias = self.add_weight( shape=(self.num_class,), initializer=Zeros(), name="global_bias") # Be sure to call this somewhere! super(PredictionLayer, self).build(input_shape)
def get_linear_logit(features, linear_feature_columns, units=1, use_bias=False, seed=1024, prefix='linear', l2_reg=0): features = features linear_feature_columns = linear_feature_columns units = 1 use_bias = False seed = 1024 prefix = 'linear' l2_reg = 0 for i in range(len(linear_feature_columns)): if linear_feature_columns[i]['feat_cat'] == 'sparse': linear_feature_columns[i]['embedding_dim'] = 3 linear_feature_columns[i]['embeddings_initializer'] = Zeros() linear_emb_list = [ input_from_feature_columns(features, linear_feature_columns, l2_reg, seed, prefix=prefix + str(i))[0] for i in range(units) ] _, dense_input_list = input_from_feature_columns(features, linear_feature_columns, l2_reg, seed, prefix=prefix) linear_logit_list = [] for i in range(units): if len(linear_emb_list[i]) > 0 and len(dense_input_list) > 0: sparse_input = concat_func(linear_emb_list[i]) dense_input = concat_func(dense_input_list) linear_logit = Linear(l2_reg, mode=2, use_bias=use_bias, seed=seed)([sparse_input, dense_input]) elif len(linear_emb_list[i]) > 0: sparse_input = concat_func(linear_emb_list[i]) linear_logit = Linear(l2_reg, mode=0, use_bias=use_bias, seed=seed)(sparse_input) elif len(dense_input_list) > 0: dense_input = concat_func(dense_input_list) linear_logit = Linear(l2_reg, mode=1, use_bias=use_bias, seed=seed)(dense_input) else: # raise NotImplementedError return add_func([]) linear_logit_list.append(linear_logit) return concat_func(linear_logit_list)
def build(self, input_shape): self.gamma = self.add_weight(name='gamma', shape=input_shape[-1:], initializer=Ones(), trainable=True) self.beta = self.add_weight(name='beta', shape=input_shape[-1:], initializer=Zeros(), trainable=True) super(LayerNormalization, self).build(input_shape)
def build(self, input_shape): if len(input_shape) != 2: raise ValueError( "Unexpected inputs dimensions %d, expect to be 2 dimensions" % (len(input_shape), )) input_dimension = int(input_shape[-1]) # Initialize expert weights (number of input features * number of units per expert * number of experts) self.expert_kernels = self.add_weight( name='expert_kernel', shape=(input_dimension, self.units, self.num_experts), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) # Initialize expert bias (number of units per expert * number of experts) self.expert_bias = self.add_weight(name='expert_bias', shape=(self.units, self.num_experts), initializer=Zeros(), trainable=True) # Initialize gate weights (number of input features * number of experts * number of tasks) self.gate_kernels = [ self.add_weight(name='gate_kernel_task_{}'.format(i), shape=(input_dimension, self.num_experts), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) for i in range(self.num_tasks) ] # Initialize gate bias (number of experts * number of tasks) self.gate_bias = [ self.add_weight(name='gate_bias_task_{}'.format(i), shape=(self.num_experts, ), initializer=Zeros(), trainable=True) for i in range(self.num_tasks) ] # Be sure to call this somewhere! super(MMoE, self).build(input_shape)
def build(self, input_shape): self.neighbor_weights = self.add_weight( shape=(self.input_dim, self.units), initializer=glorot_uniform(seed=self.seed), regularizer=l2(self.l2_reg), name="neighbor_weights") if self.use_bias: self.bias = self.add_weight(shape=(self.units), initializer=Zeros(), name="bias_weight") self.dropout = Dropout(self.dropout_rate) self.built = True
def build(self, input_shape): input_size = input_shape[-1] hidden_units = [int(input_size)] + list(self.hidden_size) self.kernels = [self.add_weight(name='kernel' + str(i), shape=(hidden_units[i], hidden_units[i+1]), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) for i in range(len(self.hidden_size))] self.bias = [self.add_weight(name='bias' + str(i), shape=(self.hidden_size[i],), initializer=Zeros(), trainable=True) for i in range(len(self.hidden_size))] super(MLP, self).build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape): if not isinstance(input_shape, list) or len(input_shape) < 2: raise ValueError('A `AttentionalFM` layer should be called ' 'on a list of at least 2 inputs') shape_set = set() reduced_input_shape = [shape.as_list() for shape in input_shape] for i in range(len(input_shape)): shape_set.add(tuple(reduced_input_shape[i])) if len(shape_set) > 1: raise ValueError('A `AttentionalFM` layer requires ' 'inputs with same shapes ' 'Got different shapes: %s' % (shape_set)) if len(input_shape[0]) != 3 or input_shape[0][1] != 1: raise ValueError('A `AttentionalFM` layer requires ' 'inputs of a list with same shape tensor like\ (None, 1, embedding_size)' 'Got different shapes: %s' % (input_shape[0])) embedding_size = int(input_shape[0][-1]) self.attention_W = self.add_weight( shape=(embedding_size, self.attention_factor), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg_w), name="attention_W") self.attention_b = self.add_weight(shape=(self.attention_factor, ), initializer=Zeros(), name="attention_b") self.projection_h = self.add_weight( shape=(self.attention_factor, 1), initializer=glorot_normal(seed=self.seed), name="projection_h") self.projection_p = self.add_weight( shape=(embedding_size, 1), initializer=glorot_normal(seed=self.seed), name="projection_p") self.dropout = tf.keras.layers.Dropout(self.dropout_rate, seed=self.seed) self.tensordot = tf.keras.layers.Lambda( lambda x: tf.tensordot(x[0], x[1], axes=(-1, 0))) # Be sure to call this somewhere! super(AFMLayer, self).build(input_shape)
def build(self, input_shape): if len(input_shape) != 2: raise ValueError("Unexpected inputs dimensions %d, expect to be 2 dimensions" % (len(input_shape),)) dim = input_shape[-1] self.kernels = [self.add_weight(name='kernel'+str(i), shape=(dim, 1), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) for i in range(self.layer_num)] self.bias = [self.add_weight(name='bias'+str(i) , shape=(dim,1), initializer=Zeros(), trainable=True) for i in range(self.layer_num)] super(CrossNet, self).build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape): self.dense_layers = [ Dense(self.input_dim, activation="relu", use_bias=True, kernel_regularizer=l2(self.l2_reg)) ] self.neighbor_weights = self.add_weight( shape=(self.input_dim * 2, self.output_dim), initializer=glorot_uniform(seed=self.seed), regularizer=l2(self.l2_reg), name="neighbor_weights") if self.use_bias: self.bias = self.add_weight(shape=(self.output_dim, ), initializer=Zeros(), name="bias_weight") self.built = True
def build(self, input_shape): if not isinstance(input_shape, list) or len(input_shape) != 2: raise ValueError('A `LocalActivationUnit` layer should be called ' 'on a list of 2 inputs') if len(input_shape[0]) != 3 or len(input_shape[1]) != 3: raise ValueError( "Unexpected inputs dimensions %d and %d, expect to be 3 dimensions" % (len(input_shape[0]), len(input_shape[1]))) if input_shape[0][-1] != input_shape[1][-1] or input_shape[0][1] != 1: raise ValueError( 'A `LocalActivationUnit` layer requires ' 'inputs of a two inputs with shape (None,1,embedding_size) and (None,T,embedding_size)' 'Got different shapes: %s,%s' % (input_shape[0], input_shape[1])) size = 4 * \ int(input_shape[0][-1] ) if len(self.hidden_units) == 0 else self.hidden_units[-1] self.kernel = self.add_weight( shape=(size, 1), initializer=glorot_normal(seed=self.seed), name="kernel") self.bias = self.add_weight(shape=(1, ), initializer=Zeros(), name="bias") self.dnn = DNN(self.hidden_units, self.activation, self.l2_reg, self.dropout_rate, self.use_bn, seed=self.seed) self.dense = tf.keras.layers.Lambda(lambda x: tf.nn.bias_add( tf.tensordot(x[0], x[1], axes=(-1, 0)), x[2])) # 理清楚tensordot和matmul之间的区别 # 1. matmul在a·b中,只能把a的最后一个维度和b的倒数第二个维度乘在一起 # 2. tensordot可以指定,其实就是把指定的维度交换到最后一个维度,然后尽心matmul # att_out和self.kernel 进行tensordot # axis作为-1和0,其实就和matmul没有任何区别 super(LocalActivationUnit, self).build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape): if not isinstance(input_shape, list) or len(input_shape) != 2: raise ValueError('A `LocalActivationUnit` layer should be called ' 'on a list of 2 inputs') if len(input_shape[0]) != 3 or len(input_shape[1]) != 3: raise ValueError( "Unexpected inputs dimensions %d and %d, expect to be 3 dimensions" % (len(input_shape[0]), len(input_shape[1]))) if input_shape[0][-1] != input_shape[1][-1] or input_shape[0][1] != 1: raise ValueError( 'A `LocalActivationUnit` layer requires ' 'inputs of a two inputs with shape (None,1,embedding_size) and (None,T,embedding_size)' 'Got different shapes: %s,%s' % (input_shape[0], input_shape[1])) # 为什么hidden_units为空时设置为embedding的四倍?因为相当于不用DNN,直接接在(原始+乘+差)后面一层 size = 4 * \ int(input_shape[0][-1] ) if len(self.hidden_units) == 0 else self.hidden_units[-1] self.kernel = self.add_weight( shape=(size, 1), initializer=glorot_normal(seed=self.seed), name="kernel") # add_weight() self.bias = self.add_weight(shape=(1, ), initializer=Zeros(), name="bias") self.dnn = DNN(self.hidden_units, self.activation, self.l2_reg, self.dropout_rate, self.use_bn, seed=self.seed) # DNN之后的一层,直接输出权重 self.dense = tf.keras.layers.Lambda(lambda x: tf.nn.bias_add( tf.tensordot(x[0], x[1], axes=(-1, 0)), x[2])) super(LocalActivationUnit, self).build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape): if len(input_shape) != 3: raise ValueError("Unexpected inputs dimensions {}, expect to be 3 dimensions".format(len(input_shape))) self.field_nums = [input_shape[1].value] for i, size in enumerate(self.layer_size): self.filters.append(self.add_weight(name='filter' + str(i), shape=[1, self.field_nums[-1] * self.field_nums[0], size], dtype=tf.float32, regularizer=l2(self.l2_reg))) self.bias.append(self.add_weight(name='bias' + str(i), shape=[size], dtype=tf.float32, initializer=Zeros())) if self.split_half: if i != len(self.layer_size) - 1 and size % 2 > 0: raise ValueError( "layer_size must be even number except for the last layer when split_half=True") self.field_nums.append(size // 2) else: self.field_nums.append(size) self.activation_layers = [Activation(self.activation) for _ in self.layer_size] super().build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape): input_size = input_shape[-1] hidden_units = [int(input_size)] + list(self.hidden_units) self.kernels = [ self.add_weight(name='kernel_{}'.format(i), shape=(hidden_units[i], hidden_units[i + 1]), initializer=glorot_uniform(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) for i in range(len(self.hidden_units)) ] self.bias = [ self.add_weight(name='bias_{}'.format(i), shape=(self.hidden_units[i], ), initializer=Zeros(), trainable=True) for i in range(len(self.hidden_units)) ] if self.use_bn: self.bn_layers = [ BatchNormalization() for _ in range(len(self.hidden_units)) ] self.dropout_layers = [ Dropout(self.dropout_rate, seed=self.seed + i) for i in range(len(self.hidden_units)) ] activations = { "leaky_relu": tf.keras.layers.LeakyReLU(), "swish": tf.nn.swish } self.activation_layer = activations[ self.activation] if self.activation in activations else Activation( self.activation) super().build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape): # if len(self.hidden_units) == 0: # raise ValueError("hidden_units is empty") input_size = input_shape[-1] hidden_units = [int(input_size)] + list(self.hidden_units) self.kernels = [ self.add_weight(name='kernel' + str(i), shape=(hidden_units[i], hidden_units[i + 1]), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) for i in range(len(self.hidden_units)) ] self.bias = [ self.add_weight(name='bias' + str(i), shape=(self.hidden_units[i], ), initializer=Zeros(), trainable=True) for i in range(len(self.hidden_units)) ] if self.use_bn: self.bn_layers = [ tf.keras.layers.BatchNormalization() for _ in range(len(self.hidden_units)) ] # AlphaDropout: 1. 均值和方差不变 2. 归一化性质也不变 self.dropout_layers = [ tf.keras.layers.AlphaDropout(self.dropout_rate, seed=self.seed + i) for i in range(len(self.hidden_units)) ] self.activation_layers = [ tf.keras.layers.Activation(self.activation) for _ in range(len(self.hidden_units)) ]
def build(self, input_shape): # if len(self.hidden_units) == 0: # raise ValueError("hidden_units is empty") input_size = input_shape[-1] print(input_size, self.udg_embedding_size) hidden_units = [int(input_size) - self.udg_embedding_size] + list( self.hidden_units) udg_units = [] for i, x in enumerate(hidden_units): tmp = [] for j in range(self.udg_embedding_layer): if j == 0: tmp.append(self.udg_embedding_size) else: tmp.append(x) udg_units.append(tmp) print(udg_units) #udg_units = [[128,384,384],[128,200,200],[128,80,80]] self.udg_kernels = [ self.add_weight(name='udg_kernel' + str(i) + '_' + str(j), shape=(udg_units[i][j], hidden_units[i]), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) for i in range(len(udg_units)) for j in range(3) ] self.udg_bias = [ self.add_weight(name='udg_bias' + str(i) + '_' + str(j), shape=(hidden_units[i], ), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) for i in range(len(udg_units)) for j in range(3) ] self.kernels = [ self.add_weight(name='kernel' + str(i), shape=(hidden_units[i], hidden_units[i + 1]), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) for i in range(len(self.hidden_units)) ] self.bias = [ self.add_weight(name='bias' + str(i), shape=(self.hidden_units[i], ), initializer=Zeros(), trainable=True) for i in range(len(self.hidden_units)) ] if self.use_bn: self.bn_layers = [ tf.keras.layers.BatchNormalization() for _ in range(len(self.hidden_units)) ] self.dropout_layers = [ tf.keras.layers.Dropout(self.dropout_rate, seed=self.seed + i) for i in range(len(self.hidden_units)) ] self.udg_activation_layers = [] for i, x in enumerate(hidden_units): for j in range(self.udg_embedding_layer): if j != self.udg_embedding_layer - 1: self.udg_activation_layers.append(activation_layer('relu')) else: self.udg_activation_layers.append( activation_layer('sigmoid')) self.activation_layers = [ activation_layer(self.activation) for _ in range(len(self.hidden_units)) ] super(DNN_UDG, self).build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape): assert len( input_shape ) >= 3, "The input Tensor should have shape=[None, input_num_capsule, input_dim_capsule]" self.input_num_capsule = int(input_shape[1]) self.input_dim_capsule = int(input_shape[2]) # transmission matrix self.reweight_W = self.add_weight( shape=[self.input_dim_capsule, self.num_capsule, self.dim_capsule], initializer=glorot_uniform(seed=self.seed), name='reweight_W') self.num_fields = self.num_capsule self.kernel_mf = self.add_weight( name='kernel_mf', shape=(int(self.num_fields * (self.num_fields - 1) / 2), 1), initializer=tf.keras.initializers.Ones(), regularizer=None, trainable=True) self.kernel_fm = self.add_weight( name='kernel_fm', shape=(self.num_fields, 1), initializer=tf.keras.initializers.Constant(value=0.5), regularizer=None, trainable=True) # self-attention self.kernel_highint = self.add_weight( name='kernel_highint', shape=(self.num_fields, 1), initializer=tf.keras.initializers.Constant(value=0.5), regularizer=None, trainable=True) self.self_attention_factor = self.dim_capsule self.self_attention_layer = 1 self.head_num = 2 # embedding_size=self.self_attention_factor * self.head_num embedding_size = self.dim_capsule self.bias_mf = self.add_weight(name='bias_mf', shape=(embedding_size), initializer=Zeros()) self.bias_fm = self.add_weight(name='bias_fm', shape=(embedding_size), initializer=Zeros()) self.routing_init = self.add_weight( name="routing_init", shape=(self.num_capsule, self.input_num_capsule), initializer=tf.keras.initializers.TruncatedNormal(seed=self.seed, stddev=10)) self.bias_highint = self.add_weight(name='bias_highint', shape=(self.self_attention_factor * self.head_num), initializer=Zeros()) self.built = True # Be sure to call this somewhere! super(CapsuleLayer, self).build(input_shape)