def build(self, input_shape): if not isinstance(input_shape, list) or len(input_shape) < 2: raise ValueError('A `AttentionalFM` layer should be called ' 'on a list of at least 2 inputs') embedding_size = int(input_shape[0][-1]) if self.bilinear_type == "all": self.W = self.add_weight(shape=(embedding_size, embedding_size), initializer=glorot_normal(seed=self.seed), name="bilinear_weight") elif self.bilinear_type == "each": self.W_list = [ self.add_weight(shape=(embedding_size, embedding_size), initializer=glorot_normal(seed=self.seed), name="bilinear_weight" + str(i)) for i in range(len(input_shape) - 1) ] elif self.bilinear_type == "interaction": self.W_list = [ self.add_weight(shape=(embedding_size, embedding_size), initializer=glorot_normal(seed=self.seed), name="bilinear_weight" + str(i) + '_' + str(j)) for i, j in itertools.combinations(range(len(input_shape)), 2) ] else: raise NotImplementedError super(BilinearInteraction, self).build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape): # input_shape: [None, field_num, embed_num] self.field_size = input_shape[1] self.embedding_size = input_shape[-1] if self.bilinear_type == "all": # 所有embedding矩阵共用一个矩阵W self.W = self.add_weight(shape=(self.embedding_size, self.embedding_size), initializer=glorot_normal(seed=self.seed), name="bilinear_weight") elif self.bilinear_type == "each": # 每个field共用一个矩阵W self.W_list = [ self.add_weight(shape=(self.embedding_size, self.embedding_size), initializer=glorot_normal(seed=self.seed), name="bilinear_weight" + str(i)) for i in range(self.field_size - 1) ] elif self.bilinear_type == "interaction": # 每个交互用一个矩阵W self.W_list = [ self.add_weight(shape=(self.embedding_size, self.embedding_size), initializer=glorot_normal(seed=self.seed), name="bilinear_weight" + str(i) + '_' + str(j)) for i, j in itertools.combinations(range(self.field_size), 2) ] else: raise NotImplementedError super(BilinearInteraction, self).build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape): # 这里 input_shape 是第一次运行 call() 时参数 inputs 的形状 if len(input_shape) != 3: raise ValueError( "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (len(input_shape))) embedding_size = int(input_shape[-1]) self.attention_W = self.add_weight( shape=(embedding_size, self.attention_factor), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg_w), name="attention_W") self.attention_b = self.add_weight(shape=(self.attention_factor, ), initializer=Zeros(), name="attention_b") self.projection_h = self.add_weight( shape=(self.attention_factor, 1), initializer=glorot_normal(seed=self.seed), name="projection_h") self.projection_p = self.add_weight( shape=(embedding_size, 1), initializer=glorot_normal(seed=self.seed), name="projection_p") self.dropout = tf.keras.layers.Dropout(self.dropout_rate, seed=self.seed) self.tensordot = tf.keras.layers.Lambda( lambda x: tf.tensordot(x[0], x[1], axes=(-1, 0))) # Be sure to call this somewhere! super(AFMLayer, self).build(input_shape)
def build(self, input_shape): if len(input_shape) < 4 or input_shape[-1] != 1: raise ValueError( 'A `SincConv` layer should be called on a 4D tensor ' 'with shape:``(batch_size,1,channel_size,sample_size)') # initialize filterbanks such that they are equally spaced in Mel scale low_hz = 4 high_hz = self.sample_rate / 2 - (self.min_low_hz + self.min_band_hz) mel = np.linspace(self.to_mel(low_hz), self.to_mel(high_hz), self.filters + 1) hz = self.to_hz(mel) # filter lower frequency (filters, 1) self.low_hz_ = self.add_weight( name='low_hz', shape=(1, self.filters), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) self.band_hz_ = self.add_weight( name='band_hz', shape=(1, self.filters), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) # filter lower frequency (1, filters) self.low_hz_.assign( tf.cast(tf.reshape(hz[:-1], [1, -1]), dtype=tf.float32)) # filter frequency band (1, filters) self.band_hz_.assign( tf.cast(tf.reshape(np.diff(hz), [1, -1]), dtype=tf.float32)) # Hamming window # computing only half of the window n_lin = tf.linspace(0., (self.kernel_size / 2) - 1, num=int(self.kernel_size / 2)) self.window_ = tf.reshape( 0.54 - 0.46 * tf.math.cos(2 * math.pi * n_lin / self.kernel_size), [-1, 1]) # (kernel_size/2, 1) n = (self.kernel_size - 1) / 2.0 # Due to symmetry, I only need half of the time axes self.n_ = 2 * math.pi * tf.reshape(tf.range(-n, 0), [-1, 1]) / self.sample_rate super(SincConv, self).build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape): input_dim = int(input_shape[-1]) self.expert_kernel = self.add_weight( name='expert_kernel', shape=(input_dim, self.num_experts * self.output_dim), dtype=tf.float32, initializer=glorot_normal(seed=self.seed)) self.gate_kernels = [] for i in range(self.num_tasks): self.gate_kernels.append(self.add_weight( name='gate_weight_'.format(i), shape=(input_dim, self.num_experts), dtype=tf.float32, initializer=glorot_normal(seed=self.seed))) super(MMOELayer, self).build(input_shape)
def build(self, input_shape): if not isinstance(input_shape, list) or len(input_shape) < 2: raise ValueError('A `AttentionalFM` layer should be called ' 'on a list of at least 2 inputs') shape_set = set() reduced_input_shape = [shape.as_list() for shape in input_shape] for i in range(len(input_shape)): shape_set.add(tuple(reduced_input_shape[i])) if len(shape_set) > 1: raise ValueError('A `AttentionalFM` layer requires ' 'inputs with same shapes ' 'Got different shapes: %s' % (shape_set)) if len(input_shape[0]) != 3 or input_shape[0][1] != 1: raise ValueError('A `AttentionalFM` layer requires ' 'inputs of a list with same shape tensor like\ (None, 1, embedding_size)' 'Got different shapes: %s' % (input_shape[0])) embedding_size = int(input_shape[0][-1]) self.attention_W = self.add_weight( shape=(embedding_size, self.attention_factor), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg_w), name="attention_W") self.attention_b = self.add_weight(shape=(self.attention_factor, ), initializer=Zeros(), name="attention_b") self.projection_h = self.add_weight( shape=(self.attention_factor, 1), initializer=glorot_normal(seed=self.seed), name="projection_h") self.projection_p = self.add_weight( shape=(embedding_size, 1), initializer=glorot_normal(seed=self.seed), name="projection_p") self.dropout = tf.keras.layers.Dropout(self.dropout_rate, seed=self.seed) self.tensordot = tf.keras.layers.Lambda( lambda x: tf.tensordot(x[0], x[1], axes=(-1, 0))) # Be sure to call this somewhere! super(AFMLayer, self).build(input_shape)
def build(self, input_shape): input_size = input_shape[-1] hidden_units = [int(input_size)] + list(self.hidden_units) # range(len(self.hidden_units)) self.kernels = [] self.bias = [] for i in range(len(self.hidden_units)): kernel = self.add_weight( name='kernel' + str(i), shape=(hidden_units[i], hidden_units[i + 1]), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True ) bias = self.add_weight( name='bias' + str(i), shape=(self.hidden_units[i],), initializer=Zeros(), trainable=True ) self.kernels.append(kernel) self.bias.append(bias) if self.use_bn: self.bn_layers = [tf.keras.layers.BatchNormalization()] * len(self.hidden_units) self.dropout_layers = [tf.keras.layers.Dropout()] * len(self.hidden_units) self.activation_layers = [activation_layer(self.activation)] * len(self.hidden_units) super(DNN, self).build(input_shape) # Be sure to call this somewhere! 传递信息?
def __init__(self, n_end, data, activation=LeakyReLU(0.1), optimizer='adam', lr=0.001, l2=0.0, l1=0.00000, is_GT=True, plot_every_n=10): self.n_end = n_end self.hsi = data self.activation = activation self.lr = lr self.l2 = l2 self.l1 = l1 self.optimizer = optimizer # self.optimizer = optimizers.Adam(lr=self.lr) self.model = None self.use_bias = False self.abundance_layer = None self.initializer = initializers.glorot_normal() self.sum_to_one = True self.is_GT = is_GT self.plot_every_n = plot_every_n self.plotS = True self.weights = None self.is_deep = False self.activation = activation
def build(self, input_shape): input_size = input_shape[-1] hidden_units = [int(input_size)] + list(self.hidden_units) self.kernels = [ self.add_weight(name='kernel' + str(i), shape=(hidden_units[i], hidden_units[i + 1]), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) for i in range(len(self.hidden_units)) ] self.bias = [ self.add_weight(name='bias' + str(i), shape=(self.hidden_units[i], ), initializer=Zeros(), trainable=True) for i in range(len(self.hidden_units)) ] if self.use_bn: self.bn_layers = [ tf.keras.layers.BatchNormalization() for _ in range(len(self.hidden_units)) ] self.dropout_layers = [ tf.keras.layers.Dropout(self.dropout_rate, seed=self.seed + i) for i in range(len(self.hidden_units)) ] self.activation_layers = [tf.keras.layers.Activation(self.activation) \ for _ in range(len(self.hidden_units))] super(DNN, self).build(input_shape)
def build(self, input_shape): if not isinstance(input_shape, list) or len(input_shape) != 2: raise ValueError('A `LocalActivationUnit` layer should be called ' 'on a list of 2 inputs') if len(input_shape[0]) != 3 or len(input_shape[1]) != 3: raise ValueError( "Unexpected inputs dimensions %d and %d, expect to be 3 dimensions" % (len(input_shape[0]), len(input_shape[1]))) if input_shape[0][-1] != input_shape[1][-1] or input_shape[0][1] != 1: raise ValueError( 'A `LocalActivationUnit` layer requires ' 'inputs of a two inputs with shape (None,1,embedding_size) and (None,T,embedding_size)' 'Got different shapes: %s,%s' % (input_shape)) size = 4 * \ int(input_shape[0][-1] ) if len(self.hidden_units) == 0 else self.hidden_units[-1] self.kernel = self.add_weight( shape=(size, 1), initializer=glorot_normal(seed=self.seed), name="kernel") self.bias = self.add_weight(shape=(1, ), initializer=Zeros(), name="bias") #self.dnn = DNN(self.hidden_units, self.activation, self.l2_reg, # self.dropout_rate, self.use_bn, seed=self.seed) super(LocalActivationUnit, self).build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape): # if len(self.hidden_units) == 0: # raise ValueError("hidden_units is empty") input_size = input_shape[-1] hidden_units = [int(input_size)] + list(self.hidden_units) self.kernels = [self.add_weight(name='kernel' + str(i), shape=(hidden_units[i], hidden_units[i + 1]), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) for i in range(len(self.hidden_units))] self.bias = [self.add_weight(name='bias' + str(i), shape=(self.hidden_units[i],), initializer=Zeros(), trainable=True) for i in range(len(self.hidden_units))] if self.use_bn: self.bn_layers = [tf.keras.layers.BatchNormalization() for _ in range(len(self.hidden_units))] self.dropout_layers = [tf.keras.layers.Dropout(self.dropout_rate, seed=self.seed + i) for i in range(len(self.hidden_units))] self.activation_layers = [activation_layer(self.activation) for _ in range(len(self.hidden_units))] if self.output_activation: self.activation_layers[-1] = activation_layer(self.output_activation) super(DNN, self).build(input_shape) # Be sure to call this somewhere!
def __init__(self, name=None): super(CustomModel, self).__init__() self._my_layers = [ layer_lib.Dense( 4096, name='dense1', kernel_initializer=initializers.glorot_normal(seed=0), use_bias=False), layer_lib.Dense( 4, name='dense2', kernel_initializer=initializers.glorot_normal(seed=0), use_bias=False), ] self.histogram_summary_layer = LayerForHistogramSummary() self.scalar_summary_layer = LayerForScalarSummary()
def fc_bn(inputs, num_output, is_relu=True, weight_decay=1e-4): out = keras.layers.Dense( num_output, kernel_initializer=initializers.glorot_normal(), kernel_regularizer=regularizers.l2(weight_decay))(inputs) out = keras.layers.BatchNormalization()(out) if is_relu is True: return keras.layers.ReLU()(out) else: return out
def build(self, input_shape): if not isinstance(input_shape, list) or len(input_shape) < 2: raise ValueError('A `AttentionalFM` layer should be called ' 'on a list of at least 2 inputs') self.filed_size = len(input_shape) self.embedding_size = input_shape[0][-1] reduction_size = max(1, self.filed_size//self.reduction_ratio) self.W_1 = self.add_weight(shape=( self.filed_size, reduction_size), initializer=glorot_normal(seed=self.seed), name="W_1") self.W_2 = self.add_weight(shape=( reduction_size, self.filed_size), initializer=glorot_normal(seed=self.seed), name="W_2") self.tensordot = tf.keras.layers.Lambda( lambda x: tf.tensordot(x[0], x[1], axes=(-1, 0))) # Be sure to call this somewhere! super(SENETLayer, self).build(input_shape)
def build(self, input_shape): if len(input_shape) != 2: raise ValueError( "Unexpected inputs dimensions %d, expect to be 2 dimensions" % (len(input_shape), )) input_dimension = int(input_shape[-1]) # Initialize expert weights (number of input features * number of units per expert * number of experts) self.expert_kernels = self.add_weight( name='expert_kernel', shape=(input_dimension, self.units, self.num_experts), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) # Initialize expert bias (number of units per expert * number of experts) self.expert_bias = self.add_weight(name='expert_bias', shape=(self.units, self.num_experts), initializer=Zeros(), trainable=True) # Initialize gate weights (number of input features * number of experts * number of tasks) self.gate_kernels = [ self.add_weight(name='gate_kernel_task_{}'.format(i), shape=(input_dimension, self.num_experts), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) for i in range(self.num_tasks) ] # Initialize gate bias (number of experts * number of tasks) self.gate_bias = [ self.add_weight(name='gate_bias_task_{}'.format(i), shape=(self.num_experts, ), initializer=Zeros(), trainable=True) for i in range(self.num_tasks) ] # Be sure to call this somewhere! super(MMoE, self).build(input_shape)
def build(self, input_shape): # input_shape [None, field_nums, embedding_dim] self.field_size = input_shape[1] self.embedding_size = input_shape[-1] # 中间层的神经单元个数 f/r reduction_size = max(1, self.field_size // self.reduction_ratio) # FC layer1和layer2的参数 self.W_1 = self.add_weight(shape=(self.field_size, reduction_size), initializer=glorot_normal(seed=self.seed), name="W_1") self.W_2 = self.add_weight(shape=(reduction_size, self.field_size), initializer=glorot_normal(seed=self.seed), name="W_2") self.tensordot = tf.keras.layers.Lambda( lambda x: tf.tensordot(x[0], x[1], axes=(-1, 0))) # Be sure to call this somewhere! super(SENETLayer, self).build(input_shape)
def build(self, input_shape): input_size = input_shape[-1] hidden_units = [int(input_size)] + list(self.hidden_size) self.kernels = [self.add_weight(name='kernel' + str(i), shape=(hidden_units[i], hidden_units[i+1]), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) for i in range(len(self.hidden_size))] self.bias = [self.add_weight(name='bias' + str(i), shape=(self.hidden_size[i],), initializer=Zeros(), trainable=True) for i in range(len(self.hidden_size))] super(MLP, self).build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape): if len(input_shape) != 2: raise ValueError("Unexpected inputs dimensions %d, expect to be 2 dimensions" % (len(input_shape),)) dim = input_shape[-1] self.kernels = [self.add_weight(name='kernel'+str(i), shape=(dim, 1), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) for i in range(self.layer_num)] self.bias = [self.add_weight(name='bias'+str(i) , shape=(dim,1), initializer=Zeros(), trainable=True) for i in range(self.layer_num)] super(CrossNet, self).build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape): if not isinstance(input_shape, list) or len(input_shape) != 2: raise ValueError('A `LocalActivationUnit` layer should be called ' 'on a list of 2 inputs') if len(input_shape[0]) != 3 or len(input_shape[1]) != 3: raise ValueError( "Unexpected inputs dimensions %d and %d, expect to be 3 dimensions" % (len(input_shape[0]), len(input_shape[1]))) if input_shape[0][-1] != input_shape[1][-1] or input_shape[0][1] != 1: raise ValueError( 'A `LocalActivationUnit` layer requires ' 'inputs of a two inputs with shape (None,1,embedding_size) and (None,T,embedding_size)' 'Got different shapes: %s,%s' % (input_shape[0], input_shape[1])) size = 4 * \ int(input_shape[0][-1] ) if len(self.hidden_units) == 0 else self.hidden_units[-1] self.kernel = self.add_weight( shape=(size, 1), initializer=glorot_normal(seed=self.seed), name="kernel") self.bias = self.add_weight(shape=(1, ), initializer=Zeros(), name="bias") self.dnn = DNN(self.hidden_units, self.activation, self.l2_reg, self.dropout_rate, self.use_bn, seed=self.seed) self.dense = tf.keras.layers.Lambda(lambda x: tf.nn.bias_add( tf.tensordot(x[0], x[1], axes=(-1, 0)), x[2])) # 理清楚tensordot和matmul之间的区别 # 1. matmul在a·b中,只能把a的最后一个维度和b的倒数第二个维度乘在一起 # 2. tensordot可以指定,其实就是把指定的维度交换到最后一个维度,然后尽心matmul # att_out和self.kernel 进行tensordot # axis作为-1和0,其实就和matmul没有任何区别 super(LocalActivationUnit, self).build(input_shape) # Be sure to call this somewhere!
def make_prediction(self, images_input): image_shape = list(images_input.shape.as_list()[2:4]) l2_reg = l2(10**(-9)) default_init = glorot_normal(seed=None) image_frame = images_input image_frame = tf.reshape(image_frame, shape=[-1] + image_shape + [3], name='image_frame_collapse') image_conv = BaseImageModel(image_frame).prediction fused_frame = image_conv frames = tf.reshape(fused_frame, [self.batch_size, -1] + fused_frame.shape.as_list()[1:], name='frame_expand') frames = Conv3D(filters=128, kernel_size=(7, 1, 1), padding='valid', kernel_initializer=default_init, activity_regularizer=l2_reg, name='conv3d_1')(frames) frames = Conv3D(filters=64, kernel_size=3, padding='valid', kernel_initializer=default_init, activity_regularizer=l2_reg, name='conv3d_2')(frames) size = frames.shape.as_list() size = size[2] * size[3] * size[4] flattened = tf.reshape(frames, [self.batch_size, -1, size], name='flatten') dense = Dense(256, activation='relu', kernel_initializer=default_init, activity_regularizer=l2_reg, name='FC_1')(flattened) logits = Dense(self.output_size, name='FC_final')(dense) logits = tf.reduce_mean(logits, axis=1, name='average') return logits
def call(self, inputs,**kwargs): deep_input = inputs deep_input = Dropout(1 - self.keep_prob)(deep_input) for l in range(len(self.hidden_size)): fc = Dense(self.hidden_size[l], activation=None, \ kernel_initializer=glorot_normal(seed=self.seed), \ kernel_regularizer=l2(self.l2_reg))(deep_input) if self.use_bn: fc = BatchNormalization()(fc) if isinstance(self.activation,str): fc = Activation(self.activation)(fc) else: fc = self.activation(fc,name=self.name+"act"+str(l)) fc = Dropout(1 - self.keep_prob)(fc) deep_input = fc return deep_input
def build(self, input_shape): if not isinstance(input_shape, list) or len(input_shape) != 2: raise ValueError('A `LocalActivationUnit` layer should be called ' 'on a list of 2 inputs') if len(input_shape[0]) != 3 or len(input_shape[1]) != 3: raise ValueError( "Unexpected inputs dimensions %d and %d, expect to be 3 dimensions" % (len(input_shape[0]), len(input_shape[1]))) if input_shape[0][-1] != input_shape[1][-1] or input_shape[0][1] != 1: raise ValueError( 'A `LocalActivationUnit` layer requires ' 'inputs of a two inputs with shape (None,1,embedding_size) and (None,T,embedding_size)' 'Got different shapes: %s,%s' % (input_shape[0], input_shape[1])) # 为什么hidden_units为空时设置为embedding的四倍?因为相当于不用DNN,直接接在(原始+乘+差)后面一层 size = 4 * \ int(input_shape[0][-1] ) if len(self.hidden_units) == 0 else self.hidden_units[-1] self.kernel = self.add_weight( shape=(size, 1), initializer=glorot_normal(seed=self.seed), name="kernel") # add_weight() self.bias = self.add_weight(shape=(1, ), initializer=Zeros(), name="bias") self.dnn = DNN(self.hidden_units, self.activation, self.l2_reg, self.dropout_rate, self.use_bn, seed=self.seed) # DNN之后的一层,直接输出权重 self.dense = tf.keras.layers.Lambda(lambda x: tf.nn.bias_add( tf.tensordot(x[0], x[1], axes=(-1, 0)), x[2])) super(LocalActivationUnit, self).build(input_shape) # Be sure to call this somewhere!
def _model_fn(features, labels, mode, config): train_flag = (mode == tf.estimator.ModeKeys.TRAIN) linear_logits = get_linear_logit(features, linear_feature_columns, l2_reg_linear=l2_reg_linear) with variable_scope(DNN_SCOPE_NAME): sparse_embedding_list, dense_value_list = input_from_feature_columns(features, dnn_feature_columns, l2_reg_embedding=l2_reg_embedding) dnn_input = combined_dnn_input(sparse_embedding_list, dense_value_list) fm_logit = FM()(concat_func(sparse_embedding_list, axis=1)) dnn_output = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, dnn_use_bn, seed)(dnn_input, training=train_flag) dnn_logit = tf.keras.layers.Dense( 1, use_bias=False, activation=None, kernel_initializer=glorot_normal(seed=seed))(dnn_output) logits = linear_logits + fm_logit + dnn_logit return deepctr_model_fn(features, mode, logits, labels, task, linear_optimizer, dnn_optimizer, training_chief_hooks =training_chief_hooks)
def build(self, input_shape): # if len(self.hidden_units) == 0: # raise ValueError("hidden_units is empty") input_size = input_shape[-1] hidden_units = [int(input_size)] + list(self.hidden_units) self.kernels = [ self.add_weight(name='kernel' + str(i), shape=(hidden_units[i], hidden_units[i + 1]), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) for i in range(len(self.hidden_units)) ] self.bias = [ self.add_weight(name='bias' + str(i), shape=(self.hidden_units[i], ), initializer=Zeros(), trainable=True) for i in range(len(self.hidden_units)) ] if self.use_bn: self.bn_layers = [ tf.keras.layers.BatchNormalization() for _ in range(len(self.hidden_units)) ] # AlphaDropout: 1. 均值和方差不变 2. 归一化性质也不变 self.dropout_layers = [ tf.keras.layers.AlphaDropout(self.dropout_rate, seed=self.seed + i) for i in range(len(self.hidden_units)) ] self.activation_layers = [ tf.keras.layers.Activation(self.activation) for _ in range(len(self.hidden_units)) ]
def call(self, inputs, **kwargs): deep_input = inputs deep_input = Dropout(1 - self.keep_prob)(deep_input) for l in range(len(self.hidden_size)): fc = Dense(self.hidden_size[l], activation=None, \ kernel_initializer=glorot_normal(seed=self.seed), \ kernel_regularizer=l2(self.l2_reg))(deep_input) if self.use_bn: fc = BatchNormalization()(fc) if isinstance(self.activation, str): fc = Activation(self.activation)(fc) elif issubclass(self.activation, Layer): fc = self.activation()(fc) else: raise ValueError( "Invalid activation of MLP,found %s.You should use a str or a Activation Layer Class." % (self.activation)) fc = Dropout(1 - self.keep_prob)(fc) deep_input = fc return deep_input
def build(self, input_shape): # if len(self.hidden_units) == 0: # raise ValueError("hidden_units is empty") input_size = input_shape[-1] print(input_size, self.udg_embedding_size) hidden_units = [int(input_size) - self.udg_embedding_size] + list( self.hidden_units) udg_units = [] for i, x in enumerate(hidden_units): tmp = [] for j in range(self.udg_embedding_layer): if j == 0: tmp.append(self.udg_embedding_size) else: tmp.append(x) udg_units.append(tmp) print(udg_units) #udg_units = [[128,384,384],[128,200,200],[128,80,80]] self.udg_kernels = [ self.add_weight(name='udg_kernel' + str(i) + '_' + str(j), shape=(udg_units[i][j], hidden_units[i]), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) for i in range(len(udg_units)) for j in range(3) ] self.udg_bias = [ self.add_weight(name='udg_bias' + str(i) + '_' + str(j), shape=(hidden_units[i], ), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) for i in range(len(udg_units)) for j in range(3) ] self.kernels = [ self.add_weight(name='kernel' + str(i), shape=(hidden_units[i], hidden_units[i + 1]), initializer=glorot_normal(seed=self.seed), regularizer=l2(self.l2_reg), trainable=True) for i in range(len(self.hidden_units)) ] self.bias = [ self.add_weight(name='bias' + str(i), shape=(self.hidden_units[i], ), initializer=Zeros(), trainable=True) for i in range(len(self.hidden_units)) ] if self.use_bn: self.bn_layers = [ tf.keras.layers.BatchNormalization() for _ in range(len(self.hidden_units)) ] self.dropout_layers = [ tf.keras.layers.Dropout(self.dropout_rate, seed=self.seed + i) for i in range(len(self.hidden_units)) ] self.udg_activation_layers = [] for i, x in enumerate(hidden_units): for j in range(self.udg_embedding_layer): if j != self.udg_embedding_layer - 1: self.udg_activation_layers.append(activation_layer('relu')) else: self.udg_activation_layers.append( activation_layer('sigmoid')) self.activation_layers = [ activation_layer(self.activation) for _ in range(len(self.hidden_units)) ] super(DNN_UDG, self).build(input_shape) # Be sure to call this somewhere!
def prediction(self, image_input): l2_reg = l2(10**(-9)) default_init = glorot_normal(seed=None) def se_net(in_block, depth): x = GlobalAveragePooling2D()(in_block) x = Dense(depth // 16, activation='relu', kernel_initializer=default_init, bias_initializer='zeros')(x) x = Dense(depth, activation='sigmoid', kernel_regularizer=l2_reg)(x) return Multiply()([in_block, x]) # single image frame processing # entry conv1 = Conv2D(32, kernel_size=3, strides=2, padding="same", activation="relu", kernel_initializer=default_init, name="initial_3x3_conv_1")(image_input) conv1 = Conv2D(32, kernel_size=3, strides=1, padding="same", activation="relu", kernel_initializer=default_init, name="initial_3x3_conv_2")(conv1) conv1 = Conv2D(32, kernel_size=3, strides=1, padding="same", activation="relu", activity_regularizer=l2_reg, kernel_initializer=default_init, name="initial_3x3_conv_3")(conv1) conv1_pool = MaxPooling2D(2, strides=2, padding='valid', name='stem_pool_1')(conv1) conv1_3 = Conv2D(32, kernel_size=3, strides=2, padding='same', activation='relu', kernel_initializer=default_init, name='conv1_reduced_1')(conv1) conv1 = Concatenate()([conv1_3, conv1_pool]) conv1_3 = Conv2D(64, kernel_size=1, strides=1, padding='same', activation='relu', kernel_initializer=default_init, name='stem_3x3_pre_conv')(conv1) conv1_3 = Conv2D(96, kernel_size=3, strides=1, padding='same', activation='relu', activity_regularizer=l2_reg, kernel_initializer=default_init, name='stem_3x3_conv')(conv1_3) conv1_7 = Conv2D(64, kernel_size=1, strides=1, padding='same', activation='relu', kernel_initializer=default_init, name='stem_7x7_pre_conv')(conv1) conv1_7 = Conv2D(64, kernel_size=[7, 1], strides=1, padding='same', activation='relu', kernel_initializer=default_init, name='stem_7x7_conv_factor_1')(conv1_7) conv1_7 = Conv2D(64, kernel_size=[1, 7], strides=1, padding='same', activation='relu', kernel_initializer=default_init, name='stem_7x7_conv_factor_2')(conv1_7) conv1_7 = Conv2D(96, kernel_size=3, strides=1, padding='same', activation='relu', activity_regularizer=l2_reg, kernel_initializer=default_init, name='stem_7x7_post_conv')(conv1_7) conv1 = Concatenate()([conv1_3, conv1_7]) conv1_pool = MaxPooling2D(2, strides=2, padding='valid', name='stem_pool_2')(conv1) conv1_3 = Conv2D(192, kernel_size=3, strides=2, padding='same', activation='relu', kernel_initializer=default_init, name='conv1_reduced_2')(conv1) conv1 = Concatenate()([conv1_3, conv1_pool]) conv1 = se_net(conv1, depth=384) # middle flow # Inception-Resnet Block A depth = 384 conv2 = conv1 for i in range(3): conv1 = Conv2D(depth, kernel_size=1, strides=1, padding='valid', activity_regularizer=l2_reg, kernel_initializer=default_init, name='block_A_base_{}'.format(i))(conv1) conv2_1 = Conv2D(128, kernel_size=1, strides=1, padding='same', activation='relu', kernel_initializer=default_init, name='block_A_1x1_conv_{}'.format(i))(conv2) conv2_3 = Conv2D(64, kernel_size=1, strides=1, padding='same', activation='relu', kernel_initializer=default_init, name='block_A_3x3_pre_conv_{}'.format(i))(conv2) conv2_3 = Conv2D(128, kernel_size=3, strides=1, padding='same', activation='relu', kernel_initializer=default_init, name='block_A_3x3_conv_{}'.format(i))(conv2_3) conv2_7 = Conv2D(32, kernel_size=1, strides=1, padding='same', activation='relu', kernel_initializer=default_init, name='block_A_7x7_pre_conv_1_{}'.format(i))(conv2) conv2_7 = Conv2D( 64, kernel_size=3, strides=1, padding='same', activation='relu', kernel_initializer=default_init, name='block_A_7x7_pre_conv_2_{}'.format(i))(conv2_7) conv2_7 = Conv2D(128, kernel_size=3, strides=1, padding='same', activation='relu', kernel_initializer=default_init, name='block_A_7x7_conv_{}'.format(i))(conv2_7) res_conv = Concatenate()([conv2_1, conv2_3, conv2_7]) res_conv = Conv2D( depth, kernel_size=1, strides=1, padding='same', activation='linear', kernel_initializer=default_init, name='block_A_res_conv_projection_{}'.format(i))(res_conv) conv1 = Add(name="block_A_final_add_{}".format(i))( [res_conv, conv1]) conv1 = se_net(conv1, depth=depth) conv2 = conv1 # Inception-Resnet Reduction A conv2_pool = MaxPooling2D(2, strides=2, padding='valid', name='red_A_pool_2')(conv2) conv2_3 = Conv2D(depth, kernel_size=3, strides=2, padding='same', activation='relu', kernel_initializer=default_init, name='red_A_conv_3')(conv2) conv2_7 = Conv2D(256, kernel_size=1, strides=1, padding='same', activation='relu', kernel_initializer=default_init, name='red_A_pre_conv_7')(conv2) conv2_7 = Conv2D(256, kernel_size=3, strides=1, padding='same', activation='relu', kernel_initializer=default_init, name='red_A_conv_7_factor_1')(conv2_7) conv2_7 = Conv2D(depth, kernel_size=3, strides=2, padding='same', activation='relu', kernel_initializer=default_init, name='red_A_conv_7_factor_2')(conv2_7) conv2 = Concatenate()([conv2_pool, conv2_3, conv2_7]) # Inception-Resnet Block B depth = 1024 conv1 = conv2 for i in range(3): conv1 = Conv2D(depth, kernel_size=1, strides=1, padding='valid', activity_regularizer=l2_reg, kernel_initializer=default_init, name='block_B_base_{}'.format(i))(conv1) conv2_1 = Conv2D(192, kernel_size=1, strides=1, padding='same', activation='relu', kernel_initializer=default_init, name='block_B_1x1_conv_{}'.format(i))(conv2) conv2_7 = Conv2D(128, kernel_size=1, strides=1, padding='same', activation='relu', kernel_initializer=default_init, name='block_B_7x7_pre_conv_1_{}'.format(i))(conv2) conv2_7 = Conv2D( 160, kernel_size=[1, 7], strides=1, padding='same', activation='relu', kernel_initializer=default_init, name='block_B_7x7_pre_conv_2_{}'.format(i))(conv2_7) conv2_7 = Conv2D(192, kernel_size=[7, 1], strides=1, padding='same', activation='relu', kernel_initializer=default_init, name='block_B_7x7_conv_{}'.format(i))(conv2_7) res_conv = Concatenate()([conv2_1, conv2_7]) res_conv = Conv2D( depth, kernel_size=1, strides=1, padding='same', activation='linear', kernel_initializer=default_init, name='block_B_res_conv_projection_{}'.format(i))(res_conv) conv1 = Add(name="block_B_final_add_{}".format(i))( [res_conv, conv1]) conv1 = se_net(conv1, depth=depth) conv2 = conv1 # Inception-Resnet Reduction B conv2_pool = MaxPooling2D(3, strides=2, padding='same', name='red_B_pool_2')(conv2) conv2_3_1 = Conv2D(256, kernel_size=1, strides=1, padding='same', activation='relu', kernel_initializer=default_init, name='red_B_pre_conv_3_1')(conv2) conv2_3_1 = Conv2D(384, kernel_size=3, strides=2, padding='same', activity_regularizer=l2_reg, activation='relu', kernel_initializer=default_init, name='red_B_conv_3_1')(conv2_3_1) conv2_3_2 = Conv2D(256, kernel_size=1, strides=1, padding='same', activation='relu', kernel_initializer=default_init, name='red_B_pre_conv_3_2')(conv2) conv2_3_2 = Conv2D(288, kernel_size=3, strides=2, padding='same', activity_regularizer=l2_reg, activation='relu', kernel_initializer=default_init, name='red_B_conv_3_2')(conv2_3_2) conv2_7 = Conv2D(256, kernel_size=1, strides=1, padding='same', activation='relu', kernel_initializer=default_init, name='red_B_pre_conv_7')(conv2) conv2_7 = Conv2D(288, kernel_size=3, strides=1, padding='same', activation='relu', kernel_initializer=default_init, name='red_B_conv_7_factor_1')(conv2_7) conv2_7 = Conv2D(320, kernel_size=3, strides=2, padding='same', activity_regularizer=l2_reg, activation='relu', kernel_initializer=default_init, name='red_B_conv_7_factor_2')(conv2_7) conv2 = Concatenate()([conv2_pool, conv2_3_1, conv2_3_2, conv2_7]) # exit return conv2
def create_model(s=2, weight_decay=1e-2, act="relu"): model = Sequential() # Block 1 model.add( Conv2D(64, (3, 3), padding='same', kernel_initializer=glorot_normal(), input_shape=x_train.shape[1:])) model.add(BatchNormalization()) model.add(Activation(act)) model.add(Dropout(0.2)) #Block 2 model.add( Conv2D(128, (3, 3), padding='same', kernel_initializer=glorot_normal())) model.add(BatchNormalization()) model.add(Activation(act)) model.add(Dropout(0.2)) # Block 3 model.add( Conv2D(128, (3, 3), padding='same', kernel_initializer=RandomNormal(stddev=0.01))) model.add(BatchNormalization()) model.add(Activation(act)) model.add(Dropout(0.2)) #Block 4 model.add( Conv2D(128, (3, 3), padding='same', kernel_initializer=RandomNormal(stddev=0.01))) model.add(BatchNormalization()) model.add(Activation(act)) # First Maxpooling model.add(MaxPooling2D(pool_size=(2, 2), strides=s)) model.add(Dropout(0.2)) # Block 5 model.add( Conv2D(128, (3, 3), padding='same', kernel_initializer=RandomNormal(stddev=0.01))) model.add(BatchNormalization()) model.add(Activation(act)) model.add(Dropout(0.2)) # Block 6 model.add( Conv2D(128, (3, 3), padding='same', kernel_initializer=glorot_normal())) model.add(BatchNormalization()) model.add(Activation(act)) model.add(Dropout(0.2)) # Block 7 model.add( Conv2D(256, (3, 3), padding='same', kernel_initializer=glorot_normal())) # Second Maxpooling model.add(MaxPooling2D(pool_size=(2, 2), strides=s)) model.add(BatchNormalization()) model.add(Activation(act)) model.add(Dropout(0.2)) # Block 8 model.add( Conv2D(256, (3, 3), padding='same', kernel_initializer=glorot_normal())) model.add(BatchNormalization()) model.add(Activation(act)) model.add(Dropout(0.2)) # Block 9 model.add( Conv2D(256, (3, 3), padding='same', kernel_initializer=glorot_normal())) model.add(BatchNormalization()) model.add(Activation(act)) model.add(Dropout(0.2)) # Third Maxpooling model.add(MaxPooling2D(pool_size=(2, 2), strides=s)) # Block 10 model.add( Conv2D(512, (3, 3), padding='same', kernel_initializer=glorot_normal())) model.add(BatchNormalization()) model.add(Activation(act)) model.add(Dropout(0.2)) # Block 11 model.add( Conv2D(2048, (1, 1), padding='same', kernel_initializer=glorot_normal())) model.add(Activation(act)) model.add(Dropout(0.2)) # Block 12 model.add( Conv2D(256, (1, 1), padding='same', kernel_initializer=glorot_normal())) model.add(Activation(act)) # Fourth Maxpooling model.add(MaxPooling2D(pool_size=(2, 2), strides=s)) model.add(Dropout(0.2)) # Block 13 model.add( Conv2D(256, (3, 3), padding='same', kernel_initializer=glorot_normal())) model.add(Activation(act)) # Fifth Maxpooling model.add(MaxPooling2D(pool_size=(2, 2), strides=s)) # Final Classifier model.add(Flatten()) model.add(Dense(num_classes, activation=Softmax(tf.nn.softmax))) return model