def build(self, input_shape):
if not isinstance(input_shape, list) or len(input_shape) < 2:
raise ValueError('A `AttentionalFM` layer should be called '
'on a list of at least 2 inputs')
embedding_size = int(input_shape[0][-1])
if self.bilinear_type == "all":
self.W = self.add_weight(shape=(embedding_size, embedding_size),
initializer=glorot_normal(seed=self.seed),
name="bilinear_weight")
elif self.bilinear_type == "each":
self.W_list = [
self.add_weight(shape=(embedding_size, embedding_size),
initializer=glorot_normal(seed=self.seed),
name="bilinear_weight" + str(i))
for i in range(len(input_shape) - 1)
]
elif self.bilinear_type == "interaction":
self.W_list = [
self.add_weight(shape=(embedding_size, embedding_size),
initializer=glorot_normal(seed=self.seed),
name="bilinear_weight" + str(i) + '_' + str(j))
for i, j in itertools.combinations(range(len(input_shape)), 2)
]
else:
raise NotImplementedError
super(BilinearInteraction,
self).build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape):
# input_shape: [None, field_num, embed_num]
self.field_size = input_shape[1]
self.embedding_size = input_shape[-1]
if self.bilinear_type == "all": # 所有embedding矩阵共用一个矩阵W
self.W = self.add_weight(shape=(self.embedding_size,
self.embedding_size),
initializer=glorot_normal(seed=self.seed),
name="bilinear_weight")
elif self.bilinear_type == "each": # 每个field共用一个矩阵W
self.W_list = [
self.add_weight(shape=(self.embedding_size,
self.embedding_size),
initializer=glorot_normal(seed=self.seed),
name="bilinear_weight" + str(i))
for i in range(self.field_size - 1)
]
elif self.bilinear_type == "interaction": # 每个交互用一个矩阵W
self.W_list = [
self.add_weight(shape=(self.embedding_size,
self.embedding_size),
initializer=glorot_normal(seed=self.seed),
name="bilinear_weight" + str(i) + '_' + str(j))
for i, j in itertools.combinations(range(self.field_size), 2)
]
else:
raise NotImplementedError
super(BilinearInteraction,
self).build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape):
# 这里 input_shape 是第一次运行 call() 时参数 inputs 的形状
if len(input_shape) != 3:
raise ValueError(
"Unexpected inputs dimensions %d, expect to be 3 dimensions" %
(len(input_shape)))
embedding_size = int(input_shape[-1])
self.attention_W = self.add_weight(
shape=(embedding_size, self.attention_factor),
initializer=glorot_normal(seed=self.seed),
regularizer=l2(self.l2_reg_w),
name="attention_W")
self.attention_b = self.add_weight(shape=(self.attention_factor, ),
initializer=Zeros(),
name="attention_b")
self.projection_h = self.add_weight(
shape=(self.attention_factor, 1),
initializer=glorot_normal(seed=self.seed),
name="projection_h")
self.projection_p = self.add_weight(
shape=(embedding_size, 1),
initializer=glorot_normal(seed=self.seed),
name="projection_p")
self.dropout = tf.keras.layers.Dropout(self.dropout_rate,
seed=self.seed)
self.tensordot = tf.keras.layers.Lambda(
lambda x: tf.tensordot(x[0], x[1], axes=(-1, 0)))
# Be sure to call this somewhere!
super(AFMLayer, self).build(input_shape)
def build(self, input_shape):
if len(input_shape) < 4 or input_shape[-1] != 1:
raise ValueError(
'A `SincConv` layer should be called on a 4D tensor '
'with shape:``(batch_size,1,channel_size,sample_size)')
# initialize filterbanks such that they are equally spaced in Mel scale
low_hz = 4
high_hz = self.sample_rate / 2 - (self.min_low_hz + self.min_band_hz)
mel = np.linspace(self.to_mel(low_hz), self.to_mel(high_hz),
self.filters + 1)
hz = self.to_hz(mel)
# filter lower frequency (filters, 1)
self.low_hz_ = self.add_weight(
name='low_hz',
shape=(1, self.filters),
initializer=glorot_normal(seed=self.seed),
regularizer=l2(self.l2_reg),
trainable=True)
self.band_hz_ = self.add_weight(
name='band_hz',
shape=(1, self.filters),
initializer=glorot_normal(seed=self.seed),
regularizer=l2(self.l2_reg),
trainable=True)
# filter lower frequency (1, filters)
self.low_hz_.assign(
tf.cast(tf.reshape(hz[:-1], [1, -1]), dtype=tf.float32))
# filter frequency band (1, filters)
self.band_hz_.assign(
tf.cast(tf.reshape(np.diff(hz), [1, -1]), dtype=tf.float32))
# Hamming window
# computing only half of the window
n_lin = tf.linspace(0., (self.kernel_size / 2) - 1,
num=int(self.kernel_size / 2))
self.window_ = tf.reshape(
0.54 - 0.46 * tf.math.cos(2 * math.pi * n_lin / self.kernel_size),
[-1, 1])
# (kernel_size/2, 1)
n = (self.kernel_size - 1) / 2.0
# Due to symmetry, I only need half of the time axes
self.n_ = 2 * math.pi * tf.reshape(tf.range(-n, 0),
[-1, 1]) / self.sample_rate
super(SincConv,
self).build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape):
input_dim = int(input_shape[-1])
self.expert_kernel = self.add_weight(
name='expert_kernel',
shape=(input_dim, self.num_experts * self.output_dim),
dtype=tf.float32,
initializer=glorot_normal(seed=self.seed))
self.gate_kernels = []
for i in range(self.num_tasks):
self.gate_kernels.append(self.add_weight(
name='gate_weight_'.format(i),
shape=(input_dim, self.num_experts),
dtype=tf.float32,
initializer=glorot_normal(seed=self.seed)))
super(MMOELayer, self).build(input_shape)
def build(self, input_shape):
if not isinstance(input_shape, list) or len(input_shape) < 2:
raise ValueError('A `AttentionalFM` layer should be called '
'on a list of at least 2 inputs')
shape_set = set()
reduced_input_shape = [shape.as_list() for shape in input_shape]
for i in range(len(input_shape)):
shape_set.add(tuple(reduced_input_shape[i]))
if len(shape_set) > 1:
raise ValueError('A `AttentionalFM` layer requires '
'inputs with same shapes '
'Got different shapes: %s' % (shape_set))
if len(input_shape[0]) != 3 or input_shape[0][1] != 1:
raise ValueError('A `AttentionalFM` layer requires '
'inputs of a list with same shape tensor like\
(None, 1, embedding_size)'
'Got different shapes: %s' % (input_shape[0]))
embedding_size = int(input_shape[0][-1])
self.attention_W = self.add_weight(
shape=(embedding_size, self.attention_factor),
initializer=glorot_normal(seed=self.seed),
regularizer=l2(self.l2_reg_w),
name="attention_W")
self.attention_b = self.add_weight(shape=(self.attention_factor, ),
initializer=Zeros(),
name="attention_b")
self.projection_h = self.add_weight(
shape=(self.attention_factor, 1),
initializer=glorot_normal(seed=self.seed),
name="projection_h")
self.projection_p = self.add_weight(
shape=(embedding_size, 1),
initializer=glorot_normal(seed=self.seed),
name="projection_p")
self.dropout = tf.keras.layers.Dropout(self.dropout_rate,
seed=self.seed)
self.tensordot = tf.keras.layers.Lambda(
lambda x: tf.tensordot(x[0], x[1], axes=(-1, 0)))
# Be sure to call this somewhere!
super(AFMLayer, self).build(input_shape)
def build(self, input_shape):
input_size = input_shape[-1]
hidden_units = [int(input_size)] + list(self.hidden_units)
# range(len(self.hidden_units))
self.kernels = []
self.bias = []
for i in range(len(self.hidden_units)):
kernel = self.add_weight(
name='kernel' + str(i),
shape=(hidden_units[i], hidden_units[i + 1]),
initializer=glorot_normal(seed=self.seed),
regularizer=l2(self.l2_reg),
trainable=True
)
bias = self.add_weight(
name='bias' + str(i),
shape=(self.hidden_units[i],),
initializer=Zeros(),
trainable=True
)
self.kernels.append(kernel)
self.bias.append(bias)
if self.use_bn:
self.bn_layers = [tf.keras.layers.BatchNormalization()] * len(self.hidden_units)
self.dropout_layers = [tf.keras.layers.Dropout()] * len(self.hidden_units)
self.activation_layers = [activation_layer(self.activation)] * len(self.hidden_units)
super(DNN, self).build(input_shape) # Be sure to call this somewhere! 传递信息？
def __init__(self,
n_end,
data,
activation=LeakyReLU(0.1),
optimizer='adam',
lr=0.001,
l2=0.0,
l1=0.00000,
is_GT=True,
plot_every_n=10):
self.n_end = n_end
self.hsi = data
self.activation = activation
self.lr = lr
self.l2 = l2
self.l1 = l1
self.optimizer = optimizer
# self.optimizer = optimizers.Adam(lr=self.lr)
self.model = None
self.use_bias = False
self.abundance_layer = None
self.initializer = initializers.glorot_normal()
self.sum_to_one = True
self.is_GT = is_GT
self.plot_every_n = plot_every_n
self.plotS = True
self.weights = None
self.is_deep = False
self.activation = activation
def build(self, input_shape):
input_size = input_shape[-1]
hidden_units = [int(input_size)] + list(self.hidden_units)
self.kernels = [
self.add_weight(name='kernel' + str(i),
shape=(hidden_units[i], hidden_units[i + 1]),
initializer=glorot_normal(seed=self.seed),
regularizer=l2(self.l2_reg),
trainable=True)
for i in range(len(self.hidden_units))
]
self.bias = [
self.add_weight(name='bias' + str(i),
shape=(self.hidden_units[i], ),
initializer=Zeros(),
trainable=True)
for i in range(len(self.hidden_units))
]
if self.use_bn:
self.bn_layers = [
tf.keras.layers.BatchNormalization()
for _ in range(len(self.hidden_units))
]
self.dropout_layers = [
tf.keras.layers.Dropout(self.dropout_rate, seed=self.seed + i)
for i in range(len(self.hidden_units))
]
self.activation_layers = [tf.keras.layers.Activation(self.activation) \
for _ in range(len(self.hidden_units))]
super(DNN, self).build(input_shape)
def build(self, input_shape):
if not isinstance(input_shape, list) or len(input_shape) != 2:
raise ValueError('A `LocalActivationUnit` layer should be called '
'on a list of 2 inputs')
if len(input_shape[0]) != 3 or len(input_shape[1]) != 3:
raise ValueError(
"Unexpected inputs dimensions %d and %d, expect to be 3 dimensions"
% (len(input_shape[0]), len(input_shape[1])))
if input_shape[0][-1] != input_shape[1][-1] or input_shape[0][1] != 1:
raise ValueError(
'A `LocalActivationUnit` layer requires '
'inputs of a two inputs with shape (None,1,embedding_size) and (None,T,embedding_size)'
'Got different shapes: %s,%s' % (input_shape))
size = 4 * \
int(input_shape[0][-1]
) if len(self.hidden_units) == 0 else self.hidden_units[-1]
self.kernel = self.add_weight(
shape=(size, 1),
initializer=glorot_normal(seed=self.seed),
name="kernel")
self.bias = self.add_weight(shape=(1, ),
initializer=Zeros(),
name="bias")
#self.dnn = DNN(self.hidden_units, self.activation, self.l2_reg,
# self.dropout_rate, self.use_bn, seed=self.seed)
super(LocalActivationUnit,
self).build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape):
# if len(self.hidden_units) == 0:
# raise ValueError("hidden_units is empty")
input_size = input_shape[-1]
hidden_units = [int(input_size)] + list(self.hidden_units)
self.kernels = [self.add_weight(name='kernel' + str(i),
shape=(hidden_units[i], hidden_units[i + 1]),
initializer=glorot_normal(seed=self.seed),
regularizer=l2(self.l2_reg),
trainable=True) for i in range(len(self.hidden_units))]
self.bias = [self.add_weight(name='bias' + str(i),
shape=(self.hidden_units[i],),
initializer=Zeros(),
trainable=True) for i in range(len(self.hidden_units))]
if self.use_bn:
self.bn_layers = [tf.keras.layers.BatchNormalization() for _ in range(len(self.hidden_units))]
self.dropout_layers = [tf.keras.layers.Dropout(self.dropout_rate, seed=self.seed + i) for i in range(len(self.hidden_units))]
self.activation_layers = [activation_layer(self.activation) for _ in range(len(self.hidden_units))]
if self.output_activation:
self.activation_layers[-1] = activation_layer(self.output_activation)
super(DNN, self).build(input_shape) # Be sure to call this somewhere!
def __init__(self, name=None):
super(CustomModel, self).__init__()
self._my_layers = [
layer_lib.Dense(
4096,
name='dense1',
kernel_initializer=initializers.glorot_normal(seed=0),
use_bias=False),
layer_lib.Dense(
4,
name='dense2',
kernel_initializer=initializers.glorot_normal(seed=0),
use_bias=False),
]
self.histogram_summary_layer = LayerForHistogramSummary()
self.scalar_summary_layer = LayerForScalarSummary()
def fc_bn(inputs, num_output, is_relu=True, weight_decay=1e-4):
out = keras.layers.Dense(
num_output,
kernel_initializer=initializers.glorot_normal(),
kernel_regularizer=regularizers.l2(weight_decay))(inputs)
out = keras.layers.BatchNormalization()(out)
if is_relu is True:
return keras.layers.ReLU()(out)
else:
return out
def build(self, input_shape):
if not isinstance(input_shape, list) or len(input_shape) < 2:
raise ValueError('A `AttentionalFM` layer should be called '
'on a list of at least 2 inputs')
self.filed_size = len(input_shape)
self.embedding_size = input_shape[0][-1]
reduction_size = max(1, self.filed_size//self.reduction_ratio)
self.W_1 = self.add_weight(shape=(
self.filed_size, reduction_size), initializer=glorot_normal(seed=self.seed), name="W_1")
self.W_2 = self.add_weight(shape=(
reduction_size, self.filed_size), initializer=glorot_normal(seed=self.seed), name="W_2")
self.tensordot = tf.keras.layers.Lambda(
lambda x: tf.tensordot(x[0], x[1], axes=(-1, 0)))
# Be sure to call this somewhere!
super(SENETLayer, self).build(input_shape)
def build(self, input_shape):
if len(input_shape) != 2:
raise ValueError(
"Unexpected inputs dimensions %d, expect to be 2 dimensions" %
(len(input_shape), ))
input_dimension = int(input_shape[-1])
# Initialize expert weights (number of input features * number of units per expert * number of experts)
self.expert_kernels = self.add_weight(
name='expert_kernel',
shape=(input_dimension, self.units, self.num_experts),
initializer=glorot_normal(seed=self.seed),
regularizer=l2(self.l2_reg),
trainable=True)
# Initialize expert bias (number of units per expert * number of experts)
self.expert_bias = self.add_weight(name='expert_bias',
shape=(self.units,
self.num_experts),
initializer=Zeros(),
trainable=True)
# Initialize gate weights (number of input features * number of experts * number of tasks)
self.gate_kernels = [
self.add_weight(name='gate_kernel_task_{}'.format(i),
shape=(input_dimension, self.num_experts),
initializer=glorot_normal(seed=self.seed),
regularizer=l2(self.l2_reg),
trainable=True) for i in range(self.num_tasks)
]
# Initialize gate bias (number of experts * number of tasks)
self.gate_bias = [
self.add_weight(name='gate_bias_task_{}'.format(i),
shape=(self.num_experts, ),
initializer=Zeros(),
trainable=True) for i in range(self.num_tasks)
]
# Be sure to call this somewhere!
super(MMoE, self).build(input_shape)
def build(self, input_shape):
# input_shape [None, field_nums, embedding_dim]
self.field_size = input_shape[1]
self.embedding_size = input_shape[-1]
# 中间层的神经单元个数 f/r
reduction_size = max(1, self.field_size // self.reduction_ratio)
# FC layer1和layer2的参数
self.W_1 = self.add_weight(shape=(self.field_size, reduction_size),
initializer=glorot_normal(seed=self.seed),
name="W_1")
self.W_2 = self.add_weight(shape=(reduction_size, self.field_size),
initializer=glorot_normal(seed=self.seed),
name="W_2")
self.tensordot = tf.keras.layers.Lambda(
lambda x: tf.tensordot(x[0], x[1], axes=(-1, 0)))
# Be sure to call this somewhere!
super(SENETLayer, self).build(input_shape)
def build(self, input_shape):
input_size = input_shape[-1]
hidden_units = [int(input_size)] + list(self.hidden_size)
self.kernels = [self.add_weight(name='kernel' + str(i),
shape=(hidden_units[i], hidden_units[i+1]),
initializer=glorot_normal(seed=self.seed),
regularizer=l2(self.l2_reg),
trainable=True) for i in range(len(self.hidden_size))]
self.bias = [self.add_weight(name='bias' + str(i),
shape=(self.hidden_size[i],),
initializer=Zeros(),
trainable=True) for i in range(len(self.hidden_size))]
super(MLP, self).build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape):
if len(input_shape) != 2:
raise ValueError("Unexpected inputs dimensions %d, expect to be 2 dimensions" % (len(input_shape),))
dim = input_shape[-1]
self.kernels = [self.add_weight(name='kernel'+str(i),
shape=(dim, 1),
initializer=glorot_normal(seed=self.seed),
regularizer=l2(self.l2_reg),
trainable=True) for i in range(self.layer_num)]
self.bias = [self.add_weight(name='bias'+str(i) ,
shape=(dim,1),
initializer=Zeros(),
trainable=True) for i in range(self.layer_num)]
super(CrossNet, self).build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape):
if not isinstance(input_shape, list) or len(input_shape) != 2:
raise ValueError('A `LocalActivationUnit` layer should be called '
'on a list of 2 inputs')
if len(input_shape[0]) != 3 or len(input_shape[1]) != 3:
raise ValueError(
"Unexpected inputs dimensions %d and %d, expect to be 3 dimensions"
% (len(input_shape[0]), len(input_shape[1])))
if input_shape[0][-1] != input_shape[1][-1] or input_shape[0][1] != 1:
raise ValueError(
'A `LocalActivationUnit` layer requires '
'inputs of a two inputs with shape (None,1,embedding_size) and (None,T,embedding_size)'
'Got different shapes: %s,%s' %
(input_shape[0], input_shape[1]))
size = 4 * \
int(input_shape[0][-1]
) if len(self.hidden_units) == 0 else self.hidden_units[-1]
self.kernel = self.add_weight(
shape=(size, 1),
initializer=glorot_normal(seed=self.seed),
name="kernel")
self.bias = self.add_weight(shape=(1, ),
initializer=Zeros(),
name="bias")
self.dnn = DNN(self.hidden_units,
self.activation,
self.l2_reg,
self.dropout_rate,
self.use_bn,
seed=self.seed)
self.dense = tf.keras.layers.Lambda(lambda x: tf.nn.bias_add(
tf.tensordot(x[0], x[1], axes=(-1, 0)), x[2]))
# 理清楚tensordot和matmul之间的区别
# 1. matmul在a·b中，只能把a的最后一个维度和b的倒数第二个维度乘在一起
# 2. tensordot可以指定，其实就是把指定的维度交换到最后一个维度，然后尽心matmul
# att_out和self.kernel 进行tensordot
# axis作为-1和0，其实就和matmul没有任何区别
super(LocalActivationUnit,
self).build(input_shape) # Be sure to call this somewhere!
def make_prediction(self, images_input):
image_shape = list(images_input.shape.as_list()[2:4])
l2_reg = l2(10**(-9))
default_init = glorot_normal(seed=None)
image_frame = images_input
image_frame = tf.reshape(image_frame,
shape=[-1] + image_shape + [3],
name='image_frame_collapse')
image_conv = BaseImageModel(image_frame).prediction
fused_frame = image_conv
frames = tf.reshape(fused_frame, [self.batch_size, -1] +
fused_frame.shape.as_list()[1:],
name='frame_expand')
frames = Conv3D(filters=128,
kernel_size=(7, 1, 1),
padding='valid',
kernel_initializer=default_init,
activity_regularizer=l2_reg,
name='conv3d_1')(frames)
frames = Conv3D(filters=64,
kernel_size=3,
padding='valid',
kernel_initializer=default_init,
activity_regularizer=l2_reg,
name='conv3d_2')(frames)
size = frames.shape.as_list()
size = size[2] * size[3] * size[4]
flattened = tf.reshape(frames, [self.batch_size, -1, size],
name='flatten')
dense = Dense(256,
activation='relu',
kernel_initializer=default_init,
activity_regularizer=l2_reg,
name='FC_1')(flattened)
logits = Dense(self.output_size, name='FC_final')(dense)
logits = tf.reduce_mean(logits, axis=1, name='average')
return logits
def call(self, inputs,**kwargs):
deep_input = inputs
deep_input = Dropout(1 - self.keep_prob)(deep_input)
for l in range(len(self.hidden_size)):
fc = Dense(self.hidden_size[l], activation=None, \
kernel_initializer=glorot_normal(seed=self.seed), \
kernel_regularizer=l2(self.l2_reg))(deep_input)
if self.use_bn:
fc = BatchNormalization()(fc)
if isinstance(self.activation,str):
fc = Activation(self.activation)(fc)
else:
fc = self.activation(fc,name=self.name+"act"+str(l))
fc = Dropout(1 - self.keep_prob)(fc)
deep_input = fc
return deep_input
def build(self, input_shape):
if not isinstance(input_shape, list) or len(input_shape) != 2:
raise ValueError('A `LocalActivationUnit` layer should be called '
'on a list of 2 inputs')
if len(input_shape[0]) != 3 or len(input_shape[1]) != 3:
raise ValueError(
"Unexpected inputs dimensions %d and %d, expect to be 3 dimensions"
% (len(input_shape[0]), len(input_shape[1])))
if input_shape[0][-1] != input_shape[1][-1] or input_shape[0][1] != 1:
raise ValueError(
'A `LocalActivationUnit` layer requires '
'inputs of a two inputs with shape (None,1,embedding_size) and (None,T,embedding_size)'
'Got different shapes: %s,%s' %
(input_shape[0], input_shape[1]))
# 为什么hidden_units为空时设置为embedding的四倍？因为相当于不用DNN，直接接在(原始+乘+差)后面一层
size = 4 * \
int(input_shape[0][-1]
) if len(self.hidden_units) == 0 else self.hidden_units[-1]
self.kernel = self.add_weight(
shape=(size, 1),
initializer=glorot_normal(seed=self.seed),
name="kernel") # add_weight()
self.bias = self.add_weight(shape=(1, ),
initializer=Zeros(),
name="bias")
self.dnn = DNN(self.hidden_units,
self.activation,
self.l2_reg,
self.dropout_rate,
self.use_bn,
seed=self.seed)
# DNN之后的一层，直接输出权重
self.dense = tf.keras.layers.Lambda(lambda x: tf.nn.bias_add(
tf.tensordot(x[0], x[1], axes=(-1, 0)), x[2]))
super(LocalActivationUnit,
self).build(input_shape) # Be sure to call this somewhere!
def _model_fn(features, labels, mode, config):
train_flag = (mode == tf.estimator.ModeKeys.TRAIN)
linear_logits = get_linear_logit(features, linear_feature_columns, l2_reg_linear=l2_reg_linear)
with variable_scope(DNN_SCOPE_NAME):
sparse_embedding_list, dense_value_list = input_from_feature_columns(features, dnn_feature_columns,
l2_reg_embedding=l2_reg_embedding)
dnn_input = combined_dnn_input(sparse_embedding_list, dense_value_list)
fm_logit = FM()(concat_func(sparse_embedding_list, axis=1))
dnn_output = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout,
dnn_use_bn, seed)(dnn_input, training=train_flag)
dnn_logit = tf.keras.layers.Dense(
1, use_bias=False, activation=None, kernel_initializer=glorot_normal(seed=seed))(dnn_output)
logits = linear_logits + fm_logit + dnn_logit
return deepctr_model_fn(features, mode, logits, labels, task, linear_optimizer, dnn_optimizer,
training_chief_hooks
=training_chief_hooks)
def build(self, input_shape):
# if len(self.hidden_units) == 0:
# raise ValueError("hidden_units is empty")
input_size = input_shape[-1]
hidden_units = [int(input_size)] + list(self.hidden_units)
self.kernels = [
self.add_weight(name='kernel' + str(i),
shape=(hidden_units[i], hidden_units[i + 1]),
initializer=glorot_normal(seed=self.seed),
regularizer=l2(self.l2_reg),
trainable=True)
for i in range(len(self.hidden_units))
]
self.bias = [
self.add_weight(name='bias' + str(i),
shape=(self.hidden_units[i], ),
initializer=Zeros(),
trainable=True)
for i in range(len(self.hidden_units))
]
if self.use_bn:
self.bn_layers = [
tf.keras.layers.BatchNormalization()
for _ in range(len(self.hidden_units))
]
# AlphaDropout: 1. 均值和方差不变 2. 归一化性质也不变
self.dropout_layers = [
tf.keras.layers.AlphaDropout(self.dropout_rate, seed=self.seed + i)
for i in range(len(self.hidden_units))
]
self.activation_layers = [
tf.keras.layers.Activation(self.activation)
for _ in range(len(self.hidden_units))
]
def call(self, inputs, **kwargs):
deep_input = inputs
deep_input = Dropout(1 - self.keep_prob)(deep_input)
for l in range(len(self.hidden_size)):
fc = Dense(self.hidden_size[l], activation=None, \
kernel_initializer=glorot_normal(seed=self.seed), \
kernel_regularizer=l2(self.l2_reg))(deep_input)
if self.use_bn:
fc = BatchNormalization()(fc)
if isinstance(self.activation, str):
fc = Activation(self.activation)(fc)
elif issubclass(self.activation, Layer):
fc = self.activation()(fc)
else:
raise ValueError(
"Invalid activation of MLP,found %s.You should use a str or a Activation Layer Class."
% (self.activation))
fc = Dropout(1 - self.keep_prob)(fc)
deep_input = fc
return deep_input
def build(self, input_shape):
# if len(self.hidden_units) == 0:
# raise ValueError("hidden_units is empty")
input_size = input_shape[-1]
print(input_size, self.udg_embedding_size)
hidden_units = [int(input_size) - self.udg_embedding_size] + list(
self.hidden_units)
udg_units = []
for i, x in enumerate(hidden_units):
tmp = []
for j in range(self.udg_embedding_layer):
if j == 0:
tmp.append(self.udg_embedding_size)
else:
tmp.append(x)
udg_units.append(tmp)
print(udg_units)
#udg_units = [[128,384,384],[128,200,200],[128,80,80]]
self.udg_kernels = [
self.add_weight(name='udg_kernel' + str(i) + '_' + str(j),
shape=(udg_units[i][j], hidden_units[i]),
initializer=glorot_normal(seed=self.seed),
regularizer=l2(self.l2_reg),
trainable=True) for i in range(len(udg_units))
for j in range(3)
]
self.udg_bias = [
self.add_weight(name='udg_bias' + str(i) + '_' + str(j),
shape=(hidden_units[i], ),
initializer=glorot_normal(seed=self.seed),
regularizer=l2(self.l2_reg),
trainable=True) for i in range(len(udg_units))
for j in range(3)
]
self.kernels = [
self.add_weight(name='kernel' + str(i),
shape=(hidden_units[i], hidden_units[i + 1]),
initializer=glorot_normal(seed=self.seed),
regularizer=l2(self.l2_reg),
trainable=True)
for i in range(len(self.hidden_units))
]
self.bias = [
self.add_weight(name='bias' + str(i),
shape=(self.hidden_units[i], ),
initializer=Zeros(),
trainable=True)
for i in range(len(self.hidden_units))
]
if self.use_bn:
self.bn_layers = [
tf.keras.layers.BatchNormalization()
for _ in range(len(self.hidden_units))
]
self.dropout_layers = [
tf.keras.layers.Dropout(self.dropout_rate, seed=self.seed + i)
for i in range(len(self.hidden_units))
]
self.udg_activation_layers = []
for i, x in enumerate(hidden_units):
for j in range(self.udg_embedding_layer):
if j != self.udg_embedding_layer - 1:
self.udg_activation_layers.append(activation_layer('relu'))
else:
self.udg_activation_layers.append(
activation_layer('sigmoid'))
self.activation_layers = [
activation_layer(self.activation)
for _ in range(len(self.hidden_units))
]
super(DNN_UDG,
self).build(input_shape) # Be sure to call this somewhere!
def prediction(self, image_input):
l2_reg = l2(10**(-9))
default_init = glorot_normal(seed=None)
def se_net(in_block, depth):
x = GlobalAveragePooling2D()(in_block)
x = Dense(depth // 16,
activation='relu',
kernel_initializer=default_init,
bias_initializer='zeros')(x)
x = Dense(depth, activation='sigmoid',
kernel_regularizer=l2_reg)(x)
return Multiply()([in_block, x])
# single image frame processing
# entry
conv1 = Conv2D(32,
kernel_size=3,
strides=2,
padding="same",
activation="relu",
kernel_initializer=default_init,
name="initial_3x3_conv_1")(image_input)
conv1 = Conv2D(32,
kernel_size=3,
strides=1,
padding="same",
activation="relu",
kernel_initializer=default_init,
name="initial_3x3_conv_2")(conv1)
conv1 = Conv2D(32,
kernel_size=3,
strides=1,
padding="same",
activation="relu",
activity_regularizer=l2_reg,
kernel_initializer=default_init,
name="initial_3x3_conv_3")(conv1)
conv1_pool = MaxPooling2D(2,
strides=2,
padding='valid',
name='stem_pool_1')(conv1)
conv1_3 = Conv2D(32,
kernel_size=3,
strides=2,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='conv1_reduced_1')(conv1)
conv1 = Concatenate()([conv1_3, conv1_pool])
conv1_3 = Conv2D(64,
kernel_size=1,
strides=1,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='stem_3x3_pre_conv')(conv1)
conv1_3 = Conv2D(96,
kernel_size=3,
strides=1,
padding='same',
activation='relu',
activity_regularizer=l2_reg,
kernel_initializer=default_init,
name='stem_3x3_conv')(conv1_3)
conv1_7 = Conv2D(64,
kernel_size=1,
strides=1,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='stem_7x7_pre_conv')(conv1)
conv1_7 = Conv2D(64,
kernel_size=[7, 1],
strides=1,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='stem_7x7_conv_factor_1')(conv1_7)
conv1_7 = Conv2D(64,
kernel_size=[1, 7],
strides=1,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='stem_7x7_conv_factor_2')(conv1_7)
conv1_7 = Conv2D(96,
kernel_size=3,
strides=1,
padding='same',
activation='relu',
activity_regularizer=l2_reg,
kernel_initializer=default_init,
name='stem_7x7_post_conv')(conv1_7)
conv1 = Concatenate()([conv1_3, conv1_7])
conv1_pool = MaxPooling2D(2,
strides=2,
padding='valid',
name='stem_pool_2')(conv1)
conv1_3 = Conv2D(192,
kernel_size=3,
strides=2,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='conv1_reduced_2')(conv1)
conv1 = Concatenate()([conv1_3, conv1_pool])
conv1 = se_net(conv1, depth=384)
# middle flow
# Inception-Resnet Block A
depth = 384
conv2 = conv1
for i in range(3):
conv1 = Conv2D(depth,
kernel_size=1,
strides=1,
padding='valid',
activity_regularizer=l2_reg,
kernel_initializer=default_init,
name='block_A_base_{}'.format(i))(conv1)
conv2_1 = Conv2D(128,
kernel_size=1,
strides=1,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='block_A_1x1_conv_{}'.format(i))(conv2)
conv2_3 = Conv2D(64,
kernel_size=1,
strides=1,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='block_A_3x3_pre_conv_{}'.format(i))(conv2)
conv2_3 = Conv2D(128,
kernel_size=3,
strides=1,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='block_A_3x3_conv_{}'.format(i))(conv2_3)
conv2_7 = Conv2D(32,
kernel_size=1,
strides=1,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='block_A_7x7_pre_conv_1_{}'.format(i))(conv2)
conv2_7 = Conv2D(
64,
kernel_size=3,
strides=1,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='block_A_7x7_pre_conv_2_{}'.format(i))(conv2_7)
conv2_7 = Conv2D(128,
kernel_size=3,
strides=1,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='block_A_7x7_conv_{}'.format(i))(conv2_7)
res_conv = Concatenate()([conv2_1, conv2_3, conv2_7])
res_conv = Conv2D(
depth,
kernel_size=1,
strides=1,
padding='same',
activation='linear',
kernel_initializer=default_init,
name='block_A_res_conv_projection_{}'.format(i))(res_conv)
conv1 = Add(name="block_A_final_add_{}".format(i))(
[res_conv, conv1])
conv1 = se_net(conv1, depth=depth)
conv2 = conv1
# Inception-Resnet Reduction A
conv2_pool = MaxPooling2D(2,
strides=2,
padding='valid',
name='red_A_pool_2')(conv2)
conv2_3 = Conv2D(depth,
kernel_size=3,
strides=2,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='red_A_conv_3')(conv2)
conv2_7 = Conv2D(256,
kernel_size=1,
strides=1,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='red_A_pre_conv_7')(conv2)
conv2_7 = Conv2D(256,
kernel_size=3,
strides=1,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='red_A_conv_7_factor_1')(conv2_7)
conv2_7 = Conv2D(depth,
kernel_size=3,
strides=2,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='red_A_conv_7_factor_2')(conv2_7)
conv2 = Concatenate()([conv2_pool, conv2_3, conv2_7])
# Inception-Resnet Block B
depth = 1024
conv1 = conv2
for i in range(3):
conv1 = Conv2D(depth,
kernel_size=1,
strides=1,
padding='valid',
activity_regularizer=l2_reg,
kernel_initializer=default_init,
name='block_B_base_{}'.format(i))(conv1)
conv2_1 = Conv2D(192,
kernel_size=1,
strides=1,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='block_B_1x1_conv_{}'.format(i))(conv2)
conv2_7 = Conv2D(128,
kernel_size=1,
strides=1,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='block_B_7x7_pre_conv_1_{}'.format(i))(conv2)
conv2_7 = Conv2D(
160,
kernel_size=[1, 7],
strides=1,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='block_B_7x7_pre_conv_2_{}'.format(i))(conv2_7)
conv2_7 = Conv2D(192,
kernel_size=[7, 1],
strides=1,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='block_B_7x7_conv_{}'.format(i))(conv2_7)
res_conv = Concatenate()([conv2_1, conv2_7])
res_conv = Conv2D(
depth,
kernel_size=1,
strides=1,
padding='same',
activation='linear',
kernel_initializer=default_init,
name='block_B_res_conv_projection_{}'.format(i))(res_conv)
conv1 = Add(name="block_B_final_add_{}".format(i))(
[res_conv, conv1])
conv1 = se_net(conv1, depth=depth)
conv2 = conv1
# Inception-Resnet Reduction B
conv2_pool = MaxPooling2D(3,
strides=2,
padding='same',
name='red_B_pool_2')(conv2)
conv2_3_1 = Conv2D(256,
kernel_size=1,
strides=1,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='red_B_pre_conv_3_1')(conv2)
conv2_3_1 = Conv2D(384,
kernel_size=3,
strides=2,
padding='same',
activity_regularizer=l2_reg,
activation='relu',
kernel_initializer=default_init,
name='red_B_conv_3_1')(conv2_3_1)
conv2_3_2 = Conv2D(256,
kernel_size=1,
strides=1,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='red_B_pre_conv_3_2')(conv2)
conv2_3_2 = Conv2D(288,
kernel_size=3,
strides=2,
padding='same',
activity_regularizer=l2_reg,
activation='relu',
kernel_initializer=default_init,
name='red_B_conv_3_2')(conv2_3_2)
conv2_7 = Conv2D(256,
kernel_size=1,
strides=1,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='red_B_pre_conv_7')(conv2)
conv2_7 = Conv2D(288,
kernel_size=3,
strides=1,
padding='same',
activation='relu',
kernel_initializer=default_init,
name='red_B_conv_7_factor_1')(conv2_7)
conv2_7 = Conv2D(320,
kernel_size=3,
strides=2,
padding='same',
activity_regularizer=l2_reg,
activation='relu',
kernel_initializer=default_init,
name='red_B_conv_7_factor_2')(conv2_7)
conv2 = Concatenate()([conv2_pool, conv2_3_1, conv2_3_2, conv2_7])
# exit
return conv2
def create_model(s=2, weight_decay=1e-2, act="relu"):
model = Sequential()
# Block 1
model.add(
Conv2D(64, (3, 3),
padding='same',
kernel_initializer=glorot_normal(),
input_shape=x_train.shape[1:]))
model.add(BatchNormalization())
model.add(Activation(act))
model.add(Dropout(0.2))
#Block 2
model.add(
Conv2D(128, (3, 3), padding='same',
kernel_initializer=glorot_normal()))
model.add(BatchNormalization())
model.add(Activation(act))
model.add(Dropout(0.2))
# Block 3
model.add(
Conv2D(128, (3, 3),
padding='same',
kernel_initializer=RandomNormal(stddev=0.01)))
model.add(BatchNormalization())
model.add(Activation(act))
model.add(Dropout(0.2))
#Block 4
model.add(
Conv2D(128, (3, 3),
padding='same',
kernel_initializer=RandomNormal(stddev=0.01)))
model.add(BatchNormalization())
model.add(Activation(act))
# First Maxpooling
model.add(MaxPooling2D(pool_size=(2, 2), strides=s))
model.add(Dropout(0.2))
# Block 5
model.add(
Conv2D(128, (3, 3),
padding='same',
kernel_initializer=RandomNormal(stddev=0.01)))
model.add(BatchNormalization())
model.add(Activation(act))
model.add(Dropout(0.2))
# Block 6
model.add(
Conv2D(128, (3, 3), padding='same',
kernel_initializer=glorot_normal()))
model.add(BatchNormalization())
model.add(Activation(act))
model.add(Dropout(0.2))
# Block 7
model.add(
Conv2D(256, (3, 3), padding='same',
kernel_initializer=glorot_normal()))
# Second Maxpooling
model.add(MaxPooling2D(pool_size=(2, 2), strides=s))
model.add(BatchNormalization())
model.add(Activation(act))
model.add(Dropout(0.2))
# Block 8
model.add(
Conv2D(256, (3, 3), padding='same',
kernel_initializer=glorot_normal()))
model.add(BatchNormalization())
model.add(Activation(act))
model.add(Dropout(0.2))
# Block 9
model.add(
Conv2D(256, (3, 3), padding='same',
kernel_initializer=glorot_normal()))
model.add(BatchNormalization())
model.add(Activation(act))
model.add(Dropout(0.2))
# Third Maxpooling
model.add(MaxPooling2D(pool_size=(2, 2), strides=s))
# Block 10
model.add(
Conv2D(512, (3, 3), padding='same',
kernel_initializer=glorot_normal()))
model.add(BatchNormalization())
model.add(Activation(act))
model.add(Dropout(0.2))
# Block 11
model.add(
Conv2D(2048, (1, 1),
padding='same',
kernel_initializer=glorot_normal()))
model.add(Activation(act))
model.add(Dropout(0.2))
# Block 12
model.add(
Conv2D(256, (1, 1), padding='same',
kernel_initializer=glorot_normal()))
model.add(Activation(act))
# Fourth Maxpooling
model.add(MaxPooling2D(pool_size=(2, 2), strides=s))
model.add(Dropout(0.2))
# Block 13
model.add(
Conv2D(256, (3, 3), padding='same',
kernel_initializer=glorot_normal()))
model.add(Activation(act))
# Fifth Maxpooling
model.add(MaxPooling2D(pool_size=(2, 2), strides=s))
# Final Classifier
model.add(Flatten())
model.add(Dense(num_classes, activation=Softmax(tf.nn.softmax)))
return model
