def model_dense_v2(drop_prob, image_pixels, runCount):
# The size of our hidden layers
hidden1_units = 500
hidden2_units = 300
hidden3_units = 0
hidden4_units = 0
output_units = 10
model = Sequential()
model.add(
Dense(hidden1_units,
name="input",
input_dim=image_pixels,
activation='relu',
bias_initializer=init.constant(
0.1))) # Fully connected layer in Keras
model.add(Dropout(drop_prob))
model.add(
Dense(hidden2_units,
name="Layer-2",
activation='relu',
bias_initializer=init.constant(
0.1))) # Fully connected layer in Keras
model.add(Dropout(drop_prob))
model.add(Dense(output_units, name="output",
activation='softmax')) # Fully connected layer in Keras
#print out a summary of the model on the first run only
if (runCount == 1):
model.summary()
return model
def id2(input_tensor, kernel_size, filters, stage, block, weight_decay,
strides):
"""The identity block is the block that has no conv layer at shortcut.
# Arguments
input_tensor: input tensor
kernel_size: defualt 3, the kernel size of middle conv layer at main path
filters: list of integers, the filterss of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
# Returns
Output tensor for the block.
"""
#print(input_tensor.shape)
kernel_reg = l2(weight_decay[0]) if weight_decay else None
bias_reg = l2(weight_decay[1]) if weight_decay else None
filters1, filters2, filters3 = filters
if K.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
x = Conv2DTranspose(filters1, (4, 4),
strides=strides,
padding='same',
name=conv_name_base + '2a',
kernel_regularizer=kernel_reg,
bias_regularizer=bias_reg,
kernel_initializer=random_normal(stddev=0.01),
bias_initializer=constant(0.0))(input_tensor)
#print(x.shape)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
x = Activation('relu')(x)
x = Conv2D(filters2,
kernel_size,
padding='same',
name=conv_name_base + '2b',
kernel_regularizer=kernel_reg,
bias_regularizer=bias_reg,
kernel_initializer=random_normal(stddev=0.01),
bias_initializer=constant(0.0))(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)
x = Activation('relu')(x)
x = Conv2D(filters3,
kernel_size,
padding='same',
name=conv_name_base + '2c',
kernel_regularizer=kernel_reg,
bias_regularizer=bias_reg,
kernel_initializer=random_normal(stddev=0.01),
bias_initializer=constant(0.0))(x)
#x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)
#x = layers.add([x, input_tensor])
#x = Activation('relu')(x)
return x
def build_CNN():
modelCNN = Sequential()
modelCNN.add(
Conv2D(4, (14, 14),
activation='relu',
input_shape=(SIZE, SIZE, 3),
bias_initializer=initializers.constant(0.1),
padding='same'))
modelCNN.add(MaxPooling2D(pool_size=(2, 2))) # pooling
modelCNN.add(
Conv2D(
8,
(8, 8), # 2nd Convolution layer with 8 channels
activation='relu',
bias_initializer=initializers.constant(0.1)))
modelCNN.add(MaxPooling2D(pool_size=(2, 2)))
# Flattening, turning to 1D
modelCNN.add(Dropout(0.25))
modelCNN.add(Flatten())
modelCNN.add(
Dense(256,
activation='relu',
bias_initializer=initializers.constant(0.1)))
modelCNN.add(Dropout(0.3))
modelCNN.add(Dense(1, activation='sigmoid')
) # sigmoid is more suitable for binary crossentropy
sgd = optimizers.SGD(lr=0.01, momentum=0.9)
modelCNN.compile(
loss=
'binary_crossentropy', # loss function used for classes that are greater than 2)
optimizer=sgd,
metrics=['accuracy'])
return modelCNN
def __init__(self,
tau,
bias=1.0,
nonlinearity='linear',
tau_learn='True',
**kwargs):
self.tau_ = constant(tau)
self.tau_learn_ = tau_learn
self.bias_ = constant(bias)
self.nonlinearity = nonlinearity
super(TauLayer, self).__init__(**kwargs)
def centernet_head(x, num_classes):
x = Dropout(rate=0.5)(x)
#-------------------------------#
# 解码器
#-------------------------------#
num_filters = 256
# 16, 16, 2048 -> 32, 32, 256 -> 64, 64, 128 -> 128, 128, 64
for i in range(3):
# 进行上采样
x = Conv2DTranspose(num_filters // pow(2, i), (4, 4),
strides=2,
use_bias=False,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(5e-4))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# 最终获得128,128,64的特征层
# hm header
y1 = Conv2D(64,
3,
padding='same',
use_bias=False,
kernel_initializer=random_normal(stddev=0.02))(x)
y1 = BatchNormalization()(y1)
y1 = Activation('relu')(y1)
y1 = Conv2D(num_classes,
1,
kernel_initializer=constant(0),
bias_initializer=constant(-2.19),
activation='sigmoid')(y1)
# wh header
y2 = Conv2D(64,
3,
padding='same',
use_bias=False,
kernel_initializer=random_normal(stddev=0.02))(x)
y2 = BatchNormalization()(y2)
y2 = Activation('relu')(y2)
y2 = Conv2D(2, 1, kernel_initializer=random_normal(stddev=0.02))(y2)
# reg header
y3 = Conv2D(64,
3,
padding='same',
use_bias=False,
kernel_initializer=random_normal(stddev=0.02))(x)
y3 = BatchNormalization()(y3)
y3 = Activation('relu')(y3)
y3 = Conv2D(2, 1, kernel_initializer=random_normal(stddev=0.02))(y3)
return y1, y2, y3
def buildModel(self, word_index, embedding_matrix, embedding_dim):
filter_sizes = [3,4,5,6]
num_filters = 16
inputs = Input(shape=(self.MAX_LEN,))
sharable_embedding = Embedding(len(word_index) + 1,
embedding_dim,
weights=[embedding_matrix],
input_length=self.MAX_LEN,
trainable= self.embedding_trainable)(inputs)
reshape = Reshape((self.MAX_LEN, embedding_dim, 1))(sharable_embedding)
conv_0 = Conv2D(num_filters, kernel_size=(filter_sizes[0], embedding_dim), padding='valid', kernel_initializer=initializers.TruncatedNormal(mean=0.0, stddev=0.1),
bias_initializer=initializers.constant(value=0.1), activation='relu')(reshape)
maxpool_0 = MaxPool2D(pool_size=(self.MAX_LEN - filter_sizes[0] + 1, 1), strides=(1,1), padding='valid')(conv_0)
conv_1 = Conv2D(num_filters, kernel_size=(filter_sizes[1], embedding_dim), padding='valid', kernel_initializer=initializers.TruncatedNormal(mean=0.0, stddev=0.1),
bias_initializer=initializers.constant(value=0.1), activation='relu')(reshape)
maxpool_1 = MaxPool2D(pool_size=(self.MAX_LEN - filter_sizes[1] + 1, 1), strides=(1,1), padding='valid')(conv_1)
conv_2 = Conv2D(num_filters, kernel_size=(filter_sizes[2], embedding_dim), padding='valid', kernel_initializer=initializers.TruncatedNormal(mean=0.0, stddev=0.1),
bias_initializer=initializers.constant(value=0.1), activation='relu')(reshape)
maxpool_2 = MaxPool2D(pool_size=(self.MAX_LEN - filter_sizes[2] + 1, 1), strides=(1,1), padding='valid')(conv_2)
conv_3 = Conv2D(num_filters, kernel_size=(filter_sizes[3], embedding_dim), padding='valid', kernel_initializer=initializers.TruncatedNormal(mean=0.0, stddev=0.1),
bias_initializer=initializers.constant(value=0.1), activation='relu')(reshape)
maxpool_3 = MaxPool2D(pool_size=(self.MAX_LEN - filter_sizes[3] + 1, 1), strides=(1,1), padding='valid')(conv_3)
concatenated_tensor = Concatenate(axis=1)([maxpool_0, maxpool_1, maxpool_2, maxpool_3])
flatten = Flatten()(concatenated_tensor)
if self.use_dropout:
dropout_layer_2 = Dropout(self.dropout_rate)(flatten)
dense_1 = Dense(self.dnn_size, activation='relu')(dropout_layer_2)
else:
dense_1 = Dense(self.dnn_size, activation='relu')(flatten)
if self.use_dropout:
dropout_layer_3 = Dropout(self.dropout_rate)(dense_1)
dense_2 = Dense(int(self.dnn_size/2), activation='relu')(dropout_layer_3)
else:
dense_2 = Dense(int(self.dnn_size/2), activation='relu')(dense_1)
dense_4 = Dense(1, activation='sigmoid')(dense_2)
model = Model(inputs=inputs, outputs = dense_4, name='text-CNN')
model.compile(loss=self.LOSS_FUNCTION,
optimizer=self.OPTIMIZER,
metrics=['accuracy'])
return model
def build(self, input_shape):
self.mu = self.add_weight('mu',
input_shape[1:],
dtype=tf.float32,
initializer=constant(self._mu_init),
trainable=False)
self.sigma = self.add_weight('sigma',
input_shape[1:],
dtype=tf.float32,
initializer=constant(self._sigma_init),
trainable=False)
super().build(input_shape)
self.built = True
def build(self, input_shape):
super(RBFLayer, self).build(input_shape)
self.centers = self.add_weight(name='centers',
shape=(self.output_dim, input_shape[1]),
initializer=initializers.RandomUniform(
0.0, 1.0),
trainable=True)
self.bias = self.add_weight(name='bias',
shape=(self.output_dim, ),
initializer=initializers.constant(1.0),
trainable=True)
self.betas = self.add_weight(name='betas',
shape=(self.output_dim, ),
initializer=initializers.constant(1.0),
trainable=True)
def build(self, input_shape):
if self.data_format == 'channels_first':
channel_axis = 1
else:
channel_axis = -1
if input_shape[channel_axis] is None:
raise ValueError('The channel dimension of the inputs '
'should be defined. Found `None`.')
input_dim = input_shape[channel_axis]
self.kernel_shape = self.kernel_size + (input_dim, self.filters)
self.kernel = self.add_weight(shape=self.kernel_shape,
initializer=self.kernel_initializer,
name='kernel',
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
# keep track of TF Variable (weights)
self.original_kernel = self.kernel
# Set the correct initial values here
# use K.set_value(x, value)
total_number_of_matrix_entries = np.prod(self.kernel_shape)
number_of_active_synapses = \
self.get_number_of_active_connections()
_pre_mask = np.zeros(total_number_of_matrix_entries, int)
_pre_mask[:number_of_active_synapses] = 1
np.random.shuffle(_pre_mask)
_pre_mask = _pre_mask.astype(bool).reshape(
self.kernel_shape).astype(float)
# set this as the mask
# K.set_value(self.mask, _pre_mask)
# self.mask = K.variable(_pre_mask, name="mask")
self.mask = self.add_weight(
shape=_pre_mask.shape,
initializer=initializers.constant(_pre_mask),
name='mask',
trainable=False)
# apply mask
self.kernel = self.kernel * self.mask
if self.connectivity_level:
self.add_update(
updates=K.update(self.original_kernel, self.kernel))
if self.use_bias:
self.bias = self.add_weight(shape=(self.filters, ),
initializer=self.bias_initializer,
name='bias',
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
else:
self.bias = None
# Set input spec.
self.input_spec = InputSpec(ndim=self.rank + 2,
axes={channel_axis: input_dim})
# Be sure to call this at the end
super(_SparseConv, self).build(input_shape)
def create_NER_model(self):
ner_model = Sequential()
# keras_contrib 2.0.8, keras 2.2.5,下 当mask_zero=True 会报
# Tensors in list passed to 'values' of 'ConcatV2' Op have types [bool, float32] that don't all match.`
# 错误。
# 改成keras 2.2.4 解决
embedding = Embedding(input_dim=VOCAB_SIZE,
output_dim=EMBED_DIM,
mask_zero=False,
embeddings_initializer=constant(
load_word2vec_embedding(config.vocab_size)))
ner_model.add(embedding)
ner_model.add(Masking(mask_value=config.src_padding, ))
ner_model.add(
Bidirectional(
LSTM(BiRNN_UNITS // 2,
return_sequences=True,
dropout=DROPOUT_RATE)))
crf = CRF(len(LABEL_DIC), sparse_target=True)
ner_model.add(crf)
# 以下两种损失和度量写法都可以
ner_model.compile(Adam(lr=LEARN_RATE, decay=1e-3),
loss=crf_loss,
metrics=[crf_accuracy])
# ner_model.compile(Adam(lr=LEARN_RATE), loss=crf.loss_function, metrics=[crf.accuracy])
return ner_model
def dilated_conv(x,
nf,
ks,
name,
weight_decay,
stride=1,
use_bias=False,
dialated_rate=2,
use_bn=False,
use_relu=True):
kernel_reg = l2(weight_decay[0]) if weight_decay else None
bias_reg = l2(weight_decay[1]) if weight_decay else None
x = Conv2D(nf, (ks, ks),
strides=stride,
padding='same',
dilation_rate=dialated_rate,
name=name,
kernel_regularizer=kernel_reg,
bias_regularizer=bias_reg,
kernel_initializer=random_normal(stddev=0.01),
bias_initializer=constant(0.0),
use_bias=use_bias)(x)
if use_bn:
bn_axis = 3 if K.image_data_format() == 'channels_last' else 1
x = BatchNormalization(axis=bn_axis,
epsilon=1.001e-5,
name=name + '_bn',
trainable=True)(x)
if use_relu:
x = relu(x)
return x
def conv(x,
nf,
ks,
name,
weight_decay,
stride=1,
use_bias=False,
use_bn=False,
use_relu=True):
kernel_reg = l2(weight_decay[0]) if weight_decay else None
bias_reg = l2(weight_decay[1]) if weight_decay else None
x = Conv2D(
nf,
(ks, ks),
strides=stride,
padding='same',
name=name,
kernel_regularizer=kernel_reg,
bias_regularizer=bias_reg,
kernel_initializer=random_normal(
stddev=0.01
), # Initializer that generates tensors with a normal distribution
bias_initializer=constant(0.0),
use_bias=use_bias)(x)
if use_bn:
bn_axis = 3 if K.image_data_format() == 'channels_last' else 1
x = BatchNormalization(axis=bn_axis, name=name + '_bn',
trainable=True)(x)
if use_relu:
x = relu(x)
return x
def build(self, input_shape):
assert len(input_shape) == 5, "Input shape is incorrect, should be "\
"[Batch size x height x width x capsules x instantiation params]"
self.input_height = input_shape[1]
self.input_width = input_shape[2]
self.input_num_capsule_types = input_shape[3]
self.input_num_caps_instantiations = input_shape[4]
weight_shape = [
self.filter_size[0], self.filter_size[1], 1,
self.num_capsule_types * self.num_caps_instantiations
]
self.weights_per_capsule_type = tf.TensorArray(
dtype=tf.float32,
size=self.input_num_capsule_types,
element_shape=tf.TensorShape(weight_shape))
for i in range(self.input_num_capsule_types):
weights = self.add_weight(
name='coupled_conv_caps_kernel_{i}'.format(i=i),
shape=weight_shape,
initializer=self.filter_initializer)
self.weights_per_capsule_type = self.weights_per_capsule_type.write(
i, weights)
self.bias = self.add_weight(
name='coupled_conv_caps_bias',
shape=[1, 1, self.num_capsule_types, self.num_caps_instantiations],
initializer=initializers.constant(0),
)
super().build(input_shape)
def infer(X, trainable=True, init=initializers.truncated_normal(stddev=0.01)):
init_w = init
init_b = initializers.constant(0.)
normed = Lambda(lambda x: x / 255., output_shape=K.int_shape(X)[1:])(X)
h_conv1 = Convolution2D(32, (8, 8), strides=(4, 4),
kernel_initializer=init_w, use_bias=False, padding='same')(normed)
h_ln1 = LayerNormalization(activation=K.relu)(h_conv1)
h_conv2 = Convolution2D(64, (4, 4), strides=(2, 2),
kernel_initializer=init_w, use_bias=False, padding='same')(h_ln1)
h_ln2 = LayerNormalization(activation=K.relu)(h_conv2)
h_conv3 = Convolution2D(64, (3, 3), strides=(1, 1),
kernel_initializer=init_w, use_bias=False, padding='same')(h_ln2)
h_ln3 = LayerNormalization(activation=K.relu)(h_conv3)
h_flat = Flatten()(h_ln3)
fc_advantage = Dense(512, use_bias=False, kernel_initializer=init_w)(h_flat)
h_ln_fc_advantage = LayerNormalization(activation=K.relu)(fc_advantage)
advantage = Dense(NUM_ACTIONS, kernel_initializer=init_w,
use_bias=False, bias_initializer=init_b)(h_ln_fc_advantage)
fc_value = Dense(512, use_bias=False, kernel_initializer=init_w)(h_flat)
h_ln_fc_value = LayerNormalization(activation=K.relu)(fc_value)
value = Dense(1, kernel_initializer=init_w, use_bias=False, bias_initializer=init_b)(h_ln_fc_value)
z = Lambda(lambda x: x[1] + x[0] - K.mean(advantage, axis=1, keepdims=True), output_shape=(NUM_ACTIONS,))([advantage, value])
# z = LayerNormalization()(fc2)
model = Model(inputs=X, outputs=z)
model.trainable = trainable
return z, model
def __init__(self, activation=None, bias_initializer=-1, **kwargs):
super(Highway, self).__init__(**kwargs)
self.activation = kact.get(activation)
self.bias_initializer = bias_initializer
if isinstance(self.bias_initializer, int):
self.bias_initializer = kinit.constant(self.bias_initializer)
self.input_spec = [InputSpec(min_ndim=2)]
def __init__(self, activation=None, bias_initializer=-1, **kwargs):
super(Highway, self).__init__(**kwargs)
self.activation = kact.get(activation)
self.bias_initializer = bias_initializer
if isinstance(self.bias_initializer, int):
self.bias_initializer = kinit.constant(self.bias_initializer)
self.input_spec = [InputSpec(min_ndim=2)]
def build(self, input_shape, x=None):
""" Create a trainable weight variable for this layer.
x must be a tensor object
input_shape - must have the shape (batch, height, width, channels) according to "channel_last" of Conv2D layer
reshape_input_shape - BHW * D
Note that the sigma values are saves as std, not as variance"""
self.mu = self.add_weight(name='mu',
shape=(self.n_clusters, input_shape[-1]),
initializer=random_normal(mean=0,
stddev=0.4,
seed=self.seed),
trainable=True)
self.std = self.add_weight(name='std',
shape=(self.n_clusters, input_shape[-1]),
initializer=random_normal(mean=0.3,
stddev=0.05,
seed=self.seed),
trainable=True,
constraint=MinValue(min_value=self.epsilon))
self.alpha = self.add_weight(
name='alpha',
shape=(self.n_clusters, ),
initializer=constant(value=(1 / self.n_clusters)),
trainable=True)
super(GMM, self).build(input_shape)
def build(self, input_shape):
assert len(input_shape) == 5, "The input Tensor should have shape=[None, input_height, input_width," \
" input_num_capsule, input_num_atoms]"
self.input_height = input_shape[1]
self.input_width = input_shape[2]
self.input_num_capsule = input_shape[3]
self.input_num_atoms = input_shape[4]
# Transform matrix
if self.upsamp_type == 'subpix':
self.W = self.add_weight(shape=[self.kernel_size, self.kernel_size,
self.input_num_atoms,
self.num_capsule * self.num_atoms * self.scaling * self.scaling],
initializer=self.kernel_initializer,
name='W')
elif self.upsamp_type == 'resize':
self.W = self.add_weight(shape=[self.kernel_size, self.kernel_size,
self.input_num_atoms, self.num_capsule * self.num_atoms],
initializer=self.kernel_initializer, name='W')
elif self.upsamp_type == 'deconv':
self.W = self.add_weight(shape=[self.kernel_size, self.kernel_size,
self.num_capsule * self.num_atoms, self.input_num_atoms],
initializer=self.kernel_initializer, name='W')
else:
raise NotImplementedError('Upsampling must be one of: "deconv", "resize", or "subpix"')
self.b = self.add_weight(shape=[1, 1, self.num_capsule, self.num_atoms],
initializer=initializers.constant(0.1),
name='b')
self.built = True
def build_neural_network(self):
"""Build single hidden layer network in Keras and return it."""
model = Sequential()
# input layer
# model.add(Dense(input_size, activation=None, bias_initializer=constant(1)))
# inputs = Input(shape=(input_size,))
# model.add(inputs)
# fan-in initialization
minval = -0.5 / self.input_size
maxval = 0.5 / self.input_size
fan_in_init = initializers.RandomUniform(minval=minval,
maxval=maxval,
seed=main.SEED)
model.add(
Dense(
self.hidden_layer_size,
input_dim=self.input_size,
activation="tanh",
bias_initializer=constant(1),
kernel_initializer=fan_in_init,
))
# output layer
minval = -0.5 / self.hidden_layer_size
maxval = 0.5 / self.hidden_layer_size
fan_in_init = initializers.RandomUniform(minval=minval,
maxval=maxval,
seed=main.SEED)
model.add(
Dense(1,
bias_initializer=constant(1),
kernel_initializer=fan_in_init))
optimizer = optimizers.SGD(lr=self.learning_rate)
if self.weights is not None:
model.set_weights(self.weights)
model.compile(optimizer=optimizer, loss="mse")
# from keras.utils import plot_model
# plot_model(
# model, to_file="model.png", show_layer_names=True, show_shapes=True
# )
self.model = model
def get_training_model(conf, weight_decay):
inputs = []
outputs = []
imsz = [x/conf.unet_rescale for x in conf.imsz]
img_input_shape = imsz + [conf.imgDim,]
img_input = Input(img_input_shape)
locs_input = Input([conf.n_classes,2])
inputs.append(img_input)
inputs.append(locs_input)
img_normalized = Lambda(lambda x: x / 256 - 0.5)(img_input) # [-0.5, 0.5]
n_stages = 6
x = img_normalized
base_filt = 32
down_layers = [x,]
for ndx in range(n_stages):
x = conv(x, 32*(2**ndx), 3, 'conv_{}_0'.format(ndx),weight_decay)
x = relu(x)
x = conv(x, 32*(2**ndx), 3, 'conv_{}_1'.format(ndx),weight_decay)
x = relu(x)
x = pooling(x,2,2,'pool_{}'.format(ndx))
down_layers.append(x)
x = conv(x, 32 * (2 ** n_stages), 3, 'top_0', weight_decay)
x = relu(x)
x = conv(x, 32 * (2 ** n_stages), 3, 'top_1', weight_decay)
x = relu(x)
up_layers = [x,]
for ndx in reversed(range(n_stages)):
x = Concatenate()([x,down_layers[ndx+1]])
x = conv(x, 32*(2**ndx), 3, 'up_conv_{}_0'.format(ndx),weight_decay)
x = relu(x)
x = conv(x, 32*(2**ndx), 3, 'up_conv_{}_1'.format(ndx),weight_decay)
x = relu(x)
x = keras.layers.UpSampling2D(size=(2,2))(x)
x_shape = x.shape.as_list()
d_shape = down_layers[ndx].shape.as_list()
x_crop = d_shape[2] - x_shape[2]
y_crop = d_shape[1] - x_shape[1]
x = keras.layers.ZeroPadding2D(padding=((0,y_crop),(0,x_crop)))(x)
up_layers.append(x)
# x = conv(x, conf.n_classes, 3, 'out_conv',weight_decay)
kernel_reg = l2(weight_decay[0]) if weight_decay else None
bias_reg = l2(weight_decay[1]) if weight_decay else None
x = Conv2D(conf.n_classes, (3, 3), padding='same', name='out_conv',
kernel_regularizer=kernel_reg,
bias_regularizer=bias_reg,
bias_initializer=constant(0.0))(x)
outputs.append(x)
model = Model(inputs=inputs, outputs=outputs)
return model
def generator(d=128, image_shape=[64, 64, 3]):
conv_options = {
'kernel_initializer': initializers.normal(mean=0.0, stddev=0.02),
}
batchnor_options = {
'gamma_initializer': initializers.normal(mean=0.1, stddev=0.02),
'beta_initializer': initializers.constant(0),
'momentum': 0.9
}
inputs = layers.Input([
100,
])
s_h, s_w = image_shape[0], image_shape[1]
s_h2, s_w2 = conv_out_size_same(s_h, 2), conv_out_size_same(s_w, 2)
s_h4, s_w4 = conv_out_size_same(s_h2, 2), conv_out_size_same(s_w2, 2)
s_h8, s_w8 = conv_out_size_same(s_h4, 2), conv_out_size_same(s_w4, 2)
s_h16, s_w16 = conv_out_size_same(s_h8, 2), conv_out_size_same(s_w8, 2)
x = layers.Dense(s_h16 * s_w16 * d * 8, **conv_options)(inputs)
x = layers.Reshape([s_h16, s_w16, d * 8])(x)
x = layers.BatchNormalization(**batchnor_options)(x)
x = layers.Activation("relu")(x)
x = layers.Conv2DTranspose(filters=d * 4,
kernel_size=4,
strides=2,
padding="same",
**conv_options)(x)
x = layers.BatchNormalization(**batchnor_options)(x)
x = layers.Activation("relu")(x)
x = layers.Conv2DTranspose(filters=d * 2,
kernel_size=4,
strides=2,
padding="same",
**conv_options)(x)
x = layers.BatchNormalization(**batchnor_options)(x)
x = layers.Activation("relu")(x)
x = layers.Conv2DTranspose(filters=d,
kernel_size=4,
strides=2,
padding="same",
**conv_options)(x)
x = layers.BatchNormalization(**batchnor_options)(x)
x = layers.Activation("relu")(x)
x = layers.Conv2DTranspose(filters=3,
kernel_size=4,
strides=2,
padding="same",
**conv_options)(x)
x = layers.Activation("tanh")(x)
model = Model(inputs, x)
return model
def conv(x, nf, ks, name, weight_decay):
kernel_reg = l2(weight_decay[0]) if weight_decay else None
bias_reg = l2(weight_decay[1]) if weight_decay else None
x = Conv2D(nf, (ks, ks), padding='same', name=name,
kernel_regularizer=kernel_reg,
bias_regularizer=bias_reg,
kernel_initializer=random_normal(stddev=0.01),
bias_initializer=constant(0.0))(x)
return x
def build(input_shape, n_classes, filter_sizes, **kwargs):
'''
filter_sizes: a list. each element is the filter for CONV
'''
n_filters = kwargs.get('n_filters', 128)
r_dropout = kwargs.get('r_dropout', 0.5)
n_text_len, n_embedding_len = input_shape
x = Input(shape=input_shape, name='input')
embedded_x = Reshape((n_text_len, n_embedding_len, 1))(x)
pooled_outputs = []
for i, filter_size in enumerate(filter_sizes):
conv = Conv2D(filters=n_filters,
kernel_size=[filter_size, n_embedding_len],
strides=1,
padding='valid',
activation='relu',
kernel_initializer=TruncatedNormal(mean=0.0, stddev=0.1),
bias_initializer=constant(value=0.1),
name=('conv_%d' % i))(embedded_x)
pool = MaxPool2D(pool_size=[n_text_len - filter_size + 1, 1],
strides=(1, 1),
padding='valid',
name=('pool_%d' % i))(conv)
pooled_outputs.append(pool)
# combine all the pooled features
n_filters_total = n_filters * len(filter_sizes)
h_pool = Concatenate(axis=3)(pooled_outputs)
h_pool_flat = Reshape([n_filters_total])(h_pool)
dropout = Dropout(r_dropout)(h_pool_flat)
output = Dense(n_classes,
kernel_initializer='glorot_normal',
bias_initializer=constant(0.1),
activation='softmax',
name='output')(dropout)
model = Model(inputs=x, outputs=output)
return model
def build(self, input_shape):
self.kernel = self.add_weight(name='kernel',
shape=(input_shape[1][2], self.units),
initializer=initializers.orthogonal(),
regularizer=regularizers.l2(5e-4))
self.bias = self.add_weight(name='bias',
shape=(self.units, ),
initializer=initializers.zeros())
if self.learn_pqr:
self.p = self.add_weight(name='p',
shape=(1, ),
initializer=initializers.constant(0))
self.q = self.add_weight(name='q',
shape=(1, ),
initializer=initializers.constant(0))
# self.trainable_weights = [self.p, self.q]
super(MyGCN, self).build(input_shape)
def conv(x, nf, ks, name, weight_decay):
kernel_reg = l2(weight_decay[0]) if weight_decay else None
bias_reg = l2(weight_decay[1]) if weight_decay else None
x = Conv2D(nf, (ks, ks), padding='same', name=name,
kernel_regularizer=kernel_reg,
bias_regularizer=bias_reg,
kernel_initializer=random_normal(stddev=0.01),
bias_initializer=constant(0.0))(x)
return x
def build(self, input_shape):
assert isinstance(input_shape, list)
# Create a trainable weights variable for this layer.
self.kernel1 = self.add_weight(name="modality_weight_1",
shape=(1, ),
initializer=constant(value=0.0),
trainable=True,
constraint=non_neg())
super(Linear, self).build(input_shape)
def network(categorical_columns_item, num_deep_numeric_feature,
num_wide_numeric_feature, bias):
input_layers = list()
embedding_layers = list()
# net categorical deep feature
for col, num in categorical_columns_item.items():
input_deep_cat_layer = Input(shape=(1, ),
name=col + "_categorical_deep_input")
embedding_layer = Embedding(
input_dim=num,
output_dim=min(10, num // 2),
embeddings_initializer=truncated_normal(mean=0,
stddev=1 / np.sqrt(num)),
input_length=1,
name=col + "_deep_embedding")(input_deep_cat_layer)
embedding_layer = (Reshape(target_shape=(min(10, num // 2), ),
name=col +
"_deep_reshape")(embedding_layer))
embedding_layer = Dropout(rate=0.15,
noise_shape=(None, 1),
name=col + "_deep_dropout")(embedding_layer)
input_layers.append(input_deep_cat_layer)
embedding_layers.append(embedding_layer)
# net numeric deep feature
input_deep_num_layer = Input(shape=(num_deep_numeric_feature, ),
name="numeric_deep_input")
input_layers.append(input_deep_num_layer)
# net numeric wide feature
input_wide_num_layer = Input(shape=(num_wide_numeric_feature, ),
name="numeric_wide_input")
input_layers.append(input_wide_num_layer)
hidden_layer = Dense(units=32,
kernel_initializer=lecun_normal(),
activation="selu")(Concatenate()([
Concatenate()(embedding_layers),
Dropout(rate=0.15)(input_deep_num_layer)
]))
hidden_layer = Dense(units=16,
kernel_initializer=lecun_normal(),
activation="selu")(hidden_layer)
hidden_layer = Dense(units=8,
kernel_initializer=lecun_normal(),
activation="selu")(hidden_layer)
hidden_layer = Concatenate()([hidden_layer, input_wide_num_layer])
output_layer = Dense(units=1,
kernel_initializer=lecun_normal(),
bias_initializer=constant(logit(bias)),
activation="sigmoid",
name="output_layer")(hidden_layer)
return Model(input_layers, output_layer)
def build(self, input_shape):
self._trainable_weights.append(prior_params)
self.kernel_mu = self.add_weight(name='kernel_mu',
shape=(input_shape[1], self.output_dim),
initializer='uniform',
trainable=True)
self.bias_mu = self.add_weight(name='bias_mu',
shape=(self.output_dim,),
initializer='uniform',
trainable=True)
self.kernel_rho = self.add_weight(name='kernel_rho',
shape=(input_shape[1], self.output_dim),
initializer=initializers.constant(0.0),
trainable=True)
self.bias_rho = self.add_weight(name='bias_rho',
shape=(self.output_dim,),
initializer=initializers.constant(0.0),
trainable=True)
super().build(input_shape)
def build(self, input_shape):
self.kernel_mu = self.add_weight(
name='kernel_mu',
shape=(input_shape[1], self.units),
initializer=initializers.normal(stddev=self.init_sigma),
trainable=True)
self.bias_mu = self.add_weight(
name='bias_mu',
shape=(self.units, ),
initializer=initializers.normal(stddev=self.init_sigma),
trainable=True)
self.kernel_rho = self.add_weight(
name='kernel_rho',
shape=(input_shape[1], self.units),
initializer=initializers.constant(0.0),
trainable=True)
self.bias_rho = self.add_weight(name='bias_rho',
shape=(self.units, ),
initializer=initializers.constant(0.0),
trainable=True)
super().build(input_shape)
def Conv(x, nf, ks, name, weight_decay):
kernel_reg = regularizers.l2(weight_decay[0]) if weight_decay else None
bias_reg = regularizers.l2(weight_decay[1]) if weight_decay else None
return layers.Conv2D(
filters=nf,
kernel_size=(ks, ks),
padding='same',
name=name,
kernel_regularizer=kernel_reg,
bias_regularizer=bias_reg,
kernel_initializer=initializers.random_normal(stddev=0.01),
bias_initializer=initializers.constant(0.0))(x)
def create_heads(num_classes, rf1, hgid):
y1 = Conv2D(256,
3,
use_bias=True,
kernel_initializer=random_normal(stddev=0.02),
padding='same',
name='hm.%d.0.conv' % hgid)(rf1)
y1 = Activation('relu', name='hm.%d.0.relu' % hgid)(y1)
y1 = Conv2D(num_classes,
1,
use_bias=True,
kernel_initializer=constant(0),
bias_initializer=constant(-2.19),
name='hm.%d.1' % hgid,
activation="sigmoid")(y1)
y2 = Conv2D(256,
3,
use_bias=True,
kernel_initializer=random_normal(stddev=0.02),
padding='same',
name='wh.%d.0.conv' % hgid)(rf1)
y2 = Activation('relu', name='wh.%d.0.relu' % hgid)(y2)
y2 = Conv2D(2, 1, use_bias=True, name='wh.%d.1' % hgid)(y2)
y3 = Conv2D(256,
3,
use_bias=True,
kernel_initializer=random_normal(stddev=0.02),
padding='same',
name='reg.%d.0.conv' % hgid)(rf1)
y3 = Activation('relu', name='reg.%d.0.relu' % hgid)(y3)
y3 = Conv2D(2,
1,
use_bias=True,
kernel_initializer=random_normal(stddev=0.02),
name='reg.%d.1' % hgid)(y3)
return [y1, y2, y3]
def __init__(self, alpha_initializer=initializers.constant(0.2),
beta_initializer=initializers.constant(5.0),
alpha_regularizer=None,
alpha_constraint=None,
beta_regularizer=None,
beta_constraint=None,
shared_axes=None,
**kwargs):
super(ParametricSoftplus, self).__init__(**kwargs)
self.supports_masking = True
self.alpha_initializer = initializers.get(alpha_initializer)
self.alpha_regularizer = regularizers.get(alpha_regularizer)
self.alpha_constraint = constraints.get(alpha_constraint)
self.beta_initializer = initializers.get(beta_initializer)
self.beta_regularizer = regularizers.get(beta_regularizer)
self.beta_constraint = constraints.get(beta_constraint)
if shared_axes is None:
self.shared_axes = None
elif not isinstance(shared_axes, (list, tuple)):
self.shared_axes = [shared_axes]
else:
self.shared_axes = list(shared_axes)
def example_network(input_shape):
im_input = Input(shape=input_shape)
t = Conv3D(64, (11, 11, 11),
padding='valid',
kernel_initializer=initializers.truncated_normal(mean=0,
stddev=0.001),
bias_initializer=initializers.constant(0.1))(im_input)
t = Activation('relu')(t)
t = MaxPool3D(pool_size=(2, 2, 2), padding='valid')(t)
t = Conv3D(128, (6, 6, 6),
padding='valid',
kernel_initializer=initializers.truncated_normal(mean=0,
stddev=0.001),
bias_initializer=initializers.constant(0.1))(t)
t = Activation('relu')(t)
t = MaxPool3D(pool_size=(2, 2, 2), padding='valid')(t)
t = Conv3D(256, (3, 3, 3),
padding="valid",
kernel_initializer=initializers.truncated_normal(mean=0,
stddev=0.001),
bias_initializer=initializers.constant(0.1))(t)
t = Activation('relu')(t)
t = Flatten()(t)
t = Dense(1000,
kernel_initializer=initializers.truncated_normal(mean=0,
stddev=1 /
np.sqrt(1000)),
bias_initializer=initializers.constant(1.0))(t)
t = Activation('relu')(t)
t = Dropout(0.5)(t)
t = Dense(500,
kernel_initializer=initializers.truncated_normal(mean=0,
stddev=1 /
np.sqrt(500)),
bias_initializer=initializers.constant(1.0))(t)
t = Activation('relu')(t)
t = Dropout(0.5)(t)
t = Dense(200,
kernel_initializer=initializers.truncated_normal(mean=0,
stddev=1 /
np.sqrt(200)),
bias_initializer=initializers.constant(1.0))(t)
t = Activation('relu')(t)
t = Dropout(0.5)(t)
t = Dense(1)(t)
output = Activation('sigmoid')(t)
model = Model(input=im_input, output=output)
return model
def build(self, input_shape):
if type(self.t_initializer) is float:
initializer = constant(self.t_initializer)
else:
initializer = self.t_initializer
self.params = self.add_weight(name='t',
initializer=initializer,
shape=(self.output_dim, ))
if self.use_M:
f = self.M * self.func(self.params / self.M) + self.M
else:
f = self.func(self.params)
self.diag = f / tf.roll(f, -1, 0)
super(Diagonal, self).build(input_shape)
def __init__(self, initial_power=1, axis=-1, **kwargs):
self.alpha_pos_initializer = initializers.constant(1.)
self.alpha_neg_initializer = initializers.constant(1.)
self.beta_pos_initializer = initializers.constant(0.)
self.beta_neg_initializer = initializers.constant(0.)
self.rho_pos_initializer = initializers.constant(initial_power)
self.rho_neg_initializer = initializers.constant(initial_power)
self.alpha_pos_constraint = None
self.alpha_neg_constraint = None
self.beta_pos_constraint = None
self.beta_neg_constraint = None
self.rho_pos_constraint = None
self.rho_neg_constraint = None
self.alpha_pos_regularizer = None
self.alpha_neg_regularizer = None
self.beta_pos_regularizer = None
self.beta_neg_regularizer = None
self.rho_pos_regularizer = None
self.rho_neg_regularizer = None
self.axis = axis
super(PowerPReLU, self).__init__(**kwargs)
