def build_model(self, input_shape): hidden_dim = self.hidden_dim output_dim = self.output_dim ''' Input :- This returns a tensor. input_shape = (number_of_times_unfolded,dimension_of_each_ouptu) ''' x = Input(batch_shape=input_shape) h_tm1 = Input(batch_shape=(input_shape[0], hidden_dim)) c_tm1 = Input(batch_shape=(input_shape[0], hidden_dim)) W1 = Dense(hidden_dim * 4, kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer, use_bias=False) W2 = Dense(output_dim, kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer,) U = Dense(hidden_dim * 4, kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer,) z = add([W1(x), U(h_tm1)]) z0, z1, z2, z3 = get_slices(z, 4) i = Activation(self.recurrent_activation)(z0) f = Activation(self.recurrent_activation)(z1) c = add([multiply([f, c_tm1]), multiply([i, Activation(self.activation)(z2)])]) o = Activation(self.recurrent_activation)(z3) h = multiply([o, Activation(self.activation)(c)]) y = Activation(self.activation)(W2(h)) return Model([x, h_tm1, c_tm1], [y, h, c]) #h_tm1 --> h(t-1) i.e h of previous timestep.
def basic_critic_model():
#track = Input(shape=INPUT_DIM[3]) Use either track or focus
#opponents = Input(shape=[INPUT_DIM[2]]) Most likely won't use as agents is racing by itself
actions = Input(shape=[ACTION_DIM])
focus = Input(shape=INPUT_DIM[0])
speed = Input(shape=INPUT_DIM[1]) # vector of speedX, speedY and speedZ
rpm = Input(shape=INPUT_DIM[2])
wheelSpinVel = Input(shape=INPUT_DIM[3])
focus = Input(shape=INPUT_DIM[0])
speed = Input(shape=INPUT_DIM[1]) # vector of speedX, speedY and speedZ
rpm = Input(shape=INPUT_DIM[2])
wheelSpinVel = Input(shape=INPUT_DIM[3])
speed_rpm = concatenate([speed, rpm])
speedh1 = Dense(150, activation='linear')(speed_rpm)
wheelSpinVelh1 = Dense(150, activation='linear')(wheelSpinVel)
combinedSpeed = add([speedh1, wheelSpinVelh1])
focush1 = Dense(150, activation='linear')(focus)
combined_layer = concatenate([focush1, combinedSpeed])
h3 = Dense(600, activation='relu')(combined_layer)
action_h = BatchNormalization()(Dense(600, activation='linear', init='glorot_normal')(actions))
combined = add([h3, action_h])
final = Dense(600, activation='relu')((combined))
Q = Dense(2, activation='linear', init='glorot_normal')((final))
model = Model(inputs=[focus, speed, rpm, wheelSpinVel, actions], outputs=[Q])
model.compile(loss='mse', optimizer=Adam(lr=1e-3))
print(model.summary())
return model
def generator_model(input_img):
# Encoder
x = Conv2D(32, (3, 3), activation='relu', padding='same')(input_img)
x = Conv2D(32, (2, 2), activation='relu', padding='same')(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block2_conv1')(x)
x = Conv2D(64, (2, 2), activation='relu', padding='same')(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block3_conv1')(x)
x = Conv2D(128, (3, 3), activation='relu', padding='same')(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block4_conv1')(x)
res = Conv2D(256, (3, 3), activation='relu', padding='same')(x)
x = layers.add([x, res])
res = Conv2D(256, (3, 3), activation='relu', padding='same')(x)
encoded = layers.add([x, res])
# Decoder
res = Conv2D(256, (3, 3), activation='relu', padding='same', name='block5_conv1')(encoded)
x = layers.add([encoded, res])
res = Conv2D(256, (3, 3), activation='relu', padding='same')(x)
x = layers.add([x, res])
res = Conv2D(256, (3, 3), activation='relu', padding='same')(x)
x = layers.add([x, res])
x = Conv2D(128, (2, 2), activation='relu', padding='same', name='block6_conv1')(x)
x = UpSampling2D((2, 2))(x)
x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block7_conv1')(x)
res = Conv2D(128, (3, 3), activation='relu', padding='same')(x)
x = layers.add([x, res])
res = Conv2D(128, (3, 3), activation='relu', padding='same')(x)
x = layers.add([x, res])
x = Conv2D(64, (2, 2), activation='relu', padding='same', name='block8_conv1')(x)
x = UpSampling2D((2, 2))(x)
x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block9_conv1')(x)
res = Conv2D(64, (3, 3), activation='relu', padding='same')(x)
x = layers.add([x, res])
res = Conv2D(64, (3, 3), activation='relu', padding='same')(x)
x = layers.add([x, res])
x = Conv2D(32, (2, 2), activation='relu', padding='same', name='block10_conv1')(x)
x = UpSampling2D((2, 2))(x)
x = Conv2D(32, (3, 3), activation='relu', padding='same', name='block11_conv1')(x)
res = Conv2D(32, (3, 3), activation='relu', padding='same')(x)
x = layers.add([x, res])
decoded = Conv2D(3, (3, 3), activation='sigmoid', padding='same')(x)
return decoded
def identity_Block(inpt, nb_filter, kernel_size, strides=(1, 1), with_conv_shortcut=False):
x = Conv2d_BN(inpt, nb_filter=nb_filter, kernel_size=kernel_size, strides=strides, padding='same')
x = Conv2d_BN(x, nb_filter=nb_filter, kernel_size=kernel_size, padding='same')
if with_conv_shortcut:
shortcut = Conv2d_BN(inpt, nb_filter=nb_filter, strides=strides, kernel_size=kernel_size)
x = layers.add([x, shortcut])
return x
else:
x = layers.add([x, inpt])
return x
def bottleneck_Block(inpt,nb_filters,strides=(1,1),with_conv_shortcut=False):
k1,k2,k3=nb_filters
x = Conv2d_BN(inpt, nb_filter=k1, kernel_size=1, strides=strides, padding='same')
x = Conv2d_BN(x, nb_filter=k2, kernel_size=3, padding='same')
x = Conv2d_BN(x, nb_filter=k3, kernel_size=1, padding='same')
if with_conv_shortcut:
shortcut = Conv2d_BN(inpt, nb_filter=k3, strides=strides, kernel_size=1)
x = layers.add([x, shortcut])
return x
else:
x = layers.add([x, inpt])
return x
def residual_block(x,planes,stride=(1,1)):
D = int(math.floor(planes * (base_width/64.0)))
C = cardinality
shortcut = x
y = Conv2D(D*C,kernel_size=(1,1),strides=(1,1),padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(weight_decay),use_bias=False)(shortcut)
y = add_common_layer(y)
y = group_conv(y,D*C,stride)
y = add_common_layer(y)
y = Conv2D(planes*expansion, kernel_size=(1,1), strides=(1,1), padding='same', kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(weight_decay),use_bias=False)(y)
y = add_common_layer(y)
if stride != (1,1) or inplanes != planes * expansion:
shortcut = Conv2D(planes * expansion, kernel_size=(1,1), strides=stride, padding='same', kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(weight_decay),use_bias=False)(x)
shortcut = BatchNormalization(momentum=0.9, epsilon=1e-5)(shortcut)
y = squeeze_excite_block(y)
y = add([y,shortcut])
y = Activation('relu')(y)
return y
def basic_actor_model():
#track = Input(shape=INPUT_DIM[3]) Use either track or focus
#opponents = Input(shape=[INPUT_DIM[2]]) Most likely won't use as agents is racing by itself
focus = Input(shape=INPUT_DIM[0])
speed = Input(shape=INPUT_DIM[1]) # vector of speedX, speedY and speedZ
rpm = Input(shape=INPUT_DIM[2])
wheelSpinVel = Input(shape=INPUT_DIM[3])
speed_rpm = concatenate([speed, rpm])
speedh1 = Dense(150, activation='linear')(speed_rpm)
wheelSpinVelh1 = Dense(150, activation='linear')(wheelSpinVel)
combinedSpeed = add([speedh1, wheelSpinVelh1])
focush1 = Dense(150, activation='linear')(focus)
combined_layer = concatenate([focush1, combinedSpeed])
h3 = Dense(600, activation='relu')(combined_layer)
steering = Dense(1, activation='tanh', init='glorot_normal')(h3) #consider adding acceleration as an input to steering
acceleration = Dense(1, activation='sigmoid', init='glorot_normal')(h3)
output = concatenate([steering, acceleration])
model = Model(inputs=[focus, speed, rpm, wheelSpinVel], outputs=[output])
model.compile(loss='mse', optimizer=Adam(lr=1e-4))
print(model.summary())
return model
def identity_block(input_tensor, kernel_size, filters, stage, block):
"""The identity block is the block that has no conv layer at shortcut.
# Arguments
input_tensor: input tensor
kernel_size: default 3, the kernel size of middle conv layer at main path
filters: list of integers, the filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
# Returns
Output tensor for the block.
"""
filters1, filters2, filters3 = filters
if K.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
x = Conv2D(filters1, (1, 1), name=conv_name_base + '2a')(input_tensor)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
x = Activation('relu')(x)
x = Conv2D(filters2, kernel_size,
padding='same', name=conv_name_base + '2b')(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)
x = Activation('relu')(x)
x = Conv2D(filters3, (1, 1), name=conv_name_base + '2c')(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)
x = layers.add([x, input_tensor])
x = Activation('relu')(x)
return x
def test_multiple_outputs_no_mask():
def func(x):
return [x * 0.2, x * 0.3]
def output_shape(input_shape):
return [input_shape, input_shape]
i = layers.Input(shape=(3, 2, 1))
o = layers.Lambda(function=func,
output_shape=output_shape)(i)
assert o[0]._keras_shape == (None, 3, 2, 1)
assert o[1]._keras_shape == (None, 3, 2, 1)
o = layers.add(o)
model = Model(i, o)
i2 = layers.Input(shape=(3, 2, 1))
o2 = model(i2)
model2 = Model(i2, o2)
x = np.random.random((4, 3, 2, 1))
out = model2.predict(x)
assert out.shape == (4, 3, 2, 1)
assert_allclose(out, x * 0.2 + x * 0.3, atol=1e-4)
def encoder(self):
""" Encoder Network """
kwargs = dict(kernel_initializer=self.kernel_initializer)
input_ = Input(shape=self.input_shape)
in_conv_filters = self.input_shape[0]
if self.input_shape[0] > 128:
in_conv_filters = 128 + (self.input_shape[0] - 128) // 4
dense_shape = self.input_shape[0] // 16
var_x = self.blocks.conv(input_, in_conv_filters, res_block_follows=True, **kwargs)
tmp_x = var_x
res_cycles = 8 if self.config.get("lowmem", False) else 16
for _ in range(res_cycles):
nn_x = self.blocks.res_block(var_x, 128, **kwargs)
var_x = nn_x
# consider adding scale before this layer to scale the residual chain
var_x = add([var_x, tmp_x])
var_x = self.blocks.conv(var_x, 128, **kwargs)
var_x = PixelShuffler()(var_x)
var_x = self.blocks.conv(var_x, 128, **kwargs)
var_x = PixelShuffler()(var_x)
var_x = self.blocks.conv(var_x, 128, **kwargs)
var_x = self.blocks.conv_sep(var_x, 256, **kwargs)
var_x = self.blocks.conv(var_x, 512, **kwargs)
if not self.config.get("lowmem", False):
var_x = self.blocks.conv_sep(var_x, 1024, **kwargs)
var_x = Dense(self.encoder_dim, **kwargs)(Flatten()(var_x))
var_x = Dense(dense_shape * dense_shape * 1024, **kwargs)(var_x)
var_x = Reshape((dense_shape, dense_shape, 1024))(var_x)
var_x = self.blocks.upscale(var_x, 512, **kwargs)
return KerasModel(input_, var_x)
def build(self):
input_encoded_m = self.encoders_m(self.input_sequence)
input_encoded_c = self.encoders_c(self.input_sequence)
question_encoded = self.encoders_question(self.question)
match = dot([input_encoded_m, question_encoded], axes=(2, 2))
match = Activation('softmax')(match)
# add the match matrix with the second input vector sequence
response = add([match, input_encoded_c]) # (samples, story_maxlen, query_maxlen)
response = Permute((2, 1))(response) # (samples, query_maxlen, story_maxlen)
# concatenate the match matrix with the question vector sequence
answer = concatenate([response, question_encoded])
# the original paper uses a matrix multiplication for this reduction step.
# we choose to use a RNN instead.
answer = LSTM(32)(answer) # (samples, 32)
# one regularization layer -- more would probably be needed.
answer = Dropout(0.3)(answer)
answer = Dense(self.vocab_size)(answer) # (samples, vocab_size)
# we output a probability distribution over the vocabulary
answer = Activation('softmax')(answer)
# build the final model
model = Model([self.input_sequence, self.question], answer)
self.model = model
return model
def test_merge_add():
i1 = layers.Input(shape=(4, 5))
i2 = layers.Input(shape=(4, 5))
i3 = layers.Input(shape=(4, 5))
o = layers.add([i1, i2, i3])
assert o._keras_shape == (None, 4, 5)
model = models.Model([i1, i2, i3], o)
add_layer = layers.Add()
o2 = add_layer([i1, i2, i3])
assert add_layer.output_shape == (None, 4, 5)
x1 = np.random.random((2, 4, 5))
x2 = np.random.random((2, 4, 5))
x3 = np.random.random((2, 4, 5))
out = model.predict([x1, x2, x3])
assert out.shape == (2, 4, 5)
assert_allclose(out, x1 + x2 + x3, atol=1e-4)
assert add_layer.compute_mask([i1, i2, i3], [None, None, None]) is None
assert np.all(K.eval(add_layer.compute_mask(
[i1, i2, i3], [K.variable(x1), K.variable(x2), K.variable(x3)])))
# Test invalid use case
with pytest.raises(ValueError):
add_layer.compute_mask([i1, i2, i3], x1)
with pytest.raises(ValueError):
add_layer.compute_mask(i1, [None, None, None])
with pytest.raises(ValueError):
add_layer.compute_mask([i1, i2, i3], [None, None])
def defineNetwork(self):
print("Setting up network...")
inputs = Input(shape=(self.nx, self.ny, self.n_diversity))
x = GaussianNoise(self.noise)(inputs)
conv = Convolution2D(self.n_filters, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(x)
x = self.residual(conv)
for i in range(self.n_conv_layers):
x = self.residual(x)
x = Convolution2D(self.n_filters, (3, 3), padding='same', kernel_initializer='he_normal')(x)
x = BatchNormalization()(x)
x = add([x, conv])
final = Convolution2D(1, (1, 1), activation='relu', padding='same', kernel_initializer='he_normal')(x)
self.model = Model(inputs=inputs, outputs=final)
json_string = self.model.to_json()
f = open('{0}_model.json'.format(self.root), 'w')
f.write(json_string)
f.close()
with open('{0}_summary.txt'.format(self.root), 'w') as f:
with redirect_stdout(f):
self.model.summary()
plot_model(self.model, to_file='{0}_model.png'.format(self.root), show_shapes=True)
def resnet_conv_block(input_tensor, kernel_size, filters, stage, block,
strides=(2, 2), bias=False):
filters1, filters2, filters3 = filters
if K.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
conv1_reduce_name = 'conv' + str(stage) + "_" + str(block) + "_1x1_reduce"
conv1_increase_name = 'conv' + str(stage) + "_" + str(
block) + "_1x1_increase"
conv1_proj_name = 'conv' + str(stage) + "_" + str(block) + "_1x1_proj"
conv3_name = 'conv' + str(stage) + "_" + str(block) + "_3x3"
x = Conv2D(filters1, (1, 1), strides=strides, use_bias=bias,
name=conv1_reduce_name)(input_tensor)
x = BatchNormalization(axis=bn_axis, name=conv1_reduce_name + "/bn")(x)
x = Activation('relu')(x)
x = Conv2D(filters2, kernel_size, padding='same', use_bias=bias,
name=conv3_name)(x)
x = BatchNormalization(axis=bn_axis, name=conv3_name + "/bn")(x)
x = Activation('relu')(x)
x = Conv2D(filters3, (1, 1), name=conv1_increase_name, use_bias=bias)(x)
x = BatchNormalization(axis=bn_axis, name=conv1_increase_name + "/bn")(x)
shortcut = Conv2D(filters3, (1, 1), strides=strides, use_bias=bias,
name=conv1_proj_name)(input_tensor)
shortcut = BatchNormalization(axis=bn_axis, name=conv1_proj_name + "/bn")(
shortcut)
x = layers.add([x, shortcut])
x = Activation('relu')(x)
return x
def senet_identity_block(input_tensor, kernel_size,
filters, stage, block, bias=False):
filters1, filters2, filters3 = filters
if K.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
conv1_reduce_name = 'conv' + str(stage) + "_" + str(block) + "_1x1_reduce"
conv1_increase_name = 'conv' + str(stage) + "_" + str(
block) + "_1x1_increase"
conv3_name = 'conv' + str(stage) + "_" + str(block) + "_3x3"
x = Conv2D(filters1, (1, 1), use_bias=bias,
name=conv1_reduce_name)(input_tensor)
x = BatchNormalization(axis=bn_axis, name=conv1_reduce_name + "/bn")(x)
x = Activation('relu')(x)
x = Conv2D(filters2, kernel_size, padding='same', use_bias=bias,
name=conv3_name)(x)
x = BatchNormalization(axis=bn_axis, name=conv3_name + "/bn")(x)
x = Activation('relu')(x)
x = Conv2D(filters3, (1, 1), name=conv1_increase_name, use_bias=bias)(x)
x = BatchNormalization(axis=bn_axis, name=conv1_increase_name + "/bn")(x)
se = senet_se_block(x, stage=stage, block=block, bias=True)
m = layers.add([x, se])
m = Activation('relu')(m)
return m
def f(xInp):
x = xInp
x = convBnAct(nInnerFilters , (1, 1))(x)
x = convBnAct(nInnerFilters , (3, 3))(x)
x = convBnAct(nExternalFilters, (3, 3))(x)
x = add([xInp, x])
return x
def build_model():
input_ = layers.Input(shape=(input_dims,))
resnet_dims = input_dims * 2
model = layers.Dense(resnet_dims,
kernel_initializer='Orthogonal',
activation=layers.advanced_activations.PReLU())(input_)
model = layers.BatchNormalization()(model)
for n in range(20):
shortcut = model
model = layers.Dense(resnet_dims,
kernel_initializer='Orthogonal')(model)
model = layers.BatchNormalization()(model)
model = layers.advanced_activations.PReLU()(model)
model = layers.Dense(resnet_dims,
kernel_initializer='Orthogonal')(model)
model = layers.BatchNormalization()(model)
model = layers.add([model, shortcut])
model = layers.advanced_activations.PReLU()(model)
#model = layers.Dropout(0.9)(model)
model = layers.Dense(16,
kernel_initializer='Orthogonal',
activation=layers.advanced_activations.PReLU())(model)
model = layers.BatchNormalization()(model)
model = layers.Dense(1,
activation='sigmoid')(model)
model = models.Model(input_, model)
model.compile(loss = 'binary_crossentropy',
optimizer = optimizers.Nadam(),
#optimizer = optimizers.SGD(),
metrics = ['binary_accuracy'])
#print(model.summary(line_length=120))
return model
def convresblock(x, nfeats=8, ksize=3, nskipped=2, elu=True):
"""The proposed residual block from [4].
Running with elu=True will use ELU nonlinearity and running with
elu=False will use BatchNorm + RELU nonlinearity. While ELU's are fast
due to the fact they do not suffer from BatchNorm overhead, they may
overfit because they do not offer the stochastic element of the batch
formation process of BatchNorm, which acts as a good regularizer.
# Arguments
x: 4D tensor, the tensor to feed through the block
nfeats: Integer, number of feature maps for conv layers.
ksize: Integer, width and height of conv kernels in first convolution.
nskipped: Integer, number of conv layers for the residual function.
elu: Boolean, whether to use ELU or BN+RELU.
# Input shape
4D tensor with shape:
`(batch, channels, rows, cols)`
# Output shape
4D tensor with shape:
`(batch, filters, rows, cols)`
"""
y0 = Conv2D(nfeats, ksize, padding='same')(x)
y = y0
for i in range(nskipped):
if elu:
y = ELU()(y)
else:
y = BatchNormalization(axis=1)(y)
y = Activation('relu')(y)
y = Conv2D(nfeats, 1, padding='same')(y)
return layers.add([y0, y])
def residual_block(y, nb_channels_in, nb_channels_out, _strides=(1, 1), _project_shortcut=False):
"""
Our network consists of a stack of residual blocks. These blocks have the same topology,
and are subject to two simple rules:
- If producing spatial maps of the same size, the blocks share the same hyper-parameters (width and filter sizes).
- Each time the spatial map is down-sampled by a factor of 2, the width of the blocks is multiplied by a factor of 2.
"""
shortcut = y
# we modify the residual building block as a bottleneck design to make the network more economical
y = layers.Conv2D(nb_channels_in, kernel_size=(1, 1), strides=(1, 1), padding='same')(y)
y = add_common_layers(y)
# ResNeXt (identical to ResNet when `cardinality` == 1)
y = grouped_convolution(y, nb_channels_in, _strides=_strides)
y = add_common_layers(y)
y = layers.Conv2D(nb_channels_out, kernel_size=(1, 1), strides=(1, 1), padding='same')(y)
# batch normalization is employed after aggregating the transformations and before adding to the shortcut
y = layers.BatchNormalization()(y)
# identity shortcuts used directly when the input and output are of the same dimensions
if _project_shortcut or _strides != (1, 1):
# when the dimensions increase projection shortcut is used to match dimensions (done by 1×1 convolutions)
# when the shortcuts go across feature maps of two sizes, they are performed with a stride of 2
shortcut = layers.Conv2D(nb_channels_out, kernel_size=(1, 1), strides=_strides, padding='same')(shortcut)
shortcut = layers.BatchNormalization()(shortcut)
y = layers.add([shortcut, y])
# relu is performed right after each batch normalization,
# expect for the output of the block where relu is performed after the adding to the shortcut
y = layers.LeakyReLU()(y)
return y
def createLayers():
x = Input(shape=env.observation_space.shape, name='x')
u = Input(shape=env.action_space.shape, name='u')
if args.batch_norm:
h = BatchNormalization()(x)
else:
h = x
for i in xrange(args.layers):
h = Dense(args.hidden_size, activation=args.activation, name='h'+str(i+1),
kernel_constraint=W_constraint, kernel_regularizer=kernel_regularizer)(h)
if args.batch_norm and i != args.layers - 1:
h = BatchNormalization()(h)
v = Dense(1, name='v', kernel_constraint=W_constraint, \
kernel_regularizer=kernel_regularizer)(h)
m = Dense(num_actuators, name='m', kernel_constraint=W_constraint, \
kernel_regularizer=kernel_regularizer)(h)
l0 = Dense(num_actuators * (num_actuators + 1)/2, name='l0',
kernel_constraint=W_constraint, kernel_regularizer=kernel_regularizer)(h)
l = Lambda(_L, output_shape=(num_actuators, num_actuators), name='l')(l0)
p = Lambda(_P, output_shape=(num_actuators, num_actuators), name='p')(l)
#a = merge([m, p, u], mode=_A, output_shape=(None, num_actuators,), name="a")
a = merge([m, p, u], mode=_A, output_shape=(num_actuators,), name="a")
#q = merge([v, a], mode=_Q, output_shape=(None, num_actuators,), name="q")
q = add([v, a], name="q")
return x, u, m, v, q, p, a
def build_model(len_words, embedding_matrix):
f_input=Input(shape=(maxlen_words,))
f_emb=Embedding(output_dim=vec_size, input_dim=len_words+1, input_length=maxlen_words, mask_zero=True, weights=[embedding_matrix], trainable=False)(f_input)
f_full=Dense(vec_size,activation='relu')(f_emb)
f_layer=LSTM(128)(f_full)
r_input=Input(shape=(maxlen_words,))
r_emb=Embedding(output_dim=vec_size, input_dim=len_words+1, input_length=maxlen_words, mask_zero=True, weights=[embedding_matrix], trainable=False)(r_input)
r_full=Dense(vec_size,activation='relu')(r_emb)
r_layer=LSTM(128)(r_full)
merged_layer=add([f_layer, r_layer])
out_layer=Dense(vec_size,activation='relu')(merged_layer)
my_model=Model([f_input, r_input], out_layer)
optimizer = RMSprop()
my_model.compile(loss='mean_squared_error', optimizer=optimizer)
return my_model
def res(inputs,channels,n_channels):
ch1 = Conv3D(n_channels, (1, 1, 1), activation='relu', padding='same')(inputs)
#ch1 = Conv3D(n_channels, 3, 3, 3, activation='relu',padding='same')(ch1)
ch1 = Conv3D(n_channels, (3, 3, 3), padding='same')(ch1)
ch1 = normalization.BatchNormalization(epsilon=2e-05, axis=1, momentum=0.9, weights=None, beta_initializer='zero', gamma_initializer='one')(ch1)
ch1 = core.Activation('relu')(ch1)
ch1 = Conv3D(n_channels, (3, 3, 3), padding='same')(ch1)
ch1 = normalization.BatchNormalization(epsilon=2e-05, axis=1, momentum=0.9, weights=None, beta_initializer='zero', gamma_initializer='one')(ch1)
ch1 = core.Activation('relu')(ch1)
ch2 = Conv3D(n_channels, (1, 1, 1), activation='relu', padding='same')(inputs)
ch2 = Conv3D(n_channels, (3, 3, 3), padding='same')(ch2)
ch2 = normalization.BatchNormalization(epsilon=2e-05, axis=1, momentum=0.9, weights=None, beta_initializer='zero', gamma_initializer='one')(ch2)
ch2 = core.Activation('relu')(ch2)
#ch2 = Conv3D(n_channels, (3, 3, 3), activation='relu', padding='same')(ch2)
ch3 = Conv3D(n_channels, (3, 3, 3), activation='relu', padding='same')(inputs)
out = concatenate([ch1,ch2,ch3], axis=1)
out = Conv3D(channels, (1, 1, 1), activation='relu', padding='same')(out)
out= add([inputs,out])
out= core.Activation('relu')(out)
return out
def shortcut(input, residual):
"""Adds a shortcut between input and residual block and merges them with "sum"
"""
# Expand channels of shortcut to match residual.
# Stride appropriately to match residual (width, height)
# Should be int if network architecture is correctly configured.
ROW_AXIS = 1
COL_AXIS = 2
CHANNEL_AXIS = 3
input_shape = K.int_shape(input)
residual_shape = K.int_shape(residual)
stride_width = int(round(input_shape[ROW_AXIS] / residual_shape[ROW_AXIS]))
stride_height = int(round(input_shape[COL_AXIS] / residual_shape[COL_AXIS]))
equal_channels = input_shape[CHANNEL_AXIS] == residual_shape[CHANNEL_AXIS]
shortcut = input
# 1 X 1 conv if shape is different. Else identity.
if stride_width > 1 or stride_height > 1 or not equal_channels:
#kernel_regularizer = l2(1e-5)
#kernel_regularizer = l2(1e-6)
kernel_regularizer = None
shortcut = Conv2D(filters=residual_shape[CHANNEL_AXIS],
kernel_size=(2, 2),
#kernel_size=(1, 1),
strides=(stride_width, stride_height),
padding="valid",
kernel_initializer="he_normal",
kernel_regularizer=kernel_regularizer)(input)
return add([shortcut, residual])
def self_attn_block(inp, n_c, squeeze_factor=8):
""" GAN Self Attention Block
Code borrows from https://github.com/taki0112/Self-Attention-GAN-Tensorflow
"""
msg = "Input channels must be >= {}, recieved nc={}".format(squeeze_factor, n_c)
assert n_c // squeeze_factor > 0, msg
var_x = inp
shape_x = var_x.get_shape().as_list()
var_f = Conv2D(n_c // squeeze_factor, 1,
kernel_regularizer=regularizers.l2(GAN22_REGULARIZER))(var_x)
var_g = Conv2D(n_c // squeeze_factor, 1,
kernel_regularizer=regularizers.l2(GAN22_REGULARIZER))(var_x)
var_h = Conv2D(n_c, 1, kernel_regularizer=regularizers.l2(GAN22_REGULARIZER))(var_x)
shape_f = var_f.get_shape().as_list()
shape_g = var_g.get_shape().as_list()
shape_h = var_h.get_shape().as_list()
flat_f = Reshape((-1, shape_f[-1]))(var_f)
flat_g = Reshape((-1, shape_g[-1]))(var_g)
flat_h = Reshape((-1, shape_h[-1]))(var_h)
var_s = Lambda(lambda var_x: K.batch_dot(var_x[0],
Permute((2, 1))(var_x[1])))([flat_g, flat_f])
beta = Softmax(axis=-1)(var_s)
var_o = Lambda(lambda var_x: K.batch_dot(var_x[0], var_x[1]))([beta, flat_h])
var_o = Reshape(shape_x[1:])(var_o)
var_o = Scale()(var_o)
out = add([var_o, inp])
return out
def linknet_decoder(conv1, enc1, enc2, enc3, enc4, enc5, filters=[64, 128, 256, 512, 512], feature_scale=4, skipFirst=False, transposed_conv=False):
decoder5 = decoder(enc5, filters[4], filters[3], name='decoder5', feature_scale=feature_scale, transposed_conv=transposed_conv)
decoder5 = add([decoder5, enc4])
decoder4 = decoder(decoder5, filters[3], filters[2], name='decoder4', feature_scale=feature_scale, transposed_conv=transposed_conv)
decoder4 = add([decoder4, enc3])
decoder3 = decoder(decoder4, filters[2], filters[1], name='decoder3', feature_scale=feature_scale, transposed_conv=transposed_conv)
decoder3 = add([decoder3, enc2])
decoder2 = decoder(decoder3, filters[1], filters[0], name='decoder2', feature_scale=feature_scale, transposed_conv=transposed_conv)
decoder2 = add([decoder2, enc1])
decoder1 = decoder(decoder2, filters[0], filters[0], name='decoder1', feature_scale=feature_scale, transposed_conv=transposed_conv)
if skipFirst:
x = concatenate([conv1, decoder1])
x = conv_bn_relu(x, 32, 3, stride=1, padding='same', name='f2_skip_1')
x = conv_bn_relu(x, 32, 3, stride=1, padding='same', name='f2_skip_2')
else:
x = conv_bn_relu(decoder1, 32, 3, stride=1, padding='same', name='f2')
return x
def residual_block(x,out_filters,increase_filter=False):
if increase_filter:
first_stride = (2,2)
else:
first_stride = (1,1)
pre_bn = BatchNormalization()(x)
pre_relu = Activation('relu')(pre_bn)
conv_1 = Conv2D(out_filters,kernel_size=(3,3),strides=first_stride,padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(self.weight_decay))(pre_relu)
bn_1 = BatchNormalization()(conv_1)
relu1 = Activation('relu')(bn_1)
conv_2 = Conv2D(out_filters, kernel_size=(3,3), strides=(1,1), padding='same', kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(self.weight_decay))(relu1)
if increase_filter or in_filters != out_filters:
projection = Conv2D(out_filters,kernel_size=(1,1),strides=first_stride,padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(self.weight_decay))(x)
block = add([conv_2, projection])
else:
block = add([conv_2,x])
return block
def fun(estimation, next_frame, t1, t2, t3):
inputs = concatenate([estimation, next_frame])
A01 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(inputs)
C11 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=2)(A01)
C12 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C11)
C13 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C12)
C14 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C13)
C14 = add([C11, C14])
C21 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C14)
C22 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C21)
C23 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C22)
C24 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C23)
C24 = add([C21, C24])
C31 = convolution(128, kernel_size=3, l2_reg=l2_reg, strides=2)(C24)
C32 = convolution(128, kernel_size=3, l2_reg=l2_reg, strides=1)(C31)
C33 = convolution(128, kernel_size=3, l2_reg=l2_reg, strides=1)(C32)
C34 = convolution(128, kernel_size=3, l2_reg=l2_reg, strides=1)(C33)
C34 = add([C31, C34])
C41 = convolution(256, kernel_size=3, l2_reg=l2_reg, strides=2)(C34)
C42 = Lambda(lambda x: K.concatenate([x,t1],axis=-1), output_shape=(nx/8,ny/8,512))(C41)
B42 = Conv2D(256, (1, 1), padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), strides=1)(C42)
C43 = convolution(256, kernel_size=3, l2_reg=l2_reg, strides=1)(B42)
C44 = convolution(256, kernel_size=3, l2_reg=l2_reg, strides=1)(C43)
C45 = convolution(256, kernel_size=3, l2_reg=l2_reg, strides=1)(C44)
C45 = add([C41, C45])
C51 = transposed_convolution(128, kernel_size=4, l2_reg=l2_reg, strides=2)(C45)
C51 = add([C51, C34])
C52 = Lambda(lambda x: K.concatenate([x,t2],axis=-1),output_shape=(nx/4,ny/4,256))(C51)
B52 = Conv2D(128, (1, 1), padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), strides=1)(C52)
C53 = convolution(128, kernel_size=3, l2_reg=l2_reg, strides=1)(B52)
C54 = convolution(128, kernel_size=3, l2_reg=l2_reg, strides=1)(C53)
C55 = convolution(128, kernel_size=3, l2_reg=l2_reg, strides=1)(C54)
C55 = add([C51, C55])
C61 = transposed_convolution(64, kernel_size=4, l2_reg=l2_reg, strides=2)(C55)
C61 = add([C61, C24])
C62 = Lambda(lambda x: K.concatenate([x,t3],axis=-1),output_shape=(nx/2,ny/2,128))(C61)
B62 = Conv2D(128, (1, 1), padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), strides=1)(C62)
C63 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(B62)
C64 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C63)
C65 = convolution(64, kernel_size=3, l2_reg=l2_reg, strides=1)(C64)
C65 = add([C61, C65])
C71 = transposed_convolution(64, kernel_size=4, l2_reg=l2_reg, strides=2)(C65)
C72 = convolution(64, kernel_size=4, l2_reg=l2_reg, strides=1)(C71)
out = convolution(1, kernel_size=3, l2_reg=l2_reg, strides=1)(C72)
return out, C43, C53, C63
def convresblock(x, nfeats=8, ksize=3, nskipped=2):
''' The proposed residual block from [4]'''
y0 = Conv2D(nfeats, ksize, padding='same')(x)
y = y0
for i in range(nskipped):
y = BatchNormalization(axis=1)(y)
y = Activation('relu')(y)
y = Conv2D(nfeats, ksize, padding='same')(y)
return layers.add([y0, y])
def residual(self, inputs):
x = Convolution2D(self.n_filters, (3, 3), padding='same', kernel_initializer='he_normal')(inputs)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Convolution2D(self.n_filters, (3, 3), padding='same', kernel_initializer='he_normal')(x)
x = BatchNormalization()(x)
x = add([x, inputs])
return x
def big_XCEPTION(input_shape, num_classes):
img_input = Input(input_shape)
x = Conv2D(32, (3, 3), strides=(2, 2), use_bias=False)(img_input)
x = BatchNormalization(name='block1_conv1_bn')(x)
x = Activation('relu', name='block1_conv1_act')(x)
x = Conv2D(64, (3, 3), use_bias=False)(x)
x = BatchNormalization(name='block1_conv2_bn')(x)
x = Activation('relu', name='block1_conv2_act')(x)
residual = Conv2D(128, (1, 1), strides=(2, 2),
padding='same', use_bias=False)(x)
residual = BatchNormalization()(residual)
x = SeparableConv2D(128, (3, 3), padding='same', use_bias=False)(x)
x = BatchNormalization(name='block2_sepconv1_bn')(x)
x = Activation('relu', name='block2_sepconv2_act')(x)
x = SeparableConv2D(128, (3, 3), padding='same', use_bias=False)(x)
x = BatchNormalization(name='block2_sepconv2_bn')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
x = layers.add([x, residual])
residual = Conv2D(256, (1, 1), strides=(2, 2),
padding='same', use_bias=False)(x)
residual = BatchNormalization()(residual)
x = Activation('relu', name='block3_sepconv1_act')(x)
x = SeparableConv2D(256, (3, 3), padding='same', use_bias=False)(x)
x = BatchNormalization(name='block3_sepconv1_bn')(x)
x = Activation('relu', name='block3_sepconv2_act')(x)
x = SeparableConv2D(256, (3, 3), padding='same', use_bias=False)(x)
x = BatchNormalization(name='block3_sepconv2_bn')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
x = layers.add([x, residual])
x = Conv2D(num_classes, (3, 3),
# kernel_regularizer=regularization,
padding='same')(x)
x = GlobalAveragePooling2D()(x)
output = Activation('softmax', name='predictions')(x)
model = Model(img_input, output)
return model
def shuffle_unit(x,
in_channels,
out_channels,
groups,
downsample,
ignore_group,
name="shuffle_unit"):
"""
ShuffleNet unit.
Parameters:
----------
x : keras.backend tensor/variable/symbol
Input tensor/variable/symbol.
in_channels : int
Number of input channels.
out_channels : int
Number of output channels.
groups : int
Number of groups in convolution layers.
downsample : bool
Whether do downsample.
ignore_group : bool
Whether ignore group value in the first convolution layer.
name : str, default 'shuffle_unit'
Unit name.
Returns
-------
keras.backend tensor/variable/symbol
Resulted tensor/variable/symbol.
"""
mid_channels = out_channels // 4
if downsample:
out_channels -= in_channels
identity = x
x = conv1x1(
x=x,
in_channels=in_channels,
out_channels=mid_channels,
groups=(1 if ignore_group else groups),
name=name + "/compress_conv1")
x = batchnorm(
x=x,
name=name + "/compress_bn1")
x = nn.Activation("relu", name=name + "/activ")(x)
x = channel_shuffle_lambda(
channels=mid_channels,
groups=groups,
name=name + "/c_shuffle")(x)
x = depthwise_conv3x3(
x=x,
channels=mid_channels,
strides=(2 if downsample else 1),
name=name + "/dw_conv2")
x = batchnorm(
x=x,
name=name + "/dw_bn2")
x = conv1x1(
x=x,
in_channels=mid_channels,
out_channels=out_channels,
groups=groups,
name=name + "/expand_conv3")
x = batchnorm(
x=x,
name=name + "/expand_bn3")
if downsample:
identity = avgpool2d(
x=identity,
pool_size=3,
strides=2,
padding=1,
name=name + "/avgpool")
x = nn.concatenate([x, identity], axis=get_channel_axis(), name=name + "/concat")
else:
x = nn.add([x, identity], name=name + "/add")
x = nn.Activation("relu", name=name + "/final_activ")(x)
return x
def lstm_model():
inputs = Input(shape=(40, 50, 150, 4))
c1 = TimeDistributed(Conv2D(32, (2, 2), activation='relu',
padding='same'))(inputs) # (None, 50, 150, 32)
x = TimeDistributed(MaxPooling2D((2, 2),
padding='same'))(c1) # (None, 25, 75, 32)
c2 = TimeDistributed(Conv2D(64, (2, 2), activation='relu',
padding='same'))(x) # (None, 25, 75, 64)
x = TimeDistributed(MaxPooling2D((2, 2),
padding='same'))(c2) # (None, 13, 38, 64)
c3 = TimeDistributed(Conv2D(128, (2, 2), activation='relu',
padding='same'))(x) # (None, 13, 38, 128)
x = TimeDistributed(MaxPooling2D((2, 2),
padding='same'))(c3) # (None, 7, 19, 128)
x = ConvLSTM2D(filters=256,
kernel_size=(3, 3),
activation='relu',
recurrent_activation='hard_sigmoid',
padding='same',
return_sequences=True)(x)
x = TimeDistributed(
Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same'))(x)
x = TimeDistributed(Cropping2D(cropping=((1, 0), (0, 0))))(x)
x = add([x, c3])
x = ConvLSTM2D(filters=128,
kernel_size=(3, 3),
activation='relu',
recurrent_activation='hard_sigmoid',
padding='same',
return_sequences=True)(x)
x = TimeDistributed(
Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same'))(x)
x = TimeDistributed(Cropping2D(cropping=((1, 0), (1, 0))))(x)
x = add([x, c2])
x = ConvLSTM2D(filters=64,
kernel_size=(3, 3),
activation='relu',
recurrent_activation='hard_sigmoid',
padding='same',
return_sequences=True)(x)
x = TimeDistributed(
Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same'))(x)
x = add([x, c1])
x = Conv3D(filters=4,
kernel_size=(3, 3, 3),
activation='linear',
padding='same',
data_format='channels_last')(x)
model = Model(inputs=inputs, outputs=x)
plot_model(model,
to_file='cnn_model.png',
show_shapes=True,
show_layer_names=False)
return model
def FCN_8S(nClasses, inputs=inpt):
conv1 = Conv2D(filters=32,
kernel_size=(3, 3),
padding='same',
activation='relu',
name='block1_conv1')(inputs)
conv1 = Conv2D(filters=32,
kernel_size=(3, 3),
padding='same',
activation='relu',
name='block1_conv2')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2), name='block1_pool')(conv1)
conv2 = Conv2D(filters=64,
kernel_size=(3, 3),
padding='same',
activation='relu',
name='block2_conv1')(pool1)
conv2 = Conv2D(filters=64,
kernel_size=(3, 3),
padding='same',
activation='relu',
name='block2_conv2')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2), name='block2_pool')(conv2)
conv3 = Conv2D(filters=128,
kernel_size=(3, 3),
padding='same',
activation='relu',
name='block3_conv1')(pool2)
conv3 = Conv2D(filters=128,
kernel_size=(3, 3),
padding='same',
activation='relu',
name='block3_conv2')(conv3)
conv3 = Conv2D(filters=128,
kernel_size=(3, 3),
padding='same',
activation='relu',
name='block3_conv3')(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2), name='block3_pool')(conv3)
score_pool3 = Conv2D(filters=3,
kernel_size=(3, 3),
padding='same',
activation='relu',
name='score_pool3')(pool3)
conv4 = Conv2D(filters=256,
kernel_size=(3, 3),
padding='same',
activation='relu',
name='block4_conv1')(pool3)
conv4 = Conv2D(filters=256,
kernel_size=(3, 3),
padding='same',
activation='relu',
name='block4_conv2')(conv4)
conv4 = Conv2D(filters=256,
kernel_size=(3, 3),
padding='same',
activation='relu',
name='block4_conv3')(conv4)
pool4 = MaxPooling2D(pool_size=(2, 2), name='block4_pool')(conv4)
score_pool4 = Conv2D(filters=3,
kernel_size=(3, 3),
padding='same',
activation='relu',
name='score_pool4')(pool4)
conv5 = Conv2D(filters=256,
kernel_size=(3, 3),
padding='same',
activation='relu',
name='block5_conv1')(pool4)
conv5 = Conv2D(filters=256,
kernel_size=(3, 3),
padding='same',
activation='relu',
name='block5_conv2')(conv5)
conv5 = Conv2D(filters=256,
kernel_size=(3, 3),
padding='same',
activation='relu',
name='block5_conv3')(conv5)
pool5 = MaxPooling2D(pool_size=(2, 2), name='block5_pool')(conv5)
fc6 = Conv2D(filters=1024,
kernel_size=(1, 1),
padding='same',
activation='relu',
name='fc6')(pool5)
fc6 = Dropout(0.3, name='dropout_3')(fc6)
fc7 = Conv2D(filters=1024,
kernel_size=(1, 1),
padding='same',
activation='relu',
name='fc7')(fc6)
fc7 = Dropout(0.3, name='dropour_2')(fc7)
score_fr = Conv2D(filters=nClasses,
kernel_size=(1, 1),
padding='same',
activation='relu',
name='score_fr')(fc7)
score2 = Conv2DTranspose(filters=nClasses,
kernel_size=(2, 2),
strides=(2, 2),
padding="valid",
activation=None,
name="score2")(score_fr)
add1 = add(inputs=[score2, score_pool4], name="add_3")
score4 = Conv2DTranspose(filters=nClasses,
kernel_size=(2, 2),
strides=(2, 2),
padding="valid",
activation=None,
name="score4")(add1)
add2 = add(inputs=[score4, score_pool3], name="add_2")
result_FCN = Conv2DTranspose(filters=nClasses,
kernel_size=(8, 8),
strides=(8, 8),
padding="valid",
name="UpSample")(add2)
# result_FCN = Activation('sigmoid')(result_FCN)
# result_FCN = Lambda(mul)(result_FCN)
# print(result_FCN)
return result_FCN
def Model(
input_layers,
output_layer,
embedded_size=None,
vocab_size=None,
embedding_matrix=None,
#maxlen = 50,
train_embedding=False,
n_lstm=32,
acti_lstm='relu',
lstm='bi',
dropout_rate=0.2,
merge='concat',
n_nn=32,
n_classif=1,
lrate=0.001,
**kwargs):
"""
Deep Learning model from Cocarascu-Toni paper:
"Identifying attack and support argumentative relations using deep learning".
It implements a two LSTM network (uni or bi directional) to predict the
relation between two arguments.
Architecture:
argA -> embedding -> lstm -> dropout |
merge -> dense -> softmax
argB -> embedding -> lstm -> dropout |
Input:
options => dict(). Keys:
embedded_size => output dimension of embeddings. Integer
vocab_size => vocabulary size (total number of attributes). Integer
maxlen => Max. Length of input text (sequences size). Integer
train_embedding => To continue training embedding or not. Boolean
embedding_matrix => Embedding from pre-trained set (eg. GloVo). Matrix
lstm => Set Unidirectional ("uni") or Bidirectional ("bi") LSTM. String
n_lstm => Output size of LSTM. Integer (values: 32,64,100,128)
acti_lstm => Activation for LSTM. String
dropout_rate => Rate for final dropout layer of individual model. Float
merge => Merge by concatenation "concat" or sum "add"
n_nn => Number of nodes from final hidden Dense layer. Integer
n_classif => Number of nodes from output layer. Defaul 3. Integer
lrate => Learning rate. Float
Returns:
model => keras.model
"""
#default_options = getOptionsCocarascu()
#if options is None:
# options = default_options
#else:
# default_options.update(options)
# options = default_options
#
##maxlen = default_options["maxlen"]
#embedded_size = default_options["embedded_size"]
#max_features = default_options["vocab_size"]
#embedding_matrix = default_options["embedding_matrix"]
#argA = Input(shape=(maxlen,))
#argB = Input(shape=(maxlen,))
argA = input_layers['premise_seq']
argB = input_layers['conclusion_seq']
word_embedding = Embedding(
vocab_size,
embedded_size,
weights=[embedding_matrix],
trainable=train_embedding,
)
pos_tagsA = input_layers['premise_pos_tags_ids']
pos_tagsB = input_layers['conclusion_pos_tags_ids']
pos_tag_embedding = Embedding(
16,
4,
embeddings_constraint=keras.constraints.MaxNorm(4.0 / embedded_size,
axis=-1))
#Model for arg1
#x = Embedding(
# vocab_size,
# embedded_size,
# weights=embedding_matrix,
# #trainable=train_embedding
# )(argA)
#x = word_embedding(argA)
x = concatenate([word_embedding(argA), pos_tag_embedding(pos_tagsA)])
if lstm == "bi":
x = Bidirectional(
LSTM(n_lstm, activation=acti_lstm, return_sequences=False))(x)
else:
x = LSTM(n_lstm, activation=acti_lstm, return_sequences=False)(x)
x = Dropout(rate=dropout_rate)(x)
x = keras.models.Model(inputs=[argA, pos_tagsA], outputs=x)
#x = keras.models.Model(inputs=argA, outputs=x)
#Model for arg2
#y = Embedding(vocab_size,
# embedded_size,
# weights=embedding_matrix,
# trainable=train_embedding)(argB)
#y = word_embedding(argB)
y = concatenate([word_embedding(argB), pos_tag_embedding(pos_tagsB)])
if lstm == "bi":
y = Bidirectional(
LSTM(n_lstm, activation=acti_lstm, return_sequences=False))(y)
else:
y = LSTM(n_lstm, activation=acti_lstm, return_sequences=False)(y)
y = Dropout(rate=dropout_rate)(y)
#y = keras.models.Model(inputs=argB, outputs=y)
y = keras.models.Model(inputs=[argB, pos_tagsB], outputs=y)
#Merge models
if merge == "concat":
combined = concatenate([x.output, y.output])
elif merge == "sum":
combined = add([x.output, y.output])
z = Dense(n_nn, activation="relu")(combined)
#z = Dense(n_classif, activation="softmax")(z)
z = Dense(n_classif, activation="sigmoid")(z)
z = output_layer(z)
model = keras.models.Model(inputs=[*x.input, *y.input], outputs=z)
adam = Adam(lr=lrate)
#model.compile(optimizer=adam, loss="categorical_crossentropy", metrics=["accuracy"])
#model.compile(optimizer=adam, loss="sparse_categorical_crossentropy", metrics=["accuracy"])
model.compile(optimizer=adam,
loss="binary_crossentropy",
metrics=["binary_accuracy"])
return model
def residual_block(inputs, filters_1, filters_2):
x = conv2d_unit(inputs, filters_1, 1, strides=1, padding='valid')
x = conv2d_unit(x, filters_2, 3, strides=1, padding='same')
x = layers.add([inputs, x])
# x = layers.Activation('linear')(x)
return x
def conv_block(input_tensor,
kernel_size,
filters,
stage,
block,
strides=(2, 2)):
'''self.conv_block is the block that has a conv layer at shortcut
# Arguments
input_tensor: input tensor
kernel_size: defualt 3, the kernel size of middle conv layer at main path
filters: list of integers, the nb_filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
Note that from stage 3, the first conv layer at main path is with subsample=(2,2)
And the shortcut should have subsample=(2,2) as well
'''
eps = 1.1e-5
nb_filter1, nb_filter2, nb_filter3 = filters
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
scale_name_base = 'scale' + str(stage) + block + '_branch'
# x = Convolution2D(nb_filter1, 1, 1, subsample=strides, name=conv_name_base + '2a', bias=False)(input_tensor)
x = Conv2D(nb_filter1,
1,
strides=strides,
name=conv_name_base + '2a',
use_bias=False)(input_tensor)
x = BatchNormalization(epsilon=eps,
axis=bn_axis,
name=bn_name_base + '2a')(x)
x = Scale(axis=bn_axis, name=scale_name_base + '2a')(x)
x = Activation('relu', name=conv_name_base + '2a_relu')(x)
x = ZeroPadding2D((1, 1), name=conv_name_base + '2b_zeropadding')(x)
# x = Convolution2D(nb_filter2, kernel_size, kernel_size, name=conv_name_base + '2b', bias=False)(x)
x = Conv2D(nb_filter2,
kernel_size,
name=conv_name_base + '2b',
use_bias=False)(x)
x = BatchNormalization(epsilon=eps,
axis=bn_axis,
name=bn_name_base + '2b')(x)
x = Scale(axis=bn_axis, name=scale_name_base + '2b')(x)
x = Activation('relu', name=conv_name_base + '2b_relu')(x)
#x = Convolution2D(nb_filter3, 1, 1, name=conv_name_base + '2c', bias=False)(x)
x = Conv2D(nb_filter3, 1, name=conv_name_base + '2c',
use_bias=False)(x)
x = BatchNormalization(epsilon=eps,
axis=bn_axis,
name=bn_name_base + '2c')(x)
x = Scale(axis=bn_axis, name=scale_name_base + '2c')(x)
#shortcut = Convolution2D(nb_filter3, 1, 1, subsample=strides, name=conv_name_base + '1', bias=False)(input_tensor)
shortcut = Conv2D(nb_filter3,
1,
strides=strides,
name=conv_name_base + '1',
use_bias=False)(input_tensor)
shortcut = BatchNormalization(epsilon=eps,
axis=bn_axis,
name=bn_name_base + '1')(shortcut)
shortcut = Scale(axis=bn_axis, name=scale_name_base + '1')(shortcut)
#x = merge([x, shortcut], mode='sum', name='res' + str(stage) + block)
x = add([x, shortcut], name='res' + str(stage) + block)
x = Activation('relu', name='res' + str(stage) + block + '_relu')(x)
return x
def nn_base(input_tensor=None, trainable=False):
# Determine proper input shape
if K.image_dim_ordering() == 'th':
input_shape = (3, None, None)
else:
input_shape = (None, None, 3)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if K.image_dim_ordering() == 'tf':
bn_axis = 3
else:
bn_axis = 1
x = Conv2D(32, (3, 3), strides=(2, 2), use_bias=False,
name='block1_conv1')(img_input)
x = BatchNormalization(name='block1_conv1_bn')(x)
x = Activation('relu', name='block1_conv1_act')(x)
x = Conv2D(64, (3, 3), use_bias=False, name='block1_conv2')(x)
x = BatchNormalization(name='block1_conv2_bn')(x)
x = Activation('relu', name='block1_conv2_act')(x)
residual = Conv2D(128, (1, 1),
strides=(2, 2),
padding='same',
use_bias=False)(x)
residual = BatchNormalization()(residual)
x = SeparableConv2D(128, (3, 3),
padding='same',
use_bias=False,
name='block2_sepconv1')(x)
x = BatchNormalization(name='block2_sepconv1_bn')(x)
x = Activation('relu', name='block2_sepconv2_act')(x)
x = SeparableConv2D(128, (3, 3),
padding='same',
use_bias=False,
name='block2_sepconv2')(x)
x = BatchNormalization(name='block2_sepconv2_bn')(x)
x = MaxPooling2D((3, 3),
strides=(2, 2),
padding='same',
name='block2_pool')(x)
x = add([x, residual])
residual = Conv2D(256, (1, 1),
strides=(2, 2),
padding='same',
use_bias=False)(x)
residual = BatchNormalization()(residual)
x = Activation('relu', name='block3_sepconv1_act')(x)
x = SeparableConv2D(256, (3, 3),
padding='same',
use_bias=False,
name='block3_sepconv1')(x)
x = BatchNormalization(name='block3_sepconv1_bn')(x)
x = Activation('relu', name='block3_sepconv2_act')(x)
x = SeparableConv2D(256, (3, 3),
padding='same',
use_bias=False,
name='block3_sepconv2')(x)
x = BatchNormalization(name='block3_sepconv2_bn')(x)
x = MaxPooling2D((3, 3),
strides=(2, 2),
padding='same',
name='block3_pool')(x)
x = add([x, residual])
residual = Conv2D(728, (1, 1),
strides=(2, 2),
padding='same',
use_bias=False)(x)
residual = BatchNormalization()(residual)
x = Activation('relu', name='block4_sepconv1_act')(x)
x = SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name='block4_sepconv1')(x)
x = BatchNormalization(name='block4_sepconv1_bn')(x)
x = Activation('relu', name='block4_sepconv2_act')(x)
x = SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name='block4_sepconv2')(x)
x = BatchNormalization(name='block4_sepconv2_bn')(x)
x = MaxPooling2D((3, 3),
strides=(2, 2),
padding='same',
name='block4_pool')(x)
x = add([x, residual])
for i in range(8):
residual = x
prefix = 'block' + str(i + 5)
x = Activation('relu', name=prefix + '_sepconv1_act')(x)
x = SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name=prefix + '_sepconv1')(x)
x = BatchNormalization(name=prefix + '_sepconv1_bn')(x)
x = Activation('relu', name=prefix + '_sepconv2_act')(x)
x = SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name=prefix + '_sepconv2')(x)
x = BatchNormalization(name=prefix + '_sepconv2_bn')(x)
x = Activation('relu', name=prefix + '_sepconv3_act')(x)
x = SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name=prefix + '_sepconv3')(x)
x = BatchNormalization(name=prefix + '_sepconv3_bn')(x)
x = add([x, residual])
residual = Conv2D(1024, (1, 1),
strides=(2, 2),
padding='same',
use_bias=False)(x)
residual = BatchNormalization()(residual)
x = Activation('relu', name='block13_sepconv1_act')(x)
x = SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name='block13_sepconv1')(x)
x = BatchNormalization(name='block13_sepconv1_bn')(x)
x = Activation('relu', name='block13_sepconv2_act')(x)
x = SeparableConv2D(1024, (3, 3),
padding='same',
use_bias=False,
name='block13_sepconv2')(x)
x = BatchNormalization(name='block13_sepconv2_bn')(x)
x = MaxPooling2D((3, 3),
strides=(2, 2),
padding='same',
name='block13_pool')(x)
x = add([x, residual])
return x
def senet_unit(x,
in_channels,
out_channels,
strides,
cardinality,
bottleneck_width,
identity_conv3x3,
name="senet_unit"):
"""
SENet unit.
Parameters:
----------
x : keras.backend tensor/variable/symbol
Input tensor/variable/symbol.
in_channels : int
Number of input channels.
out_channels : int
Number of output channels.
strides : int or tuple/list of 2 int
Strides of the convolution.
cardinality: int
Number of groups.
bottleneck_width: int
Width of bottleneck block.
identity_conv3x3 : bool, default False
Whether to use 3x3 convolution in the identity link.
name : str, default 'senet_unit'
Unit name.
Returns
-------
keras.backend tensor/variable/symbol
Resulted tensor/variable/symbol.
"""
resize_identity = (in_channels != out_channels) or (strides != 1)
if resize_identity:
if identity_conv3x3:
identity = conv3x3_block(x=x,
in_channels=in_channels,
out_channels=out_channels,
strides=strides,
activation=None,
name=name + "/identity_conv")
else:
identity = conv1x1_block(x=x,
in_channels=in_channels,
out_channels=out_channels,
strides=strides,
activation=None,
name=name + "/identity_conv")
else:
identity = x
x = senet_bottleneck(x=x,
in_channels=in_channels,
out_channels=out_channels,
strides=strides,
cardinality=cardinality,
bottleneck_width=bottleneck_width,
name=name + "/body")
x = se_block(x=x, channels=out_channels, name=name + "/se")
x = nn.add([x, identity], name=name + "/add")
activ = nn.Activation('relu', name=name + "/activ")
x = activ(x)
return x
# cuts down input size going into RNN:
inner = Dense(time_dense_size, activation=act, name='dense1')(inner)
# Two layers of bidirecitonal GRUs
# GRU seems to work as well, if not better than LSTM:
gru_1 = GRU(rnn_size,
return_sequences=True,
kernel_initializer='he_normal',
name='gru1')(inner)
gru_1b = GRU(rnn_size,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='gru1_b')(inner)
gru1_merged = add([gru_1, gru_1b])
gru_2 = GRU(rnn_size,
return_sequences=True,
kernel_initializer='he_normal',
name='gru2')(gru1_merged)
gru_2b = GRU(rnn_size,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='gru2_b')(gru1_merged)
# transforms RNN output to character activations:
inner = Dense(tiger_train.get_output_size(),
kernel_initializer='he_normal',
name='dense2')(concatenate([gru_2, gru_2b]))
y_pred = Activation('softmax', name='softmax')(inner)
def construct_model(self):
"""
Construct the :math:`1`-st order and :math:`0`-th order models, which are used to approximate the
:math:`U_1(x, C(x))` and the :math:`U_0(x)` utilities respectively. For each pair of objects in
:math:`x_i, x_j \\in Q` :math:`U_1(x, C(x))` we construct :class:`CmpNetCore` with weight sharing to
approximate a pairwise-matrix. A pairwise matrix with index (i,j) corresponds to the :math:`U_1(x_i,x_j)`
is a measure of how favorable it is to choose :math:`x_i` over :math:`x_j`. Using this matrix we calculate
the borda score for each object to calculate :math:`U_1(x, C(x))`. For `0`-th order model we construct
:math:`\\lvert Q \\lvert` sequential networks whose weights are shared to evaluate the :math:`U_0(x)` for
each object in the query set :math:`Q`. The output mode is using sigmoid activation.
Returns
-------
model: keras :class:`Model`
Neural network to learn the FETA utility score
"""
def create_input_lambda(i):
return Lambda(lambda x: x[:, i])
if self.add_zeroth_order_model:
logger.debug("Create 0th order model")
zeroth_order_outputs = []
inputs = []
for i in range(self.n_objects_fit_):
x = create_input_lambda(i)(self.input_layer)
inputs.append(x)
for hidden in self.hidden_layers_zeroth:
x = hidden(x)
zeroth_order_outputs.append(self.output_node_zeroth(x))
zeroth_order_scores = concatenate(zeroth_order_outputs)
logger.debug("0th order model finished")
logger.debug("Create 1st order model")
outputs = [list() for _ in range(self.n_objects_fit_)]
for i, j in combinations(range(self.n_objects_fit_), 2):
if self.add_zeroth_order_model:
x1 = inputs[i]
x2 = inputs[j]
else:
x1 = create_input_lambda(i)(self.input_layer)
x2 = create_input_lambda(j)(self.input_layer)
x1x2 = concatenate([x1, x2])
x2x1 = concatenate([x2, x1])
for hidden in self.hidden_layers:
x1x2 = hidden(x1x2)
x2x1 = hidden(x2x1)
merged_left = concatenate([x1x2, x2x1])
merged_right = concatenate([x2x1, x1x2])
N_g = self.output_node(merged_left)
N_l = self.output_node(merged_right)
outputs[i].append(N_g)
outputs[j].append(N_l)
# convert rows of pairwise matrix to keras layers:
outputs = [concatenate(x) for x in outputs]
# compute utility scores:
scores = [
Lambda(lambda s: K.mean(s, axis=1, keepdims=True))(x)
for x in outputs
]
scores = concatenate(scores)
logger.debug("1st order model finished")
if self.add_zeroth_order_model:
scores = add([scores, zeroth_order_scores])
scores = Activation("sigmoid")(scores)
model = Model(inputs=self.input_layer, outputs=scores)
logger.debug("Compiling complete model...")
model.compile(
loss=self.loss_function,
optimizer=self.optimizer_,
metrics=list(self.metrics),
)
return model
def build_supervised_model(seq_shape, n_classes):
# input layer
labeled_input = Input(shape=(
seq_shape[1],
seq_shape[2],
))
# build architecture
if model_type == 'cnn':
# downsample (None, 4, 128)
h = Conv1D(filters=hidden_units,
kernel_size=3,
strides=1,
padding='same')(labeled_input)
h = LeakyReLU(alpha=0.2)(h)
# downsample (None, 4, 128)
h = Conv1D(filters=hidden_units,
kernel_size=3,
strides=1,
padding='same')(h)
h = LeakyReLU(alpha=0.2)(h)
# downsample (None, 4, 128)
h = Conv1D(filters=hidden_units,
kernel_size=3,
strides=1,
padding='same')(h)
h = LeakyReLU(alpha=0.2)(h)
# fully connect (None, 512)
h = Flatten()(h)
h = Dropout(0.4)(h)
if model_type == 'lstm':
h = LSTM(hidden_units, return_sequences=True)(labeled_input)
h = LSTM(hidden_units)(h)
if model_type == 'resnet':
b1 = Conv1D(filters=hidden_units, kernel_size=8,
padding='same')(labeled_input)
b1 = BatchNormalization()(b1)
b1 = Activation('relu')(b1)
b1 = Conv1D(filters=hidden_units, kernel_size=5, padding='same')(b1)
b1 = BatchNormalization()(b1)
b1 = Activation('relu')(b1)
b1 = Conv1D(filters=hidden_units, kernel_size=3, padding='same')(b1)
b1 = BatchNormalization()(b1)
shortcut = Conv1D(filters=hidden_units, kernel_size=1,
padding='same')(labeled_input)
shortcut = BatchNormalization()(shortcut)
b1 = add([shortcut, b1])
b1 = Activation('relu')(b1)
# block 2
b2 = Conv1D(filters=hidden_units * 2, kernel_size=8,
padding='same')(b1)
b2 = BatchNormalization()(b2)
b2 = Activation('relu')(b2)
b2 = Conv1D(filters=hidden_units * 2, kernel_size=5,
padding='same')(b2)
b2 = BatchNormalization()(b2)
b2 = Activation('relu')(b2)
b2 = Conv1D(filters=hidden_units * 2, kernel_size=3,
padding='same')(b2)
b2 = BatchNormalization()(b2)
shortcut = Conv1D(filters=hidden_units * 2,
kernel_size=1,
padding='same')(b1)
shortcut = BatchNormalization()(shortcut)
b2 = add([shortcut, b2])
b2 = Activation('relu')(b2)
# block 3
b3 = Conv1D(filters=hidden_units * 2, kernel_size=8,
padding='same')(b2)
b3 = BatchNormalization()(b3)
b3 = Activation('relu')(b3)
b3 = Conv1D(filters=hidden_units * 2, kernel_size=5,
padding='same')(b3)
b3 = BatchNormalization()(b3)
b3 = Activation('relu')(b3)
b3 = Conv1D(filters=hidden_units * 2, kernel_size=3,
padding='same')(b3)
b3 = BatchNormalization()(b3)
shortcut = BatchNormalization()(b2)
b3 = add([shortcut, b3])
b3 = Activation('relu')(b3)
# output
h = GlobalAveragePooling1D()(b3)
# compile supervised model
output = Dense(n_classes, activation='softmax')(h)
supervised_model = Model(labeled_input, output)
supervised_model.compile(loss='categorical_crossentropy',
optimizer=Adam(lr=learning_rate),
metrics=['accuracy'])
return supervised_model
def test_resnet():
# for reproducibility
rand_seed = 1337
np.random.seed(rand_seed)
out_size = (49, 2)
roi_size = 128
input_shape = (roi_size, roi_size, 1)
img_input = Input(shape=input_shape)
x = Conv2D(32, (3, 3), padding='same', use_bias=False, name='conv0')(img_input)
x = BatchNormalization(name='conv0a_bn')(x)
x = Activation('relu', name='conv0_act')(x)
x = MaxPooling2D((2, 2), name='conv0_pool')(x)
x = Conv2D(64, (3, 3), padding='same', use_bias=False, name='conv1a')(x)
x = BatchNormalization(name='conv1a_bn')(x)
x = Activation('relu', name='conv1a_act')(x)
residual = Conv2D(128, (1, 1), strides=(2, 2),
padding='same', use_bias=False)(x)
residual = BatchNormalization()(residual)
x = Conv2D(128, (3, 3), padding='same', use_bias=False, name='block2_sepconv1')(x)
x = BatchNormalization(name='block2_sepconv1_bn')(x)
x = Activation('relu', name='block2_sepconv2_act')(x)
x = Conv2D(128, (3, 3), padding='same', use_bias=False, name='block2_sepconv2')(x)
x = BatchNormalization(name='block2_sepconv2_bn')(x)
x = MaxPooling2D((2, 2), padding='same', name='block2_pool')(x)
x = layers.add([x, residual])
residual = Conv2D(512, (1, 1), strides=(2, 2),
padding='same', use_bias=False)(x)
residual = BatchNormalization()(residual)
x = Activation('relu', name='block3_sepconv1_act')(x)
x = Conv2D(256, (3, 3), padding='same', use_bias=False, name='block3_sepconv1')(x)
x = BatchNormalization(name='block3_sepconv1_bn')(x)
x = Activation('relu', name='block3_sepconv2_act')(x)
x = Conv2D(512, (3, 3), padding='same', use_bias=False, name='block3_sepconv2')(x)
x = BatchNormalization(name='block3_sepconv2_bn')(x)
x = MaxPooling2D((2, 2), padding='same', name='block3_pool')(x)
x = layers.add([x, residual])
x = Conv2D(1024, (3, 3), padding='same', use_bias=False, name='block14_sepconv2')(x)
x = BatchNormalization(name='block14_sepconv2_bn')(x)
x = Activation('relu', name='block14_sepconv2_act')(x)
x = GlobalMaxPooling2D(name='avg_pool')(x)
#layers seems to be key to capture the worm shape
#i need to use ELU instead of RELU otherwise the skeletons converges to 0
x = Dense(1024, name='dense0', activation='elu')(x)
x = Dropout(0.4)(x)
x = Dense(512, name='dense1', activation='elu')(x)
x = Dropout(0.4)(x)
x = Dense(256, name='dense2', activation='elu')(x)
x = Dropout(0.2)(x)
x = Dense(128, name='dense3', activation='elu')(x)
x = Dropout(0.1)(x)
x = Dense(out_size[0]*out_size[1], activation='elu', name='skeleton')(x)
x = Reshape((out_size))(x)
model = Model(img_input, x)
optimizer = Adam(lr=1e-3)#, decay=0.05)
model.compile(loss='mean_absolute_error',
optimizer=optimizer,
metrics=['mean_absolute_error', 'mean_squared_error', 'mean_absolute_percentage_error'])
return model
def pyramid_feat2_large():
#DIDN'T CONVERNGED
out_size = (49, 2)
roi_size = 128
rand_seed = 1337
np.random.seed(rand_seed)
input_shape = (roi_size, roi_size, 1)
img_input = Input(shape=input_shape)
block1 = Conv2D(32, (3, 3), padding='same', name='conv0a')(img_input)
block1 = Activation('relu', name='conv0a_act')(block1)
block1 = Conv2D(32, (3, 3), padding='same', name='conv0b')(block1)
block1 = Activation('relu', name='conv0b_act')(block1)
block2 = MaxPooling2D((2, 2), name='conv0_pool')(block1)
block2 = Conv2D(64, (3, 3), padding='same', name='conv1a')(block2)
block2 = BatchNormalization(name='conv1a_bn')(block2)
block2 = Activation('relu', name='conv1a_act')(block2)
block2 = Conv2D(64, (3, 3), padding='same', name='conv1b')(block2)
block2 = BatchNormalization(name='conv1b_bn')(block2)
block2 = Activation('relu', name='conv1b_act')(block2)
block3 = MaxPooling2D((2, 2), name='conv1_pool')(block2)
block3 = Conv2D(128, (3, 3), padding='same', name='conv2a')(block3)
block3 = BatchNormalization(name='conv2a_bn')(block3)
block3 = Activation('relu', name='conv2a_act')(block3)
block3 = Conv2D(128, (3, 3), padding='same', name='conv2b')(block3)
block3 = BatchNormalization(name='conv2b_bn')(block3)
block3 = Activation('relu', name='conv2b_act')(block3)
block4 = MaxPooling2D((2, 2), name='conv2_pool')(block3)
block4 = Conv2D(256, (3, 3), padding='same', name='conv3a')(block4)
block4 = BatchNormalization(name='conv3a_bn')(block4)
block4 = Activation('relu', name='conv3a_act')(block4)
block4 = Conv2D(256, (3, 3), padding='same', name='conv3b')(block4)
block4 = BatchNormalization(name='conv3b_bn')(block4)
block4 = Activation('relu', name='conv3b_act')(block4)
block5 = MaxPooling2D((2, 2), name='conv3_pool')(block4)
block5 = Conv2D(512, (3, 3), padding='same', name='conv4a')(block5)
block5 = BatchNormalization(name='conv4a_bn')(block5)
block5 = Activation('relu', name='conv4a_act')(block5)
block5 = Conv2D(512, (3, 3), padding='same', name='conv4b')(block5)
block5 = BatchNormalization(name='conv4b_bn')(block5)
block5 = Activation('relu', name='conv4b_act')(block5)
feat_top = GlobalMaxPooling2D(name='avg_pool')(block5)
feat_top = Dense(1024, name='dense0', activation='elu')(feat_top)
feat_top = Dropout(0.4)(feat_top)
down_block5 = Conv2D(1, (1, 1), padding='same')(block5)
down_block5 = UpSampling2D((2,2))(down_block5)
lat_block4 = Conv2D(1, (1, 1), padding='same')(block4)
p_block4 = layers.add([lat_block4, down_block5])
down_block4 = UpSampling2D((2,2))(p_block4)
lat_block3 = Conv2D(1, (1, 1), padding='same')(block3)
p_block3 = layers.add([lat_block3, down_block4])
down_block3 = UpSampling2D((2,2))(p_block3)
lat_block2 = Conv2D(1, (1, 1), padding='same')(block2)
p_block2 = layers.add([lat_block2, down_block3])
feat_bot = Flatten()(p_block2)
feat_bot = Dense(1024, activation='elu')(feat_bot)
feat_bot = Dropout(0.4)(feat_bot)
all_feats = layers.add([feat_top, feat_bot])
all_feats = Dense(1024, activation='elu')(all_feats)
feat_bot = Dropout(0.4)(feat_bot)
skel_out = Dense(out_size[0]*out_size[1], activation='elu', name='skeleton')(all_feats)
skel_out = Reshape((out_size))(skel_out)
model = Model(img_input, skel_out)
#optimizer = Adam(lr=1e-2, decay=0.1)
optimizer = Adam(lr=1e-3, decay=0.1)
model.compile(loss='mean_absolute_error',
optimizer=optimizer,
metrics=['mean_absolute_error', 'mean_squared_error', 'mean_absolute_percentage_error'])
return model, 'pyramid_feat2_large'
def rnn(embedding_matrix, config):
if config['rnn'] == 'gru' and config['gpu']:
encode = Bidirectional(
CuDNNGRU(config['rnn_output_size'], return_sequences=True))
encode2 = Bidirectional(
CuDNNGRU(config['rnn_output_size'], return_sequences=True))
encode3 = Bidirectional(
CuDNNGRU(config['rnn_output_size'], return_sequences=True))
else:
encode = Bidirectional(
CuDNNLSTM(config['rnn_output_size'], return_sequences=True))
encode2 = Bidirectional(
CuDNNLSTM(config['rnn_output_size'] * 2, return_sequences=True))
encode3 = Bidirectional(
CuDNNGRU(config['rnn_output_size'] * 4, return_sequences=True))
q1 = Input(shape=(config['max_length'], ), dtype='int32', name='q1_input')
q2 = Input((config['max_length'], ), dtype='int32', name='q2_input')
embedding_layer = Embedding(embedding_matrix.shape[0],
embedding_matrix.shape[1],
trainable=config['embed_trainable'],
weights=[embedding_matrix]
# mask_zero=True
)
q1_embed = embedding_layer(q1)
q2_embed = embedding_layer(q2) # bsz, 1, emb_dims
q1_embed = BatchNormalization(axis=2)(q1_embed)
q2_embed = BatchNormalization(axis=2)(q2_embed)
q1_embed = SpatialDropout1D(config['spatial_dropout_rate'])(q1_embed)
q2_embed = SpatialDropout1D(config['spatial_dropout_rate'])(q2_embed)
q1_encoded = encode(q1_embed)
q2_encoded = encode(q2_embed)
q1_encoded = Dropout(0.2)(q1_encoded)
q2_encoded = Dropout(0.2)(q2_encoded)
# 双向
# q1_encoded = encode2(q1_encoded)
# q2_encoded = encode2(q2_encoded)
# resnet
rnn_layer2_input1 = concatenate([q1_embed, q1_encoded])
rnn_layer2_input2 = concatenate([q2_embed, q2_encoded])
q1_encoded2 = encode2(rnn_layer2_input1)
q2_encoded2 = encode2(rnn_layer2_input2)
# add res shortcut
res_block1 = add([q1_encoded, q1_encoded2])
res_block2 = add([q2_encoded, q2_encoded2])
rnn_layer3_input1 = concatenate([q1_embed, res_block1])
rnn_layer3_input2 = concatenate([q2_embed, res_block2])
# rnn_layer3_input1 = concatenate([q1_embed,q1_encoded,q1_encoded2])
# rnn_layer3_input2 = concatenate([q2_embed,q2_encoded,q2_encoded2])
q1_encoded3 = encode3(rnn_layer3_input1)
q2_encoded3 = encode3(rnn_layer3_input2)
# merged1 = GlobalMaxPool1D()(q1_encoded3)
# merged2 = GlobalMaxPool1D()(q2_encoded3)
# q1_encoded = concatenate([q1_encoded, q1_encoded2], axis=-1)
# q2_encoded = concatenate([q2_encoded, q2_encoded2], axis=-1)
# merged1 = concatenate([q1_encoded2, q1_embed], axis=-1)
# merged2 = concatenate([q2_encoded2, q2_embed], axis=-1)
# # TODO add attention rep , maxpooling rep
q1_encoded3 = concatenate([q1_encoded, q1_encoded2, q1_encoded3])
q2_encoded3 = concatenate([q2_encoded, q2_encoded2, q2_encoded3])
merged1 = GlobalMaxPool1D()(q1_encoded3)
merged2 = GlobalMaxPool1D()(q2_encoded3)
# avg1 = GlobalAvgPool1D()(q1_encoded3)
# avg2 = GlobalAvgPool1D()(q2_encoded3)
# merged1 = concatenate([max1,avg1])
# merged2 = concatenate([max2,avg2])
sub_rep = Lambda(lambda x: K.abs(x[0] - x[1]))([merged1, merged2])
mul_rep = Lambda(lambda x: x[0] * x[1])([merged1, merged2])
# jaccard_rep = Lambda(lambda x: x[0]*x[1]/(K.sum(x[0]**2,axis=1,keepdims=True)+K.sum(x[1]**2,axis=1,keepdims=True)-
# K.sum(K.abs(x[0]*x[1]),axis=1,keepdims=True)))([merged1,merged2])
# merged = Concatenate()([merged1, merged2, mul_rep, sub_rep,jaccard_rep])
feature_input = Input(shape=(config['feature_length'], ))
feature_dense = BatchNormalization()(feature_input)
feature_dense = Dense(config['dense_dim'],
activation='relu')(feature_dense)
merged = Concatenate()([merged1, merged2, mul_rep, sub_rep, feature_dense])
# Classifier
dense = Dropout(config['dense_dropout'])(merged)
dense = BatchNormalization()(dense)
dense = Dense(config['dense_dim'], activation='relu')(dense)
dense = Dropout(config['dense_dropout'])(dense)
dense = BatchNormalization()(dense)
predictions = Dense(1, activation='sigmoid')(dense)
model = Model(inputs=[q1, q2, feature_input], outputs=predictions)
opt = optimizers.get(config['optimizer'])
K.set_value(opt.lr, config['learning_rate'])
model.compile(optimizer=opt, loss='binary_crossentropy', metrics=[f1])
return model
def _inception_resnet_block(x,
scale,
block_type,
block_idx,
activation='relu'):
channel_axis = 3
if block_idx is None:
prefix = None
else:
prefix = '_'.join((block_type, str(block_idx)))
name_fmt = partial(_generate_layer_name, prefix=prefix)
if block_type == 'Block35':
branch_0 = conv2d_bn(x, 32, 1, name=name_fmt('Conv2d_1x1', 0))
branch_1 = conv2d_bn(x, 32, 1, name=name_fmt('Conv2d_0a_1x1', 1))
branch_1 = conv2d_bn(branch_1,
32,
3,
name=name_fmt('Conv2d_0b_3x3', 1))
branch_2 = conv2d_bn(x, 32, 1, name=name_fmt('Conv2d_0a_1x1', 2))
branch_2 = conv2d_bn(branch_2,
32,
3,
name=name_fmt('Conv2d_0b_3x3', 2))
branch_2 = conv2d_bn(branch_2,
32,
3,
name=name_fmt('Conv2d_0c_3x3', 2))
branches = [branch_0, branch_1, branch_2]
elif block_type == 'Block17':
branch_0 = conv2d_bn(x, 128, 1, name=name_fmt('Conv2d_1x1', 0))
branch_1 = conv2d_bn(x, 128, 1, name=name_fmt('Conv2d_0a_1x1', 1))
branch_1 = conv2d_bn(branch_1,
128, [1, 7],
name=name_fmt('Conv2d_0b_1x7', 1))
branch_1 = conv2d_bn(branch_1,
128, [7, 1],
name=name_fmt('Conv2d_0c_7x1', 1))
branches = [branch_0, branch_1]
elif block_type == 'Block8':
branch_0 = conv2d_bn(x, 192, 1, name=name_fmt('Conv2d_1x1', 0))
branch_1 = conv2d_bn(x, 192, 1, name=name_fmt('Conv2d_0a_1x1', 1))
branch_1 = conv2d_bn(branch_1,
192, [1, 3],
name=name_fmt('Conv2d_0b_1x3', 1))
branch_1 = conv2d_bn(branch_1,
192, [3, 1],
name=name_fmt('Conv2d_0c_3x1', 1))
branches = [branch_0, branch_1]
mixed = Concatenate(axis=channel_axis,
name=name_fmt('Concatenate'))(branches)
up = conv2d_bn(mixed,
K.int_shape(x)[channel_axis],
1,
activation=None,
use_bias=True,
name=name_fmt('Conv2d_1x1'))
up = Lambda(scaling,
output_shape=K.int_shape(up)[1:],
arguments={'scale': scale})(up)
x = add([x, up])
if activation is not None:
x = Activation(activation, name=name_fmt('Activation'))(x)
return x
def share_stream(x_shape, outputNum=256):
input = Input(x_shape)
x_a = input
conv1 = Conv2D(filters=32,
kernel_size=(5, 5),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(x_a)
conv1 = Activation('relu')(conv1)
conv1 = Conv2D(filters=32,
kernel_size=(1, 1),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(conv1)
x_a = Conv2D(filters=32,
kernel_size=(1, 1),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(x_a)
x_a = add([x_a, conv1])
attention = Conv2D(filters=32,
kernel_size=(5, 5),
activation='sigmoid',
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(x_a)
x_a = multiply([x_a, attention])
x_a = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')(x_a)
conv1 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(x_a)
conv1 = Activation('relu')(conv1)
conv1 = Conv2D(filters=64,
kernel_size=(1, 1),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(conv1)
x_a = Conv2D(filters=64,
kernel_size=(1, 1),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(x_a)
x_a = add([x_a, conv1])
x_a = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')(x_a)
x = Conv2D(filters=128,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(x_a)
temp = Multiply()([
Conv2D(filters=128,
kernel_size=(3, 3),
padding='same',
trainable=Trainable)(Lambda(lambda x: x * 10000000.)(
Activation("tanh")(Lambda(lambda x: x / 10000000.)(x)))), x
])
for jj in range(2, taylor):
temp = Add()([
Multiply()([
Conv2D(filters=128,
kernel_size=(3, 3),
padding='same',
trainable=Trainable)(Lambda(lambda x: x * 10000000.)(
Activation("tanh")(
Lambda(lambda x: x / 10000000.)(x)))),
Lambda(lambda x: (x**jj) / math.factorial(jj))(x)
]), temp
])
temp = Add()([
Conv2D(filters=128,
kernel_size=(3, 3),
padding='same',
trainable=Trainable)(Lambda(lambda x: x * 10000000.)(
Activation("tanh")(Lambda(lambda x: x / 10000000.)(x)))),
temp
])
conv2 = Lambda(lambda x: x * 10000000.)(Activation("tanh")(
Lambda(lambda x: x / 10000000.)(temp)))
conv2 = Conv2D(filters=128,
kernel_size=(1, 1),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(conv2)
x_a = Conv2D(filters=128,
kernel_size=(1, 1),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(x_a)
x_a = add([x_a, conv2])
x_a = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')(x_a)
conv3 = Conv2D(filters=outputNum,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(x_a)
conv3 = Activation('relu')(conv3)
conv3 = Conv2D(filters=outputNum,
kernel_size=(1, 1),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(conv3)
x_a = Conv2D(filters=outputNum,
kernel_size=(1, 1),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(x_a)
x_a = add([x_a, conv3])
x_a = GlobalMaxPooling2D()(x_a)
shared_layer = Model(input, x_a)
return shared_layer
def compile_context_vec(self, print_summary=False):
"""
Compiles a Language Model RNN based on the given parameters
"""
if self.parameters['token_encoding'] == 'word':
# Train word embeddings from scratch
word_inputs = Input(shape=(None, ),
name='word_indices',
dtype='int32')
embeddings = Embedding(self.parameters['vocab_size'],
self.parameters['hidden_units_size'],
trainable=True,
name='token_encoding')
inputs = embeddings(word_inputs)
# Token embeddings for Input
drop_inputs = SpatialDropout1D(
self.parameters['dropout_rate'])(inputs)
lstm_inputs = TimestepDropout(
self.parameters['word_dropout_rate'])(drop_inputs)
# Pass outputs as inputs to apply sampled softmax
next_ids = Input(shape=(None, 1), name='next_ids', dtype='float32')
previous_ids = Input(shape=(None, 1),
name='previous_ids',
dtype='float32')
elif self.parameters['token_encoding'] == 'char':
# Train character-level representation
word_inputs = Input(shape=(
None,
self.parameters['token_maxlen'],
),
dtype='int32',
name='char_indices')
inputs = self.char_level_token_encoder()(word_inputs)
# Token embeddings for Input
drop_inputs = SpatialDropout1D(
self.parameters['dropout_rate'])(inputs)
lstm_inputs = TimestepDropout(
self.parameters['word_dropout_rate'])(drop_inputs)
# Pass outputs as inputs to apply sampled softmax
next_ids = Input(shape=(None, 1), name='next_ids', dtype='float32')
previous_ids = Input(shape=(None, 1),
name='previous_ids',
dtype='float32')
# Reversed input for backward LSTMs
re_lstm_inputs = Lambda(function=Context_vec.reverse)(lstm_inputs)
mask = Lambda(function=Context_vec.reverse)(drop_inputs)
# Forward LSTMs
for i in range(self.parameters['n_lstm_layers']):
lstm = LSTM(units=self.parameters['lstm_units_size'],
return_sequences=True,
activation="tanh",
recurrent_activation='sigmoid',
kernel_constraint=MinMaxNorm(
-1 * self.parameters['cell_clip'],
self.parameters['cell_clip']),
recurrent_constraint=MinMaxNorm(
-1 * self.parameters['cell_clip'],
self.parameters['cell_clip']))(lstm_inputs)
lstm = Camouflage(mask_value=0)(inputs=[lstm, drop_inputs])
# Projection to hidden_units_size
proj = TimeDistributed(
Dense(self.parameters['hidden_units_size'],
activation='linear',
kernel_constraint=MinMaxNorm(
-1 * self.parameters['proj_clip'],
self.parameters['proj_clip'])))(lstm)
# Merge Bi-LSTMs feature vectors with the previous ones
lstm_inputs = add([proj, lstm_inputs],
name='f_block_{}'.format(i + 1))
# Apply variational drop-out between BI-LSTM layers
lstm_inputs = SpatialDropout1D(
self.parameters['dropout_rate'])(lstm_inputs)
# Backward LSTMs
for i in range(self.parameters['n_lstm_layers']):
re_lstm = LSTM(units=self.parameters['lstm_units_size'],
return_sequences=True,
activation='tanh',
recurrent_activation='sigmoid',
kernel_constraint=MinMaxNorm(
-1 * self.parameters['cell_clip'],
self.parameters['cell_clip']),
recurrent_constraint=MinMaxNorm(
-1 * self.parameters['cell_clip'],
self.parameters['cell_clip']))(re_lstm_inputs)
re_lstm = Camouflage(mask_value=0)(inputs=[re_lstm, mask])
# Projection to hidden_units_size
re_proj = TimeDistributed(
Dense(self.parameters['hidden_units_size'],
activation='linear',
kernel_constraint=MinMaxNorm(
-1 * self.parameters['proj_clip'],
self.parameters['proj_clip'])))(re_lstm)
# Merge Bi-LSTMs feature vectors with the previous ones
re_lstm_inputs = add([re_proj, re_lstm_inputs],
name='b_block_{}'.format(i + 1))
# Apply variational drop-out between BI-LSTM layers
re_lstm_inputs = SpatialDropout1D(
self.parameters['dropout_rate'])(re_lstm_inputs)
# Reverse backward LSTMs' outputs = Make it forward again
re_lstm_inputs = Lambda(function=Context_vec.reverse,
name="reverse")(re_lstm_inputs)
# Project to Vocabulary with Sampled Softmax
sampled_softmax = SampledSoftmax(
num_classes=self.parameters['vocab_size'],
num_sampled=int(self.parameters['num_sampled']),
tied_to=embeddings if self.parameters['weight_tying']
and self.parameters['token_encoding'] == 'word' else None)
outputs = sampled_softmax([lstm_inputs, next_ids])
re_outputs = sampled_softmax([re_lstm_inputs, previous_ids])
self._model = Model(inputs=[word_inputs, next_ids, previous_ids],
outputs=[outputs, re_outputs])
self._model.compile(optimizer=Adagrad(
lr=self.parameters['lr'], clipvalue=self.parameters['clip_value']),
loss=None)
if print_summary:
self._model.summary()
def model(input_shape, input_shape2):
up_0 = Input(shape=input_shape, name='up_stream_0')
up_1 = Input(shape=input_shape, name='up_stream_1')
down_0 = Input(shape=input_shape, name='down_stream_0')
down_1 = Input(shape=input_shape, name='down_stream_1')
down_02 = Input(shape=input_shape, name='down_stream_02')
down_12 = Input(shape=input_shape, name='down_stream_12')
down_2 = Input(shape=input_shape2, name='down_stream_2')
down_3 = Input(shape=input_shape2, name='down_stream_3')
up_stream = share_stream(x_shape=input_shape, outputNum=128)
down_stream = share_stream(x_shape=input_shape, outputNum=128)
down_stream3 = share_stream(x_shape=input_shape2, outputNum=128)
up_feature_0 = up_stream(up_0)
up_feature_1 = up_stream(up_1)
down_feature_0 = down_stream(down_0)
down_feature_1 = down_stream(down_1)
down_feature_02 = down_stream(down_02)
down_feature_12 = down_stream(down_12)
down_feature_2 = down_stream3(down_2)
down_feature_3 = down_stream3(down_3)
up_feature = Maximum()([up_feature_0, up_feature_1])
down_feature = Maximum()([down_feature_0, down_feature_1])
down_feature2 = Maximum()([down_feature_02, down_feature_12])
down_feature3 = Maximum()([down_feature_2, down_feature_3])
feature = concatenate(
[up_feature, down_feature, down_feature2, down_feature3])
x = Dense(units=256, use_bias=True, trainable=Trainable)(feature)
x2 = Dense(16, trainable=Trainable)(Lambda(lambda x: x * 10000000.)(
Activation("tanh")(Lambda(lambda x: x / 10000000.)(x))))
x2Shape2 = x2._keras_shape
temp = Multiply()([
Dense(256, trainable=Trainable)(Lambda(
lambda x: x * 10000000., output_shape=x2Shape2[1:])(
Activation("tanh")(Lambda(lambda x: x / 10000000.,
output_shape=x2Shape2[1:])(x2)))), x
])
for jj in range(2, taylor):
temp = Add()([
Multiply()([
Dense(256, trainable=Trainable)(
Lambda(lambda x: x * 10000000.,
output_shape=x2Shape2[1:])(Activation("tanh")(
Lambda(lambda x: x / 10000000.,
output_shape=x2Shape2[1:])(x2)))),
Lambda(lambda x: (x**jj) / math.factorial(jj))(x)
]), temp
])
temp = Add()([
Dense(256, trainable=Trainable)(Lambda(
lambda x: x * 10000000., output_shape=x2Shape2[1:])(
Activation("tanh")(Lambda(lambda x: x / 10000000.,
output_shape=x2Shape2[1:])(x2)))),
temp
])
fc_1 = Lambda(lambda x: x * 10000000.)(Activation("tanh")(
Lambda(lambda x: x / 10000000.)(temp)))
fc_1 = Dropout(0.4)(fc_1)
fc_1 = Dense(256, trainable=True)(fc_1)
feature = Dense(units=256, use_bias=True, trainable=True)(feature)
feature = add([feature, fc_1])
feature = Dropout(0.8)(feature)
fc_2 = Dense(units=128, activation='relu', use_bias=True,
trainable=True)(feature)
fc_2 = Dropout(0.4)(fc_2)
fc_2 = Dense(units=128, trainable=Trainable)(fc_2)
feature = Dense(units=128, use_bias=True, trainable=True)(feature)
feature = add([feature, fc_2])
feature = Dropout(0.3)(feature)
fc_4 = Dense(
units=55,
use_bias=True,
)(feature)
fc_4 = Activation('softmax')(fc_4)
network = Model(
input=[up_0, up_1, down_0, down_1, down_02, down_12, down_2, down_3],
outputs=fc_4)
return network
activation='relu')(encoded_summary)
# Size: 11
encoded_summary = layers.MaxPool1D(pool_size=2)(encoded_summary)
encoded_summary = layers.Activation('relu')(encoded_summary)
# Size: 5
encoded_summary = layers.Flatten()(encoded_summary)
encoded_summary = layers.RepeatVector(MAX_DOCUMENT_LENGTH)(encoded_summary)
document = layers.Input(shape=(MAX_DOCUMENT_LENGTH, ))
encoded_document = layers.Embedding(np.shape(embedding_matrix)[0],
EMBEDDINGS_SIZE,
weights=[embedding_matrix],
input_length=MAX_DOCUMENT_LENGTH,
trainable=False)(document)
merged = layers.add([encoded_summary, encoded_document])
merged = layers.Bidirectional(
layers.LSTM((int)(EMBEDDINGS_SIZE / 2), return_sequences=True))(merged)
merged = layers.Dropout(0.3)(merged)
merged = layers.Bidirectional(
layers.LSTM((int)(EMBEDDINGS_SIZE / 4), return_sequences=True))(merged)
merged = layers.Dropout(0.3)(merged)
prediction = layers.TimeDistributed(layers.Dense(
3, activation='softmax'))(merged)
model = Model([document, summary], prediction)
logging.info("Compiling the network...")
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'],
def ResBlock(self, before, features, kernel_size=(3, 3), strides=1):
conv1 = self.conv2d(before, features, kernel_size, strides)
conv2 = self.conv2d(conv1, features, kernel_size, strides)
add1 = add([before, conv2])
return add1
def test_recursion():
####################################################
# test recursion
a = Input(shape=(32, ), name='input_a')
b = Input(shape=(32, ), name='input_b')
dense = Dense(16, name='dense_1')
a_2 = dense(a)
b_2 = dense(b)
merged = layers.concatenate([a_2, b_2], name='merge')
c = Dense(64, name='dense_2')(merged)
d = Dense(5, name='dense_3')(c)
model = Model(inputs=[a, b], outputs=[c, d], name='model')
e = Input(shape=(32, ), name='input_e')
f = Input(shape=(32, ), name='input_f')
g, h = model([e, f])
# g2, h2 = model([e, f])
assert g._keras_shape == c._keras_shape
assert h._keras_shape == d._keras_shape
# test separate manipulation of different layer outputs
i = Dense(7, name='dense_4')(h)
final_model = Model(inputs=[e, f], outputs=[i, g], name='final')
assert len(final_model.inputs) == 2
assert len(final_model.outputs) == 2
assert len(final_model.layers) == 4
# we don't check names of first 2 layers (inputs) because
# ordering of same-level layers is not fixed
print('final_model layers:', [layer.name for layer in final_model.layers])
assert [layer.name
for layer in final_model.layers][2:] == ['model', 'dense_4']
print(model.compute_mask([e, f], [None, None]))
assert model.compute_mask([e, f], [None, None]) == [None, None]
print(final_model.compute_output_shape([(10, 32), (10, 32)]))
assert final_model.compute_output_shape([(10, 32), (10, 32)]) == [(10, 7),
(10, 64)]
# run recursive model
fn = K.function(final_model.inputs, final_model.outputs)
input_a_np = np.random.random((10, 32))
input_b_np = np.random.random((10, 32))
fn_outputs = fn([input_a_np, input_b_np])
assert [x.shape for x in fn_outputs] == [(10, 7), (10, 64)]
# test serialization
model_config = final_model.get_config()
print(json.dumps(model_config, indent=4))
recreated_model = Model.from_config(model_config)
fn = K.function(recreated_model.inputs, recreated_model.outputs)
input_a_np = np.random.random((10, 32))
input_b_np = np.random.random((10, 32))
fn_outputs = fn([input_a_np, input_b_np])
assert [x.shape for x in fn_outputs] == [(10, 7), (10, 64)]
####################################################
# test multi-input multi-output
j = Input(shape=(32, ), name='input_j')
k = Input(shape=(32, ), name='input_k')
m, n = model([j, k])
o = Input(shape=(32, ), name='input_o')
p = Input(shape=(32, ), name='input_p')
q, r = model([o, p])
assert n._keras_shape == (None, 5)
assert q._keras_shape == (None, 64)
s = layers.concatenate([n, q], name='merge_nq')
assert s._keras_shape == (None, 64 + 5)
# test with single output as 1-elem list
multi_io_model = Model([j, k, o, p], [s])
fn = K.function(multi_io_model.inputs, multi_io_model.outputs)
fn_outputs = fn([
np.random.random((10, 32)),
np.random.random((10, 32)),
np.random.random((10, 32)),
np.random.random((10, 32))
])
assert [x.shape for x in fn_outputs] == [(10, 69)]
# test with single output as tensor
multi_io_model = Model([j, k, o, p], s)
fn = K.function(multi_io_model.inputs, multi_io_model.outputs)
fn_outputs = fn([
np.random.random((10, 32)),
np.random.random((10, 32)),
np.random.random((10, 32)),
np.random.random((10, 32))
])
# note that the output of the K.function will still be a 1-elem list
assert [x.shape for x in fn_outputs] == [(10, 69)]
# test serialization
print('multi_io_model.layers:', multi_io_model.layers)
print('len(model.inbound_nodes):', len(model.inbound_nodes))
print('len(model.outbound_nodes):', len(model.outbound_nodes))
model_config = multi_io_model.get_config()
print(model_config)
print(json.dumps(model_config, indent=4))
recreated_model = Model.from_config(model_config)
fn = K.function(recreated_model.inputs, recreated_model.outputs)
fn_outputs = fn([
np.random.random((10, 32)),
np.random.random((10, 32)),
np.random.random((10, 32)),
np.random.random((10, 32))
])
# note that the output of the K.function will still be a 1-elem list
assert [x.shape for x in fn_outputs] == [(10, 69)]
config = model.get_config()
Model.from_config(config)
model.summary()
json_str = model.to_json()
model_from_json(json_str)
yaml_str = model.to_yaml()
model_from_yaml(yaml_str)
####################################################
# test invalid graphs
# input is not an Input tensor
j = Input(shape=(32, ), name='input_j')
j = Dense(32)(j)
k = Input(shape=(32, ), name='input_k')
m, n = model([j, k])
with pytest.raises(TypeError):
Model([j, k], [m, n])
# disconnected graph
j = Input(shape=(32, ), name='input_j')
k = Input(shape=(32, ), name='input_k')
m, n = model([j, k])
with pytest.raises(RuntimeError):
Model([j], [m, n])
# redundant outputs
j = Input(shape=(32, ), name='input_j')
k = Input(shape=(32, ), name='input_k')
m, n = model([j, k])
# this should work with a warning
Model([j, k], [m, n, n])
# redundant inputs
j = Input(shape=(32, ), name='input_j')
k = Input(shape=(32, ), name='input_k')
m, n = model([j, k])
with pytest.raises(ValueError):
Model([j, k, j], [m, n])
# i have not idea what I'm doing: garbage as inputs/outputs
j = Input(shape=(32, ), name='input_j')
k = Input(shape=(32, ), name='input_k')
m, n = model([j, k])
with pytest.raises(TypeError):
Model([j, k], [m, n, 0])
####################################################
# test calling layers/models on TF tensors
if K._BACKEND == 'tensorflow':
import tensorflow as tf
j = Input(shape=(32, ), name='input_j')
k = Input(shape=(32, ), name='input_k')
m, n = model([j, k])
tf_model = Model([j, k], [m, n])
j_tf = tf.placeholder(dtype=K.floatx())
k_tf = tf.placeholder(dtype=K.floatx())
m_tf, n_tf = tf_model([j_tf, k_tf])
assert m_tf.get_shape().as_list() == [None, 64]
assert n_tf.get_shape().as_list() == [None, 5]
# test merge
layers.concatenate([j_tf, k_tf], axis=1)
layers.add([j_tf, k_tf])
# test tensor input
x = tf.placeholder(shape=(None, 2), dtype=K.floatx())
InputLayer(input_tensor=x)
x = Input(tensor=x)
Dense(2)(x)
def YnetResNet(netpram,include_top=True, weights=None, input_tensor=None, input_shape=None,pooling=None, classes=1000):
include_top=False
pooling='max'
classes=1000
if netpram.task=='all':
num_classes=11
elif netpram.task=='binary':
num_classes=1
elif netpram.task=='parts':
num_classes=3
elif netpram.task=='instrument':
num_classes=7
#img_input=Input((224, 224, 3))
inputs_L = Input((224, 224, 3))
inputs_R = Input((224, 224, 3))
inputs=[inputs_L,inputs_R]
bn_axis = 3
x = layers.ZeroPadding2D(padding=(3, 3), name='conv1_pad')(inputs_L)
x = layers.Conv2D(64, (7, 7),
strides=(2, 2),
padding='valid',
name='conv1')(x)
x = layers.BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
x_a = layers.Activation('relu')(x)
x = layers.MaxPooling2D((3, 3), strides=(2, 2))(x_a)
x_b = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
x = identity_block(x_b, 3, [64, 64, 256], stage=2, block='b')
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')
x_c = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
x = identity_block(x_c, 3, [128, 128, 512], stage=3, block='b')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='c')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='d')
x_d = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
x = identity_block(x_d, 3, [256, 256, 1024], stage=4, block='b')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f')
x_e = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')
x = identity_block(x_e, 3, [512, 512, 2048], stage=5, block='b')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')
if include_top:
x = layers.AveragePooling2D((7, 7), name='avg_pool')(x)
x = layers.Flatten()(x)
x = layers.Dense(classes, activation='softmax', name='fc1000')(x)
else:
if pooling == 'avg':
x = layers.GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = layers.GlobalMaxPooling2D()(x)
else:
warnings.warn('The output shape of `ResNet50(include_top=False)` '
'has been changed since Keras 2.2.0.')
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
inputs = inputs_L
# Create model.
modelen1 = Model(inputs, x, name='resnet50')
# Load weights.
weights_path = utils.get_file(
'resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5',
WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
md5_hash='a268eb855778b3df3c7506639542a6af')
modelen1.load_weights(weights_path)
# for layer in modelen1.layers:
# layer.trainable=False
# Loaded enoder one- untrainable
x = layers.ZeroPadding2D(padding=(3, 3), name='enc2_conv1_pad')(inputs_R)
x = layers.Conv2D(64, (7, 7),
strides=(2, 2),
padding='valid',
name='enc2_conv1')(x)
x = layers.BatchNormalization(axis=bn_axis, name='enc2_bn_conv1')(x)
x_a_e2 = layers.Activation('relu')(x)
x = layers.MaxPooling2D((3, 3), strides=(2, 2))(x_a_e2)
x_b_e2 = conv_block(x, 3, [64, 64, 256], stage=2, block='a_enc2', strides=(1, 1))
x = identity_block(x_b_e2, 3, [64, 64, 256], stage=2, block='b_enc2')
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c_enc2')
x_c_e2 = conv_block(x, 3, [128, 128, 512], stage=3, block='a_enc2')
x = identity_block(x_c_e2, 3, [128, 128, 512], stage=3, block='b_enc2')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='c_enc2')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='d_enc2')
x_d_e2 = conv_block(x, 3, [256, 256, 1024], stage=4, block='a_enc2')
x = identity_block(x_d_e2, 3, [256, 256, 1024], stage=4, block='b_enc2')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c_enc2')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d_enc2')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e_enc2')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f_enc2')
x_e_e2 = conv_block(x, 3, [512, 512, 2048], stage=5, block='a_enc2')
x = identity_block(x_e_e2, 3, [512, 512, 2048], stage=5, block='b_enc2_enc2')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c_enc2')
if include_top:
x = layers.AveragePooling2D((7, 7), name='avg_pool_enc2')(x)
x = layers.Flatten()(x)
x = layers.Dense(classes, activation='softmax', name='fc1000_enc2')(x)
else:
if pooling == 'avg':
x = layers.GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = layers.GlobalMaxPooling2D()(x)
else:
warnings.warn('The output shape of `ResNet50(include_top=False)` '
'has been changed since Keras 2.2.0.')
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
inputs = inputs_R
# Create model.
modelen2 = Model(inputs, x, name='resnet50')
# Load weights.
weights_path = utils.get_file(
'resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5',
WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
md5_hash='a268eb855778b3df3c7506639542a6af')
modelen2.load_weights(weights_path)
#Loaded encoder 2-trainable
#center = add([xold, ynew])
#center = concatenate([xold, ynew], axis=3)
center=conv_block_decoder(add([x_e,x_e_e2]), 1024, 7, 'center', strides=(2, 2))
center=conv_block_decoder(center, 1024, 7, 'center_2', strides=(2, 2))
# 32
up4 = UpSampling2D((2, 2))(center)
#upc=concatenate([x_d,x_d_e2], axis=3)
up4 = concatenate([add([x_d,x_d_e2]), up4], axis=3)
up4=conv_block_decoder(up4, 512, 32, 'deconv1', strides=(2, 2))
# up4=conv_block_decoder(up4, 512, 32, 'deconv2', strides=(2, 2))
up4=conv_block_decoder(up4, 512, 32, 'deconv3', strides=(2, 2))
# 64
up3 = UpSampling2D((2, 2))(up4)
# upc=concatenate([down2T,down2B], axis=3)
#upc=concatenate([x_c,x_c_e2], axis=3)
up3 = concatenate([add([x_c,x_c_e2]), up3], axis=3)
up3=conv_block_decoder(up3, 512, 64, 'deconv1', strides=(2, 2))
up3=conv_block_decoder(up3, 512, 64, 'deconv2', strides=(2, 2))
up3=conv_block_decoder(up3, 512, 64, 'deconv3', strides=(2, 2))
# 128
up2 = UpSampling2D((2, 2))(up3)
#upc = concatenate([down1T,down1B], axis=3)
x_b = layers.ZeroPadding2D(padding=[1, 1], name='convdec_pad')(x_b)
x_b = layers.Cropping2D(cropping=((1, 0), (1, 0)), data_format=None)(x_b)
x_b_e2 = layers.ZeroPadding2D(padding=[1, 1], name='convdec_pad2')(x_b_e2)
x_b_e2 = layers.Cropping2D(cropping=((1, 0), (1, 0)), data_format=None)(x_b_e2)
up2 = concatenate([add([x_b,x_b_e2]), up2], axis=3)
up2=conv_block_decoder(up2, 256, 128, 'deconv1', strides=(2, 2))
# up2=conv_block_decoder(up2, 256, 128, 'deconv2', strides=(2, 2))
up2=conv_block_decoder(up2, 256, 128, 'deconv3', strides=(2, 2))
# 256
up1 = UpSampling2D((2, 2))(up2)
#upc=concatenate([down0T,down0B], axis=3)
up1 = concatenate([add([x_a,x_a_e2]), up1], axis=3)
up1=conv_block_decoder(up1, 128, 256, 'deconv1', strides=(2, 2))
up1=conv_block_decoder(up1, 128, 256, 'deconv2', strides=(2, 2))
up1=conv_block_decoder(up1, 128, 256, 'deconv3', strides=(2, 2))
# 512
up0a = UpSampling2D((2, 2))(up1)
#upc= concatenate([down0aT,down0aB],axis=3)
# up0a = concatenate([img_input, up0a], axis=3)
up0a=conv_block_decoder(up0a, 64, 512, 'deconv1', strides=(2, 2))
up0a=conv_block_decoder(up0a, 64, 512, 'deconv2', strides=(2, 2))
up0a=conv_block_decoder(up0a, 64, 512, 'deconv3', strides=(2, 2))
classify = Conv2D(num_classes, (1, 1), activation='sigmoid')(up0a)
model = Model(inputs=[inputs_L,inputs_R], outputs=classify,name='RESNetU')
# optimizerc =CustomRMSprop(lr=0.00001,multipliers = LR_mult_dict)
#lr=0.00001, F1=79.8
model.compile(optimizer=RMSprop(lr=0.0001), loss=bce_dice_loss, metrics=[dice_coeff])
# '''
return model
# output: (samples, query_maxlen, embedding_dim)
# encode input sequence and questions (which are indices)
# to sequences of dense vectors
input_encoded_m = input_encoder_m(input_sequence)
input_encoded_c = input_encoder_c(input_sequence)
question_encoded = question_encoder(question)
# compute a 'match' between the first input vector sequence
# and the question vector sequence
# shape: `(samples, story_maxlen, query_maxlen)`
match = dot([input_encoded_m, question_encoded], axes=(2, 2))
match = Activation('softmax')(match)
# add the match matrix with the second input vector sequence
response = add([match,
input_encoded_c]) # (samples, story_maxlen, query_maxlen)
response = Permute((2, 1))(response) # (samples, query_maxlen, story_maxlen)
# concatenate the match matrix with the question vector sequence
answer = concatenate([response, question_encoded])
# the original paper uses a matrix multiplication for this reduction step.
# we choose to use a RNN instead.
answer = LSTM(32)(answer) # (samples, 32)
# one regularization layer -- more would probably be needed.
answer = Dropout(0.3)(answer)
answer = Dense(vocab_size)(answer) # (samples, vocab_size)
# we output a probability distribution over the vocabulary
answer = Activation('softmax')(answer)
''' Here is how to implement a residual connection in Keras This example assumes the existence of a 4D input tensor ''' from keras import layers x = ... y = layers.Conv2D(128, 3, activation='relu', padding='same')(x) y = layers.Conv2D(128, 3, activation='relu', padding='same')(x) y = layers.MaxPooling2D(2, strides=2)(y) # uses a 1 x 1 convolution to linearly downsample the original x tensor to the same shape as y residual = layers.Conv2D(128, 1, stride=2, padding='same')(x) # adds the residual tensor bach to the output features y = layers.add([y, residual])
def test_TensorBoard_multi_input_output(tmpdir):
np.random.seed(np.random.randint(1, 1e7))
filepath = str(tmpdir / 'logs')
(X_train, y_train), (X_test, y_test) = get_test_data(
num_train=train_samples,
num_test=test_samples,
input_shape=(input_dim, input_dim),
classification=True,
num_classes=num_classes)
y_test = np_utils.to_categorical(y_test)
y_train = np_utils.to_categorical(y_train)
def data_generator(train):
if train:
max_batch_index = len(X_train) // batch_size
else:
max_batch_index = len(X_test) // batch_size
i = 0
while 1:
if train:
# simulate multi-input/output models
yield ([X_train[i * batch_size: (i + 1) * batch_size]] * 2,
[y_train[i * batch_size: (i + 1) * batch_size]] * 2)
else:
yield ([X_test[i * batch_size: (i + 1) * batch_size]] * 2,
[y_test[i * batch_size: (i + 1) * batch_size]] * 2)
i += 1
i = i % max_batch_index
inp1 = Input((input_dim, input_dim))
inp2 = Input((input_dim, input_dim))
inp_3d = add([inp1, inp2])
inp_2d = GlobalAveragePooling1D()(inp_3d)
inp_pair = Lambda(lambda x: x)([inp_3d, inp_2d]) # test a layer with a list of output tensors
hidden = dot(inp_pair, axes=-1)
hidden = Dense(num_hidden, activation='relu')(hidden)
hidden = Dropout(0.1)(hidden)
output1 = Dense(num_classes, activation='softmax')(hidden)
output2 = Dense(num_classes, activation='softmax')(hidden)
model = Model(inputs=[inp1, inp2], outputs=[output1, output2])
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
metrics=['accuracy'])
# we must generate new callbacks for each test, as they aren't stateless
def callbacks_factory(histogram_freq, embeddings_freq=1):
return [callbacks.TensorBoard(log_dir=filepath,
histogram_freq=histogram_freq,
write_images=True, write_grads=True,
embeddings_freq=embeddings_freq,
embeddings_layer_names=['dense_1'],
embeddings_data=[X_test] * 2,
batch_size=5)]
# fit without validation data
model.fit([X_train] * 2, [y_train] * 2, batch_size=batch_size,
callbacks=callbacks_factory(histogram_freq=0, embeddings_freq=0),
epochs=3)
# fit with validation data and accuracy
model.fit([X_train] * 2, [y_train] * 2, batch_size=batch_size,
validation_data=([X_test] * 2, [y_test] * 2),
callbacks=callbacks_factory(histogram_freq=1), epochs=2)
# fit generator without validation data
model.fit_generator(data_generator(True), len(X_train), epochs=2,
callbacks=callbacks_factory(histogram_freq=0,
embeddings_freq=0))
# fit generator with validation data and accuracy
model.fit_generator(data_generator(True), len(X_train), epochs=2,
validation_data=([X_test] * 2, [y_test] * 2),
callbacks=callbacks_factory(histogram_freq=1))
assert os.path.isdir(filepath)
shutil.rmtree(filepath)
assert not tmpdir.listdir()
def cnn(input_shape, num_classes, l2_regularization=0.01):
regularization = l2(l2_regularization)
# base
img_input = Input(input_shape)
x = Conv2D(8, (3, 3),
strides=(1, 1),
kernel_regularizer=regularization,
use_bias=False)(img_input)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(8, (3, 3),
strides=(1, 1),
kernel_regularizer=regularization,
use_bias=False)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# module 1
residual = Conv2D(16, (1, 1),
strides=(2, 2),
padding='same',
use_bias=False)(x)
residual = BatchNormalization()(residual)
x = SeparableConv2D(16, (3, 3),
padding='same',
kernel_regularizer=regularization,
use_bias=False)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = SeparableConv2D(16, (3, 3),
padding='same',
kernel_regularizer=regularization,
use_bias=False)(x)
x = BatchNormalization()(x)
x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
x = layers.add([x, residual])
# module 2
residual = Conv2D(32, (1, 1),
strides=(2, 2),
padding='same',
use_bias=False)(x)
residual = BatchNormalization()(residual)
x = SeparableConv2D(32, (3, 3),
padding='same',
kernel_regularizer=regularization,
use_bias=False)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = SeparableConv2D(32, (3, 3),
padding='same',
kernel_regularizer=regularization,
use_bias=False)(x)
x = BatchNormalization()(x)
x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
x = layers.add([x, residual])
# module 3
residual = Conv2D(64, (1, 1),
strides=(2, 2),
padding='same',
use_bias=False)(x)
residual = BatchNormalization()(residual)
x = SeparableConv2D(64, (3, 3),
padding='same',
kernel_regularizer=regularization,
use_bias=False)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = SeparableConv2D(64, (3, 3),
padding='same',
kernel_regularizer=regularization,
use_bias=False)(x)
x = BatchNormalization()(x)
x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
x = layers.add([x, residual])
# module 4
residual = Conv2D(128, (1, 1),
strides=(2, 2),
padding='same',
use_bias=False)(x)
residual = BatchNormalization()(residual)
x = SeparableConv2D(128, (3, 3),
padding='same',
kernel_regularizer=regularization,
use_bias=False)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = SeparableConv2D(128, (3, 3),
padding='same',
kernel_regularizer=regularization,
use_bias=False)(x)
x = BatchNormalization()(x)
x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
x = layers.add([x, residual])
x = Conv2D(
num_classes,
(3, 3),
#kernel_regularizer=regularization,
padding='same')(x)
x = GlobalAveragePooling2D()(x)
output = Activation('softmax', name='predictions')(x)
model = Model(img_input, output)
return model
def _inception_resnet_block(x,
scale,
block_type,
block_idx,
activation='relu'):
channel_axis = 1 if K.image_data_format() == 'channels_first' else 3
if block_idx is None:
prefix = None
else:
prefix = '_'.join((block_type, str(block_idx)))
name_fmt = partial(_generate_layer_name, prefix=prefix)
if block_type == 'Block35':
branch_0 = conv2d_bn(x, 32, 1, name=name_fmt('Conv2d_1x1', 0))
branch_1 = conv2d_bn(x, 32, 1, name=name_fmt('Conv2d_0a_1x1', 1))
branch_1 = conv2d_bn(branch_1,
32,
3,
name=name_fmt('Conv2d_0b_3x3', 1))
branch_2 = conv2d_bn(x, 32, 1, name=name_fmt('Conv2d_0a_1x1', 2))
branch_2 = conv2d_bn(branch_2,
32,
3,
name=name_fmt('Conv2d_0b_3x3', 2))
branch_2 = conv2d_bn(branch_2,
32,
3,
name=name_fmt('Conv2d_0c_3x3', 2))
branches = [branch_0, branch_1, branch_2]
elif block_type == 'Block17':
branch_0 = conv2d_bn(x, 128, 1, name=name_fmt('Conv2d_1x1', 0))
branch_1 = conv2d_bn(x, 128, 1, name=name_fmt('Conv2d_0a_1x1', 1))
branch_1 = conv2d_bn(branch_1,
128, [1, 7],
name=name_fmt('Conv2d_0b_1x7', 1))
branch_1 = conv2d_bn(branch_1,
128, [7, 1],
name=name_fmt('Conv2d_0c_7x1', 1))
branches = [branch_0, branch_1]
elif block_type == 'Block8':
branch_0 = conv2d_bn(x, 192, 1, name=name_fmt('Conv2d_1x1', 0))
branch_1 = conv2d_bn(x, 192, 1, name=name_fmt('Conv2d_0a_1x1', 1))
branch_1 = conv2d_bn(branch_1,
192, [1, 3],
name=name_fmt('Conv2d_0b_1x3', 1))
branch_1 = conv2d_bn(branch_1,
192, [3, 1],
name=name_fmt('Conv2d_0c_3x1', 1))
branches = [branch_0, branch_1]
else:
raise ValueError('Unknown Inception-ResNet block type. '
'Expects "Block35", "Block17" or "Block8", '
'but got: ' + str(block_type))
mixed = Concatenate(axis=channel_axis,
name=name_fmt('Concatenate'))(branches)
up = conv2d_bn(mixed,
K.int_shape(x)[channel_axis],
1,
activation=None,
use_bias=True,
name=name_fmt('Conv2d_1x1'))
up = Lambda(scaling,
output_shape=K.int_shape(up)[1:],
arguments={'scale': scale})(up)
x = add([x, up])
if activation is not None:
x = Activation(activation, name=name_fmt('Activation'))(x)
return x
def conv_block(input_tensor,
kernel_size,
filters,
stage,
block,
strides=(2, 2),
dilation_rate=1,
multigrid=[1, 2, 1],
use_se=True):
# conv filters
filters1, filters2, filters3 = filters
# compute dataformat
if K.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
# dailated rate
if dilation_rate > 1:
strides = (1, 1)
else:
multigrid = [1, 1, 1]
# forward
x = Conv2D(filters1, (1, 1),
strides=strides,
name=conv_name_base + '2a',
dilation_rate=dilation_rate * multigrid[0])(input_tensor)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
x = Activation('relu')(x)
x = Conv2D(filters2,
kernel_size,
padding='same',
name=conv_name_base + '2b',
dilation_rate=dilation_rate * multigrid[1])(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)
x = Activation('relu')(x)
x = Conv2D(filters3, (1, 1),
name=conv_name_base + '2c',
dilation_rate=dilation_rate * multigrid[2])(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)
shortcut = Conv2D(filters3, (1, 1),
strides=strides,
name=conv_name_base + '1')(input_tensor)
shortcut = BatchNormalization(axis=bn_axis,
name=bn_name_base + '1')(shortcut)
# stage after 5 squeeze and excittation
if use_se and stage < 5:
se = _squeeze_excite_block(x,
filters3,
k=1,
name=conv_name_base + '_se')
x = multiply([x, se])
x = add([x, shortcut])
x = Activation('relu')(x)
return x
x = Conv2D(64, (3, 3), use_bias=False, name='block1_conv2')(x)
x = BatchNormalization(name='block1_conv2_bn')(x)
x = Activation('relu', name='block1_conv2_act')(x)
residual = Conv2D(128, (1, 1), strides=(2, 2),
padding='same', use_bias=False)(x)
residual = BatchNormalization()(residual)
x = SeparableConv2D(128, (3, 3), padding='same', use_bias=False, name='block2_sepconv1')(x)
x = BatchNormalization(name='block2_sepconv1_bn')(x)
x = Activation('relu', name='block2_sepconv2_act')(x)
x = SeparableConv2D(128, (3, 3), padding='same', use_bias=False, name='block2_sepconv2')(x)
x = BatchNormalization(name='block2_sepconv2_bn')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='block2_pool')(x)
x = layers.add([x, residual])
residual = Conv2D(256, (1, 1), strides=(2, 2),
padding='same', use_bias=False)(x)
residual = BatchNormalization()(residual)
x = Activation('relu', name='block3_sepconv1_act')(x)
x = SeparableConv2D(256, (3, 3), padding='same', use_bias=False, name='block3_sepconv1')(x)
x = BatchNormalization(name='block3_sepconv1_bn')(x)
x = Activation('relu', name='block3_sepconv2_act')(x)
x = SeparableConv2D(256, (3, 3), padding='same', use_bias=False, name='block3_sepconv2')(x)
x = BatchNormalization(name='block3_sepconv2_bn')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='block3_pool')(x)
x = layers.add([x, residual])
def create_model(self, n_dim, r):
# load inputs
X, _, _ = self.inputs
K.set_session(self.sess)
with tf.name_scope('generator'):
x = X
L = self.layers
# dim/layer: 4096, 2048, 1024, 512, 256, 128, 64, 32,
# n_filters = [ 64, 128, 256, 384, 384, 384, 384, 384]
n_filters = [128, 256, 512, 512, 512, 512, 512, 512]
# n_filters = [ 256, 512, 512, 512, 512, 1024, 1024, 1024]
# n_filtersizes = [129, 65, 33, 17, 9, 9, 9, 9]
# n_filtersizes = [31, 31, 31, 31, 31, 31, 31, 31]
n_filtersizes = [65, 33, 17, 9, 9, 9, 9, 9, 9]
downsampling_l = []
print('building model...')
# downsampling layers
for l, nf, fs in zip(range(L), n_filters, n_filtersizes):
with tf.name_scope('downsc_conv%d' % l):
# in Keras 2
x = Conv1D(padding='same',
kernel_initializer='Orthogonal',
filters=nf,
kernel_size=fs,
activation=None)(x)
#x = (Convolution1D(nb_filter=nf, filter_length=fs,
# activation=None, border_mode='same', init=orthogonal_init,
# subsample_length=2))(x)
# if l > 0: x = BatchNormalization(mode=2)(x)
x = LeakyReLU(0.2)(x)
print('D-Block: ', x.get_shape())
downsampling_l.append(x)
# bottleneck layer
with tf.name_scope('bottleneck_conv'):
# in Keras 2
x = Conv1D(padding='same',
kernel_initializer='Orthogonal',
filters=n_filters[-1],
kernel_size=n_filtersizes[-1],
activation=None)(x)
#x = (Convolution1D(nb_filter=n_filters[-1], filter_length=n_filtersizes[-1],
# activation=None, border_mode='same', init=orthogonal_init,
# subsample_length=2))(x)
x = Dropout(rate=0.5)(x)
# x = BatchNormalization(mode=2)(x)
x = LeakyReLU(0.2)(x)
# upsampling layers
# for l, nf, fs, l_in in reversed(zip(range(L), n_filters, n_filtersizes, downsampling_l)):
for l, nf, fs, l_in in zip(reversed(range(L)), reversed(n_filters),
reversed(n_filtersizes),
reversed(downsampling_l)):
with tf.name_scope('upsc_conv%d' % l):
# (-1, n/2, 2f)
# in Keras 2
x = Conv1D(padding='same',
kernel_initializer='Orthogonal',
filters=2 * nf,
kernel_size=fs,
activation=None)(x)
#x = (Convolution1D(nb_filter=2*nf, filter_length=fs,
# activation=None, border_mode='same', init=orthogonal_init))(x)
# x = BatchNormalization(mode=2)(x)
x = Dropout(rate=0.5)(x)
x = Activation('relu')(x)
# (-1, n, f)
x = SubPixel1D(x, r=2)
# (-1, n, 2f)
# in Keras 2
x = concatenate([x, l_in]) # axis = -1 (by default)
# in Keras 1
#x = merge([x, l_in], mode='concat', concat_axis=-1)
print('U-Block: ', x.get_shape())
# final conv layer
with tf.name_scope('lastconv'):
# in Keras 2
x = Conv1D(padding='same',
kernel_initializer='he_normal',
filters=2,
kernel_size=9,
activation=None)(x)
#x = Convolution1D(nb_filter=2, filter_length=9,
# activation=None, border_mode='same', init=normal_init)(x)
x = SubPixel1D(x, r=2)
print(x.get_shape())
# in Keras 2
g = add([x, X])
# in Keras 1
#g = merge([x, X], mode='sum')
return g