def siamese_conv(pretrain=False):
input = Input(shape=(max_sequence_length, ), dtype='int32')
input_mask = Input(shape=(max_sequence_length, ), dtype='bool')
print input.get_shape()
embedding_dim = 300
with tf.device('/cpu:0'):
if pretrain:
embedding_input = Embedding(
vocab_size,
embedding_dim,
weights=[embedding_matrix],
trainable=True,
mask_zero=False,
)(input)
else:
embedding_input = Embedding(
vocab_size,
embedding_dim,
trainable=True,
init=weight_initializer,
mask_zero=False,
)(input)
# masking
#embedding_input = LSTM(256, return_sequences=True)(embedding_input)
def func(x):
# x[0] input, x[1] mask
masking = tf.cast(x[1], tf.float32)
x = x[0]
masking = tf.tile(tf.expand_dims(masking, -1), [1, 1, 300])
return tf.mul(x, masking)
embedding_input = Merge(
mode=func, output_shape=lambda x: x[0])([embedding_input, input_mask])
model = Model(input=[input, input_mask], output=embedding_input)
return model
def network_posewarp(param):
img_h = param['IMG_HEIGHT']
img_w = param['IMG_WIDTH']
n_joints = param['n_joints']
pose_dn = param['posemap_downsample']
n_limbs = param['n_limbs']
# Inputs
src_in = Input(shape=(img_h, img_w, 3))
pose_src = Input(shape=(img_h / pose_dn, img_w / pose_dn, n_joints))
pose_tgt = Input(shape=(img_h / pose_dn, img_w / pose_dn, n_joints))
src_mask_prior = Input(shape=(img_h, img_w, n_limbs + 1))
trans_in = Input(shape=(2, 3, n_limbs + 1))
# 1. FG/BG separation
print("##########################")
print(type(src_in), type(pose_src))
print(src_in.get_shape(), pose_src.get_shape())
print("##########################")
x = unet(src_in, pose_src, [64] * 2 + [128] * 9, [128] * 4 + [32])
src_mask_delta = my_conv(x, 11, activation='linear', name='mask_delta')
src_mask = keras.layers.add([src_mask_delta, src_mask_prior])
src_mask = Activation('softmax', name='mask_src')(src_mask)
# 2. Separate into fg limbs and background
warped_stack = Lambda(make_warped_stack)([src_mask, src_in, trans_in])
fg_stack = Lambda(lambda arg: arg[:, :, :, 3:],
output_shape=(img_h, img_w, 3 * n_limbs),
name='fg_stack')(warped_stack)
bg_src = Lambda(lambda arg: arg[:, :, :, 0:3],
output_shape=(img_h, img_w, 3),
name='bg_src')(warped_stack)
bg_src_mask = Lambda(lambda arg: tf.expand_dims(arg[:, :, :, 0], 3))(
src_mask)
# 3. BG/FG synthesis
x = unet(concatenate([bg_src, bg_src_mask]), pose_src,
[64] * 2 + [128] * 9, [128] * 4 + [64])
bg_tgt = my_conv(x, 3, activation='tanh', name='bg_tgt')
x = unet(fg_stack, pose_tgt, [64] * 2 + [128] * 9, [128] * 4 + [64])
# x = unet(fg_stack, pose_tgt, [64] + [128] * 3 + [256] * 7, [256, 256, 256, 128, 64])
fg_tgt = my_conv(x, 3, activation='tanh', name='fg_tgt')
fg_mask = my_conv(x, 1, activation='sigmoid', name='fg_mask_tgt')
fg_mask = concatenate([fg_mask, fg_mask, fg_mask])
bg_mask = Lambda(lambda arg: 1 - arg)(fg_mask)
# 5. Merge bg and fg
fg_tgt = keras.layers.multiply([fg_tgt, fg_mask], name='fg_tgt_masked')
bg_tgt = keras.layers.multiply([bg_tgt, bg_mask], name='bg_tgt_masked')
y = keras.layers.add([fg_tgt, bg_tgt])
model = Model(
inputs=[src_in, pose_src, pose_tgt, src_mask_prior, trans_in],
outputs=[y])
return model
def main(**kwargs):
sess = tf.Session()
img_path = int(args.image_path)
K.set_session(sess)
save_mnist_as_tfrecord()
batch_size = 32
batch_shape = [batch_size, 28, 28, 1]
epochs = 100
classes = 1
parallelism = 10
x0_train_batch, x1_train_batch, y_train_batch, angles_train_batch = read_and_decode_recordinput(
str(img_path) + 'train.mnist.tfrecord',
one_hot=True,
classes=classes,
is_train=True,
batch_shape=batch_shape,
parallelism=parallelism)
x0_test_batch, x1_test_batch, y_test_batch, angles_test_batch = read_and_decode_recordinput(
str(img_path) + 'test.mnist.tfrecord',
one_hot=True,
classes=classes,
is_train=True,
batch_shape=batch_shape,
parallelism=parallelism)
x0_batch_shape = x0_train_batch.get_shape().as_list()
x1_batch_shape = x1_train_batch.get_shape().as_list()
y_batch_shape = y_train_batch.get_shape().as_list()
x0_train_input = Input(tensor=x0_train_batch, batch_shape=x0_batch_shape)
x1_train_input = Input(tensor=x1_train_batch, batch_shape=x1_batch_shape)
input_shape = (28, 28, 1)
base_network = get_convnet(input_shape)
x0_train_out = base_network(x0_train_input)
x1_train_out = base_network(x1_train_input)
print x0_train_input.get_shape().as_list()
inputs = [x0_train_input, x1_train_input]
distance = _distance(x0_train_input, x1_train_input)
print distance.get_shape().as_list()
mse = mean_squared_error(angles_train_batch, distance)
train_model = Model(inputs=[x0_train_input, x1_train_input],
outputs=_distance(x0_train_input, x1_train_input))
writer = tf.summary.FileWriter(logs_path, graph=tf.get_default_graph())
train_model.add_loss(mse)
train_model.compile(optimizer='rmsprop', loss=None, metrics=['accuracy'])
train_model.summary()
initial_epoch = 0
train_model.fit(batch_size=batch_size, epochs=epochs,
verbose=True) # callbacks=[tensorboard])
def _build_c(self, s, a, scope, trainable):
from keras.models import Model
from keras.layers import merge, Convolution2D, MaxPooling2D, Input, Dense, Flatten, Dropout, Reshape, TimeDistributed, BatchNormalization, Merge
from keras.layers.advanced_activations import LeakyReLU
# self.tf_obs = Input(shape = (3,), dtype='float32', name="observations")
B = Input(shape=(3, ), tensor=self.tf_account)
b = Dense(5, activation="relu")(B)
inputs = [B]
merges = [b]
#S = Input(shape=[2, 60, 1])
S = Input(shape=[2, 60, 1], tensor=self.tf_obs)
inputs.append(S)
print(B.get_shape(), S.get_shape())
h = Convolution2D(2048, 3, 1, border_mode='same')(S)
h = LeakyReLU(0.001)(h)
h = Convolution2D(2048, 5, 1, border_mode='same')(S)
h = LeakyReLU(0.001)(h)
h = Convolution2D(2048, 10, 1, border_mode='same')(S)
h = LeakyReLU(0.001)(h)
h = Convolution2D(2048, 20, 1, border_mode='same')(S)
h = LeakyReLU(0.001)(h)
h = Convolution2D(2048, 40, 1, border_mode='same')(S)
h = LeakyReLU(0.001)(h)
h = Flatten()(h)
h = Dense(512)(h)
h = LeakyReLU(0.001)(h)
merges.append(h)
h = Convolution2D(2048, 60, 1, border_mode='same')(S)
h = LeakyReLU(0.001)(h)
h = Flatten()(h)
h = Dense(512)(h)
h = LeakyReLU(0.001)(h)
merges.append(h)
m = merge(merges, mode='concat', concat_axis=1)
m = Dense(1024)(m)
m = LeakyReLU(0.001)(m)
m = Dense(512)(m)
m = LeakyReLU(0.001)(m)
m = Dense(256)(m)
m = LeakyReLU(0.001)(m)
self.v = Dense(1, activation=None, name='V')(m)
with tf.variable_scope('squared_TD_error'):
self.td_error = self.r + GAMMA * self.v_ - self.v
self.loss = tf.square(
self.td_error) # TD_error = (r+gamma*V_next) - V_eval
with tf.variable_scope('train'):
self.train_op = tf.train.AdamOptimizer(self.lr).minimize(self.loss)
def build_compile_model(self, args):
X_1hot_seqsamples, X_features_floats, Y = self.trainvalid_1hot_data[
'train']
shA = X_1hot_seqsamples.shape
shB = X_features_floats.shape
inputA = Input(shape=(shA[1], shA[2]),
name='seq_%d_x_%d' % (shA[1], shA[2]))
inputB = Input(shape=(shB[1], ), name='globF_%d' % (shB[1]))
print('build_model inpA:', inputA.get_shape(), ' inpB:',
inputB.get_shape())
lstm_dim = 40
#dens_dim_first=100
dens_dim = 100
densAct = 'relu'
layerDropFrac = Constants.dropFrac
recDropFrac = layerDropFrac / 2.
print('Dens act=', densAct, ' recurDropFrac=', recDropFrac,
' layerDropFrac=', layerDropFrac, ' idx=', self.arrIdx)
#inputA is the 1-hot encoded sequences and it goes into an LSTM.
net = LSTM(lstm_dim,
activation='tanh',
recurrent_dropout=recDropFrac,
dropout=layerDropFrac,
name='A_%d' % lstm_dim,
return_sequences=True)(inputA)
net = LSTM(lstm_dim,
activation='tanh',
recurrent_dropout=recDropFrac,
dropout=layerDropFrac,
name='B_%d' % lstm_dim)(net)
#inputB is the statistical features and it goes into a Dense (CNN) network.
net2 = Dense(dens_dim, activation='tanh', name='G_4')(inputB)
net2 = Dense(dens_dim, activation='tanh', name='H_4')(net2)
#the two above get concatenated.
net = concatenate([net, net2], name='seq_glob')
net = Dense(dens_dim * 2,
activation=densAct,
name='C_%d' % (dens_dim * 2))(net)
net = Dropout(layerDropFrac, name='fr_%.1f' % layerDropFrac)(net)
net = Dense(dens_dim, activation=densAct, name='D_%d' % dens_dim)(net)
net = Dropout(layerDropFrac, name='fr_same')(net)
outputs = Dense(1, activation='sigmoid', name='score')(net)
model = Model(inputs=[inputA, inputB], outputs=outputs)
self.model = model
self.compile_model()
def build_compile_model(self, args):
XA, XB, Y = self.data['train']
shA = XA.shape
shB = XB.shape
inputA = Input(shape=(shA[1], shA[2]),
name='seq_%d_x_%d' % (shA[1], shA[2]))
inputB = Input(shape=(shB[1], ), name='globF_%d' % (shB[1]))
print('build_model inpA:', inputA.get_shape(), ' inpB:',
inputB.get_shape())
lstm_dim = 40
dens_dim = 20
densAct = 'relu'
layerDropFrac = args.dropFrac
recDropFrac = layerDropFrac / 2.
print('Dens act=', densAct, ' recurDropFrac=', recDropFrac,
' layerDropFrac=', layerDropFrac, ' idx=', args.arrIdx)
net = LSTM(lstm_dim,
activation='tanh',
recurrent_dropout=recDropFrac,
dropout=layerDropFrac,
name='A_%d' % lstm_dim,
return_sequences=True)(inputA)
net = LSTM(lstm_dim,
activation='tanh',
recurrent_dropout=recDropFrac,
dropout=layerDropFrac,
name='B_%d' % lstm_dim)(net)
net2 = Dense(4, activation='tanh', name='G_4')(inputB)
net2 = Dense(4, activation='tanh', name='H_4')(net2)
net = concatenate([net, net2], name='seq_glob')
net = Dense(dens_dim * 2,
activation=densAct,
name='C_%d' % (dens_dim * 2))(net)
net = Dropout(layerDropFrac, name='fr_%.1f' % layerDropFrac)(net)
net = Dense(dens_dim, activation=densAct, name='D_%d' % dens_dim)(net)
net = Dropout(layerDropFrac, name='fr_same')(net)
outputs = Dense(1, activation='sigmoid', name='score')(net)
model = Model(inputs=[inputA, inputB], outputs=outputs)
self.model = model
self.compile_model()
def residual_projectionNet(depth=3,
nb_filters=512,
input_shape=(64, 64, 1),
dropout=0):
# Lets make a really deep CNN
input_img = Input(shape=input_shape)
model = Convolution2D(nb_filters,
3,
3,
activation='relu',
border_mode='valid')(input_img)
if dropout > 0: model = Dropout(dropout)(model)
for i in range(depth - 2):
model = Convolution2D(nb_filters,
3,
3,
activation='relu',
border_mode='valid')(model)
if dropout > 0: model = Dropout(dropout)(model)
model = Convolution2D(1, 3, 3, border_mode='valid')(model)
crop_amount = int(int(input_img.get_shape()[1] - model.get_shape()[1]) / 2)
crop = Cropping2D(cropping=((crop_amount, crop_amount),
(crop_amount, crop_amount)))(input_img)
merge1 = merge([crop, model], mode='sum')
final_model = Model(input=[input_img], output=[merge1])
return final_model
def get_Inception_classifier():
inputs = Input((CLASSIFY_INPUT_WIDTH, CLASSIFY_INPUT_HEIGHT, CLASSIFY_INPUT_DEPTH, CLASSIFY_INPUT_CHANNEL))
print('inputs')
print(inputs.get_shape())
# Make inception base
x = inception_base(inputs)
for i in range(INCEPTION_BLOCKS):
x = inception_block(x, filters=INCEPTION_KEEP_FILTERS)
if (i + 1) % INCEPTION_REDUCTION_STEPS == 0 and i != INCEPTION_BLOCKS - 1:
x = reduction_block(x, filters=INCEPTION_KEEP_FILTERS // 2)
print('top')
x = GlobalMaxPooling3D()(x)
print(x.get_shape())
x = Dropout(INCEPTION_DROPOUT)(x)
x = Dense(2, activation='softmax')(x)
print(x.get_shape())
model_s = Model(inputs=inputs, outputs=x)
model = multi_gpu_model(model_s, gpus=4)
model.compile(optimizer=Adam(lr=TRAIN_CLASSIFY_LEARNING_RATE), loss='binary_crossentropy', metrics=['accuracy'])
return model,model_s
def dense(input_shape, embed_model, class_length=20):
"""
"""
print("input_shape = ", input_shape, " with type = ", type(input_shape))
input1 = Input(shape=[input_shape[1]])
print("input1.shape = ", input1.get_shape().as_list())
emb = embed_model(input1)
print("\nemb shape = ", emb.get_shape().as_list(), "\n")
for i in range(FLAGS.hidden_layers):
emb = TimeDistributed(Dense(FLAGS.hidden_nodes, activation='elu'))(emb)
#emb = Dense(FLAGS.hidden_nodes, activation='elu')(emb)
if FLAGS.dropout_rate != 0:
emb = Dropout(FLAGS.dropout_rate)(emb)
emb = Flatten()(emb)
predictions = Dense( #class_length,
2, activation='softmax', name="Single_Dense_Pred")(emb)
print("predictions.shape = ", predictions.get_shape().as_list())
model = Model(input1, predictions)
#opt = Adam()
opt = Adam(lr=FLAGS.learning_rate, epsilon=FLAGS.adam_epsilon)
model.compile(
optimizer=opt,
loss="binary_crossentropy",
#loss=constrained_categorical_crossentropy,
metrics=['accuracy'])
return model
def get_Inception_classifier():
inputs = Input((CLASSIFY_INPUT_WIDTH, CLASSIFY_INPUT_HEIGHT, CLASSIFY_INPUT_DEPTH, CLASSIFY_INPUT_CHANNEL))
print('inputs')
print(inputs.get_shape())
# Make inception base
x = inception_base(inputs)
for i in range(INCEPTION_BLOCKS):
x = inception_block(x, filters=INCEPTION_KEEP_FILTERS)
if (i + 1) % INCEPTION_REDUCTION_STEPS == 0 and i != INCEPTION_BLOCKS - 1:
x = reduction_block(x, filters=INCEPTION_KEEP_FILTERS // 2)
print('top')
x = GlobalMaxPooling3D()(x)
print(x.get_shape())
x = Dropout(INCEPTION_DROPOUT)(x)
x = Dense(2, activation='softmax')(x)
print(x.get_shape())
model = Model(inputs=inputs, outputs=x)
model.compile(optimizer=Adam(lr=TRAIN_CLASSIFY_LEARNING_RATE), loss='binary_crossentropy', metrics=['accuracy'])
return model
def lstm(input_shape, embed_model, class_length=20):
"""
basic LSTM model that recieves two sentences and embeds them as words and
learns their relation.
"""
print("input_shape = ", input_shape, " with type = ", type(input_shape))
input1 = Input(shape=[input_shape[1]])
print("input1.shape = ", input1.get_shape().as_list())
emb = embed_model(input1)
print("\nemb shape = ", emb.get_shape().as_list(), "\n")
#"""
if FLAGS.bidir:
emb = bilstm_stack(emb,
input_shape[-1],
input_shape[0],
identifier="1")
else:
emb = lstm_stack(emb, time_step=input_shape[1])
predictions = Dense(class_length,
activation='softmax',
name="Single_Dense")(emb)
print("predictions.shape = ", predictions.get_shape().as_list())
model = Model(input1, predictions)
#opt = Adam()
opt = Adam(lr=FLAGS.learning_rate)
model.compile(optimizer=opt,
loss="binary_crossentropy",
metrics=['accuracy', 'categorical_accuracy'])
return model
def build(self):
def tensor_product(x):
a = x[0]
b = x[1]
y = K.batch_dot(a, b, axis=1)
y = K.einsum('ijk, ikl->ijl', a, b)
return y
query = Input(name='query', shape=(self.config['text1_maxlen'], ))
print('[Input] query:\t%s' % str(query.get_shape().as_list()))
doc = Input(name='doc',
shape=(self.config['text1_maxlen'],
self.config['hist_size']))
print('[Input] doc:\t%s' % str(doc.get_shape().as_list()))
embedding = Embedding(self.config['vocab_size'],
self.config['embed_size'],
weights=[self.config['embed']],
trainable=False)
q_embed = embedding(query)
print('[Embedding] q_embed:\t%s' % str(q_embed.get_shape().as_list()))
q_w = Dense(1,
kernel_initializer=self.initializer_gate,
use_bias=False)(q_embed)
print('[Dense] q_gate:\t%s' % str(q_w.get_shape().as_list()))
q_w = Lambda(lambda x: softmax(x, axis=1),
output_shape=(self.config['text1_maxlen'], ))(q_w)
print('[Softmax] q_gate:\t%s' % str(q_w.get_shape().as_list()))
z = doc
for i in range(self.config['num_layers']):
z = Dense(self.config['hidden_sizes'][i],
kernel_initializer=self.initializer_fc)(z)
z = Activation('tanh')(z)
print('[Dense] z (full connection):\t%s' %
str(z.get_shape().as_list()))
z = Permute((2, 1))(z)
z = Reshape((self.config['text1_maxlen'], ))(z)
print('[Reshape] z (matching) :\t%s' % str(z.get_shape().as_list()))
q_w = Reshape((self.config['text1_maxlen'], ))(q_w)
print('[Reshape] q_w (gating) :\t%s' % str(q_w.get_shape().as_list()))
out_ = Dot(axes=[1, 1])([z, q_w])
print('[Dot] out_ :\t%s' % str(out_.get_shape().as_list()))
model = Model(inputs=[query, doc], outputs=[out_])
return model
def Model_Vnet3D_keras(patch_size: int, dropout: float, nbCouche: int):
input = Input(batch_shape=(None, patch_size, patch_size, patch_size, 1))
print("input.get_shape():", input.get_shape())
output = Model_Vnet3D_2(patch_size, dropout)(input)
model = Model(inputs=input, outputs=output)
model.summary()
#plot_model(model, show_shapes=True, to_file='./out/VNetBN.png')
return model
def deep_decoder2(input_shape):
encoded = Input(shape=input_shape)
print 'encoded shape:', encoded.get_shape().as_list()
x = encoded
# x = BatchNormalization(mode=2, axis=3)(encoded)
# batch_size, h, w, _ = tf.shape(x)
batch_size = tf.shape(x)[0]
# dim: (1, 1, 512)
x = Deconv2D(512,
4,
4,
output_shape=[batch_size, 4, 4, 512],
activation='relu',
border_mode='same',
subsample=(4, 4))(encoded)
x = BatchNormalization(mode=2, axis=3)(x)
# (4, 4, 512)
x = Deconv2D(256,
5,
5,
output_shape=[batch_size, 8, 8, 256],
activation='relu',
border_mode='same',
subsample=(2, 2))(x)
x = BatchNormalization(mode=2, axis=3)(x)
# dim: (8, 8, 236)
# h *= 2; w *= 2
x = Deconv2D(128,
5,
5,
output_shape=(batch_size, 16, 16, 128),
activation='relu',
border_mode='same',
subsample=(2, 2))(x)
x = BatchNormalization(mode=2, axis=3)(x)
# dim: (16, 16, 256)
x = Deconv2D(64,
5,
5,
output_shape=(batch_size, 32, 32, 64),
activation='relu',
border_mode='same',
subsample=(2, 2))(x)
x = BatchNormalization(mode=2, axis=3)(x)
# dim: (32, 32, 64)
x = Deconv2D(3,
5,
5,
output_shape=(batch_size, 32, 32, 3),
activation='linear',
border_mode='same',
subsample=(1, 1))(x)
decoded = BatchNormalization(mode=2, axis=3)(x)
return Model(encoded, decoded)
def conv(input_shape, embed_model, class_length=20):
"""
"""
print("input_shape = ", input_shape, " with type = ", type(input_shape))
input1 = Input(shape=[input_shape[1]])
print("input1.shape = ", input1.get_shape().as_list())
emb = embed_model(input1)
print("\nemb shape = ", emb.get_shape().as_list(), "\n")
conv = UpSampling1D(100)(emb)
#conv = UpSampling2D((100,0))(emb)
print("\nUpSampled shape = ", conv.get_shape().as_list(), "\n")
conv = Conv1D(150, 25, activation="elu")(conv)
conv = MaxPooling1D(2)(conv)
conv = Conv1D(50, 5, activation="elu")(conv)
conv = MaxPooling1D(2)(conv)
for i in range(FLAGS.hidden_layers):
conv = TimeDistributed(Dense(FLAGS.hidden_nodes,
activation='elu'))(conv)
if FLAGS.dropout_rate != 0:
conv = Dropout(FLAGS.dropout_rate)(conv)
conv = Flatten()(conv)
predictions = Dense(class_length,
activation='sigmoid',
name="Single_Dense")(conv)
print("predictions.shape = ", predictions.get_shape().as_list())
model = Model(input1, predictions)
#opt = RMSprop(lr=FLAGS.learning_rate)
#opt = Adam()
opt = Adam(lr=FLAGS.learning_rate)
model.compile(optimizer=opt,
loss="binary_crossentropy",
metrics=['accuracy'])
return model
def FCN(input_shape=(500, 500, 1), n_classes=124):
"""
Return an FCN architecture used for DeepScores semantic segmentation
"""
input_tensor = Input(shape=input_shape)
# Encoder
conv2 = Conv2D(filters=64,
kernel_size=(3, 3),
padding='same',
activation='relu',
name='conv2')(input_tensor)
pool2 = MaxPooling2D(pool_size=(2, 2), padding='same', name='pool2')(conv2)
dropout2 = Dropout(rate=0.15, name='dropout2')(pool2)
conv3 = Conv2D(filters=128,
kernel_size=(3, 3),
padding='same',
activation='relu',
name='conv3')(dropout2)
pool3 = MaxPooling2D(pool_size=(2, 2), padding='same', name='pool3')(conv3)
dropout3 = Dropout(rate=0.15, name='dropout3')(pool3)
conv4 = Conv2D(filters=256,
kernel_size=(3, 3),
padding='same',
activation='relu',
name='conv4')(dropout3)
pool4 = MaxPooling2D(pool_size=(2, 2), padding='same', name='pool4')(conv4)
dropout4 = Dropout(rate=0.15, name='dropout4')(pool4)
conv5 = Conv2D(filters=512,
kernel_size=(3, 3),
padding='same',
activation='relu',
name='conv5')(dropout4)
pool5 = MaxPooling2D(pool_size=(2, 2), padding='same', name='pool5')(conv5)
dropout5 = Dropout(rate=0.15, name='dropout5')(pool5)
conv6 = Conv2D(filters=512,
kernel_size=(3, 3),
padding='same',
activation='relu',
name='conv6')(dropout5)
pool6 = MaxPooling2D(pool_size=(2, 2), padding='same', name='pool6')(conv6)
dropout6 = Dropout(rate=0.15, name='dropout6')(pool6)
conv7 = Conv2D(filters=4096,
kernel_size=(3, 3),
padding='same',
name='conv7')(dropout6)
# Decoder
conv_t1 = Conv2DTranspose(filters=512,
kernel_size=(4, 4),
strides=(2, 2),
padding='same',
name='conv_t1')(conv7)
conv_t1_up = BilinearUpSampling2D(
target_size=tuple(pool5.get_shape().as_list()[1:3]))(conv_t1)
stacked_1 = concatenate(inputs=[conv_t1_up, pool5],
axis=-1,
name='stacked_1')
fuse_1_1 = Conv2D(filters=512,
kernel_size=(1, 1),
activation='relu',
padding='same',
name='fuse_1_1')(stacked_1)
fuse_1_2 = Conv2D(filters=512,
kernel_size=(1, 1),
activation='relu',
padding='same',
name='fuse_1_2')(fuse_1_1)
conv_t2 = Conv2DTranspose(filters=256,
kernel_size=(4, 4),
strides=(2, 2),
padding='same',
name='conv_t2')(fuse_1_2)
conv_t2_up = BilinearUpSampling2D(
target_size=tuple(pool4.get_shape().as_list()[1:3]))(conv_t2)
stacked_2 = concatenate(inputs=[conv_t2_up, pool4],
axis=-1,
name='stacked_2')
fuse_2_1 = Conv2D(filters=256,
kernel_size=(1, 1),
activation='relu',
padding='same',
name='fuse_2_1')(stacked_2)
fuse_2_2 = Conv2D(filters=256,
kernel_size=(1, 1),
activation='relu',
padding='same',
name='fuse_2_2')(fuse_2_1)
conv_t3 = Conv2DTranspose(filters=128,
kernel_size=(4, 4),
strides=(2, 2),
padding='same',
name='conv_t3')(fuse_2_2)
conv_t3_up = BilinearUpSampling2D(
target_size=tuple(pool3.get_shape().as_list()[1:3]))(conv_t3)
stacked_3 = concatenate(inputs=[conv_t3_up, pool3],
axis=-1,
name='stacked_3')
fuse_3_1 = Conv2D(filters=128,
kernel_size=(1, 1),
activation='relu',
padding='same',
name='fuse_3_1')(stacked_3)
fuse_3_2 = Conv2D(filters=128,
kernel_size=(1, 1),
activation='relu',
padding='same',
name='fuse_3_2')(fuse_3_1)
conv_t4 = Conv2DTranspose(filters=64,
kernel_size=(4, 4),
strides=(2, 2),
padding='same',
name='conv_t4')(fuse_3_2)
conv_t4_up = BilinearUpSampling2D(
target_size=tuple(pool2.get_shape().as_list()[1:3]))(conv_t4)
stacked_4 = concatenate(inputs=[conv_t4_up, pool2],
axis=-1,
name='stacked_4')
fuse_4_1 = Conv2D(filters=64,
kernel_size=(1, 1),
activation='relu',
padding='same',
name='fuse_4_1')(stacked_4)
fuse_4_2 = Conv2D(filters=64,
kernel_size=(1, 1),
activation='relu',
padding='same',
name='fuse_4_2')(fuse_4_1)
# Final upscaling
deconv_final = Conv2DTranspose(filters=n_classes,
kernel_size=(16, 16),
strides=(2, 2),
padding='same',
name='deconv_final')(fuse_4_2)
output = BilinearUpSampling2D(target_size=tuple(
input_tensor.get_shape().as_list()[1:3]),
name='output')(deconv_final)
return Model(inputs=[input_tensor], outputs=[output])
question_encoder.add(Dropout(0.3))
# output: (samples, query_maxlen, embedding_dim)
# encode input sequence and questions (which are indices)
# to sequences of dense vectors
input_encoded_m = input_encoder_m(
input_sequence) # Meaning that "inputs = input_sequence"
input_encoded_c = input_encoder_c(input_sequence)
question_encoded = question_encoder(question)
num_hops = 3
for i in range(num_hops):
print(i)
#DEBUG
print("input_sequence", input_sequence.get_shape())
print("question", question.get_shape())
print("input_encoded_m", input_encoded_m.get_shape())
print("input_encoded_c", input_encoded_c.get_shape())
print("question_encoded", question_encoded.get_shape())
# compute a 'match' between the first input vector sequence
# and the question vector sequence
# shape: `(samples, story_maxlen, query_maxlen)`
match = dot([input_encoded_m, question_encoded], axes=2)
print("match", match.get_shape())
match = Activation('softmax')(match)
# add the match matrix with the second input vector sequence
response = dot([match, input_encoded_c], axes=1)
# shape: (samples, story_maxlen, query_maxlen)
def get_mitosegnet(self, wmap, lr):
inputs = Input(shape=(self.img_rows, self.img_cols, 1))
print(inputs.get_shape(), type(inputs))
# core mitosegnet (modified u-net) architecture
# batchnorm architecture (batchnorm before activation)
######################################################################
conv1 = Conv2D(64, 3, padding='same',
kernel_initializer=gauss())(inputs)
print("conv1 shape:", conv1.shape)
batch1 = BatchNormalization()(conv1)
act1 = Activation("relu")(batch1)
conv1 = Conv2D(64,
3,
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 / (9 * 64))))(
act1) # conv1
print("conv1 shape:", conv1.shape)
batch1 = BatchNormalization()(conv1)
act1 = Activation("relu")(batch1)
pool1 = MaxPooling2D(pool_size=(2, 2))(act1)
print("pool1 shape:", pool1.shape)
########
########
conv2 = Conv2D(128,
3,
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 /
(9 * 64))))(pool1)
print("conv2 shape:", conv2.shape)
batch2 = BatchNormalization()(conv2)
act2 = Activation("relu")(batch2)
conv2 = Conv2D(128,
3,
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 / (9 * 128))))(
act2) # conv2
print("conv2 shape:", conv2.shape)
batch2 = BatchNormalization()(conv2)
act2 = Activation("relu")(batch2)
pool2 = MaxPooling2D(pool_size=(2, 2))(act2)
print("pool2 shape:", pool2.shape)
########
########
conv3 = Conv2D(256,
3,
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 /
(9 * 128))))(pool2)
print("conv3 shape:", conv3.shape)
batch3 = BatchNormalization()(conv3)
act3 = Activation("relu")(batch3)
conv3 = Conv2D(256,
3,
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 / (9 * 256))))(
act3) # conv3
print("conv3 shape:", conv3.shape)
batch3 = BatchNormalization()(conv3)
act3 = Activation("relu")(batch3)
pool3 = MaxPooling2D(pool_size=(2, 2))(act3)
print("pool3 shape:", pool3.shape)
########
########
conv4 = Conv2D(512,
3,
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 /
(9 * 256))))(pool3)
batch4 = BatchNormalization()(conv4)
act4 = Activation("relu")(batch4)
conv4 = Conv2D(512,
3,
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 / (9 * 512))))(
act4) # conv4
batch4 = BatchNormalization()(conv4)
act4 = Activation("relu")(batch4)
pool4 = MaxPooling2D(pool_size=(2, 2))(act4)
########
########
conv5 = Conv2D(1024,
3,
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 /
(9 * 512))))(pool4)
batch5 = BatchNormalization()(conv5)
act5 = Activation("relu")(batch5)
conv5 = Conv2D(1024,
3,
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 / (9 * 1024))))(
act5) # conv5
batch5 = BatchNormalization()(conv5)
act5 = Activation("relu")(batch5)
########
up6 = Conv2D(512,
2,
activation='relu',
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 / (9 * 1024))))(
UpSampling2D(size=(2, 2))(act5))
merge6 = concatenate([conv4, up6], axis=3)
conv6 = Conv2D(
512,
3,
activation='relu',
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 / (9 * 512))))(merge6)
conv6 = Conv2D(512,
3,
activation='relu',
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 /
(9 * 512))))(conv6)
up7 = Conv2D(256,
2,
activation='relu',
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 / (9 * 512))))(
UpSampling2D(size=(2, 2))(conv6))
merge7 = concatenate([conv3, up7], axis=3)
conv7 = Conv2D(
256,
3,
activation='relu',
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 / (9 * 256))))(merge7)
conv7 = Conv2D(256,
3,
activation='relu',
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 /
(9 * 256))))(conv7)
up8 = Conv2D(128,
2,
activation='relu',
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 / (9 * 256))))(
UpSampling2D(size=(2, 2))(conv7))
merge8 = concatenate([conv2, up8], axis=3)
conv8 = Conv2D(
128,
3,
activation='relu',
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 / (9 * 128))))(merge8)
conv8 = Conv2D(128,
3,
activation='relu',
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 /
(9 * 128))))(conv8)
up9 = Conv2D(64,
2,
activation='relu',
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 / (9 * 128))))(
UpSampling2D(size=(2, 2))(conv8))
merge9 = concatenate([conv1, up9], axis=3)
conv9 = Conv2D(64,
3,
activation='relu',
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 /
(9 * 64))))(merge9)
conv9 = Conv2D(64,
3,
activation='relu',
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 /
(9 * 64))))(conv9)
conv9 = Conv2D(2,
3,
activation='relu',
padding='same',
kernel_initializer=gauss(stddev=sqrt(2 /
(9 * 64))))(conv9)
######################################################################
conv10 = Conv2D(1,
1,
activation='sigmoid',
kernel_initializer=gauss(stddev=sqrt(2 /
(9 * 2))))(conv9)
if wmap == False:
input = inputs
loss = self.pixelwise_crossentropy()
else:
weights = Input(shape=(self.img_rows, self.img_cols, 1))
input = [inputs, weights]
loss = self.weighted_pixelwise_crossentropy(input[1])
model = Model(inputs=input, outputs=conv10)
model.compile(optimizer=Adam(lr=lr),
loss=loss,
metrics=['accuracy', self.dice_coefficient])
return model
def build(self):
def mlp_work(input_dim):
seq = Sequential()
num_hidden_layers = len(self.config['hidden_sizes'])
assert num_hidden_layers > 0
if num_hidden_layers == 1:
seq.add(
Dense(self.config['hidden_sizes'][0],
input_shape=(input_dim, )))
else:
seq.add(
Dense(self.config['hidden_sizes'][0],
activation='relu',
input_shape=(input_dim, )))
for i in range(num_hidden_layers - 2):
seq.add(
Dense(self.config['hidden_sizes'][i + 1],
activation='relu'))
seq.add(
Dense(self.config['hidden_sizes'][num_hidden_layers - 1]))
return seq
query = Input(name='query', shape=(self.config['text1_maxlen'], ))
print('[Input] query:\t%s' % str(query.get_shape().as_list()))
doc = Input(name='doc', shape=(self.config['text2_maxlen'], ))
print('[Input] doc:\t%s' % str(doc.get_shape().as_list()))
wordhashing = Embedding(self.config['vocab_size'],
self.config['embed_size'],
weights=[self.config['embed']],
trainable=self.embed_trainable)
q_embed = wordhashing(query)
print('[Embedding] query wordhash:\t%s' %
str(q_embed.get_shape().as_list()))
d_embed = wordhashing(doc)
print('[Embedding] doc wordhash:\t%s' %
str(d_embed.get_shape().as_list()))
conv1d = Convolution1D(self.config['filters'],
self.config['kernel_size'],
padding='same',
activation='relu')
q_conv = conv1d(q_embed)
print('[Conv1D] query_conv1:\t%s' % str(q_conv.get_shape().as_list()))
d_conv = conv1d(d_embed)
print('[Conv1D] doc_conv1:\t%s' % str(d_conv.get_shape().as_list()))
q_pool = MaxPooling1D(self.config['text1_maxlen'])(q_conv)
q_pool_re = Reshape((-1, ))(q_pool)
print('[MaxPooling1D] query_pool1:\t%s' %
str(q_pool.get_shape().as_list()))
d_pool = MaxPooling1D(self.config['text2_maxlen'])(d_conv)
d_pool_re = Reshape((-1, ))(d_pool)
print('[MaxPooling1D] doc_pool1:\t%s' %
str(d_pool.get_shape().as_list()))
mlp = mlp_work(self.config['embed_size'])
rq = mlp(q_pool_re)
rd = mlp(d_pool_re)
#out_ = Merge([rq, rd], mode='cos', dot_axis=1)
out_ = Dot(axes=[1, 1], normalize=True)([rq, rd])
model = Model(inputs=[query, doc], outputs=[out_])
return model
a = le3.transform([j])
b = le3.transform([k])
init_m_3[a,b] += 1
nmf = NMF(n_components=gru_output_size+embedding_dims)
init_m_3 = np.log2(init_m_3 + 1)
nmf.fit(init_m_3)
init_m_3 = nmf.components_
#%%
print('Build model...')
# Inputs
review_input = Input(shape=(maxsents,maxlen), dtype='int32')
print('review_input SHAPE')
print(review_input.get_shape())
# Embedding Layer
embedding_layer = Embedding(max_features, embedding_dims,
input_length=maxlen)
# WORD-LEVEL
sentence_input = Input(shape=(maxlen,), dtype='int32')
embedded_sequences = embedding_layer(sentence_input)
print('embedded_sequences SHAPE')
print(embedded_sequences.get_shape())
# Bidirectional GRU
l_gru = MultiplicativeLSTM(gru_output_size, return_sequences=True)(embedded_sequences)
l_dense = TimeDistributed(Dense(units=gru_output_size))(l_gru)
def CombCN(input_frame,
input_video,
video_size=None,
sampling_frame=8,
frame_net_mid_depth=4):
#frame_3DCN => total frames or jsut frame of Vk in Vin
Activ = lambda x: LeakyReLU(alpha=0.2)(x)
Bat = lambda x: BatchNormalization()(x)
video_size = None
if not video_size:
W = 320
H = 240
else:
W = video_size[0]
H = video_size[1]
if input_frame is None:
input_frame = Input(shape=(W, H, 3))
if input_video is None:
input_video = Input(shape=(sampling_frame, W / 2, H / 2, 3))
print(input_frame.get_shape())
e0 = Conv2D(filters=32, padding='same', kernel_size=5)(input_frame)
#e0 = Bat(e0)
e0 = Activ(e0)
print(e0.get_shape())
#WITH NO CONCATENATE Init_dataloader()DING IN 3DCN BUT WITH CONCAT FOR ENCODING IN combination part
e0_C = Conv2D(filters=64, padding='same', kernel_size=3, strides=2)(e0)
e0_C = Bat(e0_C)
e0_C = Activ(e0_C)
print(e0_C.get_shape())
#skip_subnet = CN3D(video_info=None , input_video = input_video, sampling_frame= 8, vid_net_mid_depth = 3)
skip_subnet = input_video
size_subnet = skip_subnet.get_shape()
# IS IT WHAT THE PAPER SAYS? NOT SURE
skip_subnet = Reshape((int(W / 2), int(H / 2),
int(size_subnet[1] * size_subnet[4])))(skip_subnet)
skip_subnet = Conv2D(filters=128,
padding='same',
kernel_size=5,
strides=1,
name='subnet')(skip_subnet)
skip_subnet = Bat(skip_subnet)
skip_subnet = Activ(skip_subnet)
skip_subnet = Conv2D(filters=64,
padding='same',
kernel_size=3,
strides=1,
name='subnet_2')(skip_subnet)
skip_subnet = Bat(skip_subnet)
skip_subnet = Activ(skip_subnet)
e0_C = Concatenate()([e0_C, skip_subnet])
print(e0_C.get_shape())
e1 = Conv2D(filters=256, padding='same', kernel_size=3)(e0_C)
e1 = Bat(e1)
e1 = Activ(e1)
print(e1.get_shape())
e1_C = Conv2D(filters=512, padding='same', kernel_size=4, strides=2)(e1)
e1_C = Bat(e1_C)
e1_C = Activ(e1_C)
print(e1_C.get_shape())
e2 = Conv2D(filters=512, padding='same', kernel_size=3)(e1_C)
e2 = Bat(e2)
e2 = Activ(e2)
print(e2.get_shape())
e2_C = Conv2D(filters=512, padding='same', kernel_size=4, strides=2)(e2)
e2_C = Bat(e2_C)
e2_C = Activ(e2_C)
print(e2_C.get_shape())
fc_mid = e2_C
fc_prev = e2_C
p_num = 2
for i in range(frame_net_mid_depth):
fc_mid = Conv2D(filters=512,
dilation_rate=p_num,
kernel_size=3,
padding='same')(fc_mid)
fc_mid = Bat(fc_mid)
fc_mid = Activ(fc_mid)
f_temp = fc_mid
fc_mid = Concatenate()([fc_mid, fc_prev])
fc_prev = f_temp
print(fc_mid.get_shape())
p_num = p_num * 2
#fc_mid = Deconvolution3D(strides=2, filters=64, kernel_size= 4, padding='same')(fc_mid)
d0 = Concatenate()([fc_prev, e2_C])
d0 = Conv2D(strides=1, filters=512, kernel_size=3, padding='same')(d0)
d0 = Bat(d0)
d0 = Activ(d0)
print(d0.get_shape())
d0_C = UpSampling2D()(d0)
d0_C = Concatenate()([d0_C, e2])
d0_C = Conv2D(strides=1, filters=512, kernel_size=4, padding='same')(d0_C)
d0_C = Bat(d0_C)
d0_C = Activ(d0_C)
print(d0_C.get_shape())
d0_CC = Concatenate()([d0_C, e1_C])
d0_CC = Conv2D(filters=512, padding='same', kernel_size=3)(d0_CC)
d0_CC = Bat(d0_CC)
d0_CC = Activ(d0_CC)
print(d0_CC.get_shape())
d1 = UpSampling2D()(d0_CC)
d1 = Concatenate()([d1, e1])
d1 = Conv2D(strides=1, filters=256, kernel_size=4, padding='same')(d1)
d1 = Bat(d1)
d1 = Activ(d1)
print(d1.get_shape())
d1_C = Concatenate()([d1, e0_C])
d1_C = Conv2D(filters=128, padding='same', kernel_size=3)(d1_C)
d1_C = Bat(d1_C)
d1_C = Activ(d1_C)
print(d1_C.get_shape())
d1_CC = UpSampling2D()(d1_C)
d1_CC = Conv2D(strides=1, filters=64, kernel_size=4, padding='same')(d1_CC)
d1_CC = Bat(d1_CC)
d1_CC = Activ(d1_CC)
print(d1_CC.get_shape())
d1_CCC = Concatenate()([d1_CC, e0])
d1_CCC = Conv2D(strides=1, filters=48, kernel_size=4,
padding='same')(d1_CCC)
d1_CCC = Bat(d1_CCC)
d1_CCC = Activ(d1_CCC)
print(d1_CCC.get_shape())
d_out = Conv2D(filters=3, padding='same', kernel_size=3)(d1_CCC)
#d_out = Bat(d_out)
d_out = Activation('tanh')(d_out)
print(d_out.get_shape())
return d_out
clstm_embed = Dropout(0.1)(clstm_embed)
#clstm_embed = MaxPooling1D(pool_length=pool_length)(clstm_embed)
print('clstm_embed max pooled', '-', clstm_embed.get_shape())
c_lstm_1 = LSTM(128, unroll=True, dropout_W=0.2, dropout_U=0.2 )(clstm_embed)
c_lstm_2 = LSTM(128, unroll=True, dropout_W=0.2, dropout_U=0.2, go_backwards=True)(clstm_embed)
clstm_out = merge([c_lstm_1, c_lstm_2], mode='concat')
c_lstm_model = Model(input=clstm_in, output=clstm_out)
chars_in = Input(shape=(input_len, char_encoder.max_wlen), name='chars_in')
c_lstm_out = TimeDistributed(c_lstm_model)(chars_in)
concatenated_input = merge([word_vecs, grammar_features_in, c_lstm_out], mode='concat')
print('word_vecs:', word_vecs.get_shape())
print('grammar_features_in:', grammar_features_in.get_shape())
print('c_lstm_out:', c_lstm_out.get_shape())
print('concatenated_input:', concatenated_input.get_shape())
lstm = LSTM(128, unroll=True, dropout_W=0.2, dropout_U=0.2, return_sequences=True)(concatenated_input)
lstm = LSTM(128, unroll=True, dropout_W=0.2, dropout_U=0.2)(lstm)
output = Dense(output_vocab_size)(lstm)
output = Activation('softmax')(output)
model = Model(input=[word_idx_in, grammar_features_in, chars_in], output=output)
optimizer = Adam(lr=0.001)
model.compile(loss='categorical_crossentropy', optimizer=optimizer)
#model = load_model('all-iter-saved.h5')
def Deeplabv3(weights='pascal_voc',
input_tensor=None,
input_shape=(512, 512, 3),
classes=21,
backbone='mobilenetv2',
OS=16,
alpha=1.):
""" Instantiates the Deeplabv3+ architecture
Optionally loads weights pre-trained
on PASCAL VOC. This model is available for TensorFlow only,
and can only be used with inputs following the TensorFlow
data format `(width, height, channels)`.
# Arguments
weights: one of 'pascal_voc' (pre-trained on pascal voc)
or None (random initialization)
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: shape of input image. format HxWxC
PASCAL VOC model was trained on (512,512,3) images
classes: number of desired classes. If classes != 21,
last layer is initialized randomly
backbone: backbone to use. one of {'xception','mobilenetv2'}
OS: determines input_shape/feature_extractor_output ratio. One of {8,16}.
Used only for xception backbone.
alpha: controls the width of the MobileNetV2 network. This is known as the
width multiplier in the MobileNetV2 paper.
- If `alpha` < 1.0, proportionally decreases the number
of filters in each layer.
- If `alpha` > 1.0, proportionally increases the number
of filters in each layer.
- If `alpha` = 1, default number of filters from the paper
are used at each layer.
Used only for mobilenetv2 backbone
# Returns
A Keras model instance.
# Raises
RuntimeError: If attempting to run this model with a
backend that does not support separable convolutions.
ValueError: in case of invalid argument for `weights` or `backbone`
"""
if not (weights in {'pascal_voc', 'cityscapes', None}):
raise ValueError(
'The `weights` argument should be either '
'`None` (random initialization), `pascal_voc`, or `cityscapes` '
'(pre-trained on PASCAL VOC)')
if K.backend() != 'tensorflow':
raise RuntimeError('The Deeplabv3+ model is only available with '
'the TensorFlow backend.')
if not (backbone in {'xception', 'mobilenetv2'}):
raise ValueError('The `backbone` argument should be either '
'`xception` or `mobilenetv2` ')
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if backbone == 'xception':
if OS == 8:
entry_block3_stride = 1
middle_block_rate = 2 # ! Not mentioned in paper, but required
exit_block_rates = (2, 4)
atrous_rates = (12, 24, 36)
else:
entry_block3_stride = 2
middle_block_rate = 1
exit_block_rates = (1, 2)
atrous_rates = (6, 12, 18)
x = Conv2D(32, (3, 3),
strides=(2, 2),
name='entry_flow_conv1_1',
use_bias=False,
padding='same')(img_input)
x = BatchNormalization(name='entry_flow_conv1_1_BN')(x)
x = Activation('relu')(x)
x = _conv2d_same(x, 64, 'entry_flow_conv1_2', kernel_size=3, stride=1)
x = BatchNormalization(name='entry_flow_conv1_2_BN')(x)
x = Activation('relu')(x)
x = _xception_block(x, [128, 128, 128],
'entry_flow_block1',
skip_connection_type='conv',
stride=2,
depth_activation=False)
x, skip1 = _xception_block(x, [256, 256, 256],
'entry_flow_block2',
skip_connection_type='conv',
stride=2,
depth_activation=False,
return_skip=True)
x = _xception_block(x, [728, 728, 728],
'entry_flow_block3',
skip_connection_type='conv',
stride=entry_block3_stride,
depth_activation=False)
for i in range(16):
x = _xception_block(x, [728, 728, 728],
'middle_flow_unit_{}'.format(i + 1),
skip_connection_type='sum',
stride=1,
rate=middle_block_rate,
depth_activation=False)
x = _xception_block(x, [728, 1024, 1024],
'exit_flow_block1',
skip_connection_type='conv',
stride=1,
rate=exit_block_rates[0],
depth_activation=False)
x = _xception_block(x, [1536, 1536, 2048],
'exit_flow_block2',
skip_connection_type='none',
stride=1,
rate=exit_block_rates[1],
depth_activation=True)
else:
OS = 8
first_block_filters = _make_divisible(32 * alpha, 8)
x = Conv2D(first_block_filters,
kernel_size=3,
strides=(2, 2),
padding='same',
use_bias=False,
name='Conv')(img_input)
x = BatchNormalization(epsilon=1e-3, momentum=0.999, name='Conv_BN')(x)
x = Activation(relu6, name='Conv_Relu6')(x)
x = _inverted_res_block(x,
filters=16,
alpha=alpha,
stride=1,
expansion=1,
block_id=0,
skip_connection=False)
x = _inverted_res_block(x,
filters=24,
alpha=alpha,
stride=2,
expansion=6,
block_id=1,
skip_connection=False)
x = _inverted_res_block(x,
filters=24,
alpha=alpha,
stride=1,
expansion=6,
block_id=2,
skip_connection=True)
x = _inverted_res_block(x,
filters=32,
alpha=alpha,
stride=2,
expansion=6,
block_id=3,
skip_connection=False)
x = _inverted_res_block(x,
filters=32,
alpha=alpha,
stride=1,
expansion=6,
block_id=4,
skip_connection=True)
x = _inverted_res_block(x,
filters=32,
alpha=alpha,
stride=1,
expansion=6,
block_id=5,
skip_connection=True)
# stride in block 6 changed from 2 -> 1, so we need to use rate = 2
x = _inverted_res_block(
x,
filters=64,
alpha=alpha,
stride=1, # 1!
expansion=6,
block_id=6,
skip_connection=False)
x = _inverted_res_block(x,
filters=64,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=7,
skip_connection=True)
x = _inverted_res_block(x,
filters=64,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=8,
skip_connection=True)
x = _inverted_res_block(x,
filters=64,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=9,
skip_connection=True)
x = _inverted_res_block(x,
filters=96,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=10,
skip_connection=False)
x = _inverted_res_block(x,
filters=96,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=11,
skip_connection=True)
x = _inverted_res_block(x,
filters=96,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=12,
skip_connection=True)
x = _inverted_res_block(
x,
filters=160,
alpha=alpha,
stride=1,
rate=2, # 1!
expansion=6,
block_id=13,
skip_connection=False)
x = _inverted_res_block(x,
filters=160,
alpha=alpha,
stride=1,
rate=4,
expansion=6,
block_id=14,
skip_connection=True)
x = _inverted_res_block(x,
filters=160,
alpha=alpha,
stride=1,
rate=4,
expansion=6,
block_id=15,
skip_connection=True)
x = _inverted_res_block(x,
filters=320,
alpha=alpha,
stride=1,
rate=4,
expansion=6,
block_id=16,
skip_connection=False)
# end of feature extractor
# branching for Atrous Spatial Pyramid Pooling
# Image Feature branch
# out_shape = int(np.ceil(input_shape[0] / OS))
# print('*****', img_input.get_shape().as_list())
b4 = AveragePooling2D(
pool_size=(int(np.ceil(img_input.get_shape().as_list()[1] / OS)),
int(np.ceil(img_input.get_shape().as_list()[2] / OS))))(x)
b4 = Conv2D(256, (1, 1),
padding='same',
use_bias=False,
name='image_pooling')(b4)
b4 = BatchNormalization(name='image_pooling_BN', epsilon=1e-5)(b4)
b4 = Activation('relu')(b4)
b4 = BilinearUpsampling((int(np.ceil(input_shape[0] / OS)),
int(np.ceil(input_shape[1] / OS))))(b4)
# simple 1x1
b0 = Conv2D(256, (1, 1), padding='same', use_bias=False, name='aspp0')(x)
b0 = BatchNormalization(name='aspp0_BN', epsilon=1e-5)(b0)
b0 = Activation('relu', name='aspp0_activation')(b0)
# there are only 2 branches in mobilenetV2. not sure why
if backbone == 'xception':
# rate = 6 (12)
b1 = SepConv_BN(x,
256,
'aspp1',
rate=atrous_rates[0],
depth_activation=True,
epsilon=1e-5)
# rate = 12 (24)
b2 = SepConv_BN(x,
256,
'aspp2',
rate=atrous_rates[1],
depth_activation=True,
epsilon=1e-5)
# rate = 18 (36)
b3 = SepConv_BN(x,
256,
'aspp3',
rate=atrous_rates[2],
depth_activation=True,
epsilon=1e-5)
# concatenate ASPP branches & project
x = Concatenate()([b4, b0, b1, b2, b3])
else:
x = Concatenate()([b4, b0])
x = Conv2D(256, (1, 1),
padding='same',
use_bias=False,
name='concat_projection')(x)
x = BatchNormalization(name='concat_projection_BN', epsilon=1e-5)(x)
x = Activation('relu')(x)
x = Dropout(0.1)(x)
# DeepLab v.3+ decoder
if backbone == 'xception':
# Feature projection
# x4 (x2) block
x = BilinearUpsampling(output_size=(int(np.ceil(input_shape[0] / 4)),
int(np.ceil(input_shape[1] /
4))))(x)
dec_skip1 = Conv2D(48, (1, 1),
padding='same',
use_bias=False,
name='feature_projection0')(skip1)
dec_skip1 = BatchNormalization(name='feature_projection0_BN',
epsilon=1e-5)(dec_skip1)
dec_skip1 = Activation('relu')(dec_skip1)
x = Concatenate()([x, dec_skip1])
x = SepConv_BN(x,
256,
'decoder_conv0',
depth_activation=True,
epsilon=1e-5)
x = SepConv_BN(x,
256,
'decoder_conv1',
depth_activation=True,
epsilon=1e-5)
# you can use it with arbitary number of classes
if classes == 21:
last_layer_name = 'logits_semantic'
else:
last_layer_name = 'custom_logits_semantic'
x = Conv2D(classes, (1, 1), padding='same', name=last_layer_name)(x)
x = BilinearUpsampling(output_size=(input_shape[0], input_shape[1]))(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
model = Model(inputs, x, name='deeplabv3plus')
# load weights
if weights == 'pascal_voc':
if backbone == 'xception':
weights_path = get_file(
'deeplabv3_xception_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH_X,
cache_subdir='models')
else:
weights_path = get_file(
'deeplabv3_mobilenetv2_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH_MOBILE,
cache_subdir='models')
model.load_weights(weights_path, by_name=True)
elif weights == 'cityscapes':
if backbone == 'xception':
weights_path = get_file(
'deeplabv3_xception_tf_dim_ordering_tf_kernels_cityscapes.h5',
WEIGHTS_PATH_X_CS,
cache_subdir='models')
else:
weights_path = get_file(
'deeplabv3_mobilenetv2_tf_dim_ordering_tf_kernels_cityscapes.h5',
WEIGHTS_PATH_MOBILE_CS,
cache_subdir='models')
model.load_weights(weights_path, by_name=True)
return model
def _build_net(self):
with tf.name_scope('inputs'):
self.tf_account = tf.placeholder(tf.float32, [None, 3],
name="account")
#self.tf_obs = tf.placeholder(tf.float32, [None, self.n_features], name="observations")
self.tf_obs = tf.placeholder(tf.float32, [None, 2, 60, 1],
name="observations")
self.tf_acts = tf.placeholder(tf.int32, [
None,
],
name="actions_num")
self.tf_vt = tf.placeholder(tf.float32, [
None,
],
name="actions_value")
from keras.models import Model
from keras.layers import merge, Convolution2D, MaxPooling2D, Input, Dense, Flatten, Dropout, Reshape, TimeDistributed, BatchNormalization, Merge
from keras.layers.advanced_activations import LeakyReLU
# self.tf_obs = Input(shape = (3,), dtype='float32', name="observations")
B = Input(shape=(3, ), tensor=self.tf_account)
b = Dense(5, activation="relu")(B)
inputs = [B]
merges = [b]
#S = Input(shape=[2, 60, 1])
S = Input(shape=[2, 60, 1], tensor=self.tf_obs)
inputs.append(S)
print(B.get_shape(), S.get_shape())
h = Convolution2D(2048, 3, 1, border_mode='same')(S)
h = LeakyReLU(0.001)(h)
h = Convolution2D(2048, 5, 1, border_mode='same')(S)
h = LeakyReLU(0.001)(h)
h = Convolution2D(2048, 10, 1, border_mode='same')(S)
h = LeakyReLU(0.001)(h)
h = Convolution2D(2048, 20, 1, border_mode='same')(S)
h = LeakyReLU(0.001)(h)
h = Convolution2D(2048, 40, 1, border_mode='same')(S)
h = LeakyReLU(0.001)(h)
h = Flatten()(h)
h = Dense(512)(h)
h = LeakyReLU(0.001)(h)
merges.append(h)
h = Convolution2D(2048, 60, 1, border_mode='same')(S)
h = LeakyReLU(0.001)(h)
h = Flatten()(h)
h = Dense(512)(h)
h = LeakyReLU(0.001)(h)
merges.append(h)
m = merge(merges, mode='concat', concat_axis=1)
m = Dense(1024)(m)
m = LeakyReLU(0.001)(m)
m = Dense(512)(m)
m = LeakyReLU(0.001)(m)
m = Dense(256)(m)
m = LeakyReLU(0.001)(m)
self.all_act = Dense(2, activation=None)(m)
self.all_act_prob = tf.nn.softmax(
self.all_act,
name='act_prob') # use softmax to convert to probability
with tf.name_scope('loss'):
# to maximize total reward (log_p * R) is to minimize -(log_p * R), and the tf only have minimize(loss)
neg_log_prob = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=self.all_act,
labels=self.tf_acts) # this is negative log of chosen action
# or in this way:
# neg_log_prob = tf.reduce_sum(-tf.log(self.all_act_prob)*tf.one_hot(self.tf_acts, self.n_actions), axis=1)
loss = tf.reduce_mean(neg_log_prob *
self.tf_vt) # reward guided loss
with tf.name_scope('train'):
self.train_op = tf.train.AdamOptimizer(self.lr).minimize(loss)
def old_model(self, input_dim = (320, 320), input_channels=1, output_channels=4, drop_out=0.0, batch_Norm=True, USE_BIAS = False, interpolation='bilinear', fullbottle=False):
n_filters = 64
inputs = Input((input_dim[0], input_dim[1], 3))
#get VGG16
vgg16 = VGG16(input_tensor=inputs, include_top=False)
# vgg16 = VGG16( input_shape=inputShape, include_top=False)
for l in vgg16.layers:
l.trainable = True
# vgg16.summary()
out_vgg16 = vgg16(inputs)
#get vgg layer outputs
block1_conv2 = vgg16.get_layer("block1_conv2").output
block2_conv2 = vgg16.get_layer("block2_conv2").output
block3_conv3 = vgg16.get_layer("block3_conv3").output
block4_conv3 = vgg16.get_layer("block4_conv3").output
block5_conv3 = vgg16.get_layer("block5_conv3").output
out_vgg16 = vgg16.get_layer("block5_pool").output
#--mid convolutions--
convMid_1 = self.double_conv2D(n_filters*16, 3, out_vgg16 , batch_norm=batch_Norm, dropout=drop_out)
if fullbottle:
convMid_1 = self.bn_conv2D(n_filters*16, 3, convMid_1)
# ------- up path ----------
# upconv_1 = Convolution2DTranspose(n_filters*8, (2, 2), strides=(2, 2), use_bias= USE_BIAS)(convMid_1)
# upconv_1 = self.SubpixelConv2D((convMid_1.get_shape()[1], convMid_1.get_shape()[2], convMid_1.get_shape()[3]), scale=2, name='subpix1')(convMid_1)
upconv_1 = UpSampling2D((2,2), interpolation=interpolation)(convMid_1)
conca_1 = concatenate([upconv_1, block5_conv3], axis=self.axis)
conv_1 = self.double_conv2D(n_filters*8, 3, conca_1, batch_norm=batch_Norm, dropout=drop_out)
conv_1 = self.bn_conv2D(n_filters*8, 3, conv_1)
# upconv_2 = Convolution2DTranspose(n_filters*8, (2, 2), strides=(2, 2), use_bias= USE_BIAS)(conv_1)
# upconv_2 = self.SubpixelConv2D((conv_1.get_shape()[1], conv_1.get_shape()[2], conv_1.get_shape()[3]), scale=2, name='subpix2')(conv_1)
upconv_2 = UpSampling2D((2,2), interpolation=interpolation)(conv_1)
conca_2 = concatenate([upconv_2, block4_conv3], axis=self.axis)
conv_2 = self.double_conv2D(n_filters*8, 3, conca_2, batch_norm=batch_Norm, dropout=drop_out)
conv_2 = self.bn_conv2D(n_filters*8, 3, conv_2)
# upconv_3 = Convolution2DTranspose(n_filters*4, (2, 2), strides=(2, 2), use_bias= USE_BIAS)(conv_2)
# upconv_3 = self.SubpixelConv2D((conv_2.get_shape()[1], conv_2.get_shape()[2], conv_2.get_shape()[3]), scale=2, name='subpix3')(conv_2)
upconv_3 = UpSampling2D((2,2), interpolation=interpolation)(conv_2)
conca_3 = concatenate([upconv_3, block3_conv3], axis=self.axis)
conv_3 = self.double_conv2D(n_filters*4, 3, conca_3, batch_norm=batch_Norm, dropout=drop_out)
conv_3 = self.bn_conv2D(n_filters*4, 3, conv_3)
# upconv_4 = Convolution2DTranspose(n_filters*2, (2, 2), strides=(2, 2), use_bias= USE_BIAS)(conv_3)
# upconv_4 = self.SubpixelConv2D((conv_3.get_shape()[1], conv_3.get_shape()[2], conv_3.get_shape()[3]), scale=2, name='subpix4')(conv_3)
upconv_4 = UpSampling2D((2,2), interpolation=interpolation)(conv_3)
conca_4 = concatenate([upconv_4, block2_conv2], axis=self.axis)
conv_4 = self.double_conv2D(n_filters*2, 3, conca_4, batch_norm=batch_Norm, dropout=drop_out)
# upconv_5 = Convolution2DTranspose(n_filters, (2, 2), strides=(2, 2), use_bias= USE_BIAS)(conv_4)
# upconv_5 = self.SubpixelConv2D((conv_4.get_shape()[1], conv_4.get_shape()[2], conv_4.get_shape()[3]), scale=2, name='subpi5')(conv_4)
upconv_5 = UpSampling2D((2,2), interpolation=interpolation)(conv_4)
conca_5 = concatenate([upconv_5, block1_conv2], axis=self.axis)
conv_5 = self.double_conv2D(n_filters, 3, conca_5, batch_norm=batch_Norm, dropout=drop_out)
in_c = Reshape((512,512,1))(Lambda(lambda x : x[:,:,:,0])(inputs))
print(in_c.get_shape())
print(inputs.get_shape())
conca_6 = concatenate([conv_5, in_c], axis=self.axis)
out = Conv2D(output_channels, (1, 1))(conca_6)
out = Activation('softmax')(out)
model = Model(input=inputs, output=out, name="unet_vgg16")
return model
def build_model():
# design the deepaclstm model
main_input = Input(shape=(700, ), dtype='float32', name='main_input')
#main_input = Masking(mask_value=23)(main_input)
x = Embedding(output_dim=21, input_dim=21, input_length=700)(main_input)
auxiliary_input = Input(shape=(700, 21), name='aux_input') #24
#auxiliary_input = Masking(mask_value=0)(auxiliary_input)
print main_input.get_shape()
print auxiliary_input.get_shape()
concat = merge([x, auxiliary_input], mode='concat', concat_axis=-1)
conv1_features = Convolution1D(42,
1,
activation='relu',
border_mode='same',
W_regularizer=l2(0.001))(concat)
# print 'conv1_features shape', conv1_features.get_shape()
conv1_features = Reshape((700, 42, 1))(conv1_features)
conv2_features = Convolution2D(42,
3,
1,
activation='relu',
border_mode='same',
W_regularizer=l2(0.001))(conv1_features)
# print 'conv2_features.get_shape()', conv2_features.get_shape()
conv2_features = Reshape((700, 42 * 42))(conv2_features)
conv2_features = Dropout(0.5)(conv2_features)
conv2_features = Dense(400, activation='relu')(conv2_features)
#, activation='tanh', inner_activation='sigmoid',dropout_W=0.5,dropout_U=0.5
lstm_f1 = LSTM(output_dim=300,
return_sequences=True,
inner_activation='sigmoid',
dropout_W=0.5,
dropout_U=0.5)(conv2_features)
lstm_b1 = LSTM(output_dim=300,
return_sequences=True,
go_backwards=True,
inner_activation='sigmoid',
dropout_W=0.5,
dropout_U=0.5)(conv2_features)
lstm_f2 = LSTM(output_dim=300,
return_sequences=True,
inner_activation='sigmoid',
dropout_W=0.5,
dropout_U=0.5)(lstm_f1)
lstm_b2 = LSTM(output_dim=300,
return_sequences=True,
go_backwards=True,
inner_activation='sigmoid',
dropout_W=0.5,
dropout_U=0.5)(lstm_b1)
concat_features = merge([lstm_f2, lstm_b2, conv2_features],
mode='concat',
concat_axis=-1)
concat_features = Dropout(0.4)(concat_features)
protein_features = Dense(600, activation='relu')(concat_features)
# protein_features = TimeDistributedDense(600,activation='relu')(concat_features)
# protein_features = TimeDistributedDense(100,activation='relu', W_regularizer=l2(0.001))(protein_features)
main_output = TimeDistributedDense(8,
activation='softmax',
name='main_output')(protein_features)
model = Model(input=[main_input, auxiliary_input], output=[main_output])
adam = Adam(lr=0.003)
model.compile(optimizer=adam,
loss={'main_output': 'categorical_crossentropy'},
metrics=['weighted_accuracy'])
model.summary()
return model
act5 = Activation('relu')(conv5)
pool3 = MaxPooling2D(pool_size=(2, 2))(act5)
conv6 = Conv2D(8, (3, 3), padding='same')(pool3)
act6 = Activation('relu')(conv6)
pool4 = MaxPooling2D(pool_size=(2, 2))(act6)
conv7 = Conv2D(16, (3, 3), padding='same')(pool4)
act7 = Activation('tanh')(conv7)
#pool5 = MaxPooling2D(pool_size=(2, 2))(act7)
#conv8 = Convolution2D(32, 3, 3,border_mode='same')(pool5)
#act8 = Activation('relu')(conv8)
print(act7.get_shape())
encoder = Model(input=[input_lr], output=[act7])
encoder.compile(optimizer=adam, loss='mse', metrics=['accuracy'])
code_lr = Input(shape=(8, 8, 16), dtype='float32', name='decode_input')
print(code_lr.get_shape())
#dgaussian = GaussianNoise(0.1)(code_lr)
#deconv0 = Convolution2D(512, 3, 3, border_mode='same')(code_lr)
#deact0 = Activation('relu')(deconv0)
#dfc1 = Dense(32*4*4, activation='relu')(code_lr)
#dfc2 = Dense(16*8*8, activation='relu')(dgaussian)
#rshp1 = Reshape((16,8,8))(dfc2)
#up0 = UpSampling2D(size=(2,2))(rshp1)
#deconv1 = Convolution2D(32, 3, 3, border_mode='same')(up0)
#deact1 = Activation('relu')(deconv1)
up1 = UpSampling2D(size=(2, 2))(code_lr)
deconv2 = Conv2D(8, (3, 3), border_mode='same')(up1)
deact2 = Activation('relu')(deconv2)
up2 = UpSampling2D(size=(2, 2))(deact2)
def _build_net(self):
with tf.name_scope('inputs'):
self.tf_account = tf.placeholder(tf.float32, [None, 3],
name="account")
#self.tf_obs = tf.placeholder(tf.float32, [None, self.n_features], name="observations")
self.tf_obs = tf.placeholder(tf.float32, [None, 2, 60, 1],
name="observations")
self.tf_acts = tf.placeholder(tf.int32, [
None,
],
name="actions_num")
self.tf_vt = tf.placeholder(tf.float32, [
None,
],
name="actions_value")
self.a = tf.placeholder(tf.int32, None, name="act")
self.td_error = tf.placeholder(tf.float32, None, name="td_error")
from keras.models import Model
from keras.layers import merge, Convolution2D, MaxPooling2D, Input, Dense, Flatten, Dropout, Reshape, TimeDistributed, BatchNormalization, Merge
from keras.layers.advanced_activations import LeakyReLU
# self.tf_obs = Input(shape = (3,), dtype='float32', name="observations")
B = Input(shape=(3, ), tensor=self.tf_account)
b = Dense(5, activation="relu")(B)
inputs = [B]
merges = [b]
#S = Input(shape=[2, 60, 1])
S = Input(shape=[2, 60, 1], tensor=self.tf_obs)
inputs.append(S)
print(B.get_shape(), S.get_shape())
h = Convolution2D(2048, 3, 1, border_mode='same')(S)
h = LeakyReLU(0.001)(h)
h = Convolution2D(2048, 5, 1, border_mode='same')(S)
h = LeakyReLU(0.001)(h)
h = Convolution2D(2048, 10, 1, border_mode='same')(S)
h = LeakyReLU(0.001)(h)
h = Convolution2D(2048, 20, 1, border_mode='same')(S)
h = LeakyReLU(0.001)(h)
h = Convolution2D(2048, 40, 1, border_mode='same')(S)
h = LeakyReLU(0.001)(h)
h = Flatten()(h)
h = Dense(512)(h)
h = LeakyReLU(0.001)(h)
merges.append(h)
h = Convolution2D(2048, 60, 1, border_mode='same')(S)
h = LeakyReLU(0.001)(h)
h = Flatten()(h)
h = Dense(512)(h)
h = LeakyReLU(0.001)(h)
merges.append(h)
m = merge(merges, mode='concat', concat_axis=1)
m = Dense(1024)(m)
m = LeakyReLU(0.001)(m)
m = Dense(512)(m)
m = LeakyReLU(0.001)(m)
m = Dense(256)(m)
m = LeakyReLU(0.001)(m)
self.all_act = Dense(2, activation=None)(m)
self.acts_prob = tf.nn.softmax(
self.all_act,
name='act_prob') # use softmax to convert to probability
with tf.variable_scope('exp_v'):
log_prob = tf.log(self.acts_prob[0, self.a])
self.exp_v = tf.reduce_mean(
log_prob * self.td_error) # advantage (TD_error) guided loss
with tf.variable_scope('train'):
self.train_op = tf.train.AdamOptimizer(self.lr).minimize(
-self.exp_v) # minimize(-exp_v) = maximize(exp_v)
def create_model():
img_input = Input(name='img_input', shape=img_input_shape, dtype='float32')
# CNN
conv_1 = Conv2D(64, (3, 3), activation='relu', padding='same',
input_shape=img_input_shape, name='conv_1')(img_input)
max_pool_1 = MaxPooling2D((2, 2), strides=(2, 2), name='pool_1')(conv_1)
conv_2 = Conv2D(128, (3, 3), activation='relu', padding='same',
name='conv_2')(max_pool_1)
max_pool_2 = MaxPooling2D((2, 2), strides=(2, 2), name='pool_2')(conv_2)
conv_3 = Conv2D(256, (3, 3), activation='relu', padding='same',
name='conv_3')(max_pool_2)
normalize_1 = BatchNormalization()(conv_3)
conv_4 = Conv2D(256, (3, 3), activation='relu', padding='same',
name='conv_4')(normalize_1)
padding_0 = ZeroPadding2D(padding=(1, 0), name='padding_0')(conv_4)
max_pool_3 = MaxPooling2D((2, 2), strides=(1, 2), name='pool_3')(padding_0)
conv_5 = Conv2D(512, (3, 3), activation='relu', padding='same',
name='conv_5')(max_pool_3)
normalize_2 = BatchNormalization()(conv_5)
conv_6 = Conv2D(512, (3, 3), activation='relu', padding='same',
name='conv_6')(normalize_2)
normalize_3 = BatchNormalization()(conv_6)
padding_1 = ZeroPadding2D(padding=(1, 0))(normalize_3)
max_pool_4 = MaxPooling2D((2, 2), strides=(1, 2),
name='pool_4')(padding_1)
conv_7 = Conv2D(512, (2, 2), activation='relu', padding='valid',
name='conv_7')(max_pool_4)
# CNN to RNN
reshape = Reshape((26, 512), input_shape=(26, 1, 512))(conv_7)
# RNN
bi_lstm_1 = Bidirectional(LSTM(256, input_shape=(26, 512),
return_sequences=True, name='bi_lstm1'))(reshape)
bi_lstm_2 = Bidirectional(LSTM(256, input_shape=(26, 512),
return_sequences=True,
name='bi_lstm2'))(bi_lstm_1)
# RNN to softmax Activations
dense = TimeDistributed(Dense(classes, activation='sigmoid',
name='dense'))(bi_lstm_2)
y_pred = Activation('softmax', name='final_softmax')(dense)
# Input specifications for CTC Function
y_true = Input(name='ground_truth', shape=[max_string_len], dtype='int64')
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
print('Shape', y_true.get_shape().as_list())
loss_out = Lambda(ctc_lambda_func, output_shape=(1,),
name='ctc')([y_true, y_pred, input_length, label_length])
# Loss function algo
# sgd = SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5)
crnn_model_train = Model(inputs=[img_input, y_true, input_length,
label_length], outputs=loss_out)
crnn_model_test = Model(inputs=[img_input], outputs=y_pred)
return crnn_model_train, crnn_model_test
def draw_cnn_model(hyper_param, embedding_matrix=None, verbose=True):
"""
Input: hyper_parameters dictionary
Construct:
input layers : x , x_pos(o), x_captialization(o)
embedding matrix : use_glove or randomly initialize
conv1 : first convolution layer
conv2 : second convolution layer
conv3 : third convolution layer
max pooling
flatten : concant maxpooled univariate vectors into one long vector
ff1, ff2: two feed forward layers
out_pred: softmax over all ner classes
Returns: keras.models.Model object
"""
# input layer(s)
x = Input(shape=(hyper_param['maxlen'], ), name='x')
if hyper_param['use_pos_tags']:
x_pos = Input(shape=(hyper_param['maxlen'], hyper_param['poslen']),
name='x_pos')
if hyper_param['use_capitalization_info']:
x_capital = Input(shape=(hyper_param['maxlen'],
hyper_param['capitallen']),
name='x_capital')
# embedding matrix
if hyper_param['use_glove']:
embed = Embedding(hyper_param['max_features'], hyper_param['embed_dim'], weights=[embedding_matrix],\
input_length=hyper_param['maxlen'], trainable=hyper_param['allow_glove_retrain'])(x)
else:
embed = Embedding(hyper_param['max_features'], hyper_param['embed_dim'], input_length=hyper_param['maxlen'],\
embeddings_initializer="random_uniform" )(x)
# concat embeddings with additional features
if hyper_param['use_pos_tags'] and hyper_param['use_capitalization_info']:
embed = Concatenate(axis=-1)([embed, x_pos, x_capital])
elif hyper_param['use_pos_tags'] and (
not hyper_param['use_capitalization_info']):
embed = Concatenate(axis=-1)([embed, x_pos])
elif (not hyper_param['use_pos_tags']
) and hyper_param['use_capitalization_info']:
embed = Concatenate(axis=-1)([embed, x_capital])
else:
embed = embed
# feed embeddings into conv1
conv1 = Conv1D( filters=hyper_param['conv1_filters'], \
kernel_size=hyper_param['conv1_kernel_size'],\
strides=hyper_param['conv1_strides'], \
padding=hyper_param['conv1_padding'],\
activation='relu', name='conv1')(embed)
# update this
# make primary capsules
conv2 = Conv1D( filters=hyper_param['conv2_filters'], \
kernel_size=hyper_param['conv2_kernel_size'],\
strides=hyper_param['conv2_strides'], \
padding=hyper_param['conv2_padding'],\
activation='relu', name='conv2')(conv1)
# update this
# make primary capsules
conv3 = Conv1D( filters=hyper_param['conv3_filters'], \
kernel_size=hyper_param['conv3_kernel_size'],\
strides=hyper_param['conv3_strides'], \
padding=hyper_param['conv3_padding'],\
activation='relu', name='conv3')(conv2)
# max pooling layer
max_pooled = MaxPooling1D(pool_size=hyper_param['max_pooling_size'], \
strides=hyper_param['max_pooling_strides'], \
padding=hyper_param['max_pooling_padding'])(conv3)
# dropout
maxpooled_dropout = Dropout(hyper_param['maxpool_dropout'])(max_pooled)
# flatten many univariate vectos into 1 long vector
flattened = Flatten()(maxpooled_dropout)
# to feed-forward layers
ff1 = Dense(hyper_param['feed_forward_1'], activation='relu')(flattened)
ff1_dropout = Dropout(hyper_param['ff1_dropout'])(ff1)
ff2 = Dense(hyper_param['feed_forward_2'], activation='relu')(ff1_dropout)
ff2_dropout = Dropout(hyper_param['ff2_dropout'])(ff2)
out_pred = Dense(hyper_param['ner_classes'],
activation='softmax',
name='out_pred')(ff2) #!
if verbose:
print("x", x.get_shape())
if hyper_param['use_pos_tags']: print("x_pos", x_pos.get_shape())
if hyper_param['use_capitalization_info']:
print("x_capital", x_capital.get_shape())
print("embed", embed.get_shape())
print("embed", embed.get_shape())
print("conv1", conv1.get_shape())
print("conv2", conv2.get_shape())
print("conv3", conv3.get_shape())
print("max_pooled", max_pooled.get_shape())
print("flattened", flattened.get_shape())
print("ff1", ff1.get_shape())
print("ff2", ff2.get_shape())
print("out_pred", out_pred.get_shape())
# return final model
if hyper_param['use_pos_tags'] and hyper_param['use_capitalization_info']:
cnnmodel = Model(inputs=[x, x_pos, x_capital], outputs=[out_pred])
elif hyper_param['use_pos_tags'] and (
not hyper_param['use_capitalization_info']):
cnnmodel = Model(inputs=[x, x_pos], outputs=[out_pred])
elif (not hyper_param['use_pos_tags']
) and hyper_param['use_capitalization_info']:
cnnmodel = Model(inputs=[x, x_capital], outputs=[out_pred])
else:
cnnmodel = Model(inputs=[x], outputs=[out_pred])
return cnnmodel
def __init__(self,
dim,
batch_norm,
dropout,
rec_dropout,
task,
target_repl=False,
deep_supervision=False,
num_classes=1,
depth=1,
input_dim=76,
embedding=False,
embed_dim=None,
n_bins=None,
seq_length=None,
vocab_size=None,
**kwargs):
print("==> not used params in network class:", kwargs.keys())
self.dim = dim
self.batch_norm = batch_norm
self.dropout = dropout
self.rec_dropout = rec_dropout
self.depth = depth
self.vocab_size = vocab_size
self.embedding = embedding
self.embed_dim = embed_dim
self.n_bins = n_bins
self.seq_length = seq_length
if task in ['decomp', 'ihm', 'ph']:
final_activation = 'sigmoid'
elif task in ['los']:
if num_classes == 1:
final_activation = 'relu'
else:
final_activation = 'softmax'
else:
raise ValueError("Wrong value for task")
# Input layers and masking
X = Input(shape=(None, input_dim), name='X')
shape = X.get_shape().as_list()
self.n_bins = shape[1]
inputs = [X]
if self.embedding:
X = Embedding(
self.vocab_size,
self.embed_dim,
input_shape=(self.n_bins, self.seq_length)
)(X) #, embeddings_regularizer=keras.regularizers.l1(0.5))(X)
mX = Masking()(X)
if self.embedding and deep_supervision:
mX = Lambda(lambda x: x, output_shape=lambda s: s)(mX)
mX = Reshape((-1, int(2 * self.embed_dim * self.seq_length)))(mX)
if self.embedding and target_repl:
mX = Lambda(lambda x: x, output_shape=lambda s: s)(mX)
mX = Reshape((-1, int(2 * self.embed_dim * self.seq_length)))(mX)
elif self.embedding and not deep_supervision:
mX = Lambda(lambda x: x, output_shape=lambda s: s)(mX)
mX = Reshape((-1, int(self.embed_dim * self.seq_length)))(mX)
if deep_supervision:
M = Input(shape=(None, ), name='M')
inputs.append(M)
# Configurations
is_bidirectional = True
if deep_supervision:
is_bidirectional = False
# Main part of the network
for i in range(depth - 1):
num_units = dim
if is_bidirectional:
num_units = num_units // 2
lstm = LSTM(units=num_units,
activation='tanh',
return_sequences=True,
recurrent_dropout=rec_dropout,
dropout=dropout)
if is_bidirectional:
mX = Bidirectional(lstm)(mX)
else:
mX = lstm(mX)
# Output module of the network
return_sequences = (target_repl or deep_supervision)
L = LSTM(units=dim,
activation='tanh',
return_sequences=return_sequences,
dropout=dropout,
recurrent_dropout=rec_dropout)(mX)
if dropout > 0:
L = Dropout(dropout)(L)
if target_repl:
y = TimeDistributed(Dense(num_classes,
activation=final_activation),
name='seq')(L)
y_last = LastTimestep(name='single')(y)
outputs = [y_last, y]
elif deep_supervision:
y = TimeDistributed(Dense(num_classes,
activation=final_activation))(L)
y = ExtendMask()([y, M]) # this way we extend mask of y to M
outputs = [y]
else:
y = Dense(num_classes, activation=final_activation)(L)
outputs = [y]
super(Network, self).__init__(inputs=inputs, outputs=outputs)
