def MyModel_Conv2D(
name='MyModel_Conv2D',
sta_num=20,
lt_shape=(3,15,20),
st_shape=(72,20),
wd_shape=(4,1),
ef_shape=(4,24),
op_shape=(1,20),
optimizer='adam',
metrics=['mae'],
loss='mse'
)->Model:
print('model name:',name)
lt=layers.Input(shape=lt_shape)
st=layers.Input(shape=st_shape)
lt_wd=layers.Input(shape=(lt_shape[0],wd_shape[1]))
st_wd=layers.Input(shape=(wd_shape[1],))
lt_ef=layers.Input(shape=(lt_shape[0],ef_shape[1]))
st_ef=layers.Input(shape=(ef_shape[1],))
y1=layers.Reshape((lt_shape[0],lt_shape[1],lt_shape[2],1))(lt)
y1=layers.ConvLSTM2D(filters=64,kernel_size=(6,6),padding='same',return_sequences=True)(y1)
y1=layers.Dropout(0.5)(y1)
y1=layers.ConvLSTM2D(filters=64,kernel_size=(6,6),padding='same',return_sequences=True)(y1)
y1=layers.Dropout(0.25)(y1)
y1=layers.Dense(1)(y1)
y1=layers.Reshape(lt_shape)(y1)
y1=layers.TimeDistributed(attention())(y1)
y1=layers.Flatten()(y1)
y2=layers.Reshape((st_shape[0],st_shape[1],1))(st)
y2=layers.Conv2D(filters=64,kernel_size=(6,6),padding='same')(y2)
y2=layers.ReLU()(y2)
y2=layers.Conv2D(filters=64,kernel_size=(6,6),padding='same')(y2)
y2=layers.ReLU()(y2)
y2=layers.Dense(1)(y2)
y2=layers.Reshape(st_shape)(y2)
y2=attention()(y2)
ltef=layers.concatenate([lt_wd,lt_ef])
stef=layers.concatenate([st_wd,st_ef])
stef=layers.Reshape((1,wd_shape[1]+ef_shape[1]))(stef)
ef=layers.concatenate([ltef,stef],axis=1)
ef=layers.LSTM(64,return_sequences=True)(ef)
ef=layers.Dropout(0.2)(ef)
ef=layers.LSTM(64,return_sequences=True)(ef)
ef=layers.Dropout(0.2)(ef)
ef=layers.ReLU()(ef)
ef=attention()(ef)
y=layers.concatenate([y1,y2,ef])
y=layers.Dense(sta_num)(y)
model=Model([lt,st,lt_wd,st_wd,lt_ef,st_ef],[y],name='MyModel')
model.compile(optimizer=optimizer,loss=loss,metrics=metrics)
return model
def build_model():
"""Build keras model.
Returns: keras model
"""
#define model input
data_shape = (3, 3, 495, 436)
prev_frames = layers.Input(shape=data_shape, name='prev_frames')
#define layers
with device('/device:GPU:0'):
convlstm_0 = layers.ConvLSTM2D(filters=32, kernel_size=(7, 7), padding='same', return_sequences=True, return_state=False,
activation='tanh', recurrent_activation='hard_sigmoid',
kernel_initializer='glorot_uniform', unit_forget_bias=True,
data_format='channels_first', name='convlstm_0')
convlstm_1 = layers.ConvLSTM2D(filters=64, kernel_size=(7, 7), padding='same', return_sequences=False, return_state=True,
activation='tanh', recurrent_activation='hard_sigmoid',
kernel_initializer='glorot_uniform', unit_forget_bias=True,
data_format='channels_first', name='convlstm_1')
with device('/device:GPU:1'):
convlstm_2 = layers.ConvLSTM2D(filters=64, kernel_size=(7, 7), padding='same', return_sequences=True, return_state=False,
activation='tanh', recurrent_activation='hard_sigmoid',
kernel_initializer='glorot_uniform', unit_forget_bias=True,
data_format='channels_first', name='convlstm_2')
convlstm_3 = layers.ConvLSTM2D(filters=3, kernel_size=(7, 7), padding='same', return_sequences=True, return_state=False,
activation='tanh', recurrent_activation='hard_sigmoid',
kernel_initializer='glorot_uniform', unit_forget_bias=True,
data_format='channels_first', name='convlstm_3')
#define model structure
#encoder
x = convlstm_0(prev_frames)
x = convlstm_1(x)[-1]
#flatten, repeat and reshape
x = layers.Flatten()(x)
x = layers.RepeatVector(3)(x)
x = layers.Reshape((3, 64, 495, 436))(x)
#decoder
x = convlstm_2(x)
x = convlstm_3(x)
#build and return model
seq_model = models.Model(inputs=prev_frames, outputs=x)
return seq_model
def load_model():
input_tensor = Input(shape=(SequenceLength, IMSIZE[0], IMSIZE[1], 3))
x = layers.ConvLSTM2D(32,
kernel_size=(7, 7),
padding='valid',
return_sequences=True)(input_tensor)
x = layers.Activation('relu')(x)
x = layers.MaxPooling3D(pool_size=(1, 2, 2))(x)
x = layers.ConvLSTM2D(64,
kernel_size=(5, 5),
padding='valid',
return_sequences=True)(x)
x = layers.MaxPooling3D(pool_size=(1, 2, 2))(x)
x = layers.ConvLSTM2D(96,
kernel_size=(3, 3),
padding='valid',
return_sequences=True)(x)
x = layers.Activation('relu')(x)
x = layers.ConvLSTM2D(96,
kernel_size=(3, 3),
padding='valid',
return_sequences=True)(x)
x = layers.Activation('relu')(x)
x = layers.ConvLSTM2D(96,
kernel_size=(3, 3),
padding='valid',
return_sequences=True)(x)
x = layers.MaxPooling3D(pool_size=(1, 2, 2))(x)
x = layers.Dense(320)(x)
x = layers.Activation('relu')(x)
x = layers.Dropout(0.5)(x)
out_shape = x.get_shape().as_list()
x = layers.Reshape(
(SequenceLength, out_shape[2] * out_shape[3] * out_shape[4]))(x)
x = layers.Bidirectional(layers.LSTM(64, return_sequences=True),
merge_mode='concat')(x)
x = layers.Dropout(0.5)(x)
x = layers.Flatten()(x)
x = layers.Dense(128, activation='relu')(x)
output_tensor = layers.Dense(N_CLASSES, activation='softmax')(x)
model = Model(input_tensor, output_tensor)
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
metrics=['accuracy'])
return model
def predict(n = 0):
h5f = h5py.File('D:/total_data_ro.h5','r')
test_data = h5f['total_2019'][:]
test_data_gen = h5f['gen_2019'][:]
h5f.close()
sc = RobustScaler()
sc = joblib.load('D:/scaler_gen.pkl')
test_data_gen = test_data_gen.reshape(-1, 1)
img_input = Input(shape=(None, 75, 75, 10), name='images')
convlstm2d = layers.ConvLSTM2D(filters=20, kernel_size=(3, 3),
input_shape=(None, 75, 75, 10),
data_format='channels_last', return_sequences=True,
padding='same')(img_input)
batch_norm = layers.BatchNormalization()(convlstm2d)
convlstm2d_1x1 = layers.ConvLSTM2D(filters=1, kernel_size=(3, 3),
data_format='channels_last',padding='same')(batch_norm)
batch_norm = layers.BatchNormalization()(convlstm2d_1x1)
flatten = layers.Flatten()(batch_norm)
Dense_1_list = [layers.Dense(units=1)(flatten) for i in range(72)]
model = Model(img_input, Dense_1_list)
model.compile(optimizer=rmsprop(lr=0.001), loss=['mae' for i in range(72)])
model.load_weights('D:/gen_pred_72hours.h5')
test_sample, gen_sample = generator(test_data, test_data_gen, 120, 72, 0, None, shuffle=True, batch_size=1)
test_sample = np.rollaxis(test_sample, 2, 5)
y_pred = model.predict(test_sample)
y_pred = np.array(y_pred).reshape(-1,1)
#y_test = np.array(gen_sample)
y_pred = sc.inverse_transform(y_pred)
sc = MinMaxScaler(feature_range=(0, 40))
plt.plot(sc.fit_transform(y_pred), label='pred')
#plt.plot(gen_sample, label='real')
plt.legend()
plt.savefig("media/test" + str(n) + ".png")
print('save img')
plt.clf()
plt.cla()
plt.close()
#plt.show()
def ConvLSTMCell(data):
conv = layers.Conv2D(64, (3, 3),
padding='same',
activation='relu',
use_bias=True)(data)
conv = ResUnit(conv, 64, (3, 3))
conv = ResUnit(conv, 64, (3, 3))
lstm_data = layers.Reshape((data.shape[-1], 20, 20, 1))(data)
lstm = layers.ConvLSTM2D(64, (3, 3),
strides=(1, 1),
padding='same',
activation='relu',
use_bias=True)(lstm_data)
lstm = layers.Conv2D(64, (1, 1),
padding='same',
activation='relu',
use_bias=True)(lstm)
conv = layers.Conv2D(64, (1, 1),
padding='same',
activation='relu',
use_bias=True)(conv)
res = layers.Add()([lstm, conv])
return res
def ConvLSTM_predictor2(num_classes=1, input_shape=(10, 500, 320, 5)):
model = Sequential()
# ConvLSTM2D (batch, steps, width, height, features)
model.add(
layers.ConvLSTM2D(30,
10,
return_sequences=True,
input_shape=input_shape))
model.add(layers.Activation("relu"))
model.add(layers.ConvLSTM2D(num_classes, 5, return_sequences=False))
model.add(layers.Activation("relu"))
model.compile(loss=euclidean_loss, optimizer=Adam(lr=1e-3))
return model
def big_conv_lstm_readout_net(feature_map_in_seqs, feature_map_size, drop_rate, gaze_prior=None):
batch_size = tf.shape(feature_map_in_seqs)[0]
n_step = tf.shape(feature_map_in_seqs)[1]
n_channel = int(feature_map_in_seqs.get_shape()[4])
feature_map = tf.reshape(feature_map_in_seqs,
[batch_size*n_step, feature_map_size[0],
feature_map_size[1], n_channel])
x = layers.Conv2D(32, (1, 1), activation='relu', name='readout_conv1')(feature_map)
x = layers.core.Dropout(drop_rate)(x)
x = layers.Conv2D(16, (1, 1), activation='relu', name='readout_conv2')(x)
x = layers.core.Dropout(drop_rate)(x)
x = layers.Conv2D(8, (1, 1), activation='relu', name='readout_conv3')(x)
x = layers.core.Dropout(drop_rate)(x)
x = layers.Conv2D(1, (1, 1), activation='relu', name='readout_conv4')(x)
x = layers.core.Dropout(drop_rate)(x)
#x = layers.core.Reshape((-1,))(x)
temp_shape = x.get_shape()[1:4]
temp_shape = [int(s) for s in temp_shape]
x = tf.reshape(x, [batch_size, n_step, temp_shape[0], temp_shape[1], temp_shape[2]])
x = layers.ConvLSTM2D(filters=1,
kernel_size=(3,3),
strides=(1,1),
padding='same',
dropout=drop_rate,
recurrent_dropout=drop_rate,
return_sequences=True)(x)
x = wps.TimeDistributed(layers.Conv2D(1, (1, 1), activation='linear'))(x)
x = tf.reshape(x, [batch_size*n_step,
feature_map_size[0], feature_map_size[1], 1])
x = GaussianSmooth(kernel_size = GAUSSIAN_KERNEL_SIZE, name='gaussian_smooth')(x)
logits = tf.reshape(x, [-1, feature_map_size[0]*feature_map_size[1]])
#gaze prior map
if gaze_prior is not None:
#predicted annotation before adding prior
pre_prior_logits = logits
gaze_prior = np.maximum(gaze_prior, EPSILON*np.ones(gaze_prior.shape))
gaze_prior = gaze_prior.astype(np.float32)
log_prior = np.log(gaze_prior)
log_prior_1d = np.reshape(log_prior, (1, -1))
log_prior_unit_tensor = tf.constant(log_prior_1d)
log_prior_tensor = tf.matmul(tf.ones((tf.shape(pre_prior_logits)[0],1)), log_prior_unit_tensor)
log_prior_tensor = tf.reshape(log_prior_tensor,
[-1, feature_map_size[0]*feature_map_size[1]])
logits = tf.add(pre_prior_logits, log_prior_tensor)
if gaze_prior is None:
return logits
else:
return logits, pre_prior_logits
def conv2dlstm_model(window_size=32, n_notes=58):
notes_model = models.Sequential()
notes_model.add(layers.ConvLSTM2D(32, 3, return_sequences=True, padding='same',
input_shape=(None, window_size, None, 3)))
notes_model.add(layers.MaxPool3D(2, 2))
notes_model.add(layers.ConvLSTM2D(64, 3, padding='same'))
notes_model.add(layers.GlobalAveragePooling2D())
beats_input = layers.Input(shape=(16,))
features = layers.concatenate([notes_model.output, beats_input])
dropout1 = layers.Dropout(0.2)(features)
fc_1 = layers.Dense(100, activation='relu')(dropout1)
dropout2 = layers.Dropout(0.2)(fc_1)
output_notes = layers.Dense(n_notes + 1, activation='softmax')(dropout2)
output_slur = layers.Dense(1, activation='sigmoid')(fc_1)
return models.Model(inputs=[notes_model.input, beats_input], outputs=[output_notes, output_slur])
def ConvLSTM_predictor1(num_classes=1, input_shape=(10, 500, 320, 5)):
model = Sequential()
#Embed down to latent space.
# ConvLSTM2D (batch, steps, width, height, features)
model.add(
layers.ConvLSTM2D(40,
10,
return_sequences=True,
input_shape=input_shape))
model.add(layers.BatchNormalization())
model.add(layers.LeakyReLU(alpha=0.2))
model.add(layers.ConvLSTM2D(30, 5, strides=2, return_sequences=True))
model.add(layers.BatchNormalization())
model.add(layers.LeakyReLU(alpha=0.2))
# Latent Space
model.add(layers.ConvLSTM2D(20, 5, return_sequences=False))
model.add(layers.BatchNormalization())
model.add(layers.LeakyReLU(alpha=0.2))
# Rebuild latent embedding into future prediction.
model.add(layers.Conv2DTranspose(40, 5))
model.add(layers.BatchNormalization())
model.add(layers.LeakyReLU(alpha=0.2))
model.add(layers.Conv2DTranspose(30, 10, strides=2))
model.add(layers.BatchNormalization())
model.add(layers.LeakyReLU(alpha=0.2))
model.add(layers.Conv2DTranspose(20, 30))
model.add(layers.BatchNormalization())
model.add(layers.LeakyReLU(alpha=0.2))
# Rebuilt future
model.add(layers.Conv2DTranspose(num_classes, 5))
model.compile(loss=euclidean_loss, optimizer=Adam(lr=1e-3))
return model
def build_model(input_shape):
model = krm.Sequential()
model.add(
krl.ConvLSTM2D(filters=64,
kernel_size=3,
return_sequences=False,
data_format="channels_last",
input_shape=input_shape[1:]))
model.add(krl.Dropout(0.2))
model.add(krl.Flatten())
model.add(krl.Dense(256, activation="relu"))
model.add(krl.Dropout(0.3))
model.add(krl.Dense(input_shape[-1], activation="softmax"))
return model
def conv_lstm(data_train, label_train, data_test, label_test, block_num):
data_train = data_train.reshape(data_train.shape[0], block_num, 1,
int(data_train.shape[1] / block_num),
data_train.shape[2])
data_test = data_test.reshape(data_test.shape[0], block_num, 1,
int(data_test.shape[1] / block_num),
data_test.shape[2])
label_train = np_utils.to_categorical(label_train - 1)
label_test = np_utils.to_categorical(label_test - 1)
#print("Model building begin:")
conv_lstm = models.Sequential()
conv_lstm.add(
layers.ConvLSTM2D(filters=128,
kernel_size=(1, 3),
activation='relu',
input_shape=(block_num, 1, data_train.shape[3],
data_train.shape[4])))
conv_lstm.add(layers.Dropout(0.5))
conv_lstm.add(layers.Flatten())
conv_lstm.add(layers.Dense(64, activation='relu'))
conv_lstm.add(layers.Dense(6, activation='softmax'))
#cnn_1d.summary()
plot_model(conv_lstm, show_shapes=True, to_file='conv.png', dpi=200)
conv_lstm.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['acc'])
history = conv_lstm.fit(data_train,
label_train,
epochs=25,
batch_size=64,
verbose=0,
validation_split=0.2)
_, te_acc = conv_lstm.evaluate(data_test,
label_test,
batch_size=32,
verbose=0)
return history, history.history['acc'][4], history.history['val_acc'][
4], te_acc, history.history['loss'][4], history.history['val_loss'][4]
def thick_conv_lstm_readout_net(feature_map_in_seqs,
feature_map_size,
drop_rate,
gaze_prior=None,
output_embedding=False):
batch_size = tf.shape(feature_map_in_seqs)[0]
n_step = tf.shape(feature_map_in_seqs)[1]
n_channel = int(feature_map_in_seqs.get_shape()[4])
feature_map = tf.reshape(feature_map_in_seqs, [
batch_size * n_step, feature_map_size[0], feature_map_size[1],
n_channel
])
x = layers.Conv2D(16, (1, 1), activation='relu',
name='readout_conv1')(feature_map)
x = layers.BatchNormalization()(x)
x = layers.core.Dropout(drop_rate)(x)
x = layers.Conv2D(32, (1, 1), activation='relu', name='readout_conv2')(x)
x = layers.BatchNormalization()(x)
x = layers.core.Dropout(drop_rate)(x)
x = layers.Conv2D(8, (1, 1), activation='relu', name='readout_conv3')(x)
x = layers.BatchNormalization()(x)
# reshape into temporal sequence
temp_shape = x.get_shape()[1:4]
temp_shape = [int(s) for s in temp_shape]
x = tf.reshape(
x, [batch_size, n_step, temp_shape[0], temp_shape[1], temp_shape[2]])
n_channel = 15
initial_c = layers.Conv2D(n_channel, (3, 3),
activation='tanh',
padding='same')(layers.core.Dropout(drop_rate)(
x[:, 0]))
initial_c = layers.core.Dropout(drop_rate)(initial_c)
initial_h = layers.Conv2D(n_channel, (3, 3),
activation='tanh',
padding='same')(layers.core.Dropout(drop_rate)(
x[:, 0]))
initial_h = layers.core.Dropout(drop_rate)(initial_h)
conv_lstm = layers.ConvLSTM2D(filters=n_channel,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
dropout=drop_rate,
recurrent_dropout=drop_rate,
return_sequences=True)
x = conv_lstm([x, initial_c, initial_h])
x = wps.TimeDistributed(
layers.Conv2D(n_channel, (1, 1), activation='linear'))(x)
x = tf.reshape(x, [
batch_size * n_step, feature_map_size[0], feature_map_size[1],
n_channel
])
x = layers.BatchNormalization()(x)
embed = x
x = layers.Conv2D(1, (1, 1), activation='linear')(x)
x = tf.reshape(
x, [batch_size * n_step, feature_map_size[0], feature_map_size[1], 1])
raw_logits = tf.reshape(x, [-1, feature_map_size[0] * feature_map_size[1]])
logits = tf.reshape(x, [-1, feature_map_size[0] * feature_map_size[1]])
#gaze prior map
if gaze_prior is not None:
#predicted annotation before adding prior
pre_prior_logits = logits
gaze_prior = np.maximum(gaze_prior,
EPSILON * np.ones(gaze_prior.shape))
gaze_prior = gaze_prior.astype(np.float32)
log_prior = np.log(gaze_prior)
log_prior_1d = np.reshape(log_prior, (1, -1))
log_prior_unit_tensor = tf.constant(log_prior_1d)
log_prior_tensor = tf.matmul(
tf.ones((tf.shape(pre_prior_logits)[0], 1)), log_prior_unit_tensor)
log_prior_tensor = tf.reshape(
log_prior_tensor, [-1, feature_map_size[0] * feature_map_size[1]])
logits = tf.add(pre_prior_logits, log_prior_tensor)
if output_embedding:
return logits, embed, raw_logits
if gaze_prior is None:
return logits
else:
return logits, pre_prior_logits
def conv_lstm_planner(peripheral_feature_map_seqs, foveal_feature_seqs,
drop_rate):
# combine feature maps
if foveal_feature_seqs is None:
feature_map_seqs = peripheral_feature_map_seqs
elif peripheral_feature_map_seqs is None:
feature_map_seqs = foveal_feature_seqs
else:
feature_map_seqs = tf.concat(
[peripheral_feature_map_seqs, foveal_feature_seqs], axis=-1)
# get the shape
batch_size = tf.shape(feature_map_seqs)[0]
n_step = tf.shape(feature_map_seqs)[1]
temp_shape = feature_map_seqs.get_shape()[2:5]
temp_shape = [int(s) for s in temp_shape]
feature_map_size = temp_shape[0:2]
n_channel = temp_shape[2]
conv_lstm = layers.ConvLSTM2D(filters=5,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
dropout=drop_rate,
recurrent_dropout=drop_rate,
return_sequences=True)
initial_c = layers.Conv2D(5, (3, 3), activation='tanh', padding='same')(
layers.core.Dropout(drop_rate)(feature_map_seqs[:, 0]))
initial_c = layers.core.Dropout(drop_rate)(initial_c)
initial_h = layers.Conv2D(5, (3, 3), activation='tanh', padding='same')(
layers.core.Dropout(drop_rate)(feature_map_seqs[:, 0]))
initial_h = layers.core.Dropout(drop_rate)(initial_h)
x = conv_lstm([feature_map_seqs, initial_c, initial_h])
# track weights
kernel_weights = conv_lstm.weights[0] # shape is [3, 3, 8+8, 5*4]
if peripheral_feature_map_seqs is None:
peripheral_weights = None
peripheral_n_channels = 0
else:
peripheral_n_channels = tf.shape(peripheral_feature_map_seqs)[-1]
peripheral_weights = kernel_weights[:, :, 0:peripheral_n_channels, :]
if foveal_feature_seqs is None:
foveal_weights = None
else:
foveal_weights = kernel_weights[:, :, peripheral_n_channels:, :]
x = tf.reshape(
x, [batch_size * n_step, feature_map_size[0], feature_map_size[1], 5])
x = layers.BatchNormalization()(x)
temp_shape = x.get_shape()[1:4]
temp_shape = [int(s) for s in temp_shape]
x = tf.reshape(
x, [batch_size, n_step, temp_shape[0] * temp_shape[1] * temp_shape[2]])
x = wps.TimeDistributed(layers.Dense(units=512, activation='linear'))(x)
logits = tf.reshape(x, [batch_size * n_step, 512])
return logits, peripheral_weights, foveal_weights
def my_model(encoder='VGG',
input_size=(256, 256, 1),
k_shot=1,
learning_rate=1e-4):
# Get the encoder
encoder_t = encoder
if encoder_t == 'VGG_b3':
encoder = EM.vgg_encoder_b3(input_size=input_size)
elif encoder_t == 'VGG_b4':
encoder = EM.vgg_encoder_b4(input_size=input_size)
elif encoder_t == 'VGG_b34':
encoder = EM.vgg_encoder_b34(input_size=input_size)
elif encoder_t == 'VGG_b5':
encoder = EM.vgg_encoder_b5(input_size=input_size)
elif encoder_t == 'VGG_b345':
encoder = EM.vgg_encoder_b345(input_size=input_size)
elif encoder_t == 'VGG_b35':
encoder = EM.vgg_encoder_b35(input_size=input_size)
elif encoder_t == 'VGG_b45':
encoder = EM.vgg_encoder_b45(input_size=input_size)
elif encoder_t == 'RN':
encoder = EM.rn_encoder(input_size=input_size)
else:
print('Encoder is not defined yet')
S_mask = layers.Input(
(k_shot, int(input_size[0] / 4), int(input_size[1] / 4), 1))
S_input = layers.Input(
(k_shot, input_size[0], input_size[1], input_size[2]))
#print(S_input.shape)
Q_input = layers.Input(input_size)
#print(S_mask.shape)
## K shot
kshot_encoder = keras.models.Sequential()
kshot_encoder.add(
layers.TimeDistributed(encoder,
input_shape=(k_shot, input_size[0],
input_size[1], input_size[2])))
## Encode support and query sample
s_encoded = kshot_encoder(S_input)
q_encoded = encoder(Q_input)
print("before")
print(s_encoded.shape)
print(q_encoded.shape)
#if encoder_t == "RN":
s_encoded = layers.TimeDistributed(
layers.Conv2D(128, (3, 3), activation='relu',
padding='same'))(s_encoded)
q_encoded = layers.Conv2D(128, (3, 3), activation='relu',
padding='same')(q_encoded)
print("after")
print(s_encoded.shape)
print(q_encoded.shape)
## Difference of Gussian Pyramid parameters
kernet_shapes = [3, 5, 7, 9]
k_value = np.power(2, 1 / 3)
sigma = 1.6
## Kernel weights for Gussian pyramid
Sigma1_kernel = get_kernel_gussian(kernel_size=kernet_shapes[0],
Sigma=sigma * np.power(k_value, 1),
in_channels=128)
Sigma2_kernel = get_kernel_gussian(kernel_size=kernet_shapes[1],
Sigma=sigma * np.power(k_value, 2),
in_channels=128)
Sigma3_kernel = get_kernel_gussian(kernel_size=kernet_shapes[2],
Sigma=sigma * np.power(k_value, 3),
in_channels=128)
Sigma4_kernel = get_kernel_gussian(kernel_size=kernet_shapes[3],
Sigma=sigma * np.power(k_value, 4),
in_channels=128)
Sigma1_layer = layers.TimeDistributed(
layers.DepthwiseConv2D(kernet_shapes[0],
use_bias=False,
padding='same'))
Sigma2_layer = layers.TimeDistributed(
layers.DepthwiseConv2D(kernet_shapes[1],
use_bias=False,
padding='same'))
Sigma3_layer = layers.TimeDistributed(
layers.DepthwiseConv2D(kernet_shapes[2],
use_bias=False,
padding='same'))
Sigma4_layer = layers.TimeDistributed(
layers.DepthwiseConv2D(kernet_shapes[3],
use_bias=False,
padding='same'))
## Gussian filtering
x1 = Sigma1_layer(s_encoded)
x2 = Sigma2_layer(s_encoded)
x3 = Sigma3_layer(s_encoded)
x4 = Sigma4_layer(s_encoded)
#S_mask = smx
#print(S_mask.shape)
#print("x1.shape")
#print(x1.shape)
#print(s_encoded.shape)
DOG1 = layers.Subtract()([s_encoded, x1])
DOG2 = layers.Subtract()([x1, x2])
DOG3 = layers.Subtract()([x2, x3])
DOG4 = layers.Subtract()([x3, x4])
#print("Dog.shape")
#print(S_mask.shape)
#print(DOG1.shape)
s_1 = ''
s_2 = ''
s_3 = ''
s_4 = ''
## Global Representation
if encoder_t == 'RN':
s_1 = RN_GlobalAveragePooling2D_r(S_mask)(DOG1)
s_2 = RN_GlobalAveragePooling2D_r(S_mask)(DOG2)
s_3 = RN_GlobalAveragePooling2D_r(S_mask)(DOG3)
s_4 = RN_GlobalAveragePooling2D_r(S_mask)(DOG4)
else:
s_1 = GlobalAveragePooling2D_r(S_mask)(DOG1)
s_2 = GlobalAveragePooling2D_r(S_mask)(DOG2)
s_3 = GlobalAveragePooling2D_r(S_mask)(DOG3)
s_4 = GlobalAveragePooling2D_r(S_mask)(DOG4)
## Common Representation of Support and Query sample
s_1 = common_representation(s_1, q_encoded)
s_2 = common_representation(s_2, q_encoded)
s_3 = common_representation(s_3, q_encoded)
s_4 = common_representation(s_4, q_encoded)
## Bidirectional Convolutional LSTM on Pyramid
s_3D = layers.concatenate([s_1, s_2, s_3, s_4], axis=1)
LSTM_f = layers.ConvLSTM2D(filters=128,
kernel_size=(3, 3),
padding='same',
return_sequences=False,
go_backwards=False,
kernel_initializer='he_normal')(s_3D)
LSTM_b = layers.ConvLSTM2D(filters=128,
kernel_size=(3, 3),
padding='same',
return_sequences=False,
go_backwards=True,
kernel_initializer='he_normal')(s_3D)
Bi_rep = layers.Add()([LSTM_f, LSTM_b])
## Decode to query segment
x = layers.Conv2D(128, 3, padding='same',
kernel_initializer='he_normal')(Bi_rep)
x = layers.BatchNormalization(axis=3)(x)
x = layers.Activation('relu')(x)
x = layers.UpSampling2D(size=(2, 2))(x)
x = layers.Conv2D(128, 3, padding='same',
kernel_initializer='he_normal')(x)
x = layers.BatchNormalization(axis=3)(x)
x = layers.Activation('relu')(x)
x = layers.UpSampling2D(size=(2, 2))(x)
x = layers.Conv2D(128, 3, padding='same',
kernel_initializer='he_normal')(x)
x = layers.BatchNormalization(axis=3)(x)
x = layers.Activation('relu')(x)
x = layers.Conv2D(64,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(x)
x = layers.Conv2D(2,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(x)
final = layers.Conv2D(1, 1, activation='sigmoid')(x)
print("AAA")
print(final.shape)
seg_model = Model(inputs=[S_input, S_mask, Q_input], outputs=final)
## Load Guassian weights and make it unlearnable
Sigma1_layer.set_weights([Sigma1_kernel])
Sigma2_layer.set_weights([Sigma2_kernel])
Sigma3_layer.set_weights([Sigma3_kernel])
Sigma4_layer.set_weights([Sigma4_kernel])
Sigma1_layer.trainable = False
Sigma2_layer.trainable = False
Sigma3_layer.trainable = False
Sigma4_layer.trainable = False
seg_model.compile(optimizer=keras.optimizers.Adam(lr=learning_rate),
loss='binary_crossentropy',
metrics=['accuracy'])
return seg_model
def ConvLSTM(optimizer, init_channel, block_num):
input_shape = (256, 256, 1)
input_1 = L.Input(shape=input_shape)
input_2 = L.Input(shape=input_shape)
input_3 = L.Input(shape=input_shape)
weights_input = L.Input(shape=input_shape)
encoder = Sequential(name='encoder')
for i in range(block_num):
if i == 0:
encoder.add(
L.Conv2D(init_channel * (2**i), (3, 3),
strides=2,
activation='relu',
padding='same',
kernel_initializer='he_normal',
input_shape=input_shape))
else:
encoder.add(
L.Conv2D(init_channel * (2**i), (3, 3),
strides=2,
activation='relu',
padding='same',
kernel_initializer='he_normal'))
encoder.add(
L.Conv2D(init_channel * (2**i), (3, 3),
strides=1,
activation='relu',
padding='same',
kernel_initializer='he_normal'))
encoded_1 = encoder(input_1)
encoded_2 = encoder(input_2)
encoded_3 = encoder(input_3)
reshape = (1, *encoder.output_shape[1:])
reshaped_1 = L.Reshape(reshape)(encoded_1)
reshaped_2 = L.Reshape(reshape)(encoded_2)
reshaped_3 = L.Reshape(reshape)(encoded_3)
concat = L.Concatenate(axis=1)([reshaped_1, reshaped_2, reshaped_3])
convlstm = L.ConvLSTM2D(init_channel * 4, (3, 3),
strides=1,
padding='same',
activation='relu',
kernel_initializer='he_normal',
return_sequences=False)(concat)
decoder_shape = (i.value for i in convlstm.get_shape()[1:])
decoder = Sequential(name='decoder')
for i in range(block_num):
if i == 0:
decoder.add(
L.Conv2DTranspose(init_channel * (2**(block_num - i)), (3, 3),
strides=2,
activation='relu',
padding='same',
kernel_initializer='he_normal',
input_shape=decoder_shape))
else:
decoder.add(
L.Conv2DTranspose(init_channel * (2**(block_num - i)), (3, 3),
strides=2,
activation='relu',
padding='same',
kernel_initializer='he_normal'))
decoder.add(
L.Conv2D(init_channel * (2**(block_num - i)), (3, 3),
strides=1,
activation='relu',
padding='same',
kernel_initializer='he_normal'))
decoder.add(
L.Conv2D(1, (3, 3),
strides=1,
activation='sigmoid',
padding='same',
kernel_initializer='he_normal'))
output = decoder(convlstm)
model = Model(inputs=[input_1, input_2, input_3, weights_input],
outputs=output)
model.compile(optimizer=optimizer, loss=utils.custom_loss(weights_input))
print(model.summary())
return model
def deeplabv3p_lstm(input_shape=(128, 128, 16),
n_class=1,
one_hot=False,
backbone_name='mobilenetv2',
backbone_weights=None,
output_stride=16,
atrous_rates=(2, 3, 5),
input_split=4):
def batch_relu(inputs):
_x = layers.BatchNormalization()(inputs)
_x = layers.Activation('relu')(_x)
return _x
assert input_shape[-1] % input_split == 0
img_input = layers.Input(shape=input_shape)
img_inputs = []
channels_per_split = input_shape[-1] // input_split
for i in range(input_split):
_x = layers.Lambda(lambda xx: xx[:, :, :, i * channels_per_split:(
i + 1) * channels_per_split])(img_input)
img_inputs.append(_x)
sub_img_input, features = build_encoder(
(input_shape[0], input_shape[1], input_shape[-1] // input_split), None,
backbone_name, backbone_weights, output_stride)
[f1, f2, f3, f4, f5] = features
feature_extractor = Model(sub_img_input, [f2, f4])
F2s = []
F4s = []
for i in range(input_split):
F24 = feature_extractor(img_inputs[i])
F2s.append(F24[0])
F4s.append(F24[1])
F2s = layers.Lambda(lambda xx: K.stack(xx, axis=1))(F2s)
F4s = layers.Lambda(lambda xx: K.stack(xx, axis=1))(F4s)
F2 = layers.ConvLSTM2D(filters=48,
kernel_size=3,
padding='same',
activation='relu',
return_sequences=False)(F2s)
F4 = layers.ConvLSTM2D(filters=48,
kernel_size=3,
padding='same',
activation='relu',
return_sequences=False)(F4s)
# 多层级特征融合
x_h = atrous_spatial_pyramid_pooling(F4, atrous_rates)
x_h = layers.Conv2D(256, kernel_size=1, use_bias=False)(x_h)
x_h = batch_relu(x_h)
x_h = layers.Dropout(0.2)(x_h)
x_h = layers.UpSampling2D((4, 4), interpolation='bilinear')(x_h)
x_l = layers.Conv2D(48, kernel_size=1, use_bias=False)(F2)
x_l = batch_relu(x_l)
x = layers.Concatenate()([x_h, x_l])
x = _sep_conv_bn(x,
256,
'decoder_conv0',
depth_activation=True,
epsilon=1e-5)
x = _sep_conv_bn(x,
256,
'decoder_conv1',
depth_activation=True,
epsilon=1e-5)
x = layers.Conv2D(n_class, kernel_size=1, use_bias=False)(x)
x = layers.UpSampling2D((4, 4), interpolation='bilinear')(x)
if n_class == 1 or (n_class == 2 and one_hot is False):
x = layers.Activation('sigmoid')(x)
else:
x = layers.Activation('softmax')(x)
return Model(img_input, x)
def getBiConvLSTM2d(input_image,filters,kernel_size,name):
L1 = KL.Bidirectional(KL.ConvLSTM2D(filters=filters, kernel_size=kernel_size,activation='relu', padding='same', return_sequences=True))(input_image)
L1 = KL.BatchNormalization(name="batchNormL_"+name)(L1)
return L1
flt_size = 5
# Training settings
opt = "adam" # Optimizer
loss = "mse"
epochs = 60
batch_size = 16
# %%
# Build the LSTM neural network
tf.keras.backend.clear_session()
flt_tp = (flt_size, flt_size)
conv_kwargs = dict(kernel_size=flt_tp, padding="same")
# Encoder
inputs = layers.Input(shape=x_train.shape[1:])
x = layers.ConvLSTM2D(flt[0], return_sequences=True, **conv_kwargs)(inputs)
x = layers.ConvLSTM2D(flt[1], return_sequences=True, **conv_kwargs)(x)
x = layers.ConvLSTM2D(flt[2], return_sequences=False, **conv_kwargs)(x)
x = layers.Conv2D(flt[3], activation=act, **conv_kwargs)(x)
out = layers.Conv2D(x_train.shape[-1],
flt_tp,
activation="linear",
padding="same")(x)
# Mount the LSTM
lstm = Model(inputs, out, name="LSTM neural network")
# %%
# Show the architecture
lstm.summary()
elif layer.__class__.__name__ in ('LSTM', 'ConvLSTM2D'):
W = [np.split(w, 4, axis=-1) for w in weights]
for w in W:
w[2], w[1] = w[1], w[2]
return sum(map(list, zip(*W)), [])
elif layer.__class__.__name__ == 'Conv2DTranspose':
return [np.transpose(weights[0], (2, 3, 0, 1)), weights[1]]
return weights
@keras_test
@pytest.mark.parametrize("layer", [
layers.GRU(2, input_shape=[3, 5]),
layers.LSTM(2, input_shape=[3, 5]),
layers.ConvLSTM2D(5, (3, 3),
input_shape=[6, 6, 6, 6],
data_format='channels_first'),
])
def test_preprocess_weights_for_loading(layer):
model = Sequential([layer])
weights1 = layer.get_weights()
weights2 = topology.preprocess_weights_for_loading(
layer, convert_weights(layer, weights1),
original_keras_version='1')
assert all([np.allclose(x, y, 1e-5)
for (x, y) in zip(weights1, weights2)])
@keras_test
@pytest.mark.parametrize("layer", [
layers.Conv2D(2, (3, 3), input_shape=[5, 5, 3]),
def resunet_lstm(input_shape=(128, 128, 16), n_class=1, n_filter=64, one_hot=False, input_split=4):
def resunet_encoder():
img_input = layers.Input(shape=(input_shape[0], input_shape[1], input_shape[-1] // input_split))
# 256 -> 128
conv1 = layers.Conv2D(n_filter * 1, (3, 3), activation=None, padding="same",
kernel_initializer=KERNEL_INITIALIZER,
kernel_regularizer=regularizers.l2(WEIGHT_DECAY)
)(img_input)
conv1 = residual_block(conv1, n_filter * 1)
conv1 = residual_block(conv1, n_filter * 1, True)
pool1 = layers.MaxPooling2D((2, 2))(conv1)
pool1 = layers.Dropout(DROPOUT)(pool1)
# 128 -> 64
conv2 = layers.Conv2D(n_filter * 2, (3, 3), activation=None, padding="same",
kernel_initializer=KERNEL_INITIALIZER,
kernel_regularizer=regularizers.l2(WEIGHT_DECAY)
)(pool1)
conv2 = residual_block(conv2, n_filter * 2)
conv2 = residual_block(conv2, n_filter * 2, True)
pool2 = layers.MaxPooling2D((2, 2))(conv2)
pool2 = layers.Dropout(DROPOUT)(pool2)
# 64 -> 32
conv3 = layers.Conv2D(n_filter * 4, (3, 3), activation=None, padding="same",
kernel_initializer=KERNEL_INITIALIZER,
kernel_regularizer=regularizers.l2(WEIGHT_DECAY)
)(pool2)
conv3 = residual_block(conv3, n_filter * 4)
conv3 = residual_block(conv3, n_filter * 4, True)
pool3 = layers.MaxPooling2D((2, 2))(conv3)
pool3 = layers.Dropout(DROPOUT)(pool3)
# 32 -> 16
conv4 = layers.Conv2D(n_filter * 8, (3, 3), activation=None, padding="same",
kernel_initializer=KERNEL_INITIALIZER,
kernel_regularizer=regularizers.l2(WEIGHT_DECAY)
)(pool3)
conv4 = residual_block(conv4, n_filter * 8)
conv4 = residual_block(conv4, n_filter * 8, True)
pool4 = layers.MaxPooling2D((2, 2))(conv4)
pool4 = layers.Dropout(DROPOUT)(pool4)
# Middle
convm = layers.Conv2D(n_filter * 16, (3, 3), activation=None, padding="same",
kernel_initializer=KERNEL_INITIALIZER,
kernel_regularizer=regularizers.l2(WEIGHT_DECAY)
)(pool4)
convm = residual_block(convm, n_filter * 16)
convm = residual_block(convm, n_filter * 16, True)
# convm = convolutional_block_attention_module(convm, hidden_size=n_filter, conv_size=7)
return models.Model(img_input, [conv1, conv2, conv3, conv4, convm])
assert input_shape[-1] % input_split == 0
img_input = layers.Input(shape=input_shape)
img_inputs = []
channels_per_split = input_shape[-1] // input_split
for i in range(input_split):
_x = layers.Lambda(lambda xx: xx[:, :, :, i * channels_per_split: (i + 1) * channels_per_split])(img_input)
img_inputs.append(_x)
feature_extractor = resunet_encoder()
F0s = []
F1s = []
F2s = []
F3s = []
F4s = []
for i in range(input_split):
F01234 = feature_extractor(img_inputs[i])
F0s.append(F01234[0])
F1s.append(F01234[1])
F2s.append(F01234[2])
F3s.append(F01234[3])
F4s.append(F01234[4])
F0s = layers.Lambda(lambda xx: K.stack(xx, axis=1))(F0s)
F1s = layers.Lambda(lambda xx: K.stack(xx, axis=1))(F1s)
F2s = layers.Lambda(lambda xx: K.stack(xx, axis=1))(F2s)
F3s = layers.Lambda(lambda xx: K.stack(xx, axis=1))(F3s)
F4s = layers.Lambda(lambda xx: K.stack(xx, axis=1))(F4s)
F0 = layers.ConvLSTM2D(filters=n_filter, kernel_size=3, padding='same', activation='relu',
return_sequences=False)(F0s)
F1 = layers.ConvLSTM2D(filters=n_filter, kernel_size=3, padding='same', activation='relu',
return_sequences=False)(F1s)
F2 = layers.ConvLSTM2D(filters=n_filter, kernel_size=3, padding='same', activation='relu',
return_sequences=False)(F2s)
F3 = layers.ConvLSTM2D(filters=n_filter, kernel_size=3, padding='same', activation='relu',
return_sequences=False)(F3s)
F4 = layers.ConvLSTM2D(filters=n_filter, kernel_size=3, padding='same', activation='relu',
return_sequences=False)(F4s)
# 16 -> 32
deconv4 = layers.UpSampling2D((2, 2))(F4)
uconv4 = layers.concatenate([deconv4, F3])
uconv4 = layers.Dropout(DROPOUT)(uconv4)
uconv4 = layers.Conv2D(n_filter * 8, (3, 3), activation=None, padding="same",
kernel_initializer=KERNEL_INITIALIZER,
kernel_regularizer=regularizers.l2(WEIGHT_DECAY)
)(uconv4)
uconv4 = residual_block(uconv4, n_filter * 8)
uconv4 = residual_block(uconv4, n_filter * 8, True)
# 32 -> 64
deconv3 = layers.UpSampling2D((2, 2))(uconv4)
uconv3 = layers.concatenate([deconv3, F2])
uconv3 = layers.Dropout(DROPOUT)(uconv3)
uconv3 = layers.Conv2D(n_filter * 4, (3, 3), activation=None, padding="same",
kernel_initializer=KERNEL_INITIALIZER,
kernel_regularizer=regularizers.l2(WEIGHT_DECAY)
)(uconv3)
uconv3 = residual_block(uconv3, n_filter * 4)
uconv3 = residual_block(uconv3, n_filter * 4, True)
# 64 -> 128
deconv2 = layers.UpSampling2D((2, 2))(uconv3)
uconv2 = layers.concatenate([deconv2, F1])
uconv2 = layers.Dropout(DROPOUT)(uconv2)
uconv2 = layers.Conv2D(n_filter * 2, (3, 3), activation=None, padding="same",
kernel_initializer=KERNEL_INITIALIZER,
kernel_regularizer=regularizers.l2(WEIGHT_DECAY)
)(uconv2)
uconv2 = residual_block(uconv2, n_filter * 2)
uconv2 = residual_block(uconv2, n_filter * 2, True)
# 128 -> 256
deconv1 = layers.UpSampling2D((2, 2))(uconv2)
uconv1 = layers.concatenate([deconv1, F0])
uconv1 = layers.Dropout(DROPOUT)(uconv1)
uconv1 = layers.Conv2D(n_filter * 1, (3, 3), activation=None, padding="same",
kernel_initializer=KERNEL_INITIALIZER,
kernel_regularizer=regularizers.l2(WEIGHT_DECAY)
)(uconv1)
uconv1 = residual_block(uconv1, n_filter * 1)
uconv1 = residual_block(uconv1, n_filter * 1, True)
x = layers.Conv2D(n_class, (1, 1), padding="same", activation=None)(uconv1)
if n_class == 1 or (n_class == 2 and one_hot is False):
x = layers.Activation('sigmoid')(x)
else:
x = layers.Activation('softmax')(x)
return models.Model(img_input, x)
