def MyModel_Conv2D( name='MyModel_Conv2D', sta_num=20, lt_shape=(3,15,20), st_shape=(72,20), wd_shape=(4,1), ef_shape=(4,24), op_shape=(1,20), optimizer='adam', metrics=['mae'], loss='mse' )->Model: print('model name:',name) lt=layers.Input(shape=lt_shape) st=layers.Input(shape=st_shape) lt_wd=layers.Input(shape=(lt_shape[0],wd_shape[1])) st_wd=layers.Input(shape=(wd_shape[1],)) lt_ef=layers.Input(shape=(lt_shape[0],ef_shape[1])) st_ef=layers.Input(shape=(ef_shape[1],)) y1=layers.Reshape((lt_shape[0],lt_shape[1],lt_shape[2],1))(lt) y1=layers.ConvLSTM2D(filters=64,kernel_size=(6,6),padding='same',return_sequences=True)(y1) y1=layers.Dropout(0.5)(y1) y1=layers.ConvLSTM2D(filters=64,kernel_size=(6,6),padding='same',return_sequences=True)(y1) y1=layers.Dropout(0.25)(y1) y1=layers.Dense(1)(y1) y1=layers.Reshape(lt_shape)(y1) y1=layers.TimeDistributed(attention())(y1) y1=layers.Flatten()(y1) y2=layers.Reshape((st_shape[0],st_shape[1],1))(st) y2=layers.Conv2D(filters=64,kernel_size=(6,6),padding='same')(y2) y2=layers.ReLU()(y2) y2=layers.Conv2D(filters=64,kernel_size=(6,6),padding='same')(y2) y2=layers.ReLU()(y2) y2=layers.Dense(1)(y2) y2=layers.Reshape(st_shape)(y2) y2=attention()(y2) ltef=layers.concatenate([lt_wd,lt_ef]) stef=layers.concatenate([st_wd,st_ef]) stef=layers.Reshape((1,wd_shape[1]+ef_shape[1]))(stef) ef=layers.concatenate([ltef,stef],axis=1) ef=layers.LSTM(64,return_sequences=True)(ef) ef=layers.Dropout(0.2)(ef) ef=layers.LSTM(64,return_sequences=True)(ef) ef=layers.Dropout(0.2)(ef) ef=layers.ReLU()(ef) ef=attention()(ef) y=layers.concatenate([y1,y2,ef]) y=layers.Dense(sta_num)(y) model=Model([lt,st,lt_wd,st_wd,lt_ef,st_ef],[y],name='MyModel') model.compile(optimizer=optimizer,loss=loss,metrics=metrics) return model
def build_model(): """Build keras model. Returns: keras model """ #define model input data_shape = (3, 3, 495, 436) prev_frames = layers.Input(shape=data_shape, name='prev_frames') #define layers with device('/device:GPU:0'): convlstm_0 = layers.ConvLSTM2D(filters=32, kernel_size=(7, 7), padding='same', return_sequences=True, return_state=False, activation='tanh', recurrent_activation='hard_sigmoid', kernel_initializer='glorot_uniform', unit_forget_bias=True, data_format='channels_first', name='convlstm_0') convlstm_1 = layers.ConvLSTM2D(filters=64, kernel_size=(7, 7), padding='same', return_sequences=False, return_state=True, activation='tanh', recurrent_activation='hard_sigmoid', kernel_initializer='glorot_uniform', unit_forget_bias=True, data_format='channels_first', name='convlstm_1') with device('/device:GPU:1'): convlstm_2 = layers.ConvLSTM2D(filters=64, kernel_size=(7, 7), padding='same', return_sequences=True, return_state=False, activation='tanh', recurrent_activation='hard_sigmoid', kernel_initializer='glorot_uniform', unit_forget_bias=True, data_format='channels_first', name='convlstm_2') convlstm_3 = layers.ConvLSTM2D(filters=3, kernel_size=(7, 7), padding='same', return_sequences=True, return_state=False, activation='tanh', recurrent_activation='hard_sigmoid', kernel_initializer='glorot_uniform', unit_forget_bias=True, data_format='channels_first', name='convlstm_3') #define model structure #encoder x = convlstm_0(prev_frames) x = convlstm_1(x)[-1] #flatten, repeat and reshape x = layers.Flatten()(x) x = layers.RepeatVector(3)(x) x = layers.Reshape((3, 64, 495, 436))(x) #decoder x = convlstm_2(x) x = convlstm_3(x) #build and return model seq_model = models.Model(inputs=prev_frames, outputs=x) return seq_model
def load_model(): input_tensor = Input(shape=(SequenceLength, IMSIZE[0], IMSIZE[1], 3)) x = layers.ConvLSTM2D(32, kernel_size=(7, 7), padding='valid', return_sequences=True)(input_tensor) x = layers.Activation('relu')(x) x = layers.MaxPooling3D(pool_size=(1, 2, 2))(x) x = layers.ConvLSTM2D(64, kernel_size=(5, 5), padding='valid', return_sequences=True)(x) x = layers.MaxPooling3D(pool_size=(1, 2, 2))(x) x = layers.ConvLSTM2D(96, kernel_size=(3, 3), padding='valid', return_sequences=True)(x) x = layers.Activation('relu')(x) x = layers.ConvLSTM2D(96, kernel_size=(3, 3), padding='valid', return_sequences=True)(x) x = layers.Activation('relu')(x) x = layers.ConvLSTM2D(96, kernel_size=(3, 3), padding='valid', return_sequences=True)(x) x = layers.MaxPooling3D(pool_size=(1, 2, 2))(x) x = layers.Dense(320)(x) x = layers.Activation('relu')(x) x = layers.Dropout(0.5)(x) out_shape = x.get_shape().as_list() x = layers.Reshape( (SequenceLength, out_shape[2] * out_shape[3] * out_shape[4]))(x) x = layers.Bidirectional(layers.LSTM(64, return_sequences=True), merge_mode='concat')(x) x = layers.Dropout(0.5)(x) x = layers.Flatten()(x) x = layers.Dense(128, activation='relu')(x) output_tensor = layers.Dense(N_CLASSES, activation='softmax')(x) model = Model(input_tensor, output_tensor) model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy']) return model
def predict(n = 0): h5f = h5py.File('D:/total_data_ro.h5','r') test_data = h5f['total_2019'][:] test_data_gen = h5f['gen_2019'][:] h5f.close() sc = RobustScaler() sc = joblib.load('D:/scaler_gen.pkl') test_data_gen = test_data_gen.reshape(-1, 1) img_input = Input(shape=(None, 75, 75, 10), name='images') convlstm2d = layers.ConvLSTM2D(filters=20, kernel_size=(3, 3), input_shape=(None, 75, 75, 10), data_format='channels_last', return_sequences=True, padding='same')(img_input) batch_norm = layers.BatchNormalization()(convlstm2d) convlstm2d_1x1 = layers.ConvLSTM2D(filters=1, kernel_size=(3, 3), data_format='channels_last',padding='same')(batch_norm) batch_norm = layers.BatchNormalization()(convlstm2d_1x1) flatten = layers.Flatten()(batch_norm) Dense_1_list = [layers.Dense(units=1)(flatten) for i in range(72)] model = Model(img_input, Dense_1_list) model.compile(optimizer=rmsprop(lr=0.001), loss=['mae' for i in range(72)]) model.load_weights('D:/gen_pred_72hours.h5') test_sample, gen_sample = generator(test_data, test_data_gen, 120, 72, 0, None, shuffle=True, batch_size=1) test_sample = np.rollaxis(test_sample, 2, 5) y_pred = model.predict(test_sample) y_pred = np.array(y_pred).reshape(-1,1) #y_test = np.array(gen_sample) y_pred = sc.inverse_transform(y_pred) sc = MinMaxScaler(feature_range=(0, 40)) plt.plot(sc.fit_transform(y_pred), label='pred') #plt.plot(gen_sample, label='real') plt.legend() plt.savefig("media/test" + str(n) + ".png") print('save img') plt.clf() plt.cla() plt.close() #plt.show()
def ConvLSTMCell(data): conv = layers.Conv2D(64, (3, 3), padding='same', activation='relu', use_bias=True)(data) conv = ResUnit(conv, 64, (3, 3)) conv = ResUnit(conv, 64, (3, 3)) lstm_data = layers.Reshape((data.shape[-1], 20, 20, 1))(data) lstm = layers.ConvLSTM2D(64, (3, 3), strides=(1, 1), padding='same', activation='relu', use_bias=True)(lstm_data) lstm = layers.Conv2D(64, (1, 1), padding='same', activation='relu', use_bias=True)(lstm) conv = layers.Conv2D(64, (1, 1), padding='same', activation='relu', use_bias=True)(conv) res = layers.Add()([lstm, conv]) return res
def ConvLSTM_predictor2(num_classes=1, input_shape=(10, 500, 320, 5)): model = Sequential() # ConvLSTM2D (batch, steps, width, height, features) model.add( layers.ConvLSTM2D(30, 10, return_sequences=True, input_shape=input_shape)) model.add(layers.Activation("relu")) model.add(layers.ConvLSTM2D(num_classes, 5, return_sequences=False)) model.add(layers.Activation("relu")) model.compile(loss=euclidean_loss, optimizer=Adam(lr=1e-3)) return model
def big_conv_lstm_readout_net(feature_map_in_seqs, feature_map_size, drop_rate, gaze_prior=None): batch_size = tf.shape(feature_map_in_seqs)[0] n_step = tf.shape(feature_map_in_seqs)[1] n_channel = int(feature_map_in_seqs.get_shape()[4]) feature_map = tf.reshape(feature_map_in_seqs, [batch_size*n_step, feature_map_size[0], feature_map_size[1], n_channel]) x = layers.Conv2D(32, (1, 1), activation='relu', name='readout_conv1')(feature_map) x = layers.core.Dropout(drop_rate)(x) x = layers.Conv2D(16, (1, 1), activation='relu', name='readout_conv2')(x) x = layers.core.Dropout(drop_rate)(x) x = layers.Conv2D(8, (1, 1), activation='relu', name='readout_conv3')(x) x = layers.core.Dropout(drop_rate)(x) x = layers.Conv2D(1, (1, 1), activation='relu', name='readout_conv4')(x) x = layers.core.Dropout(drop_rate)(x) #x = layers.core.Reshape((-1,))(x) temp_shape = x.get_shape()[1:4] temp_shape = [int(s) for s in temp_shape] x = tf.reshape(x, [batch_size, n_step, temp_shape[0], temp_shape[1], temp_shape[2]]) x = layers.ConvLSTM2D(filters=1, kernel_size=(3,3), strides=(1,1), padding='same', dropout=drop_rate, recurrent_dropout=drop_rate, return_sequences=True)(x) x = wps.TimeDistributed(layers.Conv2D(1, (1, 1), activation='linear'))(x) x = tf.reshape(x, [batch_size*n_step, feature_map_size[0], feature_map_size[1], 1]) x = GaussianSmooth(kernel_size = GAUSSIAN_KERNEL_SIZE, name='gaussian_smooth')(x) logits = tf.reshape(x, [-1, feature_map_size[0]*feature_map_size[1]]) #gaze prior map if gaze_prior is not None: #predicted annotation before adding prior pre_prior_logits = logits gaze_prior = np.maximum(gaze_prior, EPSILON*np.ones(gaze_prior.shape)) gaze_prior = gaze_prior.astype(np.float32) log_prior = np.log(gaze_prior) log_prior_1d = np.reshape(log_prior, (1, -1)) log_prior_unit_tensor = tf.constant(log_prior_1d) log_prior_tensor = tf.matmul(tf.ones((tf.shape(pre_prior_logits)[0],1)), log_prior_unit_tensor) log_prior_tensor = tf.reshape(log_prior_tensor, [-1, feature_map_size[0]*feature_map_size[1]]) logits = tf.add(pre_prior_logits, log_prior_tensor) if gaze_prior is None: return logits else: return logits, pre_prior_logits
def conv2dlstm_model(window_size=32, n_notes=58): notes_model = models.Sequential() notes_model.add(layers.ConvLSTM2D(32, 3, return_sequences=True, padding='same', input_shape=(None, window_size, None, 3))) notes_model.add(layers.MaxPool3D(2, 2)) notes_model.add(layers.ConvLSTM2D(64, 3, padding='same')) notes_model.add(layers.GlobalAveragePooling2D()) beats_input = layers.Input(shape=(16,)) features = layers.concatenate([notes_model.output, beats_input]) dropout1 = layers.Dropout(0.2)(features) fc_1 = layers.Dense(100, activation='relu')(dropout1) dropout2 = layers.Dropout(0.2)(fc_1) output_notes = layers.Dense(n_notes + 1, activation='softmax')(dropout2) output_slur = layers.Dense(1, activation='sigmoid')(fc_1) return models.Model(inputs=[notes_model.input, beats_input], outputs=[output_notes, output_slur])
def ConvLSTM_predictor1(num_classes=1, input_shape=(10, 500, 320, 5)): model = Sequential() #Embed down to latent space. # ConvLSTM2D (batch, steps, width, height, features) model.add( layers.ConvLSTM2D(40, 10, return_sequences=True, input_shape=input_shape)) model.add(layers.BatchNormalization()) model.add(layers.LeakyReLU(alpha=0.2)) model.add(layers.ConvLSTM2D(30, 5, strides=2, return_sequences=True)) model.add(layers.BatchNormalization()) model.add(layers.LeakyReLU(alpha=0.2)) # Latent Space model.add(layers.ConvLSTM2D(20, 5, return_sequences=False)) model.add(layers.BatchNormalization()) model.add(layers.LeakyReLU(alpha=0.2)) # Rebuild latent embedding into future prediction. model.add(layers.Conv2DTranspose(40, 5)) model.add(layers.BatchNormalization()) model.add(layers.LeakyReLU(alpha=0.2)) model.add(layers.Conv2DTranspose(30, 10, strides=2)) model.add(layers.BatchNormalization()) model.add(layers.LeakyReLU(alpha=0.2)) model.add(layers.Conv2DTranspose(20, 30)) model.add(layers.BatchNormalization()) model.add(layers.LeakyReLU(alpha=0.2)) # Rebuilt future model.add(layers.Conv2DTranspose(num_classes, 5)) model.compile(loss=euclidean_loss, optimizer=Adam(lr=1e-3)) return model
def build_model(input_shape): model = krm.Sequential() model.add( krl.ConvLSTM2D(filters=64, kernel_size=3, return_sequences=False, data_format="channels_last", input_shape=input_shape[1:])) model.add(krl.Dropout(0.2)) model.add(krl.Flatten()) model.add(krl.Dense(256, activation="relu")) model.add(krl.Dropout(0.3)) model.add(krl.Dense(input_shape[-1], activation="softmax")) return model
def conv_lstm(data_train, label_train, data_test, label_test, block_num): data_train = data_train.reshape(data_train.shape[0], block_num, 1, int(data_train.shape[1] / block_num), data_train.shape[2]) data_test = data_test.reshape(data_test.shape[0], block_num, 1, int(data_test.shape[1] / block_num), data_test.shape[2]) label_train = np_utils.to_categorical(label_train - 1) label_test = np_utils.to_categorical(label_test - 1) #print("Model building begin:") conv_lstm = models.Sequential() conv_lstm.add( layers.ConvLSTM2D(filters=128, kernel_size=(1, 3), activation='relu', input_shape=(block_num, 1, data_train.shape[3], data_train.shape[4]))) conv_lstm.add(layers.Dropout(0.5)) conv_lstm.add(layers.Flatten()) conv_lstm.add(layers.Dense(64, activation='relu')) conv_lstm.add(layers.Dense(6, activation='softmax')) #cnn_1d.summary() plot_model(conv_lstm, show_shapes=True, to_file='conv.png', dpi=200) conv_lstm.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc']) history = conv_lstm.fit(data_train, label_train, epochs=25, batch_size=64, verbose=0, validation_split=0.2) _, te_acc = conv_lstm.evaluate(data_test, label_test, batch_size=32, verbose=0) return history, history.history['acc'][4], history.history['val_acc'][ 4], te_acc, history.history['loss'][4], history.history['val_loss'][4]
def thick_conv_lstm_readout_net(feature_map_in_seqs, feature_map_size, drop_rate, gaze_prior=None, output_embedding=False): batch_size = tf.shape(feature_map_in_seqs)[0] n_step = tf.shape(feature_map_in_seqs)[1] n_channel = int(feature_map_in_seqs.get_shape()[4]) feature_map = tf.reshape(feature_map_in_seqs, [ batch_size * n_step, feature_map_size[0], feature_map_size[1], n_channel ]) x = layers.Conv2D(16, (1, 1), activation='relu', name='readout_conv1')(feature_map) x = layers.BatchNormalization()(x) x = layers.core.Dropout(drop_rate)(x) x = layers.Conv2D(32, (1, 1), activation='relu', name='readout_conv2')(x) x = layers.BatchNormalization()(x) x = layers.core.Dropout(drop_rate)(x) x = layers.Conv2D(8, (1, 1), activation='relu', name='readout_conv3')(x) x = layers.BatchNormalization()(x) # reshape into temporal sequence temp_shape = x.get_shape()[1:4] temp_shape = [int(s) for s in temp_shape] x = tf.reshape( x, [batch_size, n_step, temp_shape[0], temp_shape[1], temp_shape[2]]) n_channel = 15 initial_c = layers.Conv2D(n_channel, (3, 3), activation='tanh', padding='same')(layers.core.Dropout(drop_rate)( x[:, 0])) initial_c = layers.core.Dropout(drop_rate)(initial_c) initial_h = layers.Conv2D(n_channel, (3, 3), activation='tanh', padding='same')(layers.core.Dropout(drop_rate)( x[:, 0])) initial_h = layers.core.Dropout(drop_rate)(initial_h) conv_lstm = layers.ConvLSTM2D(filters=n_channel, kernel_size=(3, 3), strides=(1, 1), padding='same', dropout=drop_rate, recurrent_dropout=drop_rate, return_sequences=True) x = conv_lstm([x, initial_c, initial_h]) x = wps.TimeDistributed( layers.Conv2D(n_channel, (1, 1), activation='linear'))(x) x = tf.reshape(x, [ batch_size * n_step, feature_map_size[0], feature_map_size[1], n_channel ]) x = layers.BatchNormalization()(x) embed = x x = layers.Conv2D(1, (1, 1), activation='linear')(x) x = tf.reshape( x, [batch_size * n_step, feature_map_size[0], feature_map_size[1], 1]) raw_logits = tf.reshape(x, [-1, feature_map_size[0] * feature_map_size[1]]) logits = tf.reshape(x, [-1, feature_map_size[0] * feature_map_size[1]]) #gaze prior map if gaze_prior is not None: #predicted annotation before adding prior pre_prior_logits = logits gaze_prior = np.maximum(gaze_prior, EPSILON * np.ones(gaze_prior.shape)) gaze_prior = gaze_prior.astype(np.float32) log_prior = np.log(gaze_prior) log_prior_1d = np.reshape(log_prior, (1, -1)) log_prior_unit_tensor = tf.constant(log_prior_1d) log_prior_tensor = tf.matmul( tf.ones((tf.shape(pre_prior_logits)[0], 1)), log_prior_unit_tensor) log_prior_tensor = tf.reshape( log_prior_tensor, [-1, feature_map_size[0] * feature_map_size[1]]) logits = tf.add(pre_prior_logits, log_prior_tensor) if output_embedding: return logits, embed, raw_logits if gaze_prior is None: return logits else: return logits, pre_prior_logits
def conv_lstm_planner(peripheral_feature_map_seqs, foveal_feature_seqs, drop_rate): # combine feature maps if foveal_feature_seqs is None: feature_map_seqs = peripheral_feature_map_seqs elif peripheral_feature_map_seqs is None: feature_map_seqs = foveal_feature_seqs else: feature_map_seqs = tf.concat( [peripheral_feature_map_seqs, foveal_feature_seqs], axis=-1) # get the shape batch_size = tf.shape(feature_map_seqs)[0] n_step = tf.shape(feature_map_seqs)[1] temp_shape = feature_map_seqs.get_shape()[2:5] temp_shape = [int(s) for s in temp_shape] feature_map_size = temp_shape[0:2] n_channel = temp_shape[2] conv_lstm = layers.ConvLSTM2D(filters=5, kernel_size=(3, 3), strides=(1, 1), padding='same', dropout=drop_rate, recurrent_dropout=drop_rate, return_sequences=True) initial_c = layers.Conv2D(5, (3, 3), activation='tanh', padding='same')( layers.core.Dropout(drop_rate)(feature_map_seqs[:, 0])) initial_c = layers.core.Dropout(drop_rate)(initial_c) initial_h = layers.Conv2D(5, (3, 3), activation='tanh', padding='same')( layers.core.Dropout(drop_rate)(feature_map_seqs[:, 0])) initial_h = layers.core.Dropout(drop_rate)(initial_h) x = conv_lstm([feature_map_seqs, initial_c, initial_h]) # track weights kernel_weights = conv_lstm.weights[0] # shape is [3, 3, 8+8, 5*4] if peripheral_feature_map_seqs is None: peripheral_weights = None peripheral_n_channels = 0 else: peripheral_n_channels = tf.shape(peripheral_feature_map_seqs)[-1] peripheral_weights = kernel_weights[:, :, 0:peripheral_n_channels, :] if foveal_feature_seqs is None: foveal_weights = None else: foveal_weights = kernel_weights[:, :, peripheral_n_channels:, :] x = tf.reshape( x, [batch_size * n_step, feature_map_size[0], feature_map_size[1], 5]) x = layers.BatchNormalization()(x) temp_shape = x.get_shape()[1:4] temp_shape = [int(s) for s in temp_shape] x = tf.reshape( x, [batch_size, n_step, temp_shape[0] * temp_shape[1] * temp_shape[2]]) x = wps.TimeDistributed(layers.Dense(units=512, activation='linear'))(x) logits = tf.reshape(x, [batch_size * n_step, 512]) return logits, peripheral_weights, foveal_weights
def my_model(encoder='VGG', input_size=(256, 256, 1), k_shot=1, learning_rate=1e-4): # Get the encoder encoder_t = encoder if encoder_t == 'VGG_b3': encoder = EM.vgg_encoder_b3(input_size=input_size) elif encoder_t == 'VGG_b4': encoder = EM.vgg_encoder_b4(input_size=input_size) elif encoder_t == 'VGG_b34': encoder = EM.vgg_encoder_b34(input_size=input_size) elif encoder_t == 'VGG_b5': encoder = EM.vgg_encoder_b5(input_size=input_size) elif encoder_t == 'VGG_b345': encoder = EM.vgg_encoder_b345(input_size=input_size) elif encoder_t == 'VGG_b35': encoder = EM.vgg_encoder_b35(input_size=input_size) elif encoder_t == 'VGG_b45': encoder = EM.vgg_encoder_b45(input_size=input_size) elif encoder_t == 'RN': encoder = EM.rn_encoder(input_size=input_size) else: print('Encoder is not defined yet') S_mask = layers.Input( (k_shot, int(input_size[0] / 4), int(input_size[1] / 4), 1)) S_input = layers.Input( (k_shot, input_size[0], input_size[1], input_size[2])) #print(S_input.shape) Q_input = layers.Input(input_size) #print(S_mask.shape) ## K shot kshot_encoder = keras.models.Sequential() kshot_encoder.add( layers.TimeDistributed(encoder, input_shape=(k_shot, input_size[0], input_size[1], input_size[2]))) ## Encode support and query sample s_encoded = kshot_encoder(S_input) q_encoded = encoder(Q_input) print("before") print(s_encoded.shape) print(q_encoded.shape) #if encoder_t == "RN": s_encoded = layers.TimeDistributed( layers.Conv2D(128, (3, 3), activation='relu', padding='same'))(s_encoded) q_encoded = layers.Conv2D(128, (3, 3), activation='relu', padding='same')(q_encoded) print("after") print(s_encoded.shape) print(q_encoded.shape) ## Difference of Gussian Pyramid parameters kernet_shapes = [3, 5, 7, 9] k_value = np.power(2, 1 / 3) sigma = 1.6 ## Kernel weights for Gussian pyramid Sigma1_kernel = get_kernel_gussian(kernel_size=kernet_shapes[0], Sigma=sigma * np.power(k_value, 1), in_channels=128) Sigma2_kernel = get_kernel_gussian(kernel_size=kernet_shapes[1], Sigma=sigma * np.power(k_value, 2), in_channels=128) Sigma3_kernel = get_kernel_gussian(kernel_size=kernet_shapes[2], Sigma=sigma * np.power(k_value, 3), in_channels=128) Sigma4_kernel = get_kernel_gussian(kernel_size=kernet_shapes[3], Sigma=sigma * np.power(k_value, 4), in_channels=128) Sigma1_layer = layers.TimeDistributed( layers.DepthwiseConv2D(kernet_shapes[0], use_bias=False, padding='same')) Sigma2_layer = layers.TimeDistributed( layers.DepthwiseConv2D(kernet_shapes[1], use_bias=False, padding='same')) Sigma3_layer = layers.TimeDistributed( layers.DepthwiseConv2D(kernet_shapes[2], use_bias=False, padding='same')) Sigma4_layer = layers.TimeDistributed( layers.DepthwiseConv2D(kernet_shapes[3], use_bias=False, padding='same')) ## Gussian filtering x1 = Sigma1_layer(s_encoded) x2 = Sigma2_layer(s_encoded) x3 = Sigma3_layer(s_encoded) x4 = Sigma4_layer(s_encoded) #S_mask = smx #print(S_mask.shape) #print("x1.shape") #print(x1.shape) #print(s_encoded.shape) DOG1 = layers.Subtract()([s_encoded, x1]) DOG2 = layers.Subtract()([x1, x2]) DOG3 = layers.Subtract()([x2, x3]) DOG4 = layers.Subtract()([x3, x4]) #print("Dog.shape") #print(S_mask.shape) #print(DOG1.shape) s_1 = '' s_2 = '' s_3 = '' s_4 = '' ## Global Representation if encoder_t == 'RN': s_1 = RN_GlobalAveragePooling2D_r(S_mask)(DOG1) s_2 = RN_GlobalAveragePooling2D_r(S_mask)(DOG2) s_3 = RN_GlobalAveragePooling2D_r(S_mask)(DOG3) s_4 = RN_GlobalAveragePooling2D_r(S_mask)(DOG4) else: s_1 = GlobalAveragePooling2D_r(S_mask)(DOG1) s_2 = GlobalAveragePooling2D_r(S_mask)(DOG2) s_3 = GlobalAveragePooling2D_r(S_mask)(DOG3) s_4 = GlobalAveragePooling2D_r(S_mask)(DOG4) ## Common Representation of Support and Query sample s_1 = common_representation(s_1, q_encoded) s_2 = common_representation(s_2, q_encoded) s_3 = common_representation(s_3, q_encoded) s_4 = common_representation(s_4, q_encoded) ## Bidirectional Convolutional LSTM on Pyramid s_3D = layers.concatenate([s_1, s_2, s_3, s_4], axis=1) LSTM_f = layers.ConvLSTM2D(filters=128, kernel_size=(3, 3), padding='same', return_sequences=False, go_backwards=False, kernel_initializer='he_normal')(s_3D) LSTM_b = layers.ConvLSTM2D(filters=128, kernel_size=(3, 3), padding='same', return_sequences=False, go_backwards=True, kernel_initializer='he_normal')(s_3D) Bi_rep = layers.Add()([LSTM_f, LSTM_b]) ## Decode to query segment x = layers.Conv2D(128, 3, padding='same', kernel_initializer='he_normal')(Bi_rep) x = layers.BatchNormalization(axis=3)(x) x = layers.Activation('relu')(x) x = layers.UpSampling2D(size=(2, 2))(x) x = layers.Conv2D(128, 3, padding='same', kernel_initializer='he_normal')(x) x = layers.BatchNormalization(axis=3)(x) x = layers.Activation('relu')(x) x = layers.UpSampling2D(size=(2, 2))(x) x = layers.Conv2D(128, 3, padding='same', kernel_initializer='he_normal')(x) x = layers.BatchNormalization(axis=3)(x) x = layers.Activation('relu')(x) x = layers.Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(x) x = layers.Conv2D(2, 3, activation='relu', padding='same', kernel_initializer='he_normal')(x) final = layers.Conv2D(1, 1, activation='sigmoid')(x) print("AAA") print(final.shape) seg_model = Model(inputs=[S_input, S_mask, Q_input], outputs=final) ## Load Guassian weights and make it unlearnable Sigma1_layer.set_weights([Sigma1_kernel]) Sigma2_layer.set_weights([Sigma2_kernel]) Sigma3_layer.set_weights([Sigma3_kernel]) Sigma4_layer.set_weights([Sigma4_kernel]) Sigma1_layer.trainable = False Sigma2_layer.trainable = False Sigma3_layer.trainable = False Sigma4_layer.trainable = False seg_model.compile(optimizer=keras.optimizers.Adam(lr=learning_rate), loss='binary_crossentropy', metrics=['accuracy']) return seg_model
def ConvLSTM(optimizer, init_channel, block_num): input_shape = (256, 256, 1) input_1 = L.Input(shape=input_shape) input_2 = L.Input(shape=input_shape) input_3 = L.Input(shape=input_shape) weights_input = L.Input(shape=input_shape) encoder = Sequential(name='encoder') for i in range(block_num): if i == 0: encoder.add( L.Conv2D(init_channel * (2**i), (3, 3), strides=2, activation='relu', padding='same', kernel_initializer='he_normal', input_shape=input_shape)) else: encoder.add( L.Conv2D(init_channel * (2**i), (3, 3), strides=2, activation='relu', padding='same', kernel_initializer='he_normal')) encoder.add( L.Conv2D(init_channel * (2**i), (3, 3), strides=1, activation='relu', padding='same', kernel_initializer='he_normal')) encoded_1 = encoder(input_1) encoded_2 = encoder(input_2) encoded_3 = encoder(input_3) reshape = (1, *encoder.output_shape[1:]) reshaped_1 = L.Reshape(reshape)(encoded_1) reshaped_2 = L.Reshape(reshape)(encoded_2) reshaped_3 = L.Reshape(reshape)(encoded_3) concat = L.Concatenate(axis=1)([reshaped_1, reshaped_2, reshaped_3]) convlstm = L.ConvLSTM2D(init_channel * 4, (3, 3), strides=1, padding='same', activation='relu', kernel_initializer='he_normal', return_sequences=False)(concat) decoder_shape = (i.value for i in convlstm.get_shape()[1:]) decoder = Sequential(name='decoder') for i in range(block_num): if i == 0: decoder.add( L.Conv2DTranspose(init_channel * (2**(block_num - i)), (3, 3), strides=2, activation='relu', padding='same', kernel_initializer='he_normal', input_shape=decoder_shape)) else: decoder.add( L.Conv2DTranspose(init_channel * (2**(block_num - i)), (3, 3), strides=2, activation='relu', padding='same', kernel_initializer='he_normal')) decoder.add( L.Conv2D(init_channel * (2**(block_num - i)), (3, 3), strides=1, activation='relu', padding='same', kernel_initializer='he_normal')) decoder.add( L.Conv2D(1, (3, 3), strides=1, activation='sigmoid', padding='same', kernel_initializer='he_normal')) output = decoder(convlstm) model = Model(inputs=[input_1, input_2, input_3, weights_input], outputs=output) model.compile(optimizer=optimizer, loss=utils.custom_loss(weights_input)) print(model.summary()) return model
def deeplabv3p_lstm(input_shape=(128, 128, 16), n_class=1, one_hot=False, backbone_name='mobilenetv2', backbone_weights=None, output_stride=16, atrous_rates=(2, 3, 5), input_split=4): def batch_relu(inputs): _x = layers.BatchNormalization()(inputs) _x = layers.Activation('relu')(_x) return _x assert input_shape[-1] % input_split == 0 img_input = layers.Input(shape=input_shape) img_inputs = [] channels_per_split = input_shape[-1] // input_split for i in range(input_split): _x = layers.Lambda(lambda xx: xx[:, :, :, i * channels_per_split:( i + 1) * channels_per_split])(img_input) img_inputs.append(_x) sub_img_input, features = build_encoder( (input_shape[0], input_shape[1], input_shape[-1] // input_split), None, backbone_name, backbone_weights, output_stride) [f1, f2, f3, f4, f5] = features feature_extractor = Model(sub_img_input, [f2, f4]) F2s = [] F4s = [] for i in range(input_split): F24 = feature_extractor(img_inputs[i]) F2s.append(F24[0]) F4s.append(F24[1]) F2s = layers.Lambda(lambda xx: K.stack(xx, axis=1))(F2s) F4s = layers.Lambda(lambda xx: K.stack(xx, axis=1))(F4s) F2 = layers.ConvLSTM2D(filters=48, kernel_size=3, padding='same', activation='relu', return_sequences=False)(F2s) F4 = layers.ConvLSTM2D(filters=48, kernel_size=3, padding='same', activation='relu', return_sequences=False)(F4s) # 多层级特征融合 x_h = atrous_spatial_pyramid_pooling(F4, atrous_rates) x_h = layers.Conv2D(256, kernel_size=1, use_bias=False)(x_h) x_h = batch_relu(x_h) x_h = layers.Dropout(0.2)(x_h) x_h = layers.UpSampling2D((4, 4), interpolation='bilinear')(x_h) x_l = layers.Conv2D(48, kernel_size=1, use_bias=False)(F2) x_l = batch_relu(x_l) x = layers.Concatenate()([x_h, x_l]) x = _sep_conv_bn(x, 256, 'decoder_conv0', depth_activation=True, epsilon=1e-5) x = _sep_conv_bn(x, 256, 'decoder_conv1', depth_activation=True, epsilon=1e-5) x = layers.Conv2D(n_class, kernel_size=1, use_bias=False)(x) x = layers.UpSampling2D((4, 4), interpolation='bilinear')(x) if n_class == 1 or (n_class == 2 and one_hot is False): x = layers.Activation('sigmoid')(x) else: x = layers.Activation('softmax')(x) return Model(img_input, x)
def getBiConvLSTM2d(input_image,filters,kernel_size,name): L1 = KL.Bidirectional(KL.ConvLSTM2D(filters=filters, kernel_size=kernel_size,activation='relu', padding='same', return_sequences=True))(input_image) L1 = KL.BatchNormalization(name="batchNormL_"+name)(L1) return L1
flt_size = 5 # Training settings opt = "adam" # Optimizer loss = "mse" epochs = 60 batch_size = 16 # %% # Build the LSTM neural network tf.keras.backend.clear_session() flt_tp = (flt_size, flt_size) conv_kwargs = dict(kernel_size=flt_tp, padding="same") # Encoder inputs = layers.Input(shape=x_train.shape[1:]) x = layers.ConvLSTM2D(flt[0], return_sequences=True, **conv_kwargs)(inputs) x = layers.ConvLSTM2D(flt[1], return_sequences=True, **conv_kwargs)(x) x = layers.ConvLSTM2D(flt[2], return_sequences=False, **conv_kwargs)(x) x = layers.Conv2D(flt[3], activation=act, **conv_kwargs)(x) out = layers.Conv2D(x_train.shape[-1], flt_tp, activation="linear", padding="same")(x) # Mount the LSTM lstm = Model(inputs, out, name="LSTM neural network") # %% # Show the architecture lstm.summary()
elif layer.__class__.__name__ in ('LSTM', 'ConvLSTM2D'): W = [np.split(w, 4, axis=-1) for w in weights] for w in W: w[2], w[1] = w[1], w[2] return sum(map(list, zip(*W)), []) elif layer.__class__.__name__ == 'Conv2DTranspose': return [np.transpose(weights[0], (2, 3, 0, 1)), weights[1]] return weights @keras_test @pytest.mark.parametrize("layer", [ layers.GRU(2, input_shape=[3, 5]), layers.LSTM(2, input_shape=[3, 5]), layers.ConvLSTM2D(5, (3, 3), input_shape=[6, 6, 6, 6], data_format='channels_first'), ]) def test_preprocess_weights_for_loading(layer): model = Sequential([layer]) weights1 = layer.get_weights() weights2 = topology.preprocess_weights_for_loading( layer, convert_weights(layer, weights1), original_keras_version='1') assert all([np.allclose(x, y, 1e-5) for (x, y) in zip(weights1, weights2)]) @keras_test @pytest.mark.parametrize("layer", [ layers.Conv2D(2, (3, 3), input_shape=[5, 5, 3]),
def resunet_lstm(input_shape=(128, 128, 16), n_class=1, n_filter=64, one_hot=False, input_split=4): def resunet_encoder(): img_input = layers.Input(shape=(input_shape[0], input_shape[1], input_shape[-1] // input_split)) # 256 -> 128 conv1 = layers.Conv2D(n_filter * 1, (3, 3), activation=None, padding="same", kernel_initializer=KERNEL_INITIALIZER, kernel_regularizer=regularizers.l2(WEIGHT_DECAY) )(img_input) conv1 = residual_block(conv1, n_filter * 1) conv1 = residual_block(conv1, n_filter * 1, True) pool1 = layers.MaxPooling2D((2, 2))(conv1) pool1 = layers.Dropout(DROPOUT)(pool1) # 128 -> 64 conv2 = layers.Conv2D(n_filter * 2, (3, 3), activation=None, padding="same", kernel_initializer=KERNEL_INITIALIZER, kernel_regularizer=regularizers.l2(WEIGHT_DECAY) )(pool1) conv2 = residual_block(conv2, n_filter * 2) conv2 = residual_block(conv2, n_filter * 2, True) pool2 = layers.MaxPooling2D((2, 2))(conv2) pool2 = layers.Dropout(DROPOUT)(pool2) # 64 -> 32 conv3 = layers.Conv2D(n_filter * 4, (3, 3), activation=None, padding="same", kernel_initializer=KERNEL_INITIALIZER, kernel_regularizer=regularizers.l2(WEIGHT_DECAY) )(pool2) conv3 = residual_block(conv3, n_filter * 4) conv3 = residual_block(conv3, n_filter * 4, True) pool3 = layers.MaxPooling2D((2, 2))(conv3) pool3 = layers.Dropout(DROPOUT)(pool3) # 32 -> 16 conv4 = layers.Conv2D(n_filter * 8, (3, 3), activation=None, padding="same", kernel_initializer=KERNEL_INITIALIZER, kernel_regularizer=regularizers.l2(WEIGHT_DECAY) )(pool3) conv4 = residual_block(conv4, n_filter * 8) conv4 = residual_block(conv4, n_filter * 8, True) pool4 = layers.MaxPooling2D((2, 2))(conv4) pool4 = layers.Dropout(DROPOUT)(pool4) # Middle convm = layers.Conv2D(n_filter * 16, (3, 3), activation=None, padding="same", kernel_initializer=KERNEL_INITIALIZER, kernel_regularizer=regularizers.l2(WEIGHT_DECAY) )(pool4) convm = residual_block(convm, n_filter * 16) convm = residual_block(convm, n_filter * 16, True) # convm = convolutional_block_attention_module(convm, hidden_size=n_filter, conv_size=7) return models.Model(img_input, [conv1, conv2, conv3, conv4, convm]) assert input_shape[-1] % input_split == 0 img_input = layers.Input(shape=input_shape) img_inputs = [] channels_per_split = input_shape[-1] // input_split for i in range(input_split): _x = layers.Lambda(lambda xx: xx[:, :, :, i * channels_per_split: (i + 1) * channels_per_split])(img_input) img_inputs.append(_x) feature_extractor = resunet_encoder() F0s = [] F1s = [] F2s = [] F3s = [] F4s = [] for i in range(input_split): F01234 = feature_extractor(img_inputs[i]) F0s.append(F01234[0]) F1s.append(F01234[1]) F2s.append(F01234[2]) F3s.append(F01234[3]) F4s.append(F01234[4]) F0s = layers.Lambda(lambda xx: K.stack(xx, axis=1))(F0s) F1s = layers.Lambda(lambda xx: K.stack(xx, axis=1))(F1s) F2s = layers.Lambda(lambda xx: K.stack(xx, axis=1))(F2s) F3s = layers.Lambda(lambda xx: K.stack(xx, axis=1))(F3s) F4s = layers.Lambda(lambda xx: K.stack(xx, axis=1))(F4s) F0 = layers.ConvLSTM2D(filters=n_filter, kernel_size=3, padding='same', activation='relu', return_sequences=False)(F0s) F1 = layers.ConvLSTM2D(filters=n_filter, kernel_size=3, padding='same', activation='relu', return_sequences=False)(F1s) F2 = layers.ConvLSTM2D(filters=n_filter, kernel_size=3, padding='same', activation='relu', return_sequences=False)(F2s) F3 = layers.ConvLSTM2D(filters=n_filter, kernel_size=3, padding='same', activation='relu', return_sequences=False)(F3s) F4 = layers.ConvLSTM2D(filters=n_filter, kernel_size=3, padding='same', activation='relu', return_sequences=False)(F4s) # 16 -> 32 deconv4 = layers.UpSampling2D((2, 2))(F4) uconv4 = layers.concatenate([deconv4, F3]) uconv4 = layers.Dropout(DROPOUT)(uconv4) uconv4 = layers.Conv2D(n_filter * 8, (3, 3), activation=None, padding="same", kernel_initializer=KERNEL_INITIALIZER, kernel_regularizer=regularizers.l2(WEIGHT_DECAY) )(uconv4) uconv4 = residual_block(uconv4, n_filter * 8) uconv4 = residual_block(uconv4, n_filter * 8, True) # 32 -> 64 deconv3 = layers.UpSampling2D((2, 2))(uconv4) uconv3 = layers.concatenate([deconv3, F2]) uconv3 = layers.Dropout(DROPOUT)(uconv3) uconv3 = layers.Conv2D(n_filter * 4, (3, 3), activation=None, padding="same", kernel_initializer=KERNEL_INITIALIZER, kernel_regularizer=regularizers.l2(WEIGHT_DECAY) )(uconv3) uconv3 = residual_block(uconv3, n_filter * 4) uconv3 = residual_block(uconv3, n_filter * 4, True) # 64 -> 128 deconv2 = layers.UpSampling2D((2, 2))(uconv3) uconv2 = layers.concatenate([deconv2, F1]) uconv2 = layers.Dropout(DROPOUT)(uconv2) uconv2 = layers.Conv2D(n_filter * 2, (3, 3), activation=None, padding="same", kernel_initializer=KERNEL_INITIALIZER, kernel_regularizer=regularizers.l2(WEIGHT_DECAY) )(uconv2) uconv2 = residual_block(uconv2, n_filter * 2) uconv2 = residual_block(uconv2, n_filter * 2, True) # 128 -> 256 deconv1 = layers.UpSampling2D((2, 2))(uconv2) uconv1 = layers.concatenate([deconv1, F0]) uconv1 = layers.Dropout(DROPOUT)(uconv1) uconv1 = layers.Conv2D(n_filter * 1, (3, 3), activation=None, padding="same", kernel_initializer=KERNEL_INITIALIZER, kernel_regularizer=regularizers.l2(WEIGHT_DECAY) )(uconv1) uconv1 = residual_block(uconv1, n_filter * 1) uconv1 = residual_block(uconv1, n_filter * 1, True) x = layers.Conv2D(n_class, (1, 1), padding="same", activation=None)(uconv1) if n_class == 1 or (n_class == 2 and one_hot is False): x = layers.Activation('sigmoid')(x) else: x = layers.Activation('softmax')(x) return models.Model(img_input, x)