def forward(self, x): if K.image_data_format() == 'channels_first': channel_axis = 1 else: channel_axis = -1 branch1x1 = self.branch1x1.forward(x) branch3x3 = self.branch3x3_1.forward(x) branch3x3 = [ self.branch3x3_2a.forward(branch3x3), self.branch3x3_2b.forward(branch3x3), ] branch3x3 = concatenate( branch3x3, axis=channel_axis) #branch3x3 = torch.cat(branch3x3, 1) branch3x3dbl = self.branch3x3dbl_1.forward(x) branch3x3dbl = self.branch3x3dbl_2.forward(branch3x3dbl) branch3x3dbl = [ self.branch3x3dbl_3a.forward(branch3x3dbl), self.branch3x3dbl_3b.forward(branch3x3dbl), ] branch3x3dbl = concatenate( branch3x3dbl, axis=channel_axis) #branch3x3dbl = torch.cat(branch3x3dbl, 1) branch_pool = AveragePooling2D( pool_size=(3, 3), strides=(1, 1), padding='same' )(x ) #branch_pool = F.avg_pool2d(x, kernel_size=3, stride=1, padding=1) branch_pool = self.branch_pool.forward(branch_pool) outputs = [branch1x1, branch3x3, branch3x3dbl, branch_pool] return concatenate(outputs, axis=channel_axis) #torch.cat(outputs, 1)
def forward(self, x): layer_list = [] layer_list.append( self.squeeze) #x = self.squeeze_activation(self.squeeze(x)) layer_list.append(self.squeeze_activation) concatenate( ) #torch.cat([self.expand1x1_activation(self.expand1x1(x)),self.expand3x3_activation(self.expand3x3(x))], 1) return layer_list
def branch_attention(cost_volume_3d, cost_volume_h, cost_volume_v, cost_volume_45, cost_volume_135): feature = 4 * 9 k = 9 label = 9 cost1 = convbn(cost_volume_3d, 6, 3, 1, 1) cost1 = Activation('relu')(cost1) cost1 = convbn(cost1, 4, 3, 1, 1) cost1 = Activation('sigmoid')(cost1) cost_h = Lambda(lambda y: K.repeat_elements( K.expand_dims(y[:, :, :, :1], 1), 9, 1))(cost1) cost_h = Lambda(lambda y: K.repeat_elements(y, feature, 4))(cost_h) cost_v = Lambda(lambda y: K.repeat_elements( K.expand_dims(y[:, :, :, 1:2], 1), 9, 1))(cost1) cost_v = Lambda(lambda y: K.repeat_elements(y, feature, 4))(cost_v) cost_45 = Lambda(lambda y: K.repeat_elements( K.expand_dims(y[:, :, :, 2:3], 1), 9, 1))(cost1) cost_45 = Lambda(lambda y: K.repeat_elements(y, feature, 4))(cost_45) cost_135 = Lambda(lambda y: K.repeat_elements( K.expand_dims(y[:, :, :, 3:4], 1), 9, 1))(cost1) cost_135 = Lambda(lambda y: K.repeat_elements(y, feature, 4))(cost_135) return concatenate([ multiply([cost_h, cost_volume_h]), multiply([cost_v, cost_volume_v]), multiply([cost_45, cost_volume_45]), multiply([cost_135, cost_volume_135]) ], axis=4), cost1
def define_epinet(sz_input,sz_input2,view_n,conv_depth,filt_num,learning_rate): ''' 4-Input : Conv - Relu - Conv - BN - Relu ''' input_stack_90d = Input(shape=(sz_input,sz_input2,len(view_n)), name='input_stack_90d') input_stack_0d= Input(shape=(sz_input,sz_input2,len(view_n)), name='input_stack_0d') input_stack_45d= Input(shape=(sz_input,sz_input2,len(view_n)), name='input_stack_45d') input_stack_M45d= Input(shape=(sz_input,sz_input2,len(view_n)), name='input_stack_M45d') ''' 4-Stream layer : Conv - Relu - Conv - BN - Relu ''' mid_90d=layer1_multistream(sz_input,sz_input2,len(view_n),int(filt_num))(input_stack_90d) mid_0d=layer1_multistream(sz_input,sz_input2,len(view_n),int(filt_num))(input_stack_0d) mid_45d=layer1_multistream(sz_input,sz_input2,len(view_n),int(filt_num))(input_stack_45d) mid_M45d=layer1_multistream(sz_input,sz_input2,len(view_n),int(filt_num))(input_stack_M45d) ''' Merge layers ''' mid_merged = concatenate([mid_90d,mid_0d,mid_45d,mid_M45d], name='mid_merged') ''' Merged layer : Conv - Relu - Conv - BN - Relu ''' mid_merged_=layer2_merged(sz_input-6,sz_input2-6,int(4*filt_num),int(4*filt_num),conv_depth)(mid_merged) ''' Last Conv layer : Conv - Relu - Conv ''' output=layer3_last(sz_input-18,sz_input2-18,int(4*filt_num),int(4*filt_num))(mid_merged_) model_512 = Model(inputs = [input_stack_90d,input_stack_0d, input_stack_45d,input_stack_M45d], outputs = [output]) opt = RMSprop(lr=learning_rate) model_512.compile(optimizer=opt, loss='mae') model_512.summary() return model_512
def inference(self, mode='simple'): left_image = Input(shape=(self.model_in_height, self.model_in_width, self.model_in_depth), name='left_input') right_image = Input(shape=(self.model_in_height, self.model_in_width, self.model_in_depth), name='right_image') if mode == 'simple': concate_view = concatenate([left_image, right_image], axis=3, name='concate_view') prediction = self.FlowNetSimple(concate_view) FlowNet = Model(inputs=[left_image, right_image], outputs=[prediction]) opt = Adam(lr=self.learning_rate) FlowNet.compile(optimizer=opt, loss='mae') FlowNet.summary() return FlowNet if mode == 'correlation': prediction = self.FlowNetCorr(left_image, right_image) FlowNet = Model(inputs=[left_image, right_image], outputs=[prediction]) opt = Adam(lr=self.learning_rate) FlowNet.compile(optimizer=opt, loss='mae') FlowNet.summary() return FlowNet
def get_model(filters_count, conv_depth, learning_rate, input_shape=(512, 512, 9)): # input shape=512x512x9 ?灰度化的9个图? input_90d = Input(shape=input_shape, name='input_90d') input_0d = Input(shape=input_shape, name='input_0d') input_45d = Input(shape=input_shape, name='input_45d') input_m45d = Input(shape=input_shape, name='input_m45d') # 4 Stream layer stream_ver = layersP1_multistream(input_shape, int(filters_count))(input_90d) stream_hor = layersP1_multistream(input_shape, int(filters_count))(input_0d) stream_45d = layersP1_multistream(input_shape, int(filters_count))(input_45d) stream_m45d = layersP1_multistream(input_shape, int(filters_count))(input_m45d) # merge streams merged = concatenate([stream_ver,stream_hor, stream_45d, stream_m45d], name='merged') # layers part2: conv-relu-bn-conv-relu merged = layersP2_merged(input_shape=(input_shape[0], input_shape[1], int(filters_count) * 4), filters_count=int(filters_count) * 4, conv_depth=conv_depth)(merged) # output output = layersP3_output(input_shape=(input_shape[0], input_shape[1], int(filters_count) * 4), filters_count=int(filters_count) * 4)(merged) mymodel = Model(inputs=[input_90d,input_0d, input_45d, input_m45d], outputs=[output]) optimizer = RMSprop(lr=learning_rate) mymodel.compile(optimizer=optimizer, loss='mae') mymodel.summary() return mymodel
def decoder_block(x, y, scope, size=None, upconv=True, ksize=(3, 3), upsize=(2, 2), upstirdes=(2, 2), act_fn='relu', ep_collection='end_points', reuse=None, batch_norm=True, dropout=0.0): if size is None: base_size = x.get_shape().as_list()[-1] size = int(base_size / 2) with tf.variable_scope(scope, scope, [x], reuse=reuse) as sc: x = ThresholdedReLU(theta=0.0)(x) uped = Conv2DTranspose(size, upsize, strides=upstirdes, padding='same')(x) if upconv else x uped, y = reconcile_feature_size(uped, y) up = concatenate([uped, y], axis=3) tf.add_to_collection(ep_collection, up) conv = Conv2D(size, ksize, activation=act_fn, padding='same')(up) tf.add_to_collection(ep_collection, conv) conv = Conv2D(size, ksize, activation=act_fn, padding='same')(conv) tf.add_to_collection(ep_collection, conv) if batch_norm: conv = BatchNormalization()(conv, training=True) tf.add_to_collection(ep_collection, conv) if dropout > 0.0: conv = Dropout(dropout)(conv) tf.add_to_collection(ep_collection, conv) return conv
def forward(self, x): if K.image_data_format() == 'channels_first': channel_axis = 1 else: channel_axis = -1 branch1x1 = self.branch1x1.forward(x) branch7x7 = self.branch7x7_1.forward(x) branch7x7 = self.branch7x7_2.forward(branch7x7) branch7x7 = self.branch7x7_3.forward(branch7x7) branch7x7dbl = self.branch7x7dbl_1.forward(x) branch7x7dbl = self.branch7x7dbl_2.forward(branch7x7dbl) branch7x7dbl = self.branch7x7dbl_3.forward(branch7x7dbl) branch7x7dbl = self.branch7x7dbl_4.forward(branch7x7dbl) branch7x7dbl = self.branch7x7dbl_5.forward(branch7x7dbl) branch_pool = AveragePooling2D( pool_size=(3, 3), strides=(1, 1), padding='same' )(x ) #branch_pool = F.avg_pool2d(x, kernel_size=3, stride=1, padding=1) branch_pool = self.branch_pool.forward(branch_pool) outputs = [branch1x1, branch7x7, branch7x7dbl, branch_pool] return concatenate(outputs, axis=channel_axis) #torch.cat(outputs, 1)
def feature_extraction(sz_input, sz_input2): i = Input(shape=(sz_input, sz_input2, 1)) firstconv = convbn(i, 4, 3, 1, 1) firstconv = Activation('relu')(firstconv) firstconv = convbn(firstconv, 4, 3, 1, 1) firstconv = Activation('relu')(firstconv) layer1 = _make_layer(firstconv, 4, 2, 1, 1) # (?, 32, 32, 4) layer2 = _make_layer(layer1, 8, 8, 1, 1) # (?, 32, 32, 8) layer3 = _make_layer(layer2, 16, 2, 1, 1) # (?, 32, 32, 16) layer4 = _make_layer(layer3, 16, 2, 1, 2) # (?, 32, 32, 16) layer4_size = (layer4.get_shape().as_list()[1], layer4.get_shape().as_list()[2]) branch1 = AveragePooling2D((2, 2), (2, 2), 'same', data_format='channels_last')(layer4) branch1 = convbn(branch1, 4, 1, 1, 1) branch1 = Activation('relu')(branch1) branch1 = UpSampling2DBilinear(layer4_size)(branch1) branch2 = AveragePooling2D((4, 4), (4, 4), 'same', data_format='channels_last')(layer4) branch2 = convbn(branch2, 4, 1, 1, 1) branch2 = Activation('relu')(branch2) branch2 = UpSampling2DBilinear(layer4_size)(branch2) branch3 = AveragePooling2D((8, 8), (8, 8), 'same', data_format='channels_last')(layer4) branch3 = convbn(branch3, 4, 1, 1, 1) branch3 = Activation('relu')(branch3) branch3 = UpSampling2DBilinear(layer4_size)(branch3) branch4 = AveragePooling2D((16, 16), (16, 16), 'same', data_format='channels_last')(layer4) branch4 = convbn(branch4, 4, 1, 1, 1) branch4 = Activation('relu')(branch4) branch4 = UpSampling2DBilinear(layer4_size)(branch4) output_feature = concatenate( [layer2, layer4, branch4, branch3, branch2, branch1], ) lastconv = convbn(output_feature, 16, 3, 1, 1) lastconv = Activation('relu')(lastconv) lastconv = Conv2D(4, 1, (1, 1), 'same', data_format='channels_last', use_bias=False)(lastconv) print(lastconv.get_shape()) model = Model(inputs=[i], outputs=[lastconv]) return model
def Fire_modele(x, squeeze=16, expand=64): x = Conv2D(squeeze, (1, 1), padding='valid')(x) x = Activation('relu')(x) left = Conv2D(expand, (1, 1), padding='valid')(x) left = Activation('relu')(left) right = Conv2D(expand, (3, 3), padding='same')(x) right = Activation('relu')(right) x = concatenate([left, right], axis=3) return x
def inference(self): ''' 4-Input : Conv - Relu - Conv - BN - Relu ''' input_stack_90d = Input(shape=(self.img_height, self.img_width, len(self.view_n)), name='input_stack_90d') input_stack_0d = Input(shape=(self.img_height, self.img_width, len(self.view_n)), name='input_stack_0d') input_stack_45d = Input(shape=(self.img_height, self.img_width, len(self.view_n)), name='input_stack_45d') input_stack_M45d = Input(shape=(self.img_height, self.img_width, len(self.view_n)), name='input_stack_M45d') ''' 4-Stream layer : Conv - Relu - Conv - BN - Relu ''' mid_90d = self.layer1_multistream(self.img_height, self.img_width, len(self.view_n), int(self.filt_num))(input_stack_90d) mid_0d = self.layer1_multistream(self.img_height, self.img_width, len(self.view_n), int(self.filt_num))(input_stack_0d) mid_45d = self.layer1_multistream(self.img_height, self.img_width, len(self.view_n), int(self.filt_num))(input_stack_45d) mid_M45d = self.layer1_multistream( self.img_height, self.img_width, len(self.view_n), int(self.filt_num))(input_stack_M45d) ''' Merge layers ''' mid_merged = concatenate([mid_90d, mid_0d, mid_45d, mid_M45d], name='mid_merged') ''' Merged layer : Conv - Relu - Conv - BN - Relu ''' mid_merged_ = self.layer2_merged(self.img_height - 6, self.img_width - 6, int(4 * self.filt_num), int(4 * self.filt_num), self.conv_depth)(mid_merged) ''' Last Conv layer : Conv - Relu - Conv ''' output = self.layer3_last(self.img_height - 18, self.img_width - 18, int(4 * self.filt_num), int(4 * self.filt_num))(mid_merged_) epinet = Model(inputs=[ input_stack_90d, input_stack_0d, input_stack_45d, input_stack_M45d ], outputs=[output]) opt = RMSprop(lr=self.learning_rate) epinet.compile(optimizer=opt, loss='mae') epinet.summary() return epinet
def forward(self, x): if K.image_data_format() == 'channels_first': channel_axis = 1 else: channel_axis = -1 x = self.BN1(x) x = self.relu(x) x = self.conv(x) x = self.BN2(x) x = self.relu2(x) new_features = self.conv2(x) #new_features = super(_DenseLayer, self).forward(x) if self.drop_rate > 0: new_features = Dropout(self.drop_rate)(new_features) #new_features = F.dropout(new_features, p=self.drop_rate, training=self.training) return concatenate([x, new_features], axis=channel_axis) #torch.cat([x, new_features], 1)
def forward(self, x): if K.image_data_format() == 'channels_first': channel_axis = 1 else: channel_axis = -1 #layer_list = [] x = self.squeeze_activation( self.squeeze(x)) #x = self.squeeze_activation(self.squeeze(x)) #layer_list.append(self.squeeze_activation) return concatenate([ self.expand1x1_activation(self.expand1x1(x)), self.expand3x3_activation(self.expand3x3(x)) ], axis=channel_axis)
def forward(self, x): if K.image_data_format() == 'channels_first': channel_axis = 1 else: channel_axis = -1 branch3x3 = self.branch3x3.forward(x) branch3x3dbl = self.branch3x3dbl_1.forward(x) branch3x3dbl = self.branch3x3dbl_2.forward(branch3x3dbl) branch3x3dbl = self.branch3x3dbl_3.forward(branch3x3dbl) branch_pool = MaxPooling2D(pool_size=(3, 3), strides=(2, 2))( x) #branch_pool = F.max_pool2d(x, kernel_size=3, stride=2) outputs = [branch3x3, branch3x3dbl, branch_pool] return concatenate(outputs, axis=channel_axis) #torch.cat(outputs, 1)
def define_cepinet(sz_input,sz_input2,view_n,conv_depth,filt_num,learning_rate,for_vis = False): global feats if for_vis: feats = [] else: feats = None ''' 4-Input : Conv - Relu - Conv - BN - Relu ''' input_stack_90d = Input(shape=(sz_input,sz_input2,len(view_n)), name='input_stack_90d') input_stack_0d= Input(shape=(sz_input,sz_input2,len(view_n)), name='input_stack_0d') # input_stack_45d= Input(shape=(sz_input,sz_input2,len(view_n)), name='input_stack_45d') input_stack_M45d= Input(shape=(sz_input,sz_input2,len(view_n)), name='input_stack_M45d') num_stacks = 3 with tf.variable_scope("4-Stream"): ''' 4-Stream layer : Conv - Relu - Conv - BN - Relu ''' mid_90d = layer1_multistream(sz_input,sz_input2,len(view_n),int(filt_num),do_vis=True,name="90d")(input_stack_90d) mid_0d = layer1_multistream(sz_input,sz_input2,len(view_n),int(filt_num),do_vis=True,name="0d")(input_stack_0d) # mid_45d=layer1_multistream(sz_input,sz_input2,len(view_n),int(filt_num))(input_stack_45d) mid_M45d = layer1_multistream(sz_input,sz_input2,len(view_n),int(filt_num),do_vis=True,name="M45d")(input_stack_M45d) with tf.variable_scope("Merge"): ''' Merge layers ''' mid_merged = concatenate([mid_90d,mid_0d,mid_M45d], name='mid_merged') ''' Merged layer : Conv - Relu - Conv - BN - Relu ''' mid_merged_=layer2_merged(sz_input-6,sz_input2-6,int(num_stacks*filt_num),int(num_stacks*filt_num),conv_depth)(mid_merged) with tf.variable_scope("Last"): ''' Last Conv layer : Conv - Relu - Conv ''' output=layer3_last(sz_input-18,sz_input2-18,int(num_stacks*filt_num),int(num_stacks*filt_num))(mid_merged_) if for_vis: feat_outs90d = [feat(input_stack_90d) for feat in feats[0:6]] feat_outs0d = [feat(input_stack_0d) for feat in feats[6:12]] feat_outsM45d = [feat(input_stack_M45d) for feat in feats[12:18]] outputs = feat_outs90d + feat_outs0d + feat_outsM45d + [output] else: outputs = [output] model_512 = Model(inputs = [input_stack_90d,input_stack_0d, # input_stack_45d, input_stack_M45d], outputs = outputs) opt = RMSprop(lr=learning_rate) model_512.compile(optimizer=opt, loss='mae') model_512.summary() return model_512, feat_names
def fire_module(x, fire_id, squeeze=16, expand=64): s_id = 'fire' + str(fire_id) + '/' if K.image_data_format() == 'channels_first': channel_axis = 1 else: channel_axis = 3 x = Convolution2D(squeeze, (1, 1), padding='valid', name=s_id + sq1x1)(x) x = Activation('relu', name=s_id + relu + sq1x1)(x) left = Convolution2D(expand, (1, 1), padding='valid', name=s_id + exp1x1)(x) left = Activation('relu', name=s_id + relu + exp1x1)(left) right = Convolution2D(expand, (3, 3), padding='same', name=s_id + exp3x3)(x) right = Activation('relu', name=s_id + relu + exp3x3)(right) x = concatenate([left, right], axis=channel_axis, name=s_id + 'concat') return x
print(vision_model(image_input)) print(vision_model(image_input2)) print(vision_model.inputs) print(vision_model.outputs) encoded_image = vision_model(image_input) # Next, let's define a language model to encode the question into a vector. # Each question will be at most 100 word long, # and we will index words as integers from 1 to 9999. question_input = Input(shape=(100, ), dtype='int32') temp = Embedding(input_dim=10000, output_dim=256, input_length=100) embedded_question = temp(question_input) encoded_question = LSTM(256)(embedded_question) # Let's concatenate the question vector and the image vector: merged = concatenate([encoded_question, encoded_image]) # And let's train a logistic regression over 1000 words on top: output = Dense(1000, activation='softmax')(merged) # This is our final model: vqa_model = Model(inputs=[image_input, question_input], outputs=output) print(isinstance(vqa_model, Layer)) print(isinstance(Dense(1000, activation='softmax'), Layer)) #isinstance(Vehicle(), Vehicle)
def FlowNetCorr(self, input_left, input_right): left_Conv1 = Conv2D(64, (7, 7), (2, 2), padding='same', activation='relu', name='left_Conv1')(input_left) left_Conv2 = Conv2D(128, (5, 5), (2, 2), padding='same', activation='relu', name='left_Conv2')(left_Conv1) left_Conv3 = Conv2D(256, (5, 5), (2, 2), padding='same', activation='relu', name='left_Conv3')(left_Conv2) right_Conv1 = Conv2D(64, (7, 7), (2, 2), padding='same', activation='relu', name='right_Conv1')(input_right) right_Conv2 = Conv2D(128, (5, 5), (2, 2), padding='same', activation='relu', name='right_Conv2')(right_Conv1) right_Conv3 = Conv2D(256, (5, 5), (2, 2), padding='same', activation='relu', name='right_Conv3')(right_Conv2) max_disp = 10 layer_list = [] dotLayer = Lambda(lambda x: tf.reduce_sum( tf.multiply(x[0], x[1]), axis=-1, keepdims=True), name='dotLayer') for i in range(-2 * max_disp, 2 * max_disp + 2, 2): for j in range(-2 * max_disp, 2 * max_disp + 2, 2): slice_height = int(self.model_in_height / 8) - abs(j) slice_width = int(self.model_in_width / 8) - abs(i) start_y = abs(j) if j < 0 else 0 start_x = abs(i) if i < 0 else 0 top_pad = j if (j > 0) else 0 bottom_pad = start_y left_pad = i if (i > 0) else 0 right_pad = start_x gather_layer = Lambda(lambda x: tf.pad( tf.slice(x, begin=[0, start_y, start_x, 0], size=[-1, slice_height, slice_width, -1]), paddings=[[0, 0], [top_pad, bottom_pad], [left_pad, right_pad], [0, 0]]), name='gather_{}_{}'.format( i, j))(right_Conv3) current_layer = dotLayer([left_Conv3, gather_layer]) layer_list.append(current_layer) Corr_441 = Lambda(lambda x: tf.concat(x, 3), name='Corr_441')(layer_list) Conv_redir = Conv2D(32, (1, 1), (1, 1), padding='same', activation='relu', name='Conv_redir')(left_Conv3) Corr = concatenate([Corr_441, Conv_redir], axis=3, name='Corr') Conv3_1 = Conv2D(256, (3, 3), padding='same', activation='relu', name='Conv3_1')(Corr) Conv4 = Conv2D(512, (3, 3), (2, 2), padding='same', activation='relu', name='Conv4')(Conv3_1) Conv4_1 = Conv2D(512, (3, 3), padding='same', activation='relu', name='Conv4_1')(Conv4) Conv5 = Conv2D(512, (3, 3), (2, 2), padding='same', activation='relu', name='Conv5')(Conv4_1) Conv5_1 = Conv2D(512, (3, 3), padding='same', activation='relu', name='Conv5_1')(Conv5) Conv6 = Conv2D(1024, (3, 3), (2, 2), padding='same', activation='relu', name='Conv6')(Conv5_1) ''' deconvolution layers : deconv5(flow5) - deconv4(flow4) - deconv3(flow3) - deconv2 - prediction ''' deconv5 = Conv2DTranspose(512, (3, 3), (2, 2), padding='same', activation='relu', name='deconv5')(Conv6) concat1 = concatenate([deconv5, Conv5_1], axis=3, name='concat1') flow5 = Conv2D(2, (3, 3), padding='same', name='flow5')(concat1) flow5_up = Conv2DTranspose(2, (3, 3), (2, 2), padding='same', name='flow5_up')(flow5) deconv4 = Conv2DTranspose(256, (3, 3), (2, 2), padding='same', activation='relu', name='deconv4')(concat1) concat2 = concatenate([deconv4, Conv4_1, flow5_up], axis=3, name='concat2') flow4 = Conv2D(1, (3, 3), padding='same', name='flow4')(concat2) flow4_up = Conv2DTranspose(2, (3, 3), (2, 2), padding='same', name='flow4_up')(flow4) deconv3 = Conv2DTranspose(128, (5, 5), (2, 2), padding='same', activation='relu', name='deconv3')(concat2) concat3 = concatenate([deconv3, Conv3_1, flow4_up], axis=3, name='concat3') flow3 = Conv2D(1, (3, 3), padding='same', name='flow3')(concat3) flow3_up = Conv2DTranspose(2, (3, 3), (2, 2), padding='same', name='flow3_up')(flow3) deconv2 = Conv2DTranspose(64, (5, 5), (2, 2), padding='same', activation='relu', name='deconv2')(concat3) concat4 = concatenate([deconv2, left_Conv2, flow3_up], axis=3, name='concat4') prediction = Conv2D(1, (3, 3), padding='same', name='prediction')(concat4) return prediction
def FlowNetSimple(self, input): ''' convolution layers : Conv1 - Conv2 - Conv3 - Conv3_1 - Conv4 - Conv4_1 - Conv5 - Conv5_1 - Conv6 ''' Conv1 = Conv2D(64, (7, 7), (2, 2), padding='same', activation='relu', name='Conv1')(input) Conv2 = Conv2D(128, (5, 5), (2, 2), padding='same', activation='relu', name='Conv2')(Conv1) Conv3 = Conv2D(256, (5, 5), (2, 2), padding='same', activation='relu', name='Conv3')(Conv2) Conv3_1 = Conv2D(256, (3, 3), padding='same', activation='relu', name='Conv3_1')(Conv3) Conv4 = Conv2D(512, (3, 3), (2, 2), padding='same', activation='relu', name='Conv4')(Conv3_1) Conv4_1 = Conv2D(512, (3, 3), padding='same', activation='relu', name='Conv4_1')(Conv4) Conv5 = Conv2D(512, (3, 3), (2, 2), padding='same', activation='relu', name='Conv5')(Conv4_1) Conv5_1 = Conv2D(512, (3, 3), padding='same', activation='relu', name='Conv5_1')(Conv5) Conv6 = Conv2D(1024, (3, 3), (2, 2), padding='same', activation='relu', name='Conv6')(Conv5_1) ''' deconvolution layers : deconv5(flow5) - deconv4(flow4) - deconv3(flow3) - deconv2 - prediction ''' deconv5 = Conv2DTranspose(512, (3, 3), (2, 2), padding='same', activation='relu', name='deconv5')(Conv6) concat1 = concatenate([deconv5, Conv5_1], axis=3, name='concat1') flow5 = Conv2D(2, (3, 3), padding='same', name='flow5')(concat1) flow5_up = Conv2DTranspose(2, (3, 3), (2, 2), padding='same', name='flow5_up')(flow5) deconv4 = Conv2DTranspose(256, (3, 3), (2, 2), padding='same', activation='relu', name='deconv4')(concat1) concat2 = concatenate([deconv4, Conv4_1, flow5_up], axis=3, name='concat2') flow4 = Conv2D(1, (3, 3), padding='same', name='flow4')(concat2) flow4_up = Conv2DTranspose(2, (3, 3), (2, 2), padding='same', name='flow4_up')(flow4) deconv3 = Conv2DTranspose(128, (5, 5), (2, 2), padding='same', activation='relu', name='deconv3')(concat2) concat3 = concatenate([deconv3, Conv3_1, flow4_up], axis=3, name='concat3') flow3 = Conv2D(1, (3, 3), padding='same', name='flow3')(concat3) flow3_up = Conv2DTranspose(2, (3, 3), (2, 2), padding='same', name='flow3_up')(flow3) deconv2 = Conv2DTranspose(64, (5, 5), (2, 2), padding='same', activation='relu', name='deconv2')(concat3) concat4 = concatenate([deconv2, Conv2, flow3_up], axis=3, name='concat4') prediction = Conv2D(1, (3, 3), padding='same', name='prediction')(concat4) return prediction
def define_epinet(sz_input, sz_input2, view_n, conv_depth, filt_num, learning_rate): global feats ''' 4-Input : Conv - Relu - Conv - BN - Relu ''' input_stack_90d = Input(shape=(sz_input, sz_input2, len(view_n)), name='input_stack_90d') input_stack_0d = Input(shape=(sz_input, sz_input2, len(view_n)), name='input_stack_0d') input_stack_45d = Input(shape=(sz_input, sz_input2, len(view_n)), name='input_stack_45d') input_stack_M45d = Input(shape=(sz_input, sz_input2, len(view_n)), name='input_stack_M45d') ''' 4-Stream layer : Conv - Relu - Conv - BN - Relu ''' mid_90d = layer1_multistream(sz_input, sz_input2, len(view_n), int(filt_num), do_vis=True, name="90d")(input_stack_90d) mid_0d = layer1_multistream(sz_input, sz_input2, len(view_n), int(filt_num), do_vis=True, name="0d")(input_stack_0d) mid_45d = layer1_multistream(sz_input, sz_input2, len(view_n), int(filt_num), do_vis=True, name="45d")(input_stack_45d) mid_M45d = layer1_multistream(sz_input, sz_input2, len(view_n), int(filt_num), do_vis=True, name="M45d")(input_stack_M45d) ''' Merge layers ''' mid_merged = concatenate([mid_90d, mid_0d, mid_45d, mid_M45d], name='mid_merged') ''' Merged layer : Conv - Relu - Conv - BN - Relu ''' mid_merged_ = layer2_merged(sz_input - 6, sz_input2 - 6, int(4 * filt_num), int(4 * filt_num), conv_depth)(mid_merged) ''' Last Conv layer : Conv - Relu - Conv ''' output = layer3_last(sz_input - 18, sz_input2 - 18, int(4 * filt_num), int(4 * filt_num))(mid_merged_) feat_outs90d = [feat(input_stack_90d) for feat in feats[0:1]] feat_outs0d = [feat(input_stack_0d) for feat in feats[1:2]] feat_outs45d = [feat(input_stack_45d) for feat in feats[2:3]] feat_outsM45d = [feat(input_stack_M45d) for feat in feats[3:4]] outputs = feat_outs90d + feat_outs0d + feat_outs45d + feat_outsM45d + [ output ] model_512 = Model(inputs=[ input_stack_90d, input_stack_0d, input_stack_45d, input_stack_M45d ], outputs=outputs) opt = RMSprop(lr=learning_rate) model_512.compile(optimizer=opt, loss='mae') model_512.summary() return model_512, feat_names
def build(self): gru = GRU(units=self.input_shape[-1], return_sequences=False, name='gru') bn = BatchNormalization() decoder_1 = Dense(units=128, activation='sigmoid', name='decoder-1') decoder_2 = Dense(units=self.output_dim, activation='sigmoid', name='decoder-2') self.layers.append(gru) self.layers.append(decoder_1) self.layers.append(decoder_2) activations = {} for i in range(self.historical_len): s_input = self.input_s[i] t_input = self.input_t[i] if i == 0: # hs = bn(s_input) hs = gru(s_input) activations['s-r-{}-o'.format(i)] = hs hs = Reshape((1, -1))(hs) activations['s-r-{}'.format(i)] = hs # ht = bn(t_input) ht = gru(t_input) activations['t-r-{}-o'.format(i)] = ht ht = Reshape((1, -1))(ht) activations['t-r-{}'.format(i)] = ht else: # hs = bn(s_input) hs = gru( Add()([activations['s-r-{}-o'.format(i - 1)], s_input])) activations['s-r-{}-o'.format(i)] = hs hs = Reshape((1, -1))(hs) activations['s-r-{}'.format(i)] = hs # ht = bn(t_input) ht = gru( Add()([activations['t-r-{}-o'.format(i - 1)], t_input])) activations['s-r-{}-o'.format(i)] = ht ht = Reshape((1, -1))(ht) activations['t-r-{}'.format(i)] = ht for i in range(self.historical_len): idx = self.historical_len - 1 - i s_input = self.input_s[idx] t_input = self.input_t[idx] if i == 0: # hs = bn(s_input) hs = gru(s_input) activations['s-l-{}-o'.format(i)] = hs hs = Reshape((1, -1))(hs) activations['s-l-{}'.format(i)] = hs # ht = bn(t_input) ht = gru(t_input) activations['t-l-{}-o'.format(i)] = ht ht = Reshape((1, -1))(ht) activations['t-l-{}'.format(i)] = ht else: # hs = bn(s_input) hs = gru( Add()([activations['s-l-{}-o'.format(i - 1)], s_input])) activations['s-l-{}-o'.format(i)] = hs hs = Reshape((1, -1))(hs) activations['s-l-{}'.format(i)] = hs # ht = bn(t_input) ht = gru( Add()([activations['t-r-{}-o'.format(i - 1)], t_input])) activations['s-l-{}-o'.format(i)] = ht ht = Reshape((1, -1))(ht) activations['t-l-{}'.format(i)] = ht s_h_r = concatenate([ activations['s-r-{}'.format(i)] for i in range(self.historical_len) ], axis=1) s_h_l = concatenate([ activations['s-l-{}'.format(i)] for i in range(self.historical_len) ], axis=1) t_h_r = concatenate([ activations['t-r-{}'.format(i)] for i in range(self.historical_len) ], axis=1) t_h_l = concatenate([ activations['t-l-{}'.format(i)] for i in range(self.historical_len) ], axis=1) s_c = concatenate([s_h_r, s_h_l], axis=1) t_c = concatenate([t_h_r, t_h_l], axis=1) s_c = Activation('relu')(Reshape((-1, ))(s_c)) t_c = Activation('relu')(Reshape((-1, ))(t_c)) # diff = tf.square(tf.subtract(s_c, t_c)) # tmp = diff.get_shape().as_list()[-1] # self.weight = RepeatVector(tmp)(self.weight) # self.weight = tf.reshape(self.weight, [-1, tmp]) # self.diff = tf.multiply(diff, self.weight) self.output_s = decoder_2(decoder_1(s_c)) self.output_t = decoder_2(decoder_1(t_c)) self.variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) self.vars = {var.name: var for var in self.variables} self._loss() self._gradients() self.opt_op = self.optimizer.minimize(self.loss)