def model(input_shape, input_shape2): up_0 = Input(shape=input_shape, name='up_stream_0') up_1 = Input(shape=input_shape, name='up_stream_1') down_0 = Input(shape=input_shape, name='down_stream_0') down_1 = Input(shape=input_shape, name='down_stream_1') down_02 = Input(shape=input_shape, name='down_stream_02') down_12 = Input(shape=input_shape, name='down_stream_12') down_2 = Input(shape=input_shape2, name='down_stream_2') down_3 = Input(shape=input_shape2, name='down_stream_3') up_stream = share_stream(x_shape=input_shape, outputNum=128) down_stream = share_stream(x_shape=input_shape, outputNum=128) down_stream3 = share_stream(x_shape=input_shape2, outputNum=128) up_feature_0 = up_stream(up_0) up_feature_1 = up_stream(up_1) down_feature_0 = down_stream(down_0) down_feature_1 = down_stream(down_1) down_feature_02 = down_stream(down_02) down_feature_12 = down_stream(down_12) down_feature_2 = down_stream3(down_2) down_feature_3 = down_stream3(down_3) up_feature = Maximum()([up_feature_0, up_feature_1]) down_feature = Maximum()([down_feature_0, down_feature_1]) down_feature2 = Maximum()([down_feature_02, down_feature_12]) down_feature3 = Maximum()([down_feature_2, down_feature_3]) feature = concatenate( [up_feature, down_feature, down_feature2, down_feature3]) x = Dense(units=256, use_bias=True, trainable=Trainable)(feature) x2 = Dense(16, trainable=Trainable)(Lambda(lambda x: x * 10000000.)( Activation("tanh")(Lambda(lambda x: x / 10000000.)(x)))) x2Shape2 = x2._keras_shape temp = Multiply()([ Dense(256, trainable=Trainable)(Lambda( lambda x: x * 10000000., output_shape=x2Shape2[1:])( Activation("tanh")(Lambda(lambda x: x / 10000000., output_shape=x2Shape2[1:])(x2)))), x ]) for jj in range(2, taylor): temp = Add()([ Multiply()([ Dense(256, trainable=Trainable)( Lambda(lambda x: x * 10000000., output_shape=x2Shape2[1:])(Activation("tanh")( Lambda(lambda x: x / 10000000., output_shape=x2Shape2[1:])(x2)))), Lambda(lambda x: (x**jj) / math.factorial(jj))(x) ]), temp ]) temp = Add()([ Dense(256, trainable=Trainable)(Lambda( lambda x: x * 10000000., output_shape=x2Shape2[1:])( Activation("tanh")(Lambda(lambda x: x / 10000000., output_shape=x2Shape2[1:])(x2)))), temp ]) fc_1 = Lambda(lambda x: x * 10000000.)(Activation("tanh")( Lambda(lambda x: x / 10000000.)(temp))) fc_1 = Dropout(0.4)(fc_1) fc_1 = Dense(256, trainable=True)(fc_1) feature = Dense(units=256, use_bias=True, trainable=True)(feature) feature = add([feature, fc_1]) feature = Dropout(0.8)(feature) fc_2 = Dense(units=128, activation='relu', use_bias=True, trainable=True)(feature) fc_2 = Dropout(0.4)(fc_2) fc_2 = Dense(units=128, trainable=Trainable)(fc_2) feature = Dense(units=128, use_bias=True, trainable=True)(feature) feature = add([feature, fc_2]) feature = Dropout(0.3)(feature) fc_4 = Dense( units=55, use_bias=True, )(feature) fc_4 = Activation('softmax')(fc_4) network = Model( input=[up_0, up_1, down_0, down_1, down_02, down_12, down_2, down_3], outputs=fc_4) return network
# state transition function messages = Dense(atom_features, activation='relu')(messages) messages = Dense(atom_features)(messages) atom_state = Add()([original_atom_state, messages]) return atom_state, bond_state for i in range(num_messages): atom_state, bond_state = message_block(atom_state, bond_state, connectivity, i) bond_state = Dense(1)(bond_state) bond_state = Add()([bond_state, bond_mean]) symb_inputs = [ mol_type, node_graph_indices, bond_graph_indices, atom_types, bond_types, connectivity ] model = GraphModel(symb_inputs, [bond_state]) epochs = 500 model.compile(optimizer=keras.optimizers.Adam(lr=lr, decay=decay), loss=masked_mean_absolute_error) if not os.path.exists(model_name): os.makedirs(model_name)
# state transition function messages = Dense(atom_features, activation='softplus')(messages) messages = Dense(atom_features)(messages) atom_state = Add()([atom_state, messages]) return atom_state, bond_state for _ in range(3): atom_state, bond_state = message_block(atom_state, bond_state, connectivity) atom_state = Dense(atom_features // 2, activation='softplus')(atom_state) atom_state = Dense(1)(atom_state) atom_state = Add()([atom_state, atomwise_energy]) output = ReduceAtomToMol(reducer='mean')([atom_state, snode_graph_indices]) model = GraphModel( [node_graph_indices, atom_types, distance_rbf, connectivity], [output]) lr = 1E-4 epochs = 500 model.compile(optimizer=keras.optimizers.Adam(lr=lr, decay=1E-5), loss='mae') model.summary() if not os.path.exists(model_name): os.makedirs(model_name)
def build_model(self): ### Build VGG-16 ### content_input = Input(shape=self.batch_shape) # Block 1 conv1_1 = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv1')(content_input) conv1_2 = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv2')(conv1_1) pool1_1 = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(conv1_2) # Block 2 conv2_1 = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv1')(pool1_1) conv2_2 = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv2')(conv2_1) pool2_1 = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(conv2_2) # Block 3 conv3_1 = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv1')(pool2_1) conv3_2 = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv2')(conv3_1) conv3_3 = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv3')(conv3_2) pool3_1 = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(conv3_3) # Block 4 conv4_1 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv1')(pool3_1) conv4_2 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv2')(conv4_1) conv4_3 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv3')(conv4_2) pool4_1 = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(conv4_3) # Block 5 conv5_1 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv1')(pool4_1) conv5_2 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv2')(conv5_1) conv5_3 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv3')(conv5_2) pool5_1 = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(conv5_3) self.vgg = Model(input=content_input, output=pool5_1) ### Build FCN-8s ### # Fully-Connected layers conv6 = Conv2D(4096, (7, 7), activation='relu', padding='same', name='fcn_fc1')(pool5_1) drop6 = Dropout(0.5)(conv6) conv7 = Conv2D(4096, (1, 1), activation='relu', padding='same', name='fcn_fc2')(drop6) drop7 = Dropout(0.5)(conv7) # Classifier # conv8 = Conv2D(7, (1, 1), strides=(1, 1), kernel_initializer='he_normal', activation='linear', padding='valid')(drop7) # up8 = Conv2DTranspose(7, (64, 64), strides=(32, 32), padding='same', name='deconv')(conv8) # up8 = Conv2DTranspose(7, (32, 32), strides=(16, 16), padding='same', use_bias=False, name='deconv9_1')(conv8) # pad9 = ZeroPadding2D(padding=(1, 1), name='padding9')(up8) # up10 = Conv2DTranspose(7, (4, 4), strides=(2, 2), padding='same', use_bias=False, name='deconv9_2')(pad9) # output = Activation('softmax')(up10) conv8 = Conv2D(7, (1, 1), strides=(1, 1), kernel_initializer='he_normal', activation='linear', padding='valid', name='conv8')(drop7) up8 = Conv2DTranspose(7, (4, 4), strides=(2, 2), padding='same', use_bias=False, name='deconv8_1')(conv8) scaled_pool4 = Lambda(lambda x: x * 1, name='scaled_pool4')(pool4_1) # 0.01 before conv9 = Conv2D(7, (1, 1), strides=(1, 1), kernel_initializer='he_normal', padding='valid', name='conv9')(scaled_pool4) o1, o2 = self.crop(up8, conv9, content_input) add9 = Add()([o1, o2]) up9 = Conv2DTranspose(7, (4, 4), strides=(2, 2), padding='same', use_bias=False, name='deconv9_1')(add9) scaled_pool3 = Lambda(lambda x: x * 0.1, name='scaled_pool3')(pool3_1) # 0.0001 before conv10 = Conv2D(7, (1, 1), strides=(1, 1), kernel_initializer='he_normal', padding='valid', name='conv10')(scaled_pool3) o1, o2 = self.crop(up9, conv10, content_input) add10 = Add()([o1, o2]) up11 = Conv2DTranspose(7, (16, 16), strides=(8, 8), padding='same', use_bias=False, name='deconv10_1')(add10) output = Activation('softmax')(up11) self.model = Model(input=content_input, output=output) if self.mode == 'train': print('Loading VGG16 weights...') self.vgg.load_weights(self.vgg_path, by_name=True) for layer in self.vgg.layers: layer.trainable = False print('VGG16 weights loaded!') elif self.mode == 'eval': print('Loading model from {}...'.format(self.model_path)) self.model.load_weights(self.model_path, by_name=True) print('Model weights loaded!') return self.model
def get_decoder(self, inputs, name='decoder'): """Builds the decoder of a LinkNet architecture. Args: name (string, optional): The encoder model name. Default: 'decoder'. Returns: The decoder as a Keras model instance. """ # Decoder inputs encoder4 = Input(shape=int_shape(inputs[0])[1:], name='encoder4') encoder3 = Input(shape=int_shape(inputs[1])[1:], name='encoder3') encoder2 = Input(shape=int_shape(inputs[2])[1:], name='encoder2') encoder1 = Input(shape=int_shape(inputs[3])[1:], name='encoder1') initial2 = Input(shape=int_shape(inputs[4])[1:], name='initial2') initial1 = inputs[5] # Decoder blocks decoder4 = self.decoder_block( encoder4, self.initial_block_filters * 4, strides=2, output_shape=int_shape(encoder3)[1:], bias=self.bias, name=name + '/4' ) decoder4 = Add(name=name + '/shortcut_e3_d4')([encoder3, decoder4]) decoder3 = self.decoder_block( decoder4, self.initial_block_filters * 2, strides=2, output_shape=int_shape(encoder2)[1:], bias=self.bias, name=name + '/3' ) decoder3 = Add(name=name + '/shortcut_e2_d3')([encoder2, decoder3]) decoder2 = self.decoder_block( decoder3, self.initial_block_filters, strides=2, output_shape=int_shape(encoder1)[1:], bias=self.bias, name=name + '/2' ) decoder2 = Add(name=name + '/shortcut_e1_d2')([encoder1, decoder2]) decoder1 = self.decoder_block( decoder2, self.initial_block_filters, strides=1, output_shape=int_shape(initial2)[1:], bias=self.bias, name=name + '/1' ) decoder1 = Add(name=name + '/shortcut_init_d1')([initial2, decoder1]) # Final block # Build the output shape of the next layer - same width and height # as initial1 shape = ( int_shape(initial1)[1], int_shape(initial1)[2], self.initial_block_filters // 2, ) final = Conv2DTranspose( self.initial_block_filters // 2, kernel_size=3, strides=2, padding='same', output_shape=shape, use_bias=self.bias, name=name + '/0/transposed2d_1' )(decoder1) final = BatchNormalization(name=name + '/0/bn_1')(final) final = Activation('relu', name=name + '/0/relu_1')(final) final = Conv2D( self.initial_block_filters // 2, kernel_size=3, padding='same', use_bias=self.bias, name=name + '/0/conv2d_1' )(final) final = BatchNormalization(name=name + '/0/bn_2')(final) final = Activation('relu', name=name + '/0/relu_2')(final) logits = Conv2DTranspose( self.num_classes, kernel_size=2, strides=2, padding='same', output_shape=self.output_shape, use_bias=self.bias, name=name + '/0/transposed2d_2' )(final) prediction = Softmax(name=name + '/0/softmax')(logits) return Model( inputs=[ encoder4, encoder3, encoder2, encoder1, initial2 ], outputs=prediction, name=name )
def get_effnet_model(save_path, model_res=1024, image_size=256, depth=1, size=3, activation='elu', loss='logcosh', optimizer='adam'): if os.path.exists(save_path): print('Loading model') return load_model(save_path) # Build model print('Building model') model_scale = int(2 * (math.log(model_res, 2) - 1)) # For example, 1024 -> 18 if (size <= 0): effnet = EfficientNetB0(include_top=False, weights='imagenet', input_shape=(image_size, image_size, 3)) if (size == 1): effnet = EfficientNetB1(include_top=False, weights='imagenet', input_shape=(image_size, image_size, 3)) if (size == 2): effnet = EfficientNetB2(include_top=False, weights='imagenet', input_shape=(image_size, image_size, 3)) if (size >= 3): effnet = EfficientNetB3(include_top=False, weights='imagenet', input_shape=(image_size, image_size, 3)) layer_size = model_scale * 8 * 8 * 8 if is_square(layer_size): # work out layer dimensions layer_l = int(math.sqrt(layer_size) + 0.5) layer_r = layer_l else: layer_m = math.log(math.sqrt(layer_size), 2) layer_l = 2**math.ceil(layer_m) layer_r = layer_size // layer_l layer_l = int(layer_l) layer_r = int(layer_r) x_init = None inp = Input(shape=(image_size, image_size, 3)) x = effnet(inp) if (size < 1): x = Conv2D(model_scale * 8, 1, activation=activation)(x) # scale down if (depth > 0): x = Reshape((layer_r, layer_l))( x ) # See https://github.com/OliverRichter/TreeConnect/blob/master/cifar.py - TreeConnect inspired layers instead of dense layers. else: if (depth < 1): depth = 1 if (size <= 2): x = Conv2D(model_scale * 8 * 4, 1, activation=activation)(x) # scale down a bit x = Reshape((layer_r * 2, layer_l * 2))( x ) # See https://github.com/OliverRichter/TreeConnect/blob/master/cifar.py - TreeConnect inspired layers instead of dense layers. else: x = Reshape((384, 256))(x) # full size for B3 while (depth > 0): x = LocallyConnected1D(layer_r, 1, activation=activation)(x) x = Permute((2, 1))(x) x = LocallyConnected1D(layer_l, 1, activation=activation)(x) x = Permute((2, 1))(x) if x_init is not None: x = Add()([x, x_init]) # add skip connection x_init = x depth -= 1 if ( size >= 2 ): # add unshared layers at end for different sections of the latent space x_init = x if layer_r % 3 == 0 and layer_l % 3 == 0: a = LocallyConnected1D(layer_r, 1, activation=activation)(x) b = LocallyConnected1D(layer_r, 1, activation=activation)(x) c = LocallyConnected1D(layer_r, 1, activation=activation)(x) a = Permute((2, 1))(a) b = Permute((2, 1))(b) c = Permute((2, 1))(c) a = LocallyConnected1D(layer_l // 3, 1, activation=activation)(a) b = LocallyConnected1D(layer_l // 3, 1, activation=activation)(b) c = LocallyConnected1D(layer_l // 3, 1, activation=activation)(c) x = Concatenate()([a, b, c]) else: a = LocallyConnected1D(layer_l, 1, activation=activation)(x) b = LocallyConnected1D(layer_l, 1, activation=activation)(x) a = Permute((2, 1))(a) b = Permute((2, 1))(b) a = LocallyConnected1D(layer_r // 2, 1, activation=activation)(a) b = LocallyConnected1D(layer_r // 2, 1, activation=activation)(b) x = Concatenate()([a, b]) x = Add()([x, x_init]) # add skip connection x = Reshape((model_scale, 512))(x) # train against all dlatent values model = Model(inputs=inp, outputs=x) model.compile(loss=loss, metrics=[], optimizer=optimizer ) # By default: adam optimizer, logcosh used for loss. return model
def get_resnet_model(save_path, model_res=1024, image_size=256, depth=2, size=0, activation='elu', loss='logcosh', optimizer='adam'): # Build model if os.path.exists(save_path): print('Loading model') return load_model(save_path) print('Building model') model_scale = int(2*(math.log(model_res, 2)-1)) # For example, 1024 -> 18 if size <= 0: from keras.applications.resnet50 import ResNet50 resnet = ResNet50(include_top=False, pooling=None, weights='imagenet', input_shape=(image_size, image_size, 3)) else: from keras_applications.resnet_v2 import ResNet50V2, ResNet101V2, ResNet152V2 if size == 1: resnet = ResNet50V2(include_top=False, pooling=None, weights='imagenet', input_shape=(image_size, image_size, 3), backend=keras.backend, layers=keras.layers, models=keras.models, utils=keras.utils) if size == 2: resnet = ResNet101V2(include_top=False, pooling=None, weights='imagenet', input_shape=(image_size, image_size, 3), backend=keras.backend, layers=keras.layers, models=keras.models, utils=keras.utils) if size >= 3: resnet = ResNet152V2(include_top=False, pooling=None, weights='imagenet', input_shape=(image_size, image_size, 3), backend=keras.backend, layers=keras.layers, models=keras.models, utils=keras.utils) layer_size = model_scale*8*8*8 if is_square(layer_size): # work out layer dimensions layer_l = int(math.sqrt(layer_size)+0.5) layer_r = layer_l else: layer_m = math.log(math.sqrt(layer_size), 2) layer_l = 2**math.ceil(layer_m) layer_r = layer_size // layer_l layer_l = int(layer_l) layer_r = int(layer_r) x_init = None inp = Input(shape=(image_size, image_size, 3)) x = resnet(inp) if depth < 0: depth = 1 if size <= 1: if size <= 0: x = Conv2D(model_scale*8, 1, activation=activation)(x) # scale down x = Reshape((layer_r, layer_l))(x) else: x = Conv2D(model_scale*8*4, 1, activation=activation)(x) # scale down a little x = Reshape((layer_r*2, layer_l*2))(x) else: if size == 2: x = Conv2D(1024, 1, activation=activation)(x) # scale down a bit x = Reshape((256, 256))(x) else: x = Reshape((256, 512))(x) # all weights used # See https://github.com/OliverRichter/TreeConnect/blob/master/cifar.py # - TreeConnect inspired layers instead of dense layers. while depth > 0: x = LocallyConnected1D(layer_r, 1, activation=activation)(x) x = Permute((2, 1))(x) x = LocallyConnected1D(layer_l, 1, activation=activation)(x) x = Permute((2, 1))(x) if x_init is not None: x = Add()([x, x_init]) # add skip connection x_init = x depth -= 1 x = Reshape((model_scale, 512))(x) # train against all dlatent values model = Model(inputs=inp, outputs=x) model.compile(loss=loss, metrics=[], optimizer=optimizer) # by default: adam optimizer, logcosh used for loss. return model
def green_block(inp, filters, data_format='channels_first', name=None): """ green_block(inp, filters, name=None) ------------------------------------ Implementation of the special residual block used in the paper. The block consists of two (GroupNorm --> ReLu --> 3x3x3 non-strided Convolution) units, with a residual connection from the input `inp` to the output. Used internally in the model. Can be used independently as well. Parameters ---------- `inp`: An keras.layers.layer instance, required The keras layer just preceding the green block. `filters`: integer, required No. of filters to use in the 3D convolutional block. The output layer of this green block will have this many no. of channels. `data_format`: string, optional The format of the input data. Must be either 'chanels_first' or 'channels_last'. Defaults to `channels_first`, as used in the paper. `name`: string, optional The name to be given to this green block. Defaults to None, in which case, keras uses generated names for the involved layers. If a string is provided, the names of individual layers are generated by attaching a relevant prefix from [GroupNorm_, Res_, Conv3D_, Relu_, ], followed by _1 or _2. Returns ------- `out`: A keras.layers.Layer instance The output of the green block. Has no. of channels equal to `filters`. The size of the rest of the dimensions remains same as in `inp`. """ inp_res = Conv3D(filters=filters, kernel_size=(1, 1, 1), strides=1, data_format=data_format, name=f'Res_{name}' if name else None)(inp) # axis=1 for channels_first data format # No. of groups = 8, as given in the paper x = GroupNormalization(groups=8, axis=1 if data_format == 'channels_first' else 0, name=f'GroupNorm_1_{name}' if name else None)(inp) x = Activation('relu', name=f'Relu_1_{name}' if name else None)(x) x = Conv3D(filters=filters, kernel_size=(3, 3, 3), strides=1, padding='same', data_format=data_format, name=f'Conv3D_1_{name}' if name else None)(x) x = GroupNormalization(groups=8, axis=1 if data_format == 'channels_first' else 0, name=f'GroupNorm_2_{name}' if name else None)(x) x = Activation('relu', name=f'Relu_2_{name}' if name else None)(x) x = Conv3D(filters=filters, kernel_size=(3, 3, 3), strides=1, padding='same', data_format=data_format, name=f'Conv3D_2_{name}' if name else None)(x) out = Add(name=f'Out_{name}' if name else None)([x, inp_res]) return out
def __init__(self): # Input shape self.img_rows = 256 self.img_cols = 256 self.channels_in = 3 self.channels_out = 3 self.img_shape_in = (self.img_rows, self.img_cols, self.channels_in) self.img_shape_out = (self.img_rows, self.img_cols, self.channels_out) vgg19 = VGG19() selectedLayers = [4, 5, 7] selectedOutputs = [vgg19.layers[i].output for i in selectedLayers] for i in np.arange(len(vgg19.layers)): vgg19.layers[i].trainable = False self.lossModel = Model(vgg19.inputs, selectedOutputs) self.lossModel.summary() # Configure data loader self.dataset_name = 'faces_bald_InsNorm_4x4_D2' self.data_loader = DataLoader() # Calculate output shape of D (PatchGAN) patch = int(self.img_rows / 2**4) self.disc_patch = (patch, patch, 1) # Number of filters in the first layer of G and D self.gf = 64 self.df = 64 optimizer = Adam(0.0002, 0.5) # Build and compile the discriminator self.discriminator = self.build_discriminator() self.discriminator.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) self.discriminator_mask = self.build_discriminator() self.discriminator_mask.compile(loss='mse', loss_weights=[2], optimizer=optimizer, metrics=['accuracy']) # ------------------------- # Construct Computational # Graph of Generator # ------------------------- # Build the generator self.generator = self.build_generator() self.generator.summary() # Input images and their conditioning images img_A = Input(shape=self.img_shape_out) img_B = Input(shape=self.img_shape_in) # By conditioning on B generate a fake version of A fake_A = self.generator(img_B) # find perceptual loss between fake_A and img_A fake_A_preproc = Lambda(lambda x: ((x + 1.) * 127.5))(fake_A) img_A_preproc = Lambda(lambda x: ((x + 1.) * 127.5))(img_A) fake_A_preproc = Lambda(lambda x: preprocess_input(x))(fake_A_preproc) img_A_preproc = Lambda(lambda x: preprocess_input(x))(img_A_preproc) embeddings_fake_A = self.lossModel(fake_A_preproc) embeddings_img_A = self.lossModel(img_A_preproc) diffs = [] for emb_fake_A, emb_img_A in zip(embeddings_fake_A, embeddings_img_A): l2_dist = Lambda(lambda x: L2(x[0], x[1]))([emb_fake_A, emb_img_A]) diffs.append(l2_dist) diffs = Add()(diffs) # For the combined model we will only train the generator self.discriminator.trainable = False self.discriminator_mask.trainable = False # Discriminators determines validity of translated images / condition pairs valid = self.discriminator([fake_A, img_B]) valid_mask = self.discriminator_mask([fake_A, img_B]) def empty_loss(y_true, y_pred): return y_pred self.combined = Model(inputs=[img_A, img_B], outputs=[valid, valid_mask, fake_A, diffs]) self.combined.compile(loss=['mse', 'mse', 'mae', empty_loss], loss_weights=[1, 2, 1, 0.00001], optimizer=optimizer)
def simple_resnet_1113( input_shape=(75, 75, 3), KernelSize=(5, 5), Momentum=0.99): X_input = Input(input_shape) #input_CNN = ZeroPadding2D((0, 0))(X_input) input_CNN = BatchNormalization(momentum=Momentum)(X_input) ## Input Layer input_CNN = Conv2D(32, kernel_size=KernelSize, padding='same', name='c11')(input_CNN) input_CNN = BatchNormalization(momentum=Momentum, name='b11')(input_CNN) input_CNN = Activation('elu')(input_CNN) input_CNN = MaxPooling2D((2, 2), strides=(2, 2), name='m11')(input_CNN) #input_CNN = Dropout(0.25)(input_CNN) input_CNN = Conv2D(64, kernel_size=KernelSize, padding='same', name='c12')(input_CNN) input_CNN = BatchNormalization(momentum=Momentum, name='b12')(input_CNN) input_CNN = Activation('elu')(input_CNN) input_CNN = MaxPooling2D((2, 2), strides=(2, 2), name='m12')(input_CNN) #input_CNN = Dropout(0.25)(input_CNN) ## First Residual input_CNN_residual = BatchNormalization(momentum=Momentum)(input_CNN) input_CNN_residual = Conv2D(128, kernel_size=KernelSize, padding='same')(input_CNN_residual) input_CNN_residual = BatchNormalization( momentum=Momentum)(input_CNN_residual) input_CNN_residual = Activation('elu')(input_CNN_residual) input_CNN_residual = Dropout(0.25)(input_CNN_residual) input_CNN_residual = Conv2D(64, kernel_size=KernelSize, padding='same')(input_CNN_residual) input_CNN_residual = BatchNormalization( momentum=Momentum)(input_CNN_residual) input_CNN_residual = Activation('elu')(input_CNN_residual) input_CNN_residual = Dropout(0.25)(input_CNN_residual) input_CNN_residual = Add()([input_CNN_residual, input_CNN]) ## Top CNN top_CNN = Conv2D(128, kernel_size=KernelSize, padding='same')(input_CNN_residual) top_CNN = BatchNormalization(momentum=Momentum)(top_CNN) top_CNN = Activation('elu')(top_CNN) top_CNN = MaxPooling2D((2, 2), strides=(2, 2))(top_CNN) top_CNN = Conv2D(256, kernel_size=KernelSize, padding='same')(top_CNN) top_CNN = BatchNormalization(momentum=Momentum)(top_CNN) top_CNN = Activation('elu')(top_CNN) top_CNN = Dropout(0.25)(top_CNN) top_CNN = MaxPooling2D((2, 2), strides=(2, 2))(top_CNN) top_CNN = Conv2D(512, kernel_size=KernelSize, padding='same')(top_CNN) top_CNN = BatchNormalization(momentum=Momentum)(top_CNN) top_CNN = Activation('elu')(top_CNN) top_CNN = Dropout(0.25)(top_CNN) top_CNN = MaxPooling2D((2, 2), strides=(2, 2))(top_CNN) top_CNN = GlobalMaxPooling2D()(top_CNN) #Dense Layers #X = Flatten()(top_CNN) X = Dense(512)(top_CNN) X = BatchNormalization(momentum=Momentum)(X) X = Activation('elu')(X) X = Dropout(0.25)(X) X = Dense(256)(X) X = BatchNormalization(momentum=Momentum)(X) X = Activation('elu')(X) X = Dropout(0.25)(X) X = Dense(1, activation='sigmoid')(X) model = Model(inputs=X_input, outputs=X, name='simple_resnet') return model
def build_model(input_shape=(4, 160, 192, 128), output_channels=3, weight_L2=0.1, weight_KL=0.1, dice_e=1e-8): """ build_model(input_shape=(4, 160, 192, 128), output_channels=3, weight_L2=0.1, weight_KL=0.1) ------------------------------------------- Creates the model used in the BRATS2018 winning solution by Myronenko A. (https://arxiv.org/pdf/1810.11654.pdf) Parameters ---------- `input_shape`: A 4-tuple, optional. Shape of the input image. Must be a 4D image of shape (c, H, W, D), where, each of H, W and D are divisible by 2^4, and c is divisible by 4. Defaults to the crop size used in the paper, i.e., (4, 160, 192, 128). `output_channels`: An integer, optional. The no. of channels in the output. Defaults to 3 (BraTS 2018 format). `weight_L2`: A real number, optional The weight to be given to the L2 loss term in the loss function. Adjust to get best results for your task. Defaults to 0.1. `weight_KL`: A real number, optional The weight to be given to the KL loss term in the loss function. Adjust to get best results for your task. Defaults to 0.1. `dice_e`: Float, optional A small epsilon term to add in the denominator of dice loss to avoid dividing by zero and possible gradient explosion. This argument will be passed to loss_gt function. Returns ------- `model`: A keras.models.Model instance The created model. """ c, H, W, D = input_shape assert len(input_shape) == 4, "Input shape must be a 4-tuple" assert (c % 4) == 0, "The no. of channels must be divisible by 4" assert (H % 16) == 0 and (W % 16) == 0 and (D % 16) == 0, \ "All the input dimensions must be divisible by 16" # ------------------------------------------------------------------------- # Encoder # ------------------------------------------------------------------------- ## Input Layer inp = Input(input_shape) ## The Initial Block x = Conv3D(filters=32, kernel_size=(3, 3, 3), strides=1, padding='same', data_format='channels_first', name='Input_x1')(inp) ## Dropout (0.2) x = SpatialDropout3D(0.2, data_format='channels_first')(x) ## Green Block x1 (output filters = 32) x1 = green_block(x, 32, name='x1') x = Conv3D(filters=32, kernel_size=(3, 3, 3), strides=2, padding='same', data_format='channels_first', name='Enc_DownSample_32')(x1) ## Green Block x2 (output filters = 64) x = green_block(x, 64, name='Enc_64_1') x2 = green_block(x, 64, name='x2') x = Conv3D(filters=64, kernel_size=(3, 3, 3), strides=2, padding='same', data_format='channels_first', name='Enc_DownSample_64')(x2) ## Green Blocks x2 (output filters = 128) x = green_block(x, 128, name='Enc_128_1') x3 = green_block(x, 128, name='x3') x = Conv3D(filters=128, kernel_size=(3, 3, 3), strides=2, padding='same', data_format='channels_first', name='Enc_DownSample_128')(x3) ## Green Blocks x4 (output filters = 256) x = green_block(x, 256, name='Enc_256_1') x = green_block(x, 256, name='Enc_256_2') x = green_block(x, 256, name='Enc_256_3') x4 = green_block(x, 256, name='x4') # ------------------------------------------------------------------------- # Decoder # ------------------------------------------------------------------------- ## GT (Groud Truth) Part # ------------------------------------------------------------------------- ### Green Block x1 (output filters=128) x = Conv3D(filters=128, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_GT_ReduceDepth_128')(x4) x = UpSampling3D(size=2, data_format='channels_first', name='Dec_GT_UpSample_128')(x) x = Add(name='Input_Dec_GT_128')([x, x3]) x = green_block(x, 128, name='Dec_GT_128') ### Green Block x1 (output filters=64) x = Conv3D(filters=64, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_GT_ReduceDepth_64')(x) x = UpSampling3D(size=2, data_format='channels_first', name='Dec_GT_UpSample_64')(x) x = Add(name='Input_Dec_GT_64')([x, x2]) x = green_block(x, 64, name='Dec_GT_64') ### Green Block x1 (output filters=32) x = Conv3D(filters=32, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_GT_ReduceDepth_32')(x) x = UpSampling3D(size=2, data_format='channels_first', name='Dec_GT_UpSample_32')(x) x = Add(name='Input_Dec_GT_32')([x, x1]) x = green_block(x, 32, name='Dec_GT_32') ### Blue Block x1 (output filters=32) x = Conv3D(filters=32, kernel_size=(3, 3, 3), strides=1, padding='same', data_format='channels_first', name='Input_Dec_GT_Output')(x) ### Output Block out_GT = Conv3D( filters=output_channels, # No. of tumor classes is 3 kernel_size=(1, 1, 1), strides=1, data_format='channels_first', activation='sigmoid', name='Dec_GT_Output')(x) ## VAE (Variational Auto Encoder) Part # ------------------------------------------------------------------------- ### VD Block (Reducing dimensionality of the data) x = GroupNormalization(groups=8, axis=1, name='Dec_VAE_VD_GN')(x4) x = Activation('relu', name='Dec_VAE_VD_relu')(x) x = Conv3D(filters=16, kernel_size=(3, 3, 3), strides=2, padding='same', data_format='channels_first', name='Dec_VAE_VD_Conv3D')(x) # Not mentioned in the paper, but the author used a Flattening layer here. x = Flatten(name='Dec_VAE_VD_Flatten')(x) x = Dense(256, name='Dec_VAE_VD_Dense')(x) ### VDraw Block (Sampling) z_mean = Dense(128, name='Dec_VAE_VDraw_Mean')(x) z_var = Dense(128, name='Dec_VAE_VDraw_Var')(x) x = Lambda(sampling, name='Dec_VAE_VDraw_Sampling')([z_mean, z_var]) ### VU Block (Upsizing back to a depth of 256) x = Dense((c // 4) * (H // 16) * (W // 16) * (D // 16))(x) x = Activation('relu')(x) x = Reshape(((c // 4), (H // 16), (W // 16), (D // 16)))(x) x = Conv3D(filters=256, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_VAE_ReduceDepth_256')(x) x = UpSampling3D(size=2, data_format='channels_first', name='Dec_VAE_UpSample_256')(x) ### Green Block x1 (output filters=128) x = Conv3D(filters=128, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_VAE_ReduceDepth_128')(x) x = UpSampling3D(size=2, data_format='channels_first', name='Dec_VAE_UpSample_128')(x) x = green_block(x, 128, name='Dec_VAE_128') ### Green Block x1 (output filters=64) x = Conv3D(filters=64, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_VAE_ReduceDepth_64')(x) x = UpSampling3D(size=2, data_format='channels_first', name='Dec_VAE_UpSample_64')(x) x = green_block(x, 64, name='Dec_VAE_64') ### Green Block x1 (output filters=32) x = Conv3D(filters=32, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_VAE_ReduceDepth_32')(x) x = UpSampling3D(size=2, data_format='channels_first', name='Dec_VAE_UpSample_32')(x) x = green_block(x, 32, name='Dec_VAE_32') ### Blue Block x1 (output filters=32) x = Conv3D(filters=32, kernel_size=(3, 3, 3), strides=1, padding='same', data_format='channels_first', name='Input_Dec_VAE_Output')(x) ### Output Block out_VAE = Conv3D(filters=4, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_VAE_Output')(x) # Build and Compile the model out = out_GT model = Model(inp, outputs=[out, out_VAE]) # Create the model model.compile(adam(lr=1e-4), [ loss_gt(dice_e), loss_VAE(input_shape, z_mean, z_var, weight_L2=weight_L2, weight_KL=weight_KL) ], metrics=[dice_coefficient]) return model
def get_lattice_max(cnn_all): maxed_cnn_all = [tf.reduce_max(t, axis=-2) for t in cnn_all] result = Add()(maxed_cnn_all) return result
def get_model(wordMatrix, model_param=None): if model_param is None: model_param = {'layer_num': 1} if not 'pooling_mode' in model_param: model_param['pooling_mode'] = 'gat_2' #'gcn' if not 'if_d' in model_param: model_param['if_d'] = True if not 'n_param' in model_param: model_param['n_param'] = 1024 # print(json.dumps(model_param)) # {"layer_num": 1, "pooling_mode": "gcn", "if_d": true} qu_l = 50 pre_l = 20 pad_l = 5 """ inputs """ qu_ids = Input(shape=(qu_l, ), dtype='int32', name='qu') pre_ids = Input(shape=(pre_l, ), dtype='int32', name='pre') input_layers = [qu_ids, pre_ids] # (?, 50) (?, 20) qu_mid_r = Input(shape=(qu_l, ), dtype='int32', name='qu_mid') qu_head = Input(shape=(qu_l, pad_l), dtype='int32', name='qu_head') qu_tail = Input(shape=(qu_l, pad_l), dtype='int32', name='qu_tail') pre_mid_r = Input(shape=(pre_l, ), dtype='int32', name='pre_mid') pre_head = Input(shape=(pre_l, pad_l), dtype='int32', name='pre_head') pre_tail = Input(shape=(pre_l, pad_l), dtype='int32', name='pre_tail') qu_i_ids = [qu_head, qu_mid_r, qu_tail] pre_i_ids = [pre_head, pre_mid_r, pre_tail] input_layers += [qu_head, qu_mid_r, qu_tail, pre_head, pre_mid_r, pre_tail] expand_dim = Lambda(lambda x: K.expand_dims(x, -1), output_shape=lambda x: x + (1, )) qu_i_ids[1] = expand_dim(qu_i_ids[1]) pre_i_ids[1] = expand_dim(pre_i_ids[1]) # Tensor("lambda_1/ExpandDims:0", shape=(?, 50, 1), dtype=int32) # Tensor("lambda_1_1/ExpandDims:0", shape=(?, 20, 1), dtype=int32) """ embedding """ input_dim, output_dim = np.shape(wordMatrix) embedding = Embedding(input_dim, output_dim, weights=[wordMatrix], name='embedding') qu_words = embedding(qu_ids) pre_words = embedding(pre_ids) # (?, 50, 300) (?, 20, 300) """ convolution """ n_param = model_param['n_param'] #1024 cnn1_head = Dense(n_param, name='cnn1_head') cnn1_mid = Dense(n_param, name='cnn1_mid') if model_param['if_d'] else cnn1_head cnn1_tail = Dense(n_param, name='cnn1_tail') if model_param['if_d'] else cnn1_head cnn1 = [cnn1_head, cnn1_mid, cnn1_tail] gate_dense = None if model_param['pooling_mode'] == 'gat': base_gate = Dense(1, name='gate1_head', activation='sigmoid') gate_dense = [ base_gate, Dense(1, name='gate1_mid', activation='sigmoid') if model_param['if_d'] else base_gate, Dense(1, name='gate1_tail', activation='sigmoid') if model_param['if_d'] else base_gate, ] elif model_param['pooling_mode'] in ['gat_n', 'gat_2']: base_gate = Dense(1, name='gate1_head', activation='sigmoid') gate_dense = [ base_gate, Dense(1, name='gate1_mid', activation=None) if model_param['if_d'] else base_gate, Dense(1, name='gate1_tail', activation=None) if model_param['if_d'] else base_gate, ] qu_results = gcn_layer(qu_i_ids, qu_words, cnn1, mode=model_param['pooling_mode'], gate_dense=gate_dense) pre_results = gcn_layer(pre_i_ids, pre_words, cnn1, mode=model_param['pooling_mode'], gate_dense=gate_dense) # (?, 80, 1024) # (?, 160, 1024) for i in range(model_param['layer_num'] - 1): cnn2_head = Dense(n_param, name='cnn%d_head' % (i + 2)) cnn2_mid = Dense(n_param, name='cnn%d_mid' % (i + 2)) if model_param['if_d'] else cnn2_head cnn2_tail = Dense(n_param, name='cnn%d_tail' % (i + 2)) if model_param['if_d'] else cnn2_head cnn2 = [cnn2_head, cnn2_mid, cnn2_tail] gate_dense_2 = None if model_param['pooling_mode'] == 'gat': base_gate_2 = Dense(1, name='gate%d_head' % (i + 2), activation='sigmoid') gate_dense_2 = [ base_gate_2, Dense(1, name='gate%d_mid' % (i + 2), activation='sigmoid') if model_param['if_d'] else base_gate_2, Dense(1, name='gate%d_tail' % (i + 2), activation='sigmoid') if model_param['if_d'] else base_gate_2, ] elif model_param['pooling_mode'] in ['gat_n', 'gat_2']: base_gate_2 = Dense(1, name='gate%d_head' % (i + 2), activation='sigmoid') gate_dense_2 = [ base_gate_2, Dense(1, name='gate%d_mid' % (i + 2), activation=None) if model_param['if_d'] else base_gate_2, Dense(1, name='gate%d_tail' % (i + 2), activation=None) if model_param['if_d'] else base_gate_2, ] qu_results_2 = gcn_layer(qu_i_ids, qu_results, cnn2, active=None, mode=model_param['pooling_mode'], gate_dense=gate_dense_2) pre_results_2 = gcn_layer(pre_i_ids, pre_results, cnn2, active=None, mode=model_param['pooling_mode'], gate_dense=gate_dense_2) # (?, 80, 1024) # (?, 160, 1024) qu_results_2 = Activation('relu')(Add()([qu_results, qu_results_2])) pre_results_2 = Activation('relu')(Add()([pre_results, pre_results_2])) qu_results = qu_results_2 pre_results = pre_results_2 """ merge """ label = count_similarity(qu_results, pre_results) # (?, 1) model = Model(inputs=input_layers, outputs=[label]) model.compile(loss={'label': 'binary_crossentropy'}, optimizer='adadelta') return model
def share_stream(x_shape, outputNum=256): input = Input(x_shape) x_a = input conv1 = Conv2D(filters=32, kernel_size=(5, 5), strides=(1, 1), padding='same', use_bias=use_bias, trainable=Trainable)(x_a) conv1 = Activation('relu')(conv1) conv1 = Conv2D(filters=32, kernel_size=(1, 1), strides=(1, 1), padding='same', use_bias=use_bias, trainable=Trainable)(conv1) x_a = Conv2D(filters=32, kernel_size=(1, 1), strides=(1, 1), padding='same', use_bias=use_bias, trainable=Trainable)(x_a) x_a = add([x_a, conv1]) attention = Conv2D(filters=32, kernel_size=(5, 5), activation='sigmoid', strides=(1, 1), padding='same', use_bias=use_bias, trainable=Trainable)(x_a) x_a = multiply([x_a, attention]) x_a = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')(x_a) conv1 = Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='same', use_bias=use_bias, trainable=Trainable)(x_a) conv1 = Activation('relu')(conv1) conv1 = Conv2D(filters=64, kernel_size=(1, 1), strides=(1, 1), padding='same', use_bias=use_bias, trainable=Trainable)(conv1) x_a = Conv2D(filters=64, kernel_size=(1, 1), strides=(1, 1), padding='same', use_bias=use_bias, trainable=Trainable)(x_a) x_a = add([x_a, conv1]) x_a = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')(x_a) x = Conv2D(filters=128, kernel_size=(3, 3), strides=(1, 1), padding='same', use_bias=use_bias, trainable=Trainable)(x_a) temp = Multiply()([ Conv2D(filters=128, kernel_size=(3, 3), padding='same', trainable=Trainable)(Lambda(lambda x: x * 10000000.)( Activation("tanh")(Lambda(lambda x: x / 10000000.)(x)))), x ]) for jj in range(2, taylor): temp = Add()([ Multiply()([ Conv2D(filters=128, kernel_size=(3, 3), padding='same', trainable=Trainable)(Lambda(lambda x: x * 10000000.)( Activation("tanh")( Lambda(lambda x: x / 10000000.)(x)))), Lambda(lambda x: (x**jj) / math.factorial(jj))(x) ]), temp ]) temp = Add()([ Conv2D(filters=128, kernel_size=(3, 3), padding='same', trainable=Trainable)(Lambda(lambda x: x * 10000000.)( Activation("tanh")(Lambda(lambda x: x / 10000000.)(x)))), temp ]) conv2 = Lambda(lambda x: x * 10000000.)(Activation("tanh")( Lambda(lambda x: x / 10000000.)(temp))) conv2 = Conv2D(filters=128, kernel_size=(1, 1), strides=(1, 1), padding='same', use_bias=use_bias, trainable=Trainable)(conv2) x_a = Conv2D(filters=128, kernel_size=(1, 1), strides=(1, 1), padding='same', use_bias=use_bias, trainable=Trainable)(x_a) x_a = add([x_a, conv2]) x_a = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')(x_a) conv3 = Conv2D(filters=outputNum, kernel_size=(3, 3), strides=(1, 1), padding='same', use_bias=use_bias, trainable=Trainable)(x_a) conv3 = Activation('relu')(conv3) conv3 = Conv2D(filters=outputNum, kernel_size=(1, 1), strides=(1, 1), padding='same', use_bias=use_bias, trainable=Trainable)(conv3) x_a = Conv2D(filters=outputNum, kernel_size=(1, 1), strides=(1, 1), padding='same', use_bias=use_bias, trainable=Trainable)(x_a) x_a = add([x_a, conv3]) x_a = GlobalMaxPooling2D()(x_a) shared_layer = Model(input, x_a) return shared_layer
def dense_gener(self): def residual_block(layer_input, filters): """Residual block described in paper""" d = Conv2D(filters, kernel_size=(3, 3), strides=1, padding='same')(layer_input) d = Activation('relu')(d) # d = BatchNormalization(momentum=0.8)(d) d = Conv2D(filters, kernel_size=(3, 3), strides=1, padding='same')(d) d = Activation('relu')(d) # d = BatchNormalization(momentum=0.8)(d) d = Add()([d, layer_input]) return d fil = 1 # Low resolution image input img_lr = Input(shape=self.lr_shape) # Pre-residual block c0 = Conv2D(fil, kernel_size=3, strides=1, padding='same')(img_lr) c0 = LeakyReLU(alpha=0.2)(c0) # e0 = Conv2D(1, kernel_size=3, strides=1, padding='same')(c0) # e0 = BatchNormalization(momentum=0.8)(e0) # e0 = LeakyReLU(alpha=0.2)(e0) rr1 = residual_block(c0, fil) for _ in range(self.n_residual_blocks - 1): rr1 = residual_block(rr1, fil) rr2 = Add()([c0, rr1]) rr2 = residual_block(rr2, fil) for _ in range(self.n_residual_blocks - 1): rr2 = residual_block(rr2, fil) rr3 = Add()([rr1, rr2]) rr3 = residual_block(rr3, fil) for _ in range(self.n_residual_blocks - 1): rr3 = residual_block(rr3, fil) # l0 = Conv2D(16, kernel_size=9, strides=1, padding='same')(img_lr) # l0 = LeakyReLU(alpha=0.2)(l0) # # e1 = Conv2D(16, kernel_size=3, strides=1, padding='same')(l0) # e1 = BatchNormalization(momentum=0.8)(e1) # e1 = LeakyReLU(alpha=0.2)(e1) # # rr2 = residual_block(e1, fil) # for _ in range(self.n_residual_blocks - 1): # rr2 = residual_block(rr2, fil) # # ll0 = Conv2D(16, kernel_size=27, strides=1, padding='same')(img_lr) # ll0 = LeakyReLU(alpha=0.2)(ll0) # # ee1 = Conv2D(16, kernel_size=3, strides=1, padding='same')(ll0) # ee1 = BatchNormalization(momentum=0.8)(ee1) # ee1 = LeakyReLU(alpha=0.2)(ee1) # rr3 = residual_block(ee1, fil) # for _ in range(self.n_residual_blocks - 1): # rr3 = residual_block(rr3, fil) dens0 = Concatenate()([rr3, rr2, rr1, c0, img_lr]) # for _ in range(self.n_residual_blocks - 1): # e0 = residual_block(e0, self.gf) # fusion f0 = Conv2D(64, kernel_size=(3, self.lr_width), strides=1, padding="same")(dens0) f0 = Activation('relu')(f0) # f0 = BatchNormalization(momentum=0.8)(f0) # f0 = Reshape((128,128,64))(f0) # r = residual_block(f0, self.gf) # for _ in range(self.n_residual_blocks - 1): # r = residual_block(r, self.gf) # f1 = Add()([f0,r]) # c1 = Conv2D(64, kernel_size=(3, 3), strides=1, padding='same')(f0) # c1 = Activation('relu')(c1) # c1 = BatchNormalization(momentum=0.8)(c1) # Propogate through residual blocks # r = residual_block(c1, self.gf) # for _ in range(self.n_residual_blocks - 1): # r = residual_block(r, self.gf) # ,,,, # Post-residual block # c2 = Conv2D(64, kernel_size=(3, 3), strides=1, padding='same')(f0) # c2 = Activation('relu')(c2) # c2 = BatchNormalization(momentum=0.8)(c2) # # d0 = Conv2D(32, kernel_size=3, strides=1, padding="same")(c2) # d0 = Activation('relu')(d0) # d0 = BatchNormalization(momentum=0.8)(d0) # ,,,, # generate # # g2 = Conv2D(128, kernel_size=3, strides=1, padding="same")(d0) # g2 = BatchNormalization(momentum=0.8)(g2) # g2 = Activation('relu')(g2) # # g3 = Conv2D(128, kernel_size=3, strides=1, padding="same")(g2) # g3 = BatchNormalization(momentum=0.8)(g3) # g3 = Activation('relu')(g3) # Generate high resolution output gen_hr = Conv2D(self.channels, kernel_size=3, strides=1, padding='same', activation='tanh')(f0) return Model(img_lr, gen_hr)
def get_crfrnn_model_def(): """ Returns Keras CRN-RNN model definition. Currently, only 500 x 500 images are supported. However, one can get this to work with different image sizes by adjusting the parameters of the Cropping2D layers below. """ channels, height, weight = 3, 500, 500 # Input input_shape = (height, weight, 3) img_input = Input(shape=input_shape) # Add plenty of zero padding x = ZeroPadding2D(padding=(100, 100))(img_input) # VGG-16 convolution block 1 x = Conv2D(64, (3, 3), activation='relu', padding='valid', name='conv1_1')(x) x = Conv2D(64, (3, 3), activation='relu', padding='same', name='conv1_2')(x) x = MaxPooling2D((2, 2), strides=(2, 2), name='pool1')(x) # VGG-16 convolution block 2 x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv2_1')(x) x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv2_2')(x) x = MaxPooling2D((2, 2), strides=(2, 2), name='pool2', padding='same')(x) # VGG-16 convolution block 3 x = Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3_1')(x) x = Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3_2')(x) x = Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3_3')(x) x = MaxPooling2D((2, 2), strides=(2, 2), name='pool3', padding='same')(x) pool3 = x # VGG-16 convolution block 4 x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_1')(x) x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_2')(x) x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_3')(x) x = MaxPooling2D((2, 2), strides=(2, 2), name='pool4', padding='same')(x) pool4 = x # VGG-16 convolution block 5 x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_1')(x) x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_2')(x) x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_3')(x) x = MaxPooling2D((2, 2), strides=(2, 2), name='pool5', padding='same')(x) # Fully-connected layers converted to convolution layers x = Conv2D(4096, (7, 7), activation='relu', padding='valid', name='fc6')(x) x = Dropout(0.5)(x) x = Conv2D(4096, (1, 1), activation='relu', padding='valid', name='fc7')(x) x = Dropout(0.5)(x) x = Conv2D(21, (1, 1), padding='valid', name='score-fr')(x) # Deconvolution score2 = Conv2DTranspose(21, (4, 4), strides=2, name='score2')(x) # Skip connections from pool4 score_pool4 = Conv2D(21, (1, 1), name='score-pool4')(pool4) score_pool4c = Cropping2D((5, 5))(score_pool4) score_fused = Add()([score2, score_pool4c]) score4 = Conv2DTranspose(21, (4, 4), strides=2, name='score4', use_bias=False)(score_fused) # Skip connections from pool3 score_pool3 = Conv2D(21, (1, 1), name='score-pool3')(pool3) score_pool3c = Cropping2D((9, 9))(score_pool3) # Fuse things together score_final = Add()([score4, score_pool3c]) # Final up-sampling and cropping upsample = Conv2DTranspose(21, (16, 16), strides=8, name='upsample', use_bias=False)(score_final) upscore = Cropping2D(((31, 37), (31, 37)))(upsample) output = CrfRnnLayer(image_dims=(height, weight), num_classes=21, theta_alpha=160., theta_beta=3., theta_gamma=3., num_iterations=10, name='crfrnn')([upscore, img_input]) # Build the model model = Model(img_input, output, name='crfrnn_net') return model
def build_generator(self): def residual_block(layer_input, filters): """Residual block described in paper""" d = Conv2D(filters, kernel_size=3, strides=1, padding='same')(layer_input) d = Activation('relu')(d) d = BatchNormalization(momentum=0.8)(d) d = Conv2D(filters, kernel_size=3, strides=1, padding='same')(d) d = BatchNormalization(momentum=0.8)(d) d = Add()([d, layer_input]) return d def deconv2d(layer_input): """Layers used during upsampling""" u = UpSampling2D(size=2)(layer_input) u = Conv2D(256, kernel_size=3, strides=1, padding='same')(u) u = Activation('relu')(u) return u # Low resolution image input img_lr = Input(shape=self.lr_shape) # Pre-residual block c0 = Conv2D(64, kernel_size=9, strides=1, padding='same')(img_lr) c0 = Activation('relu')(c0) d1 = Conv2D(64, kernel_size=(3, self.lr_width), strides=1, padding="same")(c0) d1 = Activation('relu')(d1) c1 = Conv2D(64, kernel_size=3, strides=1, padding='same')(d1) c1 = Activation('relu')(c1) # Propogate through residual blocks r = residual_block(c1, self.gf) for _ in range(self.n_residual_blocks - 1): r = residual_block(r, self.gf) # Post-residual block c2 = Conv2D(64, kernel_size=3, strides=1, padding='same')(r) c2 = Activation('relu')(c2) c2 = BatchNormalization(momentum=0.8)(c2) c2 = Add()([c2, c1]) # Upsampling # u1 = deconv2d(c2) # u2 = deconv2d(u1) u1 = Conv2D(256, kernel_size=3, strides=1, padding="same")(c2) u1 = Activation('relu')(u1) u2 = Conv2D(256, kernel_size=3, strides=1, padding="same")(u1) u2 = Activation('relu')(u2) # Generate high resolution output gen_hr = Conv2D(self.channels, kernel_size=9, strides=1, padding='same', activation='tanh')(u2) return Model(img_lr, gen_hr)
def create_model(self, shape, stddev=0.1, skip_shortcuts=False, deconvolutional=False, loss='mse'): ''' Skip shortcuts: https://arxiv.org/pdf/1606.08921.pdf ''' inputs = Input(shape=shape) inputs_noizy = inputs inputs_noizy = GaussianNoise(stddev)(inputs_noizy) x1 = Conv2D(self.kernel_size, self.filter_size, padding='same', activation='relu')(inputs_noizy) x2 = Conv2D(self.kernel_size, self.filter_size, padding='same', activation='relu')(x1) x2 = BatchNormalization()(x2) x2 = Dropout(0.2)(x2) x3 = Conv2D(self.kernel_size, self.filter_size, padding='same', activation='relu')(x2) x4 = Conv2D(self.kernel_size, self.filter_size, padding='same', activation='relu')(x3) x4 = BatchNormalization()(x4) x4 = Dropout(0.2)(x4) if deconvolutional == False: x5 = Conv2D(self.kernel_size, self.filter_size, padding='same', activation='relu')(x4) x6 = Conv2D(self.kernel_size, self.filter_size, padding='same', activation='relu')(x5) x6 = BatchNormalization()(x6) x6 = Dropout(0.2)(x6) x7 = Conv2D(self.kernel_size, self.filter_size, padding='same', activation='relu')(x6) x8 = Conv2D(self.kernel_size, self.filter_size, padding='same', activation='relu')(x7) x8 = BatchNormalization()(x8) x8 = Dropout(0.2)(x8) outputs = Conv2D(3, self.filter_size, padding='same', activation='sigmoid')(x8) else: x5 = Conv2DTranspose(self.kernel_size, self.filter_size, padding='same', activation='relu')(x4) x6 = Conv2DTranspose(self.kernel_size, self.filter_size, padding='same', activation='relu')(x5) x6 = BatchNormalization()(x6) x6 = Dropout(0.2)(x6) x7 = Conv2DTranspose(self.kernel_size, self.filter_size, padding='same', activation='relu')(x6) x8 = Conv2DTranspose(self.kernel_size, self.filter_size, padding='same', activation='relu')(x7) x8 = BatchNormalization()(x8) x8 = Dropout(0.2)(x8) outputs = Conv2DTranspose(shape[-1], self.filter_size, padding='same', activation='sigmoid')(x8) if skip_shortcuts == True: y = BatchNormalization()(inputs) y = LeakyReLU()(y) outputs = Add([outputs, y]) model = Model(inputs=inputs, outputs=outputs) model.summary() model.compile(loss=loss, optimizer=Adam(lr=self.learn_rate), metrics=['accuracy']) return model
def convolutional_block(X, f, filters, stage, block, s=2): """ Implementation of the convolutional block as defined in Figure 4 Arguments: X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev) f -- integer, specifying the shape of the middle CONV's window for the main path filters -- python list of integers, defining the number of filters in the CONV layers of the main path stage -- integer, used to name the layers, depending on their position in the network block -- string/character, used to name the layers, depending on their position in the network s -- Integer, specifying the stride to be used Returns: X -- output of the convolutional block, tensor of shape (n_H, n_W, n_C) """ # defining name basis conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' # Retrieve Filters F1, F2, F3 = filters # Save the input value X_shortcut = X ##### MAIN PATH ##### # First component of main path X = Conv2D(F1, (1, 1), strides=(s, s), name=conv_name_base + '2a', kernel_initializer=glorot_uniform(seed=0))(X) X = BatchNormalization(axis=3, name=bn_name_base + '2a')(X) X = Activation('relu')(X) ### START CODE HERE ### # Second component of main path (≈3 lines) X = Conv2D(F2, (f, f), strides=(1, 1), padding='same', name=conv_name_base + '2b', kernel_initializer=glorot_uniform(seed=0))(X) X = BatchNormalization(axis=3, name=bn_name_base + '2b')(X) X = Activation('relu')(X) # Third component of main path (≈2 lines) X = Conv2D(F3, (1, 1), strides=(1, 1), padding='valid', name=conv_name_base + '2c', kernel_initializer=glorot_uniform(seed=0))(X) X = BatchNormalization(axis=3, name=bn_name_base + '2c')(X) ##### SHORTCUT PATH #### (≈2 lines) X_shortcut = Conv2D(F3, (1, 1), strides=(s, s), padding='valid', name=conv_name_base + '1', kernel_initializer=glorot_uniform(seed=0))(X_shortcut) X_shortcut = BatchNormalization(axis=3, name=bn_name_base + '1')(X_shortcut) # Final step: Add shortcut value to main path, and pass it through a RELU activation (≈2 lines) X = Add()([X, X_shortcut]) X = Activation('relu')(X) ### END CODE HERE ### return X
def get_predict_model(config): h, w = config.IMAGE_SHAPE[:2] if h / 2**6 != int(h / 2**6) or w / 2**6 != int(w / 2**6): raise Exception("Image size must be dividable by 2 at least 6 times " "to avoid fractions when downscaling and upscaling." "For example, use 256, 320, 384, 448, 512, ... etc. ") # 输入进来的图片必须是2的6次方以上的倍数 input_image = Input(shape=[None, None, config.IMAGE_SHAPE[2]], name="input_image") # meta包含了一些必要信息 input_image_meta = Input(shape=[config.IMAGE_META_SIZE], name="input_image_meta") # 输入进来的先验框 input_anchors = Input(shape=[None, 4], name="input_anchors") # 获得Resnet里的压缩程度不同的一些层 _, C2, C3, C4, C5 = get_resnet(input_image, stage5=True, train_bn=config.TRAIN_BN) # 组合成特征金字塔的结构 # P5长宽共压缩了5次 # Height/32,Width/32,256 P5 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c5p5')(C5) # P4长宽共压缩了4次 # Height/16,Width/16,256 P4 = Add(name="fpn_p4add")([ UpSampling2D(size=(2, 2), name="fpn_p5upsampled")(P5), Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c4p4')(C4) ]) # P4长宽共压缩了3次 # Height/8,Width/8,256 P3 = Add(name="fpn_p3add")([ UpSampling2D(size=(2, 2), name="fpn_p4upsampled")(P4), Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c3p3')(C3) ]) # P4长宽共压缩了2次 # Height/4,Width/4,256 P2 = Add(name="fpn_p2add")([ UpSampling2D(size=(2, 2), name="fpn_p3upsampled")(P3), Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c2p2')(C2) ]) # 各自进行一次256通道的卷积,此时P2、P3、P4、P5通道数相同 # Height/4,Width/4,256 P2 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p2")(P2) # Height/8,Width/8,256 P3 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p3")(P3) # Height/16,Width/16,256 P4 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p4")(P4) # Height/32,Width/32,256 P5 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p5")(P5) # 在建议框网络里面还有一个P6用于获取建议框 # Height/64,Width/64,256 P6 = MaxPooling2D(pool_size=(1, 1), strides=2, name="fpn_p6")(P5) # P2, P3, P4, P5, P6可以用于获取建议框 rpn_feature_maps = [P2, P3, P4, P5, P6] # P2, P3, P4, P5用于获取mask信息 mrcnn_feature_maps = [P2, P3, P4, P5] anchors = input_anchors # 建立RPN模型 rpn = build_rpn_model(len(config.RPN_ANCHOR_RATIOS), config.TOP_DOWN_PYRAMID_SIZE) rpn_class_logits, rpn_class, rpn_bbox = [], [], [] # 获得RPN网络的预测结果,进行格式调整,把五个特征层的结果进行堆叠 for p in rpn_feature_maps: # [P2, P3, P4, P5, P6] logits, classes, bbox = rpn([p]) rpn_class_logits.append(logits) rpn_class.append(classes) rpn_bbox.append(bbox) rpn_class_logits = Concatenate(axis=1, name="rpn_class_logits")(rpn_class_logits) rpn_class = Concatenate(axis=1, name="rpn_class")(rpn_class) rpn_bbox = Concatenate(axis=1, name="rpn_bbox")(rpn_bbox) # 此时获得的rpn_class_logits、rpn_class、rpn_bbox的维度是 # rpn_class_logits : Batch_size, num_anchors, 2 # rpn_class : Batch_size, num_anchors, 2 # rpn_bbox : Batch_size, num_anchors, 4 proposal_count = config.POST_NMS_ROIS_INFERENCE # Batch_size, proposal_count, 4 # 对先验框进行解码 rpn_rois = ProposalLayer(proposal_count=proposal_count, nms_threshold=config.RPN_NMS_THRESHOLD, name="ROI", config=config)([rpn_class, rpn_bbox, anchors]) # 获得classifier的结果 mrcnn_class_logits, mrcnn_class, mrcnn_bbox =\ fpn_classifier_graph(rpn_rois, mrcnn_feature_maps, input_image_meta, config.POOL_SIZE, config.NUM_CLASSES, train_bn=config.TRAIN_BN, fc_layers_size=config.FPN_CLASSIF_FC_LAYERS_SIZE) # 最终预测框 detections = DetectionLayer( config, name="mrcnn_detection" )( # RPN层的rpn_rois建议框, mrcnn_class, mrcnn_bbox, input_image_meta [rpn_rois, mrcnn_class, mrcnn_bbox, input_image_meta]) detection_boxes = Lambda(lambda x: x[..., :4])(detections) # 获得mask的结果 mrcnn_mask = build_fpn_mask_graph(detection_boxes, mrcnn_feature_maps, input_image_meta, config.MASK_POOL_SIZE, config.NUM_CLASSES, train_bn=config.TRAIN_BN) # 汇总,作为输出 model = Model([input_image, input_image_meta, input_anchors], [ detections, mrcnn_class, mrcnn_bbox, mrcnn_mask, rpn_rois, rpn_class, rpn_bbox ], name='mask_rcnn') return model
for i in range(0, 1): #num of blocks for k in range( 0, 9 ): #num of dilations/resolutions (increase them to get RF > sample_len for the encoder) z = x x = Convolution1D(512, 1, padding='same')(x) x = LeakyReLU()(x) x = BatchNormalization()(x) x = DepthwiseConv1D(x, 3, dilation_rate=2**k) x = LeakyReLU()(x) x = BatchNormalization()(x) t = Convolution1D(128, 1, padding='same')(x) if (i != 0) or (k != 0): s = Add()( [t, s] ) #skip connection, gradually adding the masks from each dilated block else: s = t if (i != 2) or (k != 8): x = Convolution1D(128, 1, padding='same')( x) #1x1 conv to reshape the feature map. x = Add()( [z, x]) #output connection, with a residual block to the input x = LeakyReLU()(s) x = Convolution1D(512, 1, padding='same', activation='sigmoid')(x) x = Multiply()([e, x]) #multiplies the masks t = Conv1DTranspose(x, 1, 16, strides=8, padding='same')
def get_train_model(config): h, w = config.IMAGE_SHAPE[:2] if h / 2**6 != int(h / 2**6) or w / 2**6 != int(w / 2**6): raise Exception("Image size must be dividable by 2 at least 6 times " "to avoid fractions when downscaling and upscaling." "For example, use 256, 320, 384, 448, 512, ... etc. ") # 输入进来的图片必须是2的6次方以上的倍数 input_image = Input(shape=[None, None, config.IMAGE_SHAPE[2]], name="input_image") # meta包含了一些必要信息 input_image_meta = Input(shape=[config.IMAGE_META_SIZE], name="input_image_meta") # RPN建议框网络的真实框信息 input_rpn_match = Input(shape=[None, 1], name="input_rpn_match", dtype=tf.int32) input_rpn_bbox = Input(shape=[None, 4], name="input_rpn_bbox", dtype=tf.float32) # 种类信息 input_gt_class_ids = Input(shape=[None], name="input_gt_class_ids", dtype=tf.int32) # 框的位置信息 input_gt_boxes = Input(shape=[None, 4], name="input_gt_boxes", dtype=tf.float32) # 标准化到0-1之间 gt_boxes = Lambda(lambda x: norm_boxes_graph(x, K.shape(input_image)[1:3]))( input_gt_boxes) # mask语义分析信息 # [batch, height, width, MAX_GT_INSTANCES] if config.USE_MINI_MASK: input_gt_masks = Input( shape=[config.MINI_MASK_SHAPE[0], config.MINI_MASK_SHAPE[1], None], name="input_gt_masks", dtype=bool) else: input_gt_masks = Input( shape=[config.IMAGE_SHAPE[0], config.IMAGE_SHAPE[1], None], name="input_gt_masks", dtype=bool) # 获得Resnet里的压缩程度不同的一些层 _, C2, C3, C4, C5 = get_resnet(input_image, stage5=True, train_bn=config.TRAIN_BN) # 组合成特征金字塔的结构 # P5长宽共压缩了5次 # Height/32,Width/32,256 P5 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c5p5')(C5) # P4长宽共压缩了4次 # Height/16,Width/16,256 P4 = Add(name="fpn_p4add")([ UpSampling2D(size=(2, 2), name="fpn_p5upsampled")(P5), Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c4p4')(C4) ]) # P4长宽共压缩了3次 # Height/8,Width/8,256 P3 = Add(name="fpn_p3add")([ UpSampling2D(size=(2, 2), name="fpn_p4upsampled")(P4), Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c3p3')(C3) ]) # P4长宽共压缩了2次 # Height/4,Width/4,256 P2 = Add(name="fpn_p2add")([ UpSampling2D(size=(2, 2), name="fpn_p3upsampled")(P3), Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c2p2')(C2) ]) # 各自进行一次256通道的卷积,此时P2、P3、P4、P5通道数相同 # Height/4,Width/4,256 P2 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p2")(P2) # Height/8,Width/8,256 P3 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p3")(P3) # Height/16,Width/16,256 P4 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p4")(P4) # Height/32,Width/32,256 P5 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3), padding="SAME", name="fpn_p5")(P5) # 在建议框网络里面还有一个P6用于获取建议框 # Height/64,Width/64,256 P6 = MaxPooling2D(pool_size=(1, 1), strides=2, name="fpn_p6")(P5) # P2, P3, P4, P5, P6可以用于获取建议框 rpn_feature_maps = [P2, P3, P4, P5, P6] # P2, P3, P4, P5用于获取mask信息 mrcnn_feature_maps = [P2, P3, P4, P5] anchors = get_anchors(config, config.IMAGE_SHAPE) # 拓展anchors的shape,第一个维度拓展为batch_size anchors = np.broadcast_to(anchors, (config.BATCH_SIZE, ) + anchors.shape) # 将anchors转化成tensor的形式 anchors = Lambda(lambda x: tf.Variable(anchors), name="anchors")(input_image) # 建立RPN模型 rpn = build_rpn_model(len(config.RPN_ANCHOR_RATIOS), config.TOP_DOWN_PYRAMID_SIZE) rpn_class_logits, rpn_class, rpn_bbox = [], [], [] # 获得RPN网络的预测结果,进行格式调整,把五个特征层的结果进行堆叠 for p in rpn_feature_maps: logits, classes, bbox = rpn([p]) rpn_class_logits.append(logits) rpn_class.append(classes) rpn_bbox.append(bbox) rpn_class_logits = Concatenate(axis=1, name="rpn_class_logits")(rpn_class_logits) rpn_class = Concatenate(axis=1, name="rpn_class")(rpn_class) rpn_bbox = Concatenate(axis=1, name="rpn_bbox")(rpn_bbox) # 此时获得的rpn_class_logits、rpn_class、rpn_bbox的维度是 # rpn_class_logits : Batch_size, num_anchors, 2 # rpn_class : Batch_size, num_anchors, 2 # rpn_bbox : Batch_size, num_anchors, 4 proposal_count = config.POST_NMS_ROIS_TRAINING # Batch_size, proposal_count, 4 rpn_rois = ProposalLayer(proposal_count=proposal_count, nms_threshold=config.RPN_NMS_THRESHOLD, name="ROI", config=config)([rpn_class, rpn_bbox, anchors]) active_class_ids = Lambda(lambda x: parse_image_meta_graph(x)[ "active_class_ids"])(input_image_meta) if not config.USE_RPN_ROIS: # 使用外部输入的建议框 input_rois = Input(shape=[config.POST_NMS_ROIS_TRAINING, 4], name="input_roi", dtype=np.int32) # Normalize coordinates target_rois = Lambda( lambda x: norm_boxes_graph(x, K.shape(input_image)[1:3]))(input_rois) else: # 利用预测到的建议框进行下一步的操作 target_rois = rpn_rois """找到建议框的ground_truth Inputs: proposals: [batch, N, (y1, x1, y2, x2)]建议框 gt_class_ids: [batch, MAX_GT_INSTANCES]每个真实框对应的类 gt_boxes: [batch, MAX_GT_INSTANCES, (y1, x1, y2, x2)]真实框的位置 gt_masks: [batch, height, width, MAX_GT_INSTANCES]真实框的语义分割情况 Returns: rois: [batch, TRAIN_ROIS_PER_IMAGE, (y1, x1, y2, x2)]内部真实存在目标的建议框 target_class_ids: [batch, TRAIN_ROIS_PER_IMAGE]每个建议框对应的类 target_deltas: [batch, TRAIN_ROIS_PER_IMAGE, (dy, dx, log(dh), log(dw)]每个建议框应该有的调整参数 target_mask: [batch, TRAIN_ROIS_PER_IMAGE, height, width]每个建议框语义分割情况 """ rois, target_class_ids, target_bbox, target_mask =\ DetectionTargetLayer(config, name="proposal_targets")([ target_rois, input_gt_class_ids, gt_boxes, input_gt_masks]) # 找到合适的建议框的classifier预测结果 mrcnn_class_logits, mrcnn_class, mrcnn_bbox =\ fpn_classifier_graph(rois, mrcnn_feature_maps, input_image_meta, config.POOL_SIZE, config.NUM_CLASSES, train_bn=config.TRAIN_BN, fc_layers_size=config.FPN_CLASSIF_FC_LAYERS_SIZE) # 找到合适的建议框的mask预测结果 mrcnn_mask = build_fpn_mask_graph(rois, mrcnn_feature_maps, input_image_meta, config.MASK_POOL_SIZE, config.NUM_CLASSES, train_bn=config.TRAIN_BN) output_rois = Lambda(lambda x: x * 1, name="output_rois")(rois) # Losses rpn_class_loss = Lambda(lambda x: rpn_class_loss_graph(*x), name="rpn_class_loss")( [input_rpn_match, rpn_class_logits]) rpn_bbox_loss = Lambda(lambda x: rpn_bbox_loss_graph(config, *x), name="rpn_bbox_loss")( [input_rpn_bbox, input_rpn_match, rpn_bbox]) class_loss = Lambda(lambda x: mrcnn_class_loss_graph(*x), name="mrcnn_class_loss")([ target_class_ids, mrcnn_class_logits, active_class_ids ]) bbox_loss = Lambda(lambda x: mrcnn_bbox_loss_graph(*x), name="mrcnn_bbox_loss")( [target_bbox, target_class_ids, mrcnn_bbox]) mask_loss = Lambda(lambda x: mrcnn_mask_loss_graph(*x), name="mrcnn_mask_loss")( [target_mask, target_class_ids, mrcnn_mask]) # Model inputs = [ input_image, input_image_meta, input_rpn_match, input_rpn_bbox, input_gt_class_ids, input_gt_boxes, input_gt_masks ] if not config.USE_RPN_ROIS: inputs.append(input_rois) outputs = [ rpn_class_logits, rpn_class, rpn_bbox, mrcnn_class_logits, mrcnn_class, mrcnn_bbox, mrcnn_mask, rpn_rois, output_rois, rpn_class_loss, rpn_bbox_loss, class_loss, bbox_loss, mask_loss ] model = Model(inputs, outputs, name='mask_rcnn') return model
def encoder_basic_block( self, input, out_filters, kernel_size=3, strides=1, padding='same', bias=False, name='' ): """Creates a basic encoder block. Main brach architecture: 1. Conv2D 2. BatchNormalization 3. ReLU 4. Conv2D 5. BatchNormalization Residual branch architecture: 1. Conv2D, if `strides` is greater than 1 The output of the main and residual branches are then added together with ReLU activation. Args: input (tensor): A tensor or variable. out_filters (int): The number of filters in the block output. kernel_size (int, tuple, list, optional): A tuple/list of 2 integers, specifying the height and width of the 2D kernel window. In case it's a single integer, it's value is used for all spatial dimensions. Default: 3. strides (int, tuple, list, optional): A tuple/list of 2 integers, specifying the strides along the height and width of the 2D input. In case it's a single integer, it's value is used for all spatial dimensions. Default: 1. padding (str, optional): One of "valid" or "same" (case-insensitive). Default: "same". bias (bool, optional): If ``True``, adds a learnable bias. Default: ``False``. name (string, optional): A string to identify this block. Default: Empty string. Returns: The output tensor of the block. """ residual = input x = Conv2D( out_filters, kernel_size=kernel_size, strides=strides, padding=padding, use_bias=bias, name=name + '/main/conv2d_1' )(input) x = BatchNormalization(name=name + '/main/bn_1')(x) x = Activation('relu', name=name + '/main/relu_1')(x) x = Conv2D( out_filters, kernel_size=kernel_size, strides=1, padding=padding, use_bias=bias, name=name + '/main/conv2d_2' )(x) x = BatchNormalization(name=name + '/main/bn_2')(x) if strides > 1: residual = Conv2D( out_filters, kernel_size=1, strides=strides, padding=padding, use_bias=bias, name=name + '/res/conv2d_1' )(residual) residual = BatchNormalization(name=name + '/res/bn_1')(residual) x = Add(name=name + '/add')([x, residual]) x = Activation('relu', name=name + '/relu_1')(x) return x
def identity_block_3D(X, f, filters, stage, block): """ Implementation of the identity block as defined in Figure 3 Arguments: X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev) f -- integer, specifying the shape of the middle CONV's window for the main path filters -- python list of integers, defining the number of filters in the CONV layers of the main path stage -- integer, used to name the layers, depending on their position in the network block -- string/character, used to name the layers, depending on their position in the network Returns: X -- output of the identity block, tensor of shape (n_H, n_W, n_C) """ # defining name basis conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' # Retrieve Filters F1, F2, F3 = filters # Save the input value. You'll need this later to add back to the main path. X_shortcut = X # First component of main path X = Conv3D(filters=F1, kernel_size=(1, 1, 1), strides=(1, 1, 1), padding='valid', name=conv_name_base + '2a', kernel_initializer=glorot_uniform(seed=0))(X) X = BatchNormalization(axis=4, name=bn_name_base + '2a')(X) X = Activation('relu')(X) ### START CODE HERE ### # Second component of main path (≈3 lines) X = Conv3D(filters=F2, kernel_size=(f, f, f), strides=(1, 1, 1), padding='same', name=conv_name_base + '2b', kernel_initializer=glorot_uniform(seed=0))(X) X = BatchNormalization(axis=4, name=bn_name_base + '2b')(X) X = Activation('relu')(X) # Third component of main path (≈2 lines) X = Conv3D(filters=F3, kernel_size=(1, 1, 1), strides=(1, 1, 1), padding='valid', name=conv_name_base + '2c', kernel_initializer=glorot_uniform(seed=0))(X) X = BatchNormalization(axis=4, name=bn_name_base + '2c')(X) # Final step: Add shortcut value to main path, and pass it through a RELU activation (≈2 lines) X = Add()([X, X_shortcut]) X = Activation('relu')(X) ### END CODE HERE ### return X
# print("p[0].shape:", p[0].shape) # print("p[1].shape:", p[1].shape) # exit() u_bias = Embedding(N, 1, embeddings_regularizer=l2(reg))(u) # (N, 1, 1) m_bias = Embedding(M, 1, embeddings_regularizer=l2(reg))(m) # (N, 1, 1) x = Dot(axes=2)([u_embedding, m_embedding]) # (N, 1, 1) # submodel = Model([u, m], x) # user_ids = df_train.userId.values[0:5] # movie_ids = df_train.movie_idx.values[0:5] # p = submodel.predict([user_ids, movie_ids]) # print("p.shape:", p.shape) # exit() x = Add()([x, u_bias, m_bias]) x = Flatten()(x) # (N, 1) model = Model(inputs=[u, m], outputs=x) model.compile( loss='mse', # optimizer='adam', # optimizer=Adam(lr=0.01), optimizer=SGD(lr=0.08, momentum=0.9), metrics=['mse'], ) r = model.fit( x=[df_train.userId.values, df_train.movie_idx.values], y=df_train.rating.values - mu, epochs=epochs,
def standard_func_wbond(model, _emb_atoms, _emb_bonds, n_filters, adj, reg, drop, step, prefix): emb_shape = _emb_atoms.get_shape().as_list() bond_shape = _emb_bonds.get_shape().as_list() # print(emb_shape, bond_shape, adj.shape) n_filters = n_filters kernel_size_1, kernel_size_2 = (1, emb_shape[2]), (1, bond_shape[2] + emb_shape[2]) strides_1, strides_2 = (1, emb_shape[2]), (1, bond_shape[2] + emb_shape[2]) kreg, breg = None, None # if reg != 0: # kreg, breg = regularizers.l2(reg), regularizers.l2(reg) # else: # kreg, breg = None, None x1 = Conv2D(filters=n_filters, kernel_size=kernel_size_1, strides=strides_1, padding='same', data_format='channels_last', activation=None, kernel_regularizer=kreg, bias_regularizer=breg, name=str(step) + '_emb_conv')( Lambda(lambda t: expand_last_dim(t))(_emb_atoms)) if reg != 0: x1 = GaussianNoise(reg)(x1) x1 = Lambda(lambda t: squeeze(t, -2))(x1) # xbis = RepeatVector4D(emb_bonds.shape[1])(emb_atoms) # xbis = Lambda(lambda t: dynamic_repeat4D(t), # output_shape=(None, None, emb_shape[2]))(emb_atoms) xbis = Lambda(lambda t: dynamic_repeataxis(t, 1, rep=None), name=str(step) + 'x2repeat')(_emb_atoms) # print("xbis", xbis) x2 = Concatenate(axis=-1, name=str(step) + 'x2concat')([xbis, _emb_bonds]) # print("1 x2", x2) x2bis = Lambda(lambda t: expand_last_dim(t), name=str(step) + 'x2expand')(x2) # print("1bis x2", x2bis) x2_a = Conv2D(filters=n_filters, kernel_size=kernel_size_2, strides=strides_2, padding='same', data_format='channels_last', activation=None, kernel_regularizer=kreg, bias_regularizer=breg, name=str(step) + 'x2_emb_nei')(x2bis) x2_b = Lambda(lambda t: squeeze(t, -2), name=str(step) + 'x2squeeze')(x2_a) # print('2 x2', x2_b) x2_c = Lambda(lambda args: K.batch_dot(args[0], args[1], axes=(2, 1)))( [adj, x2_b]) # print('3 x2', x2_c) # print("x1", x1) if reg != 0: x2_c = GaussianNoise(reg)(x2_c) emb_atoms = Activation('relu')(Add()([x1, x2_c])) if drop != 0: emb_atoms = Dropout(drop, noise_shape=None, seed=None)(emb_atoms) # print('emb_atoms', emb_atoms) return emb_atoms
def build(num_users=7000, num_movies=5000, latent_dimension=120, is_regularized=True, is_biased=True, **kwargs): lamda = kwargs["lamda"] if "lamda" in kwargs else 1e-5 user_id_input = Input(shape=(1, ), name="UserID") movie_id_input = Input(shape=(1, ), name="MovieID") user_latent = Embedding( num_users, latent_dimension, input_length=1, embeddings_initializer="random_uniform", embeddings_regularizer=l2(lamda) if is_regularized else None, name="UserLatent")(user_id_input) movie_latent = Embedding( num_movies, latent_dimension, input_length=1, embeddings_initializer="random_uniform", embeddings_regularizer=l2(lamda) if is_regularized else None, name="MovieLatent")(movie_id_input) user_latent = Flatten(name="FlattenedUserLatent")(user_latent) movie_latent = Flatten(name="FlattenedMovieLatent")(movie_latent) output = Dot(1, name="NoBiasedRating")([user_latent, movie_latent]) model = Model([user_id_input, movie_id_input], output) if is_biased: bias_input = Input(shape=(1, ), name="GlobalBiasCoef") bias = Embedding(1, 1, input_length=1, embeddings_initializer="random_uniform", name="GlobalBias")(bias_input) user_bias = Embedding( num_users, 1, input_length=1, embeddings_initializer="random_uniform", embeddings_regularizer=l2(lamda) if is_regularized else None, name="UserBias")(user_id_input) movie_bias = Embedding( num_movies, 1, input_length=1, embeddings_initializer="random_uniform", embeddings_regularizer=l2(lamda) if is_regularized else None, name="MovieBias")(movie_id_input) bias = Flatten(name="FlattenedGlobalBias")(bias) user_bias = Flatten(name="FlattenedUserBias")(user_bias) movie_bias = Flatten(name="FlattenedMovieBias")(movie_bias) output = Add(name="Rating")([output, bias, user_bias, movie_bias]) model = Model([user_id_input, movie_id_input, bias_input], output) model.compile(optimizer="adam", loss="mse", metrics=["mse"]) return model
def standard_func(model, _emb_atoms, n_filters, adj, reg, drop, step, prefix): emb_shape = _emb_atoms.get_shape().as_list() # print(emb_shape, adj.shape) kernel_size, strides = (1, emb_shape[2]), (1, emb_shape[2]) kreg, breg = None, None # if reg != 0: # kreg, breg = regularizers.l2(reg), regularizers.l2(reg) # else: # kreg, breg = None, None x1 = Conv2D(filters=n_filters, kernel_size=kernel_size, strides=strides, padding='same', data_format='channels_last', activation=None, kernel_regularizer=kreg, bias_regularizer=breg, name=str(step) + prefix + '_emb_conv')( Lambda(lambda t: expand_last_dim(t))(_emb_atoms)) if reg != 0: x1 = GaussianNoise(reg)(x1) x1 = Lambda(lambda t: squeeze(t, -2), name=str(step) + prefix + "_squeeze_x1")(x1) ##### new # print("1 x2", emb_atoms) x2_a = Conv2D(filters=n_filters, kernel_size=kernel_size, strides=strides, padding='same', data_format='channels_last', activation=None, kernel_regularizer=kreg, bias_regularizer=breg, name=str(step) + prefix + '_emb_nei')( Lambda(lambda t: expand_last_dim(t))(_emb_atoms)) x2_b = Lambda(lambda t: squeeze(t, -2), name=str(step) + prefix + "_squeeze_x2")(x2_a) # print('2 x2', x2_b) x2_c = Lambda(lambda arg: batch_dot_layer(arg[0], arg[1], axes=(2, 1)), name=str(step) + prefix + "_emb_nei_sum")([adj, x2_b]) if reg != 0: x2_c = GaussianNoise(reg)(x2_c) # print('3 x2', x2_c) # print("x1", x1) ##### old # x2 = Lambda(lambda arg: batch_dot_layer(arg[0], arg[1], (2, 1)), # name=str(step) + '_emb_nei_sum')([adj, emb_atoms]) # print('1 x2', x2) # x2 = Conv2D(filters=n_filters, kernel_size=kernel_size, strides=strides, # padding='same', data_format='channels_last', # activation=None, kernel_regularizer=kreg, bias_regularizer=breg, # name=str(step) + '_nei_conv')(Lambda(lambda t: expand_last_dim(t))(x2)) # print('2 x2', x2) # x2 = Multiply()([norm, x2]) # , name=str(step) + '_emb_nei_sum_norm') # print('3 x2', x2) # x3 = x1 + x2 ; x3 = K.squeeze(ReLU()(x3), axis=-2) ; print('x3.shape', x3.shape) emb_atoms = \ (Activation('relu', name=str(step) + prefix + '_emb_atoms')(Add(name="")([x1, x2_c]))) if drop != 0: emb_atoms = Dropout(drop, noise_shape=None, seed=None)(emb_atoms) else: print('mol drop is 0 ###########') return emb_atoms
def _main(args): config_path = os.path.expanduser(args.config_path) weights_path = os.path.expanduser(args.weights_path) assert config_path.endswith('.cfg'), '{} is not a .cfg file'.format( config_path) assert weights_path.endswith( '.weights'), '{} is not a .weights file'.format(weights_path) output_path = os.path.expanduser(args.output_path) assert output_path.endswith( '.h5'), 'output path {} is not a .h5 file'.format(output_path) output_root = os.path.splitext(output_path)[0] # Load weights and config. print('Loading weights.') weights_file = open(weights_path, 'rb') major, minor, revision = np.ndarray(shape=(3, ), dtype='int32', buffer=weights_file.read(12)) if (major * 10 + minor) >= 2 and major < 1000 and minor < 1000: seen = np.ndarray(shape=(1, ), dtype='int64', buffer=weights_file.read(8)) else: seen = np.ndarray(shape=(1, ), dtype='int32', buffer=weights_file.read(4)) print('Weights Header: ', major, minor, revision, seen) print('Parsing Darknet config.') unique_config_file = unique_config_sections(config_path) cfg_parser = configparser.ConfigParser() cfg_parser.read_file(unique_config_file) print('Creating Keras model.') input_layer = Input(shape=(None, None, 3)) prev_layer = input_layer all_layers = [] weight_decay = float(cfg_parser['net_0']['decay'] ) if 'net_0' in cfg_parser.sections() else 5e-4 count = 0 out_index = [] for section in cfg_parser.sections(): print('Parsing section {}'.format(section)) if section.startswith('convolutional'): filters = int(cfg_parser[section]['filters']) size = int(cfg_parser[section]['size']) stride = int(cfg_parser[section]['stride']) pad = int(cfg_parser[section]['pad']) activation = cfg_parser[section]['activation'] batch_normalize = 'batch_normalize' in cfg_parser[section] padding = 'same' if pad == 1 and stride == 1 else 'valid' # Setting weights. # Darknet serializes convolutional weights as: # [bias/beta, [gamma, mean, variance], conv_weights] prev_layer_shape = K.int_shape(prev_layer) weights_shape = (size, size, prev_layer_shape[-1], filters) darknet_w_shape = (filters, weights_shape[2], size, size) weights_size = np.product(weights_shape) print('conv2d', 'bn' if batch_normalize else ' ', activation, weights_shape) conv_bias = np.ndarray(shape=(filters, ), dtype='float32', buffer=weights_file.read(filters * 4)) count += filters if batch_normalize: bn_weights = np.ndarray(shape=(3, filters), dtype='float32', buffer=weights_file.read(filters * 12)) count += 3 * filters bn_weight_list = [ bn_weights[0], # scale gamma conv_bias, # shift beta bn_weights[1], # running mean bn_weights[2] # running var ] conv_weights = np.ndarray(shape=darknet_w_shape, dtype='float32', buffer=weights_file.read(weights_size * 4)) count += weights_size # DarkNet conv_weights are serialized Caffe-style: # (out_dim, in_dim, height, width) # We would like to set these to Tensorflow order: # (height, width, in_dim, out_dim) conv_weights = np.transpose(conv_weights, [2, 3, 1, 0]) conv_weights = [conv_weights] if batch_normalize else [ conv_weights, conv_bias ] # Handle activation. act_fn = None if activation == 'leaky': pass # Add advanced activation later. elif activation != 'linear': raise ValueError( 'Unknown activation function `{}` in section {}'.format( activation, section)) # Create Conv2D layer if stride > 1: # Darknet uses left and top padding instead of 'same' mode prev_layer = ZeroPadding2D(((1, 0), (1, 0)))(prev_layer) conv_layer = (Conv2D(filters, (size, size), strides=(stride, stride), kernel_regularizer=l2(weight_decay), use_bias=not batch_normalize, weights=conv_weights, activation=act_fn, padding=padding))(prev_layer) if batch_normalize: conv_layer = (BatchNormalization( weights=bn_weight_list))(conv_layer) prev_layer = conv_layer if activation == 'linear': all_layers.append(prev_layer) elif activation == 'leaky': act_layer = LeakyReLU(alpha=0.1)(prev_layer) prev_layer = act_layer all_layers.append(act_layer) elif section.startswith('route'): ids = [int(i) for i in cfg_parser[section]['layers'].split(',')] layers = [all_layers[i] for i in ids] if len(layers) > 1: print('Concatenating route layers:', layers) concatenate_layer = Concatenate()(layers) all_layers.append(concatenate_layer) prev_layer = concatenate_layer else: skip_layer = layers[0] # only one layer to route all_layers.append(skip_layer) prev_layer = skip_layer elif section.startswith('maxpool'): size = int(cfg_parser[section]['size']) stride = int(cfg_parser[section]['stride']) all_layers.append( MaxPooling2D(pool_size=(size, size), strides=(stride, stride), padding='same')(prev_layer)) prev_layer = all_layers[-1] elif section.startswith('shortcut'): index = int(cfg_parser[section]['from']) activation = cfg_parser[section]['activation'] assert activation == 'linear', 'Only linear activation supported.' all_layers.append(Add()([all_layers[index], prev_layer])) prev_layer = all_layers[-1] elif section.startswith('upsample'): stride = int(cfg_parser[section]['stride']) assert stride == 2, 'Only stride=2 supported.' all_layers.append(UpSampling2D(stride)(prev_layer)) prev_layer = all_layers[-1] elif section.startswith('yolo'): out_index.append(len(all_layers) - 1) all_layers.append(None) prev_layer = all_layers[-1] elif section.startswith('net'): pass else: raise ValueError( 'Unsupported section header type: {}'.format(section)) # Create and save model. if len(out_index) == 0: out_index.append(len(all_layers) - 1) model = Model(inputs=input_layer, outputs=[all_layers[i] for i in out_index]) print(model.summary()) if args.weights_only: model.save_weights('{}'.format(output_path)) print('Saved Keras weights to {}'.format(output_path)) else: model.save('{}'.format(output_path)) print('Saved Keras model to {}'.format(output_path)) # Check to see if all weights have been read. remaining_weights = len(weights_file.read()) / 4 weights_file.close() print('Read {} of {} from Darknet weights.'.format( count, count + remaining_weights)) if remaining_weights > 0: print('Warning: {} unused weights'.format(remaining_weights)) if args.plot_model: plot(model, to_file='{}.png'.format(output_root), show_shapes=True) print('Saved model plot to {}.png'.format(output_root))
def network_builder(topology, weight_decay, weight_constraint, dropout, stage='block_optimization'): if weight_decay is not None: weight_decay = regularizers.l2(weight_decay) if weight_constraint is not None: weight_constraint = constraints.max_norm(weight_constraint, axis=0) input_dim = topology[0] output_dim = topology[-1][0] nb_hidden_layer = len(topology) - 2 inputs = Input((input_dim, ), name='inputs') hiddens = inputs for layer_iter in range(nb_hidden_layer): hidden_blocks = [] for block_iter in range(len(topology[1 + layer_iter])): dense_output = Dense(topology[1 + layer_iter][block_iter], kernel_initializer='he_normal', kernel_regularizer=weight_decay, kernel_constraint=weight_constraint, name='dense%d_%d' % (layer_iter, block_iter))(hiddens) bn_output = BN(name='bn%d_%d' % (layer_iter, block_iter))(dense_output) act_output = Activation('relu')(bn_output) if (stage=='block_optimization' and\ layer_iter == nb_hidden_layer -1 and\ block_iter == len(topology[1+layer_iter])-1 and\ dropout is not None) or\ (stage=='finetune' and dropout is not None): dropout_output = Dropout(dropout)(act_output) hidden_blocks.append(dropout_output) else: hidden_blocks.append(act_output) # if last layer, attach output prediction block if layer_iter == nb_hidden_layer - 1: output_blocks = [] for block_idx, hidden_block in enumerate(hidden_blocks): output_blocks.append( Dense(output_dim, kernel_initializer='he_normal', kernel_regularizer=weight_decay, kernel_constraint=weight_constraint, name='output%d_%d' % (nb_hidden_layer - 1, block_idx))(hidden_block)) # if not last hidden layer, concatenate all hidden blocks else: if len(hidden_blocks) > 1: hiddens = Concatenate(axis=-1)(hidden_blocks) else: hiddens = hidden_blocks[0] if len(output_blocks) > 1: outputs = Add()(output_blocks) else: outputs = output_blocks[0] output_activation = topology[-1][-1] pre_predictions = Activation('linear', name='pre_predictions')(outputs) if output_activation is not None: predictions = Activation(output_activation, name='predictions')(pre_predictions) else: predictions = Activation('linear', name='predictions')(pre_predictions) model = Model(inputs=inputs, outputs=predictions) return model