def fcn_8s(num_classes, input_shape, lr_init, lr_decay, vgg_weight_path=None): img_input = Input(input_shape) # Block 1 x = Conv2D(64, (3, 3), padding='same', name='block1_conv1')(img_input) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(64, (3, 3), padding='same', name='block1_conv2')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = MaxPooling2D()(x) # Block 2 x = Conv2D(128, (3, 3), padding='same', name='block2_conv1')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(128, (3, 3), padding='same', name='block2_conv2')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = MaxPooling2D()(x) # Block 3 x = Conv2D(256, (3, 3), padding='same', name='block3_conv1')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(256, (3, 3), padding='same', name='block3_conv2')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(256, (3, 3), padding='same', name='block3_conv3')(x) x = BatchNormalization()(x) x = Activation('relu')(x) block_3_out = MaxPooling2D()(x) # Block 4 x = Conv2D(512, (3, 3), padding='same', name='block4_conv1')(block_3_out) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(512, (3, 3), padding='same', name='block4_conv2')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(512, (3, 3), padding='same', name='block4_conv3')(x) x = BatchNormalization()(x) x = Activation('relu')(x) block_4_out = MaxPooling2D()(x) # Block 5 x = Conv2D(512, (3, 3), padding='same', name='block5_conv1')(block_4_out) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(512, (3, 3), padding='same', name='block5_conv2')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(512, (3, 3), padding='same', name='block5_conv3')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = MaxPooling2D()(x) # Load pretrained weights. if vgg_weight_path is not None: vgg16 = Model(img_input, x) vgg16.load_weights(vgg_weight_path, by_name=True) # Convolutinalized fully connected layer. x = Conv2D(4096, (7, 7), activation='relu', padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(4096, (1, 1), activation='relu', padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) # Classifying layers. x = Conv2D(num_classes, (1, 1), strides=(1, 1), activation='linear')(x) x = BatchNormalization()(x) block_3_out = Conv2D(num_classes, (1, 1), strides=(1, 1), activation='linear')(block_3_out) block_3_out = BatchNormalization()(block_3_out) block_4_out = Conv2D(num_classes, (1, 1), strides=(1, 1), activation='linear')(block_4_out) block_4_out = BatchNormalization()(block_4_out) x = Lambda(lambda x: tf.image.resize_images(x, (x.shape[1] * 2, x.shape[2] * 2)))(x) x = Add()([x, block_4_out]) x = Activation('relu')(x) x = Lambda(lambda x: tf.image.resize_images(x, (x.shape[1] * 2, x.shape[2] * 2)))(x) x = Add()([x, block_3_out]) x = Activation('relu')(x) x = Lambda(lambda x: tf.image.resize_images(x, (x.shape[1] * 8, x.shape[2] * 8)))(x) x = Activation('softmax')(x) model = Model(img_input, x) model.compile(optimizer=Adam(lr=lr_init, decay=lr_decay), loss='categorical_crossentropy', metrics=[dice_coef]) return model
def modelv2(nbChannels, stride=2, nbFilters=[64, 64, 128, 128, 256], nbEmbeddingDim=32, nbQuantizationBins=16, nbResidualLayers=1, **kwargs): inputs = Input(shape=(None, nbChannels)) x = inputs kernelSize = 5 with K.name_scope('encoder'): for i, n in enumerate(nbFilters): # Non-linear convolution followed by linear downsampling x = Conv1D(n, kernel_size=kernelSize, strides=1, activation='linear', padding='same', name='codec-conv-ds-%d' % (i + 1))(x) x = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(x) # Residual blocks for _ in range(nbResidualLayers): fx = Conv1D(n, kernel_size=kernelSize, strides=1, activation='linear', padding='same')(x) fx = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(fx) fx = Conv1D(n, kernel_size=kernelSize, strides=1, activation='linear', padding='same')(fx) x = Add()([fx, x]) x = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(x) x = Resample1D(scale=1.0 / stride, method='linear')(x) x = Conv1D(nbEmbeddingDim, kernel_size=kernelSize, strides=1, activation='linear', padding='same', name='codec-conv-embedding')(x) x = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(x) # Quantization if nbQuantizationBins is not None: quantization = SoftmaxQuantization(nbQuantizationBins, name='quantization') x = quantization(x) # Dequantization if nbQuantizationBins is not None: x = SoftmaxDequantization(nbQuantizationBins)(x) kernelSize = 5 with K.name_scope('decoder'): # Upsampling with resize convolutions for i, n in enumerate(reversed(nbFilters)): # Linear upsampling followed by non-linear convolution x = Resample1D(stride, method='linear')(x) x = Conv1D(n, kernel_size=kernelSize, strides=1, activation='linear', padding='same', name='codec-conv-us-%d' % (i + 1))(x) x = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(x) # Residual blocks for _ in range(nbResidualLayers): fx = Conv1D(n, kernel_size=kernelSize, strides=1, activation='linear', padding='same')(x) fx = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(fx) fx = Conv1D(n, kernel_size=kernelSize, strides=1, activation='linear', padding='same')(fx) x = Add()([fx, x]) x = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(x) xr = Conv1D(nbChannels, kernelSize, strides=1, activation='linear', padding='same', name='codec-conv-out')(x) outputs = xr return Model(inputs, outputs, **kwargs)
def generate_model(weight_decay=0.0005): ''' define the model structure --------------------------------------------------------------------------- INPUT: weight_decay: all the weights in the layer would be decayed by this factor OUTPUT: model: the model structure after being defined # References - [An Improved Deep Learning Architecture for Person Re-Identification] --------------------------------------------------------------------------- ''' def upsample_neighbor_function(input_x): #向input_x填充八个0值 input_x_pad = K.spatial_2d_padding(input_x, padding=((2,2),(2,2))) x_length = K.int_shape(input_x)[1] y_length = K.int_shape(input_x)[2] output_x_list = [] output_y_list = [] for i_x in range(2, x_length + 2): for i_y in range(2, y_length + 2): output_y_list.append(input_x_pad[:,i_x-2:i_x+3,i_y-2:i_y+3,:]) output_x_list.append(K.concatenate(output_y_list, axis=2)) output_y_list = [] return K.concatenate(output_x_list, axis=1) max_pooling = MaxPooling2D() x1_input = Input(shape=(160,60,3)) x2_input = Input(shape=(160,60,3)) #kernel_regularizer: 卷积核化的正则化,可选. share_conv_1 = Conv2D(20, 5, kernel_regularizer=l2(weight_decay), activation="relu") x1 = share_conv_1(x1_input) x2 = share_conv_1(x2_input) x1 = max_pooling(x1) x2 = max_pooling(x2) share_conv_2 = Conv2D(25, 5, kernel_regularizer=l2(weight_decay), activation="relu") x1 = share_conv_2(x1) x2 = share_conv_2(x2) x1 = max_pooling(x1) x2 = max_pooling(x2) upsample_same = UpSampling2D(size=(5, 5)) x1_up = upsample_same(x1) x2_up = upsample_same(x2) upsample_neighbor = Lambda(upsample_neighbor_function) x1_nn = upsample_neighbor(x1) x2_nn = upsample_neighbor(x2) negative = Lambda(lambda x: -x) x1_nn = negative(x1_nn) x2_nn = negative(x2_nn) x1 = Add()([x1_up, x2_nn]) x2 = Add()([x2_up, x1_nn]) conv_3_1 = Conv2D(25, 5, strides=(5, 5), kernel_regularizer=l2(weight_decay), activation="relu") conv_3_2 = Conv2D(25, 5, strides=(5, 5), kernel_regularizer=l2(weight_decay), activation="relu") x1 = conv_3_1(x1) x2 = conv_3_2(x2) conv_4_1 = Conv2D(25, 3, kernel_regularizer=l2(weight_decay), activation="relu") conv_4_2 = Conv2D(25, 3, kernel_regularizer=l2(weight_decay), activation="relu") x1 = conv_4_1(x1) x2 = conv_4_2(x2) x1 = max_pooling(x1) x2 = max_pooling(x2) y = Concatenate()([x1, x2]) y = Flatten()(y) y = Dense(500, kernel_regularizer=l2(weight_decay), activation='relu')(y) y = Dense(2, kernel_regularizer=l2(weight_decay), activation='softmax')(y) model = Model(inputs=[x1_input, x2_input], outputs=[y]) model.summary() return model
def unet2D( x_in, input_shape, out_im_chans, nf_enc=[64, 64, 128, 128, 256, 256, 512], nf_dec=None, regularizer=None, initializer=None, layer_prefix='unet', n_convs_per_stage=1, use_residuals=False, use_maxpool=False, concat_at_stages=None, do_last_conv=True, ks=3, ): reg_params = {} if regularizer == 'l1': reg = regularizers.l1(1e-6) else: reg = None if initializer == 'zeros': reg_params['kernel_initializer'] = initializers.Zeros() x = x_in encodings = [] for i in range(len(nf_enc)): if not use_maxpool and i > 0: x = LeakyReLU(0.2)(x) for j in range(n_convs_per_stage): if nf_enc[ i] is not None: # in case we dont want to convolve at the first resolution x = Conv2D(nf_enc[i], kernel_regularizer=reg, kernel_size=ks, strides=(1, 1), padding='same', name='{}_enc_conv2D_{}_{}'.format( layer_prefix, i, j + 1))(x) if concat_at_stages and concat_at_stages[i] is not None: x = Concatenate(axis=-1)([x, concat_at_stages[i]]) if j == 0 and use_residuals: residual_input = x elif j == n_convs_per_stage - 1 and use_residuals: x = Add()([residual_input, x]) x = LeakyReLU(0.2)(x) if i < len(nf_enc) - 1: encodings.append(x) if use_maxpool: x = MaxPooling2D(pool_size=(2, 2), padding='same', name='{}_enc_maxpool_{}'.format( layer_prefix, i))(x) else: x = Conv2D(nf_enc[i], kernel_size=ks, strides=(2, 2), padding='same', name='{}_enc_conv2D_{}'.format(layer_prefix, i))(x) print('Encodings to concat later: {}'.format(encodings)) if nf_dec is None: nf_dec = list(reversed(nf_enc[1:])) print('Decoder channels: {}'.format(nf_dec)) for i in range(len(nf_dec)): curr_shape = x.get_shape().as_list()[1:-1] # if we're not at full resolution, keep upsampling if np.any(list(curr_shape[:2]) < list(input_shape[:2])): x = UpSampling2D(size=(2, 2), name='{}_dec_upsamp_{}'.format(layer_prefix, i))(x) # if we still have things to concatenate, do that if (i + 1) <= len(encodings): curr_shape = x.get_shape().as_list()[1:-1] concat_with_shape = encodings[-i - 1].get_shape().as_list()[1:-1] x = _pad_or_crop_to_shape(x, curr_shape, concat_with_shape) x = Concatenate()([x, encodings[-i - 1]]) residual_input = x for j in range(n_convs_per_stage): x = Conv2D(nf_dec[i], kernel_regularizer=reg, kernel_size=ks, strides=(1, 1), padding='same', name='{}_dec_conv2D_{}_{}'.format(layer_prefix, i, j))(x) if j == 0 and use_residuals: residual_input = x elif j == n_convs_per_stage - 1 and use_residuals: x = Add()([residual_input, x]) x = LeakyReLU(0.2)(x) #x = Concatenate()([x, encodings[0]]) ''' for j in range(n_convs_per_stage - 1): x = Conv2D(out_im_chans, kernel_regularizer=reg, kernel_size=ks, strides=(1, 1), padding='same', name='{}_dec_conv2D_last_{}'.format(layer_prefix, j))(x) x = LeakyReLU(0.2)(x) ''' if do_last_conv: y = Conv2D(out_im_chans, kernel_size=1, padding='same', kernel_regularizer=reg, name='{}_dec_conv2D_final'.format(layer_prefix))( x) # add your own activation after this model else: y = x # add your own activation after this model return y
def unet3D( x_in, img_shape, out_im_chans, nf_enc=[64, 64, 128, 128, 256, 256, 512], nf_dec=None, regularizer=None, initializer=None, layer_prefix='unet', n_convs_per_stage=1, include_residual=False, use_maxpool=True, max_time_downsample=None, n_tasks=1, use_dropout=False, do_unpool=False, do_last_conv=True, ): ks = 3 if max_time_downsample is None: max_time_downsample = len( nf_enc) # downsample in time all the way down encoding_im_sizes = np.asarray([( int(np.ceil(img_shape[0] / 2.0**i)), int(np.ceil(img_shape[1] / 2.0**i)), int(np.ceil(img_shape[2] / 2.0**i)), ) for i in range(0, len(nf_enc) + 1)]) else: encoding_im_sizes = np.asarray([( int(np.ceil(img_shape[0] / 2.0**i)), int(np.ceil(img_shape[1] / 2.0**i)), max(int(np.ceil(img_shape[2] / 2.0**(max_time_downsample))), int(np.ceil(img_shape[2] / 2.0**i))), ) for i in range(0, len(nf_enc) + 1)]) reg_params = {} if regularizer == 'l1': reg = regularizers.l1(1e-6) else: reg = None if initializer == 'zeros': reg_params['kernel_initializer'] = initializers.Zeros() x = x_in encodings = [] encoding_im_sizes = [] for i in range(len(nf_enc)): if not use_maxpool and i > 0: x = LeakyReLU(0.2)(x) for j in range(n_convs_per_stage): if nf_enc[ i] is not None: # in case we dont want to convovle at max resolution x = Conv3D(nf_enc[i], kernel_regularizer=reg, kernel_size=ks, strides=(1, 1, 1), padding='same', name='{}_enc_conv3D_{}_{}'.format( layer_prefix, i, j + 1))(x) #if use_dropout: # x = Dropout(0.2)(x) if j == 0 and include_residual: residual_input = x elif j == n_convs_per_stage - 1 and include_residual: x = Add()([residual_input, x]) x = LeakyReLU(0.2)(x) encodings.append(x) encoding_im_sizes.append(np.asarray(x.get_shape().as_list()[1:-1])) # only downsample if we haven't reached the max if i >= max_time_downsample: ds = (2, 2, 1) else: ds = (2, 2, 2) if i < len(nf_enc) - 1: if use_maxpool: x = MaxPooling3D(pool_size=ds, padding='same', name='{}_enc_maxpool_{}'.format( layer_prefix, i))(x) #x, pool_idxs = Lambda(lambda x:._max_pool_3d_with_argmax(x, ksize=ks, strides=(2, 2, 2), padding='same'), name='{}_enc_maxpool3dwithargmax_{}'.format(layer_prefix, i))(x) else: x = Conv3D(nf_enc[i], kernel_size=ks, strides=ds, padding='same', name='{}_enc_conv3D_{}'.format(layer_prefix, i))(x) if nf_dec is None: nf_dec = list(reversed(nf_enc[1:])) decoder_outputs = [] x_encoded = x print(encoding_im_sizes) print(nf_dec) for ti in range(n_tasks): decoding_im_sizes = [] x = x_encoded for i in range(len(nf_dec)): curr_shape = x.get_shape().as_list()[1:-1] print('Current shape {}, img shape {}'.format( x.get_shape().as_list(), img_shape)) # only do upsample if we are not yet at max resolution if np.any(curr_shape < list(img_shape[:len(curr_shape)])): # TODO: fix this for time ''' if i < len(nf_dec) - max_time_downsample + 1 \ or curr_shape[-1] >= encoding_im_sizes[-i-2][-1]: # if we are already at the correct time scale us = (2, 2, 1) else: ''' us = (2, 2, 2) #decoding_im_sizes.append(encoding_im_sizes[-i-1] * np.asarray(us)) x = UpSampling3D(size=us, name='{}_dec{}_upsamp_{}'.format( layer_prefix, ti, i))(x) # just concatenate the final layer here if i <= len(encodings) - 2: x = _pad_or_crop_to_shape_3D( x, np.asarray(x.get_shape().as_list()[1:-1]), encoding_im_sizes[-i - 2]) x = Concatenate(axis=-1)([x, encodings[-i - 2]]) #x = LeakyReLU(0.2)(x) residual_input = x for j in range(n_convs_per_stage): x = Conv3D(nf_dec[i], kernel_regularizer=reg, kernel_size=ks, strides=(1, 1, 1), padding='same', name='{}_dec{}_conv3D_{}_{}'.format( layer_prefix, ti, i, j))(x) if use_dropout and i < 2: x = Dropout(0.2)(x) if j == 0 and include_residual: residual_input = x elif j == n_convs_per_stage - 1 and include_residual: x = Add()([residual_input, x]) x = LeakyReLU(0.2)(x) if do_last_conv: y = Conv3D(out_im_chans, kernel_size=1, padding='same', kernel_regularizer=reg, name='{}_dec{}_conv3D_final'.format(layer_prefix, ti))( x) # add your own activation after this model else: y = x decoder_outputs.append(y) # add your own activation after this model if n_tasks == 1: return y else: return decoder_outputs
def create_deepconn_dp(self): dotproduct = Dot(axes=1)([self.towerU, self.towerM]) output = Add()([self.outNeuron, dotproduct]) self.model = Model(inputs=[self.inputU, self.inputM], outputs=[output]) self.model.compile(optimizer='Adam', loss='mse')
def encoder3D(x, img_shape, conv_chans=None, n_convs_per_stage=1, min_h=5, min_c=None, prefix='vte', ks=3, return_skips=False, use_residuals=False, use_maxpool=False, max_time_downsample=None): skip_layers = [] concat_skip_sizes = [] if max_time_downsample is None: # do not attempt to downsample beyond 1 max_time_downsample = int(np.floor(np.log2(img_shape[-2]))) - 1 print('Max downsamples in time: {}'.format(max_time_downsample)) if conv_chans is None: n_convs = int(np.floor(np.log2(img_shape[0] / min_h))) conv_chans = [min_c * 2] * (n_convs - 1) + [min_c] elif not type(conv_chans) == list: n_convs = int(np.floor(np.log2(img_shape[0] / min_h))) conv_chans = [conv_chans] * (n_convs - 1) + [min_c] for i in range(len(conv_chans)): if n_convs_per_stage is not None and n_convs_per_stage > 1 or use_maxpool and n_convs_per_stage is not None: for ci in range(n_convs_per_stage): x = Conv3D(conv_chans[i], kernel_size=ks, padding='same', name='{}_enc_conv3D_{}_{}'.format( prefix, i, ci + 1))(x) if ci == 0 and use_residuals: residual_input = x elif ci == n_convs_per_stage - 1 and use_residuals: x = Add(name='{}_enc_{}_add_residual'.format(prefix, i))( [residual_input, x]) x = LeakyReLU(0.2, name='{}_enc_leakyrelu_{}_{}'.format( prefix, i, ci + 1))(x) if return_skips: skip_layers.append(x) concat_skip_sizes.append(x.get_shape().as_list()[1:-1]) # only downsample if we are below the max number of downsamples in time if i < max_time_downsample: strides = (2, 2, 2) else: strides = (2, 2, 1) if use_maxpool: x = MaxPooling3D(pool_size=strides, name='{}_enc_maxpool_{}'.format(prefix, i))(x) else: x = Conv3D(conv_chans[i], kernel_size=ks, strides=strides, padding='same', name='{}_enc_conv3D_{}'.format(prefix, i))(x) if i < len(conv_chans) - 1: # no activation on last convolution x = LeakyReLU(0.2, name='{}_enc_leakyrelu_{}'.format(prefix, i))(x) if min_c is not None and min_c > 0: if n_convs_per_stage is not None and n_convs_per_stage > 1: for ci in range(n_convs_per_stage): x = Conv3D(min_c, kernel_size=ks, padding='same', name='{}_enc_conv3D_last_{}'.format(prefix, ci + 1))(x) if ci == 0 and use_residuals: residual_input = x elif ci == n_convs_per_stage - 1 and use_residuals: x = Add( name='{}_enc_{}_add_residual'.format(prefix, 'last'))( [residual_input, x]) x = LeakyReLU(0.2, name='{}_enc_leakyrelu_last'.format(prefix))(x) x = Conv3D(min_c, kernel_size=ks, strides=(1, 1, 1), padding='same', name='{}_enc_conv3D_last'.format(prefix))(x) if return_skips: skip_layers.append(x) concat_skip_sizes.append(x.get_shape().as_list()[1:-1]) if return_skips: return x, skip_layers, concat_skip_sizes else: return x
def train(self, trainingdata, validation_data, pretrained=None): in_dim = len( ConfigurationVector(Configuration([], None), None).get_vector()) # Shared model if pretrained: base_model = load_pretrained_base_model(pretrained) else: base_model = make_base_model(in_dim) self.base_model = base_model # All golden decisions = ONE SEQUENCE golden = [] golden_inputs = [] for i in range(self.k): name = "Golden" + str(i) # Configuration feature vector input state_input = Input(shape=(in_dim, ), name=name) golden_inputs.append(state_input) # Decision input decision_input = Input(shape=(4, )) golden_inputs.append(decision_input) # Probabilities of decisions under model distribution = base_model(state_input) # Decision activation output = Dot(axes=0)([distribution, decision_input]) golden.append(output) # Sum all probabilities in golden sequence golden_sum = Add()(golden) # All decisions in the beam = MULTIPLE SEQUENCES beam = [] beam_inputs = [] for i in range(self.b): sequence = [] for j in range(self.k): name = "Beam{i}.{j}".format(i=i, j=j) # Configuration feature vector input state_input = Input(shape=(in_dim, ), name=name) beam_inputs.append(state_input) # Decision input decision_input = Input(shape=(4, )) beam_inputs.append(decision_input) # Probabilities of decisions under model distribution = base_model(state_input) # Decision activation output = Dot(axes=0)([distribution, decision_input]) sequence.append(output) beam.append(sequence) # Make the inner sums over all sequences in the beam inner_sequence_sums = list(map(Add(), beam)) # Apply exp to all inner sums exp_activation = Activation(K.exp) inner_sequence_sums = list(map(exp_activation, inner_sequence_sums)) if len(inner_sequence_sums) > 1: # Sum all beam sequences together outer_sum = Add()(inner_sequence_sums) else: outer_sum = inner_sequence_sums # Log activation outer_sum = Activation(K.log)(outer_sum) # Negate golden sum golden_sum = Activation(negativeActivation)(golden_sum) output = Add()([golden_sum, outer_sum]) # Add beam inputs to all inputs inputs = golden_inputs + beam_inputs model = Model(inputs, output) self.machine = model self.graph = tf.get_default_graph() model.compile(loss=global_norm_loss, optimizer=SGD()) model.fit_generator( self.generate_training_data(trainingdata), verbose=1, epochs=100, steps_per_epoch=20, callbacks=[earlyStopping, csv_logger], validation_data=self.generate_training_data(validation_data), validation_steps=20, max_q_size=1) self.save()
def segnet_crf_res(nClasses, optimizer=None, input_height=360, input_width=480): kernel = 3 filter_size = 64 pad = 1 pool_size = 2 img_input = Input(shape=(input_height, input_width, 3)) # encoder x = ZeroPadding2D(padding=(pad, pad))(img_input) x = Convolution2D(filter_size, (kernel, kernel), padding='valid')(x) x = BatchNormalization()(x) x = Activation('relu')(x) l1 = x x = MaxPooling2D(pool_size=(pool_size, pool_size))(x) x = ZeroPadding2D(padding=(pad, pad))(x) x = Convolution2D(128, (kernel, kernel), padding='valid')(x) x = BatchNormalization()(x) x = Activation('relu')(x) l2 = x x = MaxPooling2D(pool_size=(pool_size, pool_size))(x) x = ZeroPadding2D(padding=(pad, pad))(x) x = Convolution2D(256, (kernel, kernel), padding='valid')(x) x = BatchNormalization()(x) x = Activation('relu')(x) l3 = x x = MaxPooling2D(pool_size=(pool_size, pool_size))(x) x = ZeroPadding2D(padding=(pad, pad))(x) x = Convolution2D(512, (kernel, kernel), padding='valid')(x) x = BatchNormalization()(x) l4 = x x = Activation('relu')(x) # decoder x = ZeroPadding2D(padding=(pad, pad))(x) x = Convolution2D(512, (kernel, kernel), padding='valid')(x) x = BatchNormalization()(x) x = Add()([l4, x]) x = UpSampling2D(size=(pool_size, pool_size))(x) x = ZeroPadding2D(padding=(pad, pad))(x) x = Convolution2D(256, (kernel, kernel), padding='valid')(x) x = BatchNormalization()(x) x = Add()([l3, x]) x = UpSampling2D(size=(pool_size, pool_size))(x) x = ZeroPadding2D(padding=(pad, pad))(x) x = Convolution2D(128, (kernel, kernel), padding='valid')(x) x = BatchNormalization()(x) x = Add()([l2, x]) x = UpSampling2D(size=(pool_size, pool_size))(x) x = ZeroPadding2D(padding=(pad, pad))(x) x = Convolution2D(filter_size, (kernel, kernel), padding='valid')(x) x = BatchNormalization()(x) x = Add()([l1, x]) x = Convolution2D(nClasses, (1, 1), padding='valid')(x) out = CrfRnnLayer(image_dims=(input_height, input_width), num_classes=nClasses, theta_alpha=160., theta_beta=3., theta_gamma=3., num_iterations=5, name='crfrnn')([x, img_input]) # out = x a = Model(inputs=img_input, outputs=out) model = [] a.outputHeight = a.output_shape[1] a.outputWidth = a.output_shape[2] out = Reshape((a.outputHeight * a.outputWidth, nClasses), input_shape=(nClasses, a.outputHeight, a.outputWidth))(out) out = Activation('softmax')(out) # if not optimizer is None: # model.compile(loss="categorical_crossentropy", optimizer=optimizer, metrics=['accuracy']) model = Model(inputs=img_input, outputs=out) model.outputHeight = a.outputHeight model.outputWidth = a.outputWidth return model
def gossip_conv_block(self, images, masks, nfilters): cfs = self.conv_filter_size conv = lambda: Conv2D(nfilters, cfs, activation='linear', padding=self.conv_padding, kernel_initializer=glorot_uniform(seed=42)) if self.shared: cc = conv() conv = lambda: cc images = [conv()(l) for l in images] """ constant = Constant(value=1. / float(cfs[0] * cfs[1])) mask_conv = Conv2D(1, cfs, activation='linear', padding=self.conv_padding, kernel_initializer=constant, bias_initializer=Zeros(), trainable=False) mask_conv.trainable = False masks = [mask_conv(l) for l in masks] """ if self.conv_padding == 'valid': crop_cfs = int(floor(cfs[0] / 2)) crop_cfs = (crop_cfs, crop_cfs) crop = Cropping2D(cropping=(crop_cfs, crop_cfs)) masks = [crop(m) for m in masks] # Merge connections act_diff = [ Lambda(lambda_diff)([this_, other_]) for this_, other_ in zip(images, images[::-1]) ] #this_high = [Activation('relu')(d) for d in act_diff] other_high = [ Activation('relu')(Lambda(lambda_neg)(d)) for d in act_diff ] # FIXME: without relu #other_high = [Lambda(lambda_neg)(d) for d in act_diff] stimuli = [ Lambda(lambda_neg)(Multiply()([m, d])) for m, d in zip(masks, other_high) ] """ stimuli = [Lambda(lambda_diff)([p, n]) for p, n in zip(pos_stimuli, neg_stimuli)] #stimuli = Lambda(lambda_diff)(stimuli) #stimuli = [stimuli, Lambda(lambda_neg)(stimuli)] """ #stimuli = [Lambda(lambda x: self.reciprocal_stimuli * x)(s) #for s in stimuli] # Second merge images = [Add()([i, s]) for i, s in zip(images, stimuli)] #images = [Activation('relu')(i) for i in images] return images, masks
def resblock(x, kernel_size, resample, nfilters, norm=BatchNormalization): assert resample in ["UP", "SAME", "DOWN"] if resample == "UP": shortcut = UpSampling2D(size=(2, 2))(x) shortcut = Conv2D(nfilters, kernel_size, padding='same', kernel_initializer='he_uniform', use_bias=True)(shortcut) convpath = norm()(x) convpath = Activation('relu')(convpath) convpath = UpSampling2D(size=(2, 2))(convpath) convpath = Conv2D(nfilters, kernel_size, kernel_initializer='he_uniform', use_bias=False, padding='same')(convpath) convpath = norm()(convpath) convpath = Activation('relu')(convpath) convpath = Conv2D(nfilters, kernel_size, kernel_initializer='he_uniform', use_bias=True, padding='same')(convpath) y = Add()([shortcut, convpath]) elif resample == "SAME": shortcut = Conv2D(nfilters, kernel_size, padding='same', kernel_initializer='he_uniform', use_bias=True)(x) convpath = norm()(x) convpath = Activation('relu')(convpath) convpath = Conv2D(nfilters, kernel_size, kernel_initializer='he_uniform', use_bias=False, padding='same')(convpath) convpath = norm()(convpath) convpath = Activation('relu')(convpath) convpath = Conv2D(nfilters, kernel_size, kernel_initializer='he_uniform', use_bias=True, padding='same')(convpath) y = Add()([shortcut, convpath]) else: shortcut = AveragePooling2D(pool_size=(2, 2))(x) shortcut = Conv2D(nfilters, kernel_size, kernel_initializer='he_uniform', padding='same', use_bias=True)(shortcut) convpath = norm()(x) convpath = Activation('relu')(convpath) convpath = Conv2D(nfilters, kernel_size, kernel_initializer='he_uniform', use_bias=False, padding='same')(convpath) convpath = AveragePooling2D(pool_size=(2, 2))(convpath) convpath = norm()(convpath) convpath = Activation('relu')(convpath) convpath = Conv2D(nfilters, kernel_size, kernel_initializer='he_uniform', use_bias=True, padding='same')(convpath) y = Add()([shortcut, convpath]) return y
def separable_residual_block(inputs, n_filters_list=[256, 256, 256], block_id="entry_block2", skip_type="sum", stride=1, rate=1, weight_decay=1e-4, kernel_initializer="he_normal", bn_epsilon=1e-3, bn_momentum=0.99): """ separable residual block :param inputs: 4-D tensor, shape of (batch_size, height, width, channel). :param n_filters_list: list of int, numbers of filters in the separable convolutions, default [256, 256, 256]. :param block_id: string, default "entry_block2". :param skip_type: string, one of {"sum", "conv", "none"}, default "sum". :param stride: int, default 1. :param rate: int, default 1. :param weight_decay: float, default 1e-4. :param kernel_initializer: string, default "he_normal". :param bn_epsilon: float, default 1e-3. :param bn_momentum: float, default 0.99. :return: 4-D tensor, shape of (batch_size, height, width, channel). """ x = Activation("relu", name=block_id + "sepconv1_act")(inputs) x = SeparableConv2D(n_filters_list[0], (3, 3), padding='same', use_bias=False, name=block_id + '_sepconv1', dilation_rate=rate, kernel_initializer=kernel_initializer, kernel_regularizer=l2(weight_decay))(x) x = BatchNormalization(name=block_id + '_sepconv1_bn', epsilon=bn_epsilon, momentum=bn_momentum)(x) x = Activation('relu', name=block_id + '_sepconv2_act')(x) x = SeparableConv2D(n_filters_list[1], (3, 3), padding='same', use_bias=False, name=block_id + '_sepconv2', dilation_rate=rate, kernel_initializer=kernel_initializer, kernel_regularizer=l2(weight_decay))(x) x = BatchNormalization(name=block_id + '_sepconv2_bn', epsilon=bn_epsilon, momentum=bn_momentum)(x) x = Activation("relu", name=block_id + "_sepconv3_act")(x) x = SeparableConv2D(n_filters_list[2], (3, 3), padding="same", use_bias=False, strides=stride, name=block_id + "_sepconv3", dilation_rate=rate, kernel_initializer=kernel_initializer, kernel_regularizer=l2(weight_decay))(x) x = BatchNormalization(name=block_id + "_sepconv3_bn", epsilon=bn_epsilon, momentum=bn_momentum)(x) if skip_type == "sum": x = Add()([inputs, x]) elif skip_type == "conv": shortcut = Conv2D(n_filters_list[2], (1, 1), strides=stride, padding='same', use_bias=False, kernel_initializer=kernel_initializer, kernel_regularizer=l2(weight_decay))(inputs) shortcut = BatchNormalization(epsilon=bn_epsilon, momentum=bn_momentum)(shortcut) x = Add()([shortcut, x]) else: x = x return x
def stresnet(c_conf=(3, 2, 32, 32), p_conf=(3, 2, 32, 32), t_conf=(3, 2, 32, 32), external_dim=8, nb_residual_unit=3): ''' C - Temporal Closeness P - Period T - Trend conf = (len_seq, nb_flow, map_height, map_width) external_dim ''' # main input main_inputs = [] outputs = [] # c_conf, p_conf, t_conf # for conf in [ c_conf]: # if conf is not None: # # base 模型 # len_seq, nb_flow, map_height, map_width = conf # input = Input(shape=(nb_flow * len_seq, map_height, map_width)) # main_inputs.append(input) # # Conv1 # # conv1 = Convolution2D(nb_filter=64, nb_row=3, nb_col=3, border_mode="same")(input) # conv1 = Conv2D(padding="same", kernel_size=(stride, stride), filters=64)(input) # # [nb_residual_unit] Residual Units # residual_output = ResUnits(_residual_unit, nb_filter=64, # repetations=nb_residual_unit,)(conv1) # # Conv2 # activation = Activation('relu')(residual_output) # # conv2 = Convolution2D( # # nb_filter=nb_flow, nb_row=3, nb_col=3, border_mode="same")(activation) # conv2 = Convolution2D( # nb_filter=nb_flow, nb_row=3, nb_col=3, border_mode="same")(activation) # outputs.append(conv2) for conf in [c_conf]: if conf is not None: # base 模型 # len_seq, nb_flow, map_height, map_width = conf # input = Input(shape=(nb_flow * len_seq, map_height, map_width)) # main_inputs.append(input) # # Conv1 # # conv1 = Convolution2D(nb_filter=64, nb_row=3, nb_col=3, border_mode="same")(input) # conv1 = Conv2D(padding="same", kernel_size=(3, 3), filters=64)(input) # # [nb_residual_unit] Residual Units # residual_output = ResUnits(_residual_unit, nb_filter=64, # repetations=nb_residual_unit)(conv1) # # Conv2 # activation = Activation('relu')(residual_output) # # conv2 = Convolution2D( # # nb_filter=nb_flow, nb_row=3, nb_col=3, border_mode="same")(activation) # conv2 = Convolution2D( # nb_filter=output_nb_flow, nb_row=3, nb_col=3, border_mode="same")(activation) # outputs.append(conv2) # 我的模型 len_seq, nb_flow, map_height, map_width = conf # input=main_inputs[0] input = Input(shape=(nb_flow * len_seq, map_height, map_width)) # main_inputs[1:3]=main_inputs[0:2] # main_inputs[0]=input main_inputs.append(input) input = Reshape((len_seq, nb_flow, map_height, map_width))(input) timeSliceOutputs = [] print(input.shape[1]) # 定义共用的CNN模型 # conv1=Reshape((nb_flow, map_height, map_width))(input[:,timeSlice]) # Conv1 aa = Conv2D(padding="same", kernel_size=(stride, stride), filters=64) # conv1 = Conv2D(padding="same", kernel_size=(3, 3), filters=64) # [nb_residual_unit] Residual Units nb_residual_unit = 5 resIntput = Input(shape=(64, map_height, map_width)) # resIntput=Input(shape=aa.output_shape) bb = ResUnits(_residual_unit, nb_filter=64, repetations=nb_residual_unit, pool=True)(resIntput) resModel = Model(resIntput, bb) resModel.summary() plot_model(resModel, to_file='resModel.png') # Conv2 cc = Activation('relu') # conv2 = Convolution2D( # nb_filter=nb_flow, nb_row=3, nb_col=3, border_mode="same")(activation) dd = Convolution2D( nb_filter=128, nb_row=2, nb_col=2, border_mode="same") ee = Reshape((1, -1)) for timeSlice in range(len_seq): ii = sliceLayer(1, timeSlice)(input) # ii=input[:, timeSlice] conv2 = ee(dd(cc(resModel(aa(ii))))) timeSliceOutputs.append(conv2) convOutput = Concatenate(axis=1)(timeSliceOutputs) # lstm = GRU(output_nb_flow * map_height * map_width)(convOutput) # lstm = GRU(2 * map_height * map_width)(convOutput) lstm = LSTM(600)(convOutput) # input+lstm out1 = sliceLayer(1, len_seq - 1)(input) def antirectifier(x): # x=np.reshape(x,[-1,18,2,64,64]) # x = tf.reshape(x, [-1, 18, 2, 64, 64]) # x = tf.reduce_sum(x, axis=1) x = K.reshape(x, [-1, 18, 2, map_height, map_width]) x = K.sum(x, axis=1) return x def antirectifier_output_shape(input_shape): return (input_shape[0], 2, map_height, map_width) # out1 = Lambda(antirectifier, # output_shape=antirectifier_output_shape)(out1) # out2 = Flatten()(lstm) sig = Dense(output_dim=nb_flow * map_height * map_width, activation='sigmoid')(lstm) sig = Reshape((nb_flow, map_height, map_width))(sig) tan = Dense(output_dim=nb_flow * map_height * map_width, activation='tanh')(lstm) tan = Reshape((nb_flow, map_height, map_width))(tan) den = Dense(output_dim=nb_flow * map_height * map_width)(lstm) den = Reshape((nb_flow, map_height, map_width))(den) # out2=Add()([Multiply()([outputs[0],sig]),tan,out1]) mul=Multiply()([out1, sig]) out2 = Add()([mul, den]) # output = Add()([out1, tan]) # outputs[0]=out2 outputs.append(out2) # len_seq, nb_flow, map_height, map_width = c_conf # input = main_inputs[0] # # input = Input(shape=(nb_flow * len_seq, map_height, map_width)) # input2 = Reshape((len_seq, nb_flow * map_height * map_width))(input) # lstm = LSTM(output_nb_flow * map_height * map_width)(input) # output = Reshape((output_nb_flow, map_height, map_width))(lstm) # outputs.append(output) # outputs=outputs[1:] # parameter-matrix-based fusion if len(outputs) == 1: main_output = outputs[0] else: new_outputs = [] for output in outputs: # new_outputs.append(iLayer()(output)) new_outputs.append(mulLayer()(output)) main_output = Add()(new_outputs) # main_output=Add()(outputs) # fusing with external component # if external_dim != None and external_dim > 0: # # external input # external_input = Input(shape=(external_dim,)) # main_inputs.append(external_input) # embedding = Dense(output_dim=10)(external_input) # embedding = Activation('relu')(embedding) # # h1 = Dense(output_dim=nb_flow * map_height * map_width)(embedding) # h1 = Dense(output_dim=nb_flow * map_height * map_width)(embedding) # activation = Activation('relu')(h1) # # external_output = Reshape((nb_flow, map_height, map_width))(activation) # external_output = Reshape((nb_flow, map_height, map_width))(activation) # main_output = merge([main_output, external_output], mode='sum') # else: # print('external_dim:', external_dim) main_output = Activation('tanh')(main_output) def antirectifier2(x): #x=np.reshape(x,[-1,18,2,64,64]) x = K.reshape(x,[-1,18,2,map_height,map_width]) x = K.sum(x, axis=1) return x # def antirectifier_output_shape(input_shape): # # return (input_shape[0],2,64,64) # # main_output=Lambda(antirectifier2 )(main_output) model = Model(input=main_inputs, output=main_output) return model
def _shortcut(input, residual): # return merge([input, residual], mode='sum') return Add()([input, residual])
def encoder(x, img_shape, conv_chans=None, n_convs_per_stage=1, min_h=5, min_c=None, prefix='', ks=3, return_skips=False, use_residuals=False, use_maxpool=False, use_batchnorm=False): skip_layers = [] concat_skip_sizes = [] n_dims = len(img_shape) - 1 # assume img_shape includes spatial dims, followed by channels if conv_chans is None: n_convs = int(np.floor(np.log2(img_shape[0] / min_h))) conv_chans = [min_c * 2] * (n_convs - 1) + [min_c] elif not type(conv_chans) == list: n_convs = int(np.floor(np.log2(img_shape[0] / min_h))) conv_chans = [conv_chans] * (n_convs - 1) + [min_c] for i in range(len(conv_chans)): #if n_convs_per_stage is not None and n_convs_per_stage > 1 or use_maxpool and n_convs_per_stage is not None: for ci in range(n_convs_per_stage): x = myConv(nf=conv_chans[i], ks=ks, strides=1, n_dims=n_dims, prefix='{}_enc'.format(prefix), suffix='{}_{}'.format(i, ci + 1))(x) if ci == 0 and use_residuals: residual_input = x elif ci == n_convs_per_stage - 1 and use_residuals: x = Add(name='{}_enc_{}_add_residual'.format(prefix, i))([residual_input, x]) if use_batchnorm: x = BatchNormalization()(x) x = LeakyReLU(0.2, name='{}_enc_leakyrelu_{}_{}'.format(prefix, i, ci + 1))(x) if return_skips: skip_layers.append(x) concat_skip_sizes.append(np.asarray(x.get_shape().as_list()[1:-1])) if use_maxpool and i < len(conv_chans) - 1: # changed 5/30/19, don't pool after our last conv x = myPool(n_dims=n_dims, prefix=prefix, suffix=i)(x) else: x = myConv(conv_chans[i], ks=ks, strides=2, n_dims=n_dims, prefix='{}_enc'.format(prefix), suffix=i)(x) # don't activate right after a maxpool, it makes no sense if i < len(conv_chans) - 1: # no activation on last convolution x = LeakyReLU(0.2, name='{}_enc_leakyrelu_{}'.format(prefix, i))(x) if min_c is not None and min_c > 0: # if the last number of channels is specified, convolve to that if n_convs_per_stage is not None and n_convs_per_stage > 1: for ci in range(n_convs_per_stage): x = myConv(min_c, ks=ks, n_dims=n_dims, strides=1, prefix='{}_enc'.format(prefix), suffix='last_{}'.format(ci + 1))(x) if ci == 0 and use_residuals: residual_input = x elif ci == n_convs_per_stage - 1 and use_residuals: x = Add(name='{}_enc_{}_add_residual'.format(prefix, 'last'))([residual_input, x]) x = LeakyReLU(0.2, name='{}_enc_leakyrelu_last'.format(prefix))(x) x = myConv(min_c, ks=ks, strides=1, n_dims=n_dims, prefix='{}_enc'.format(prefix), suffix='_last')(x) if return_skips: skip_layers.append(x) concat_skip_sizes.append(np.asarray(x.get_shape().as_list()[1:-1])) if return_skips: return x, skip_layers, concat_skip_sizes else: return x
def get_DPCNN(embedding_matrix, sequence_length, dropout_rate, recurrent_units, dense_size, filter_sizes=3, repeat=4): input_layer = Input(shape=(sequence_length,)) num_filters = embedding_matrix.shape[1] embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1], weights=[embedding_matrix], trainable=False)(input_layer) z = SpatialDropout1D(rate=dropout_rate)(embedding_layer) # x = Bidirectional(CuDNNGRU(recurrent_units, return_sequences=True))(x) conv = Convolution1D(filters=num_filters, kernel_size=filter_sizes, padding="same", activation="relu", strides=1, kernel_regularizer=regularizers.l2(0.0001), activity_regularizer=regularizers.l2(0.0001), bias_regularizer=regularizers.l2(0.0001) )(z) conv = Convolution1D(filters=num_filters, kernel_size=filter_sizes, padding="same", activation="relu", strides=1, kernel_regularizer=regularizers.l2(0.0001), activity_regularizer=regularizers.l2(0.0001), bias_regularizer=regularizers.l2(0.0001) )(conv) # conv = Dropout(dropout_rate)(conv) conv = Add()([conv, z]) for i in range(repeat): pool = MaxPooling1D(pool_size=3, strides=2, padding="same")(conv) # conv = SpatialDropout1D(rate=dropout_rate)(pool) conv = Convolution1D(filters=num_filters, kernel_size=filter_sizes, padding="same", activation="relu", strides=1, kernel_regularizer=regularizers.l2(0.0001), activity_regularizer=regularizers.l2(0.0001), bias_regularizer=regularizers.l2(0.0001) )(pool) conv = Convolution1D(filters=num_filters, kernel_size=filter_sizes, padding="same", activation="relu", strides=1, kernel_regularizer=regularizers.l2(0.0001), activity_regularizer=regularizers.l2(0.0001), bias_regularizer=regularizers.l2(0.0001) )(conv) # conv = Dropout(dropout_rate)(conv) conv = Add()([conv, pool]) #z1 = GlobalMaxPooling1D()(conv) #z2 = GlobalAveragePooling1D()(conv) #z = Concatenate()([z1, z2]) z = MaxPooling1D(3)(conv) z = Flatten()(z) z = Dense(dense_size, activation="relu", kernel_regularizer=regularizers.l2(0.0001), activity_regularizer=regularizers.l2(0.0001), bias_regularizer=regularizers.l2(0.0001))(z) # z = Dropout(0.5)(z) output_layer = Dense(6, activation="sigmoid", kernel_regularizer=regularizers.l2(0.0001), activity_regularizer=regularizers.l2(0.0001), bias_regularizer=regularizers.l2(0.0001))(z) model = Model(inputs=input_layer, outputs=output_layer) model.compile(loss='binary_crossentropy', optimizer=Nadam(), metrics=['accuracy']) return model
def net_module(input_shape, num_outputs): """Builds a net architecture. Args: input_shape: The input shape in the form (nb_rows, nb_cols, nb_channels) num_outputs: The number of outputs at final softmax layer Returns: The keras `Model`. """ CHANNEL_AXIS = 3 handle_dim_ordering() if len(input_shape) != 3: raise Exception( "Input shape should be a tuple (nb_rows, nb_cols, nb_channels)") # Permute dimension order if necessary if K.image_dim_ordering() != 'tf': input_shape = (input_shape[2], input_shape[0], input_shape[1]) input_img0 = Input(shape=input_shape, name="input_img0") input_img1 = Input(shape=input_shape, name="input_img1") concatenate = Concatenate(axis=CHANNEL_AXIS, name="concatenate")([input_img0, input_img1]) base_channel = 24 block_conv_1 = conv_bn_leakyrelu_repetition_block( filters=1 * base_channel, kernel_size=(3, 3), repetitions=2, first_layer_down_size=False, alpha=0.0, name="conv_block1")(concatenate) block_pool_2 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid', data_format=None, name="pool_block2")(block_conv_1) block_conv_2 = conv_bn_leakyrelu_repetition_block( filters=2 * base_channel, kernel_size=(3, 3), repetitions=2, first_layer_down_size=False, alpha=0.0, name="conv_block2")(block_pool_2) block_pool_4 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid', data_format=None, name="pool_block4")(block_conv_2) block_conv_4 = conv_bn_leakyrelu_repetition_block( filters=4 * base_channel, kernel_size=(3, 3), repetitions=2, first_layer_down_size=False, alpha=0.0, name="conv_block4")(block_pool_4) block_pool_8 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid', data_format=None, name="pool_block8")(block_conv_4) block_conv_8 = conv_bn_leakyrelu_repetition_block( filters=8 * base_channel, kernel_size=(3, 3), repetitions=2, first_layer_down_size=False, alpha=0.0, name="conv_block8")(block_pool_8) block_pool_16 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid', data_format=None, name="pool_block16")(block_conv_8) block_conv_16 = conv_bn_leakyrelu_repetition_block( filters=16 * base_channel, kernel_size=(3, 3), repetitions=2, first_layer_down_size=False, alpha=0.0, name="conv_block16")(block_pool_16) block_up_8 = UpSampling2D(size=(2, 2), name="up_block8")(block_conv_16) block_concat_8 = Concatenate(axis=CHANNEL_AXIS, name="concat8")([block_up_8, block_conv_8]) block_expan_conv_8 = conv_bn_leakyrelu_repetition_block( filters=8 * base_channel, kernel_size=(3, 3), repetitions=2, first_layer_down_size=False, alpha=0.0, name="expan_conv_block8")(block_concat_8) block_up_4 = UpSampling2D(size=(2, 2), name="up_block4")(block_expan_conv_8) block_concat_4 = Concatenate(axis=CHANNEL_AXIS, name="concat4")([block_up_4, block_conv_4]) block_expan_conv_4 = conv_bn_leakyrelu_repetition_block( filters=4 * base_channel, kernel_size=(3, 3), repetitions=2, first_layer_down_size=False, alpha=0.0, name="expan_conv_block4")(block_concat_4) block_up_2 = UpSampling2D(size=(2, 2), name="up_block2")(block_expan_conv_4) block_concat_2 = Concatenate(axis=CHANNEL_AXIS, name="concat2")([block_up_2, block_conv_2]) block_expan_conv_2 = conv_bn_leakyrelu_repetition_block( filters=2 * base_channel, kernel_size=(3, 3), repetitions=2, first_layer_down_size=False, alpha=0.0, name="expan_conv_block2")(block_concat_2) block_up_1 = UpSampling2D(size=(2, 2), name="up_block1")(block_expan_conv_2) block_concat_1 = Concatenate(axis=CHANNEL_AXIS, name="concat1")([block_up_1, block_conv_1]) block_expan_conv_1 = conv_bn_leakyrelu_repetition_block( filters=1 * base_channel, kernel_size=(3, 3), repetitions=2, first_layer_down_size=False, alpha=0.0, name="expan_conv_block1")(block_concat_1) block_seg_4 = Conv2D(filters=num_outputs, kernel_size=(1, 1), strides=(1, 1), padding="same", data_format=None, dilation_rate=(1, 1), activation=None, use_bias=True, kernel_initializer="he_normal", bias_initializer="zeros", kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, name="seg_block4")(block_expan_conv_4) block_seg_2 = Conv2D(filters=num_outputs, kernel_size=(1, 1), strides=(1, 1), padding="same", data_format=None, dilation_rate=(1, 1), activation=None, use_bias=True, kernel_initializer="he_normal", bias_initializer="zeros", kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, name="seg_block2")(block_expan_conv_2) block_seg_1 = Conv2D(filters=num_outputs, kernel_size=(1, 1), strides=(1, 1), padding="same", data_format=None, dilation_rate=(1, 1), activation=None, use_bias=True, kernel_initializer="he_normal", bias_initializer="zeros", kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, name="seg_block1")(block_expan_conv_1) block_seg_up_2 = UpSampling2D(size=(2, 2), name="seg_up_block2")(block_seg_4) block_add_2 = Add(name="add_block2")([block_seg_up_2, block_seg_2]) block_seg_up_1 = UpSampling2D(size=(2, 2), name="seg_up_block1")(block_add_2) output = Add(name="output")([block_seg_up_1, block_seg_1]) model = Model(inputs=[input_img0, input_img1], outputs=output) return model
wr = K.variable(value=0.25, name="wr") for j in range(MAX_SENTENCE): hj = Lambda((lambda X: X[:, j, :]), output_shape=(2 * HIDDEN_SIZE, ))(h) content = Dense(1, activation='sigmoid', name="c" + str(j))(hj) salience = Dot(axes=1, normalize=True)([d, hj]) salience = Activation('sigmoid')(salience) weighted_content = Lambda(lambda X: wc * X)(content) weighted_salience = Lambda(lambda X: ws * X)(salience) if j == 0: sj = Lambda((lambda X: 0 * X), output_shape=(2 * HIDDEN_SIZE, ))(hj) redundancy = Dot(axes=1, normalize=True)([hj, sj]) redundancy = Activation('sigmoid')(redundancy) weighted_redundancy = Lambda(lambda X: ws * X)(redundancy) weighted_redundancy = Lambda(lambda X: -1 * X)(redundancy) score = Add()( [weighted_content, weighted_salience, weighted_redundancy]) p = Dense(2, activation='softmax', name="p" + str(j))(score) prob = Reshape((1, 2))(p) else: hj_minus_one = Lambda((lambda X: X[:, j - 1, :]), output_shape=(2 * HIDDEN_SIZE, ))(h) pi_minus_one = Lambda((lambda X: X[:, j - 1, 1]), output_shape=(1, ))(prob) pi_minus_one = Reshape((1, ))(pi_minus_one) sj_minus_one = Lambda(lambda X: pi_minus_one * X)(hj_minus_one) sj = Add()([sj_minus_one, sj]) redundancy = Dot(axes=1, normalize=True)([hj, sj]) redundancy = Activation('sigmoid')(redundancy) weighted_redundancy = Lambda(lambda X: ws * X)(redundancy) weighted_redundancy = Lambda((lambda X: -1 * X))(redundancy) score = Add()(
def encoder(x, img_shape, conv_chans=None, n_convs_per_stage=1, min_h=5, min_c=None, prefix='', ks=3, return_skips=False, use_residuals=False, use_maxpool=False, use_batchnorm=False, kernel_initializer=None, bias_initializer=None): skip_layers = [] concat_skip_sizes = [] n_dims = len( img_shape ) - 1 # assume img_shape includes spatial dims, followed by channels if conv_chans is None: n_convs = int(np.floor(np.log2(img_shape[0] / min_h))) conv_chans = [min_c * 2] * (n_convs - 1) + [min_c] elif not type(conv_chans) == list: n_convs = int(np.floor(np.log2(img_shape[0] / min_h))) conv_chans = [conv_chans] * (n_convs - 1) + [min_c] else: n_convs = len(conv_chans) if isinstance(ks, list): assert len(ks) == (n_convs + 1 ) # specify for each conv, as well as the last one else: ks = [ks] * (n_convs + 1) for i in range(len(conv_chans)): #if n_convs_per_stage is not None and n_convs_per_stage > 1 or use_maxpool and n_convs_per_stage is not None: for ci in range(n_convs_per_stage): x = myConv(nf=conv_chans[i], ks=ks[i], strides=1, n_dims=n_dims, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, prefix='{}_enc'.format(prefix), suffix='{}_{}'.format(i, ci + 1))(x) if ci == 0 and use_residuals: residual_input = x elif ci == n_convs_per_stage - 1 and use_residuals: x = Add(name='{}_enc_{}_add_residual'.format(prefix, i))( [residual_input, x]) if use_batchnorm: x = BatchNormalization()(x) x = LeakyReLU(0.2, name='{}_enc_leakyrelu_{}_{}'.format( prefix, i, ci + 1))(x) if return_skips: skip_layers.append(x) concat_skip_sizes.append(np.asarray(x.get_shape().as_list()[1:-1])) if use_maxpool and i < len(conv_chans) - 1: # changed 5/30/19, don't pool after our last conv x = myPool(n_dims=n_dims, prefix=prefix, suffix=i)(x) else: x = myConv(conv_chans[i], ks=ks[i], strides=2, n_dims=n_dims, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, prefix='{}_enc'.format(prefix), suffix=i)(x) # don't activate right after a maxpool, it makes no sense if i < len(conv_chans) - 1: # no activation on last convolution x = LeakyReLU(0.2, name='{}_enc_leakyrelu_{}'.format(prefix, i))(x) if min_c is not None and min_c > 0: # if the last number of channels is specified, convolve to that if n_convs_per_stage is not None and n_convs_per_stage > 1: for ci in range(n_convs_per_stage): # TODO: we might not have enough ks for this x = myConv(min_c, ks=ks[-1], n_dims=n_dims, strides=1, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, prefix='{}_enc'.format(prefix), suffix='last_{}'.format(ci + 1))(x) if ci == 0 and use_residuals: residual_input = x elif ci == n_convs_per_stage - 1 and use_residuals: x = Add( name='{}_enc_{}_add_residual'.format(prefix, 'last'))( [residual_input, x]) x = LeakyReLU(0.2, name='{}_enc_leakyrelu_last'.format(prefix))(x) x = myConv(min_c, ks=ks[-1], strides=1, n_dims=n_dims, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, prefix='{}_enc'.format(prefix), suffix='_last')(x) if return_skips: skip_layers.append(x) concat_skip_sizes.append(np.asarray(x.get_shape().as_list()[1:-1])) if return_skips: return x, skip_layers, concat_skip_sizes else: return x
def resunet(shape, n_filters): # Convolutional block: BN -> ReLU -> Conv3x3 def conv_block(inputs, n_filters, kernel_size=(3, 3), strides=(1, 1), activation='relu', batch_norm=True, padding='same'): if batch_norm: x = BatchNormalization()(inputs) else: x = inputs if activation: x = Activation('relu')(x) x = Conv2D(filters=n_filters, kernel_size=kernel_size, strides=strides, padding=padding)(x) return x inputs = Input((*shape, 1)) # Contracting path short1 = inputs conv1 = conv_block(inputs, n_filters, activation=None, batch_norm=False) conv1 = conv_block(conv1, n_filters) short1 = conv_block(short1, n_filters, activation=None) conv1 = Add()([conv1, short1]) short2 = conv1 conv2 = conv_block(conv1, n_filters * 2, strides=(2, 2)) conv2 = conv_block(conv2, n_filters * 2) short2 = conv_block(short2, n_filters * 2, strides=(2, 2), activation=None) conv2 = Add()([conv2, short2]) short3 = conv2 conv3 = conv_block(conv2, n_filters * 4, strides=(2, 2)) conv3 = conv_block(conv3, n_filters * 4) short3 = conv_block(short3, n_filters * 4, strides=(2, 2), activation=None) conv3 = Add()([conv3, short3]) # Bridge short4 = conv3 conv4 = conv_block(conv3, n_filters * 8, strides=(2, 2)) conv4 = conv_block(conv4, n_filters * 8) short4 = conv_block(short4, n_filters * 8, strides=(2, 2), activation=None) conv4 = Add()([conv4, short4]) # Expansive path up5 = Conv2DTranspose(filters=n_filters * 4, kernel_size=(3, 3), strides=(2, 2), padding='same')(conv4) up5 = concatenate([up5, conv3]) short5 = up5 conv5 = conv_block(up5, n_filters * 4) conv5 = conv_block(conv5, n_filters * 4) short5 = conv_block(short5, n_filters * 4, activation=None) conv5 = Add()([conv5, short5]) up6 = Conv2DTranspose(filters=n_filters * 2, kernel_size=(3, 3), strides=(2, 2), padding='same')(conv5) up6 = concatenate([up6, conv2]) short6 = up6 conv6 = conv_block(up6, n_filters * 2) conv6 = conv_block(conv6, n_filters * 2) short6 = conv_block(short6, n_filters * 2, activation=None) conv6 = Add()([conv6, short6]) up7 = Conv2DTranspose(filters=n_filters, kernel_size=(3, 3), strides=(2, 2), padding='same')(conv6) up7 = concatenate([up7, conv1]) short7 = up7 conv7 = conv_block(up7, n_filters) conv7 = conv_block(conv7, n_filters) short7 = conv_block(short7, n_filters, activation=None) conv7 = Add()([conv7, short7]) outputs = Conv2D(filters=1, kernel_size=(1, 1), activation='sigmoid')(conv7) model = Model(inputs=[inputs], outputs=[outputs]) return model
def decoder( x, output_shape, encoded_shape, conv_chans=None, min_h=5, min_c=4, prefix='vte_dec', n_convs_per_stage=1, include_dropout=False, ks=3, include_skips=None, use_residuals=False, use_upsample=False, use_batchnorm=False, target_vol_sizes=None, n_samples=1, kernel_initializer=None, bias_initializer=None, ): n_dims = len(output_shape) - 1 if conv_chans is None: n_convs = int(np.floor(np.log2(output_shape[0] / min_h))) conv_chans = [min_c * 2] * n_convs elif not type(conv_chans) == list: n_convs = int(np.floor(np.log2(output_shape[0] / min_h))) conv_chans = [conv_chans] * n_convs elif type(conv_chans) == list: n_convs = len(conv_chans) if isinstance(ks, list): assert len(ks) == (n_convs + 1 ) # specify for each conv, as well as the last one else: ks = [ks] * (n_convs + 1) print('Decoding with conv filters {}'.format(conv_chans)) # compute default sizes that we want on the way up, mainly in case we have more convs than stages # and we upsample past the output size if n_dims == 2: # just upsample by a factor of 2 and then crop the final volume to the desired volume default_target_vol_sizes = np.asarray( [(int(encoded_shape[0] * 2.**(i + 1)), int(encoded_shape[1] * 2.**(i + 1))) for i in range(n_convs - 1)] + [output_shape[:2]]) else: print(output_shape) print(encoded_shape) # just upsample by a factor of 2 and then crop the final volume to the desired volume default_target_vol_sizes = np.asarray( [(min(output_shape[0], int(encoded_shape[0] * 2.**(i + 1))), min(output_shape[1], int(encoded_shape[1] * 2.**(i + 1))), min(output_shape[2], int(encoded_shape[2] * 2.**(i + 1)))) for i in range(n_convs - 1)] + [output_shape[:3]]) # automatically stop when we reach the desired image shape for vi, vs in enumerate(default_target_vol_sizes): if np.all(vs >= output_shape[:-1]): default_target_vol_sizes[vi] = output_shape[:-1] print('Automatically computed target output sizes: {}'.format( default_target_vol_sizes)) if target_vol_sizes is None: target_vol_sizes = default_target_vol_sizes else: print('Target concat vols to match shapes to: {}'.format( target_vol_sizes)) # TODO: check that this logic makes sense for more convs # fill in any Nones that we might have in our target_vol_sizes filled_target_vol_sizes = [None] * len(target_vol_sizes) for i in range(n_convs): if i < len(target_vol_sizes) and target_vol_sizes[i] is not None: filled_target_vol_sizes[i] = target_vol_sizes[i] target_vol_sizes = filled_target_vol_sizes if include_skips is not None: print('Concatentating padded/cropped shapes {} with skips {}'.format( target_vol_sizes, include_skips)) curr_shape = np.asarray(encoded_shape[:n_dims]) for i in range(n_convs): print(target_vol_sizes[i]) if i < len(target_vol_sizes) and target_vol_sizes[i] is not None: x = _pad_or_crop_to_shape(x, curr_shape, target_vol_sizes[i]) curr_shape = np.asarray( target_vol_sizes[i]) # we will upsample first thing next stage # if we want to concatenate with another volume (e.g. from encoder, or a downsampled input)... if include_skips is not None and i < len( include_skips) and include_skips[i] is not None: x_shape = x.get_shape().as_list() skip_shape = include_skips[i].get_shape().as_list() print( 'Attempting to concatenate current layer {} with previous skip connection {}' .format(x_shape, skip_shape)) # input size might not match in time dimension, so just tile it if n_samples > 1: tile_factor = [1] + [n_samples] + [1] * (len(x_shape) - 1) print('Tiling by {}'.format(tile_factor)) print(target_vol_sizes[i]) skip = Lambda(lambda y: K.expand_dims(y, axis=1))( include_skips[i]) skip = Lambda(lambda y: tf.tile(y, tile_factor), name='{}_lambdatilesamples_{}'.format(prefix, i), output_shape=[n_samples] + skip_shape[1:])(skip) skip = Lambda(lambda y: tf.reshape(y, [-1] + skip_shape[1:]), output_shape=skip_shape[1:])(skip) else: skip = include_skips[i] x = Concatenate(axis=-1, name='{}_concatskip_{}'.format(prefix, i))([x, skip]) for ci in range(n_convs_per_stage): x = myConv( conv_chans[i], ks=ks[i], strides=1, n_dims=n_dims, prefix=prefix, suffix='{}_{}'.format(i, ci + 1), kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, )(x) if use_batchnorm: # TODO: check to see if this should go before the residual x = BatchNormalization()(x) # if we want residuals, store them here if ci == 0 and use_residuals: residual_input = x elif ci == n_convs_per_stage - 1 and use_residuals: x = Add(name='{}_{}_add_residual'.format(prefix, i))( [residual_input, x]) x = LeakyReLU(0.2, name='{}_leakyrelu_{}_{}'.format(prefix, i, ci + 1))(x) if include_dropout and i < 2: x = Dropout(0.3)(x) # if we are not at the output resolution yet, upsample or do a transposed convolution if not np.all(curr_shape == output_shape[:len(curr_shape)]): if not use_upsample: # if we have convolutional filters left at the end, just apply them at full resolution x = myConvTranspose( conv_chans[i], n_dims=n_dims, ks=ks[i], strides=2, prefix=prefix, suffix=i, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, )(x) if use_batchnorm: x = BatchNormalization()(x) x = LeakyReLU(0.2, name='{}_leakyrelu_{}'.format(prefix, i))( x) # changed 5/15/2018, will break old models else: x = myUpsample(size=2, n_dims=n_dims, prefix=prefix, suffix=i)(x) curr_shape *= 2 # last stage of convolutions, no more upsampling x = myConv( output_shape[-1], ks=ks[-1], n_dims=n_dims, strides=1, prefix=prefix, suffix='final', kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, )(x) return x
def RESNET_BILSTM_model(self, conf, arm_shape): n_feature_maps = 8 #when you have large dataset increase it to 16 or 32 or 64 , experiment road_num = arm_shape[0] A = arm_shape[1] input_x = Input((road_num, conf.observe_length, 2)) #changed here input_ram = Input(arm_shape) input_veh_type = Input((road_num, conf.observe_length, 1)) input_engine = Input((road_num, conf.observe_length, 1)) input_weight = Input((road_num, conf.observe_length, 1)) print('input_x.shape', input_x.shape) print('input_ram.shape', input_ram.shape) print('road_num.shape', road_num) # BLOCK 1 veh_type_embd = Embedding(5, 3, mask_zero=False)(input_veh_type) engine_embd = Embedding(63, 10, mask_zero=False)(input_engine) weight_embd = Embedding(10, 5, mask_zero=False)(input_weight) squeezer = Lambda(lambda x: squeeze(x, axis=-2)) veh_type_embd = squeezer(veh_type_embd) engine_embd = squeezer(engine_embd) weight_embd = squeezer(weight_embd) concat_x = Concatenate()( [input_x, veh_type_embd, engine_embd, weight_embd]) conv_x = Lookup(conf.batch_size)([concat_x, input_ram]) conv_x = Conv3D(n_feature_maps, (1, A, 2), activation='relu')(conv_x) conv_x = BatchNormalization()(conv_x) conv_x = LookUpSqueeze()(conv_x) conv_y = Lookup(conf.batch_size)([conv_x, input_ram]) conv_y = Conv3D(n_feature_maps, (1, A, 2), activation='relu')(conv_y) conv_y = BatchNormalization()(conv_y) conv_y = LookUpSqueeze()(conv_y) conv_z = Lookup(conf.batch_size)([conv_y, input_ram]) conv_z = Conv3D(n_feature_maps, (1, A, 2))(conv_z) conv_z = BatchNormalization()(conv_z) conv_z = LookUpSqueeze()(conv_z) # expand channels for the sum shortcut_y = Lookup(conf.batch_size)([concat_x, input_ram]) shortcut_y = Conv3D(n_feature_maps, (1, A, 4), activation='relu')(shortcut_y) shortcut_y = BatchNormalization()(shortcut_y) shortcut_y = LookUpSqueeze()(shortcut_y) output_block_1 = keras.layers.add([shortcut_y, conv_z]) output_block_1 = Activation('relu')(output_block_1) # BLOCK 2 conv_x = Lookup(conf.batch_size)([output_block_1, input_ram]) conv_x = Conv3D(n_feature_maps * 2, (1, A, 2), activation='relu')(conv_x) conv_x = BatchNormalization()(conv_x) conv_x = LookUpSqueeze()(conv_x) conv_y = Lookup(conf.batch_size)([conv_x, input_ram]) conv_y = Conv3D(n_feature_maps * 2, (1, A, 2), activation='relu')(conv_y) conv_y = BatchNormalization()(conv_y) conv_y = LookUpSqueeze()(conv_y) conv_z = Lookup(conf.batch_size)([conv_y, input_ram]) conv_z = Conv3D(n_feature_maps * 2, (1, A, 2))(conv_z) conv_z = BatchNormalization()(conv_z) conv_z = LookUpSqueeze()(conv_z) # expand channels for the sum shortcut_y = Lookup(conf.batch_size)([output_block_1, input_ram]) shortcut_y = Conv3D(n_feature_maps * 2, (1, A, 4), activation='relu')(shortcut_y) shortcut_y = BatchNormalization()(shortcut_y) shortcut_y = LookUpSqueeze()(shortcut_y) print('conv_z.shape', conv_z.shape) output_block_2 = keras.layers.add([shortcut_y, conv_z]) output_block_2 = Activation('relu')(output_block_2) # BLOCK 3 conv_x = Lookup(conf.batch_size)([output_block_2, input_ram]) conv_x = Conv3D(n_feature_maps * 2, (1, A, 2), activation='relu')(conv_x) conv_x = BatchNormalization()(conv_x) conv_x = LookUpSqueeze()(conv_x) conv_y = Lookup(conf.batch_size)([conv_x, input_ram]) conv_y = Conv3D(n_feature_maps * 2, (1, A, 2), activation='relu')(conv_y) conv_y = BatchNormalization()(conv_y) conv_y = LookUpSqueeze()(conv_y) conv_z = Lookup(conf.batch_size)([conv_y, input_ram]) conv_z = Conv3D(n_feature_maps * 2, (1, A, 2))(conv_z) conv_z = BatchNormalization()(conv_z) conv_z = LookUpSqueeze()(conv_z) # need to expand shortcut_y = Lookup(conf.batch_size)([output_block_2, input_ram]) shortcut_y = Conv3D(n_feature_maps * 2, (1, A, 4), activation='relu')(shortcut_y) shortcut_y = BatchNormalization()(shortcut_y) shortcut_y = LookUpSqueeze()(shortcut_y) output_block_3 = keras.layers.add([shortcut_y, conv_z]) output_block_3 = Activation('relu')(output_block_3) print('output_block_3', output_block_3.shape) to_lstm = Lookup(conf.batch_size)([output_block_3, input_ram]) print('to_lstm BEFORE EXTERNAL #############', to_lstm.shape) if conf.use_externel: to_lstm = Conv3D(n_feature_maps, (1, A, 38), activation='relu')(to_lstm) else: to_lstm = Conv3D(n_feature_maps, (1, A, 1), activation='relu')(to_lstm) to_lstm = LookUpSqueeze()(to_lstm) to_lstm = Lambda(lambda y: squeeze(y, 0))(to_lstm) # output_block_3 = Activation('relu')(output_block_3) #FINAL # gap_layer = GlobalAveragePooling2D()(output_block_3) # gap_layer = Dropout(rate=.25)(gap_layer) # gap_layer = Dense(50, activation='sigmoid')(gap_layer) # gap_layer = Dropout(rate=.25)(gap_layer) # time_distibuted = TimeDistributed(output_block_3) print('to_lstm.shape', to_lstm.shape) # output = SimpleRNN(5)(to_lstm) # output = MyReshape(conf.batch_size)(gap_layer) output = Bidirectional( LSTM(10, return_sequences=True, dropout=0.5, recurrent_dropout=0.2))(to_lstm) print('lstm out.shape', output.shape) #output = MyReshape(conf.batch_size)(output) inputs = [ input_x, input_ram, input_veh_type, input_engine, input_weight ] if conf.use_externel: output = Dense(1, activation='relu')(output) output = MyInverseReshape2(conf.batch_size)(output) print('dense.output.shape', output.shape) input_e, output_e = self.__E_input_output(conf, arm_shape) print('output_e shape', output_e.shape) if isinstance(input_e, list): inputs += input_e else: inputs += [input_e] if conf.use_matrix_fuse: outputs = [matrixLayer()(output)] print('outputs', outputs) print('outputs.e', output_e) outputs.append(matrixLayer2()(output_e)) print('outputs.shape', outputs) output = Add()(outputs) else: output = Add()([output, output_e]) output = Activation('tanh')(output) else: output = Dense(1, activation='tanh')(to_lstm) output = MyInverseReshape2(conf.batch_size)(output) output = Dense(conf.predict_length, activation='tanh')(output) print('final layer', output.shape) model = Model(inputs=inputs, outputs=output) return model
def segnet2D(x_in, img_shape, out_im_chans, nf_enc=[64, 64, 128, 128, 256, 256, 512], nf_dec=None, regularizer=None, initializer=None, layer_prefix='segnet', n_convs_per_stage=1, include_residual=False, concat_at_stages=None): ks = 3 encoding_im_sizes = np.asarray([(int(np.ceil(img_shape[0]/2.0**i)), int(np.ceil(img_shape[1]/2.0**i))) \ for i in range(0, len(nf_enc) + 1)]) reg_params = {} if regularizer == 'l1': reg = regularizers.l1(1e-6) else: reg = None if initializer == 'zeros': reg_params['kernel_initializer'] = initializers.Zeros() x = x_in # start with the input channels encoder_pool_idxs = [] for i in range(len(nf_enc)): for j in range(n_convs_per_stage): x = Conv2D(nf_enc[i], kernel_regularizer=reg, kernel_size=ks, strides=(1, 1), padding='same', name='{}_enc_conv2D_{}_{}'.format( layer_prefix, i, j + 1))(x) if concat_at_stages and concat_at_stages[i] is not None: x = Concatenate(axis=-1)([x, concat_at_stages[i]]) if j == 0 and include_residual: residual_input = x elif j == n_convs_per_stage - 1 and include_residual: x = Add()([residual_input, x]) x = LeakyReLU(0.2)(x) x, pool_idxs = MaxPoolingWithArgmax2D(pool_size=(ks, ks), padding='same', name='{}_enc_maxpool_{}'.format( layer_prefix, i))(x) encoder_pool_idxs.append(pool_idxs) if nf_dec is None: nf_dec = list(reversed(nf_enc[1:])) decoding_im_sizes = [encoding_im_sizes[-1] * 2] for i in range(len(nf_dec)): x = MaxUnpooling2D()([x, encoder_pool_idxs[-1 - i]]) x = _pad_or_crop_to_shape(x, decoding_im_sizes[-1], encoding_im_sizes[-i - 2]) decoding_im_sizes.append( encoding_im_sizes[-i - 2] * 2) # the next deconv layer will produce this image height residual_input = x for j in range(n_convs_per_stage): x = Conv2D(nf_dec[i], kernel_regularizer=reg, kernel_size=ks, strides=(1, 1), padding='same', name='{}_dec_conv2D_{}_{}'.format(layer_prefix, i, j))(x) if j == 0 and include_residual: residual_input = x elif j == n_convs_per_stage - 1 and include_residual: x = Add()([residual_input, x]) x = LeakyReLU(0.2)(x) if i < len(nf_dec) - 1: # an extra conv compared to unet, so that the unpool op gets the right number of filters x = Conv2D(nf_dec[i + 1], kernel_regularizer=reg, kernel_size=ks, strides=(1, 1), padding='same', name='{}_dec_conv2D_{}_extra'.format(layer_prefix, i))(x) x = LeakyReLU(0.2)(x) y = Conv2D(out_im_chans, kernel_size=1, padding='same', kernel_regularizer=reg, name='{}_dec_conv2D_last_last'.format(layer_prefix))( x) # add your own activation after this model # add your own activation after this model return y
def NIC(max_token_length, vocabulary_size, embedding_matrix, rnn='lstm', num_image_features=2048, hidden_size=512, embedding_size=512, regularizer=1e-8): embedding_layer = Embedding(vocabulary_size, 50, weights=[embedding_matrix], input_length=max_token_length, trainable=False) # word embedding text_input = Input(shape=(max_token_length, ), name='text') text_dropout = embedding_layer(text_input) #text_mask = Masking(mask_value=0.0, name='text_mask')(text_input) #text_to_embedding = TimeDistributed(Dense(units=embedding_size, # kernel_regularizer=l2(regularizer), # name='text_embedding'))(text_mask) #text_dropout = Dropout(.5, name='text_dropout')(text_to_embedding) # image embedding image_input = Input(shape=(max_token_length, num_image_features), name='image') image_embedding = TimeDistributed( Dense(units=embedding_size, kernel_regularizer=l2(regularizer), name='image_embedding'))(image_input) image_dropout = Dropout(.5, name='image_dropout')(image_embedding) # language model recurrent_inputs = [text_dropout, image_dropout] merged_input = Add()(recurrent_inputs) if rnn == 'lstm': recurrent_network = LSTM(units=hidden_size, recurrent_regularizer=l2(regularizer), kernel_regularizer=l2(regularizer), bias_regularizer=l2(regularizer), return_sequences=True, name='recurrent_network')(merged_input) elif rnn == 'gru': recurrent_network = GRU(units=hidden_size, recurrent_regularizer=l2(regularizer), kernel_regularizer=l2(regularizer), bias_regularizer=l2(regularizer), return_sequences=True, name='recurrent_network')(merged_input) else: raise Exception('Invalid rnn name') output = TimeDistributed(Dense(units=vocabulary_size, kernel_regularizer=l2(regularizer), activation='softmax'), name='output')(recurrent_network) inputs = [text_input, image_input] model = Model(inputs=inputs, outputs=output) return model
def model(nbChannels, fs, stride=2, embeddingRate=30.0, nbEmbeddingDim=32, nbQuantizationBins=16, nbResidualLayers=2, **kwargs): inputs = Input(shape=(None, nbChannels)) x = inputs nbFilters = 64 kernelSize = 3 with K.name_scope('encoder'): i = 0 fs = fs while fs > embeddingRate: fs = int(np.ceil(fs / stride)) # Projection shortcut using resampling xs = Resample1D(scale=1.0 / stride)(x) # Residual blocks for _ in range(nbResidualLayers): fx = Conv1D(nbFilters, kernel_size=kernelSize, strides=1, activation='linear', padding='same')(x) fx = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(fx) fx = Conv1D(nbFilters, kernel_size=kernelSize, strides=1, activation='linear', padding='same')(fx) x = Add()([fx, x]) x = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(x) # Non-linear convolution followed by linear downsampling x = Conv1D(nbFilters, kernel_size=kernelSize, strides=1, activation='linear', padding='same', name='codec-conv-ds-%d' % (i + 1))(x) x = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(x) x = Resample1D(scale=1.0 / stride, method='linear')(x) x = Add()([x, xs]) i += 1 nbDownsamplingLayers = i x = Conv1D(nbEmbeddingDim, kernel_size=3, strides=1, activation='linear', padding='same', name='codec-conv-embedding')(x) x = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(x) # Quantization if nbQuantizationBins is not None: quantization = SoftmaxQuantization(nbQuantizationBins, name='quantization') x = quantization(x) # Dequantization if nbQuantizationBins is not None: x = SoftmaxDequantization(nbQuantizationBins)(x) with K.name_scope('decoder'): x = Conv1D(nbFilters, kernel_size=1, strides=1, activation='linear', padding='same')(x) x = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(x) # Upsampling with resize convolutions for i in range(nbDownsamplingLayers): # Projection shortcut xs = Resample1D(stride, method='linear')(x) # Linear upsampling followed by non-linear convolution x = Resample1D(stride, method='linear')(x) x = Conv1D(nbFilters, kernel_size=kernelSize, strides=1, activation='linear', padding='same', name='codec-conv-us-%d' % (i + 1))(x) x = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(x) # Residual blocks for _ in range(nbResidualLayers): fx = Conv1D(nbFilters, kernel_size=kernelSize, strides=1, activation='linear', padding='same')(x) fx = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(fx) fx = Conv1D(nbFilters, kernel_size=kernelSize, strides=1, activation='linear', padding='same')(fx) x = Add()([fx, x]) x = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(x) x = Add()([x, xs]) xr = Conv1D(nbChannels, kernelSize, strides=1, activation='linear', padding='same', name='codec-conv-out')(x) outputs = xr return Model(inputs, outputs, **kwargs)
def add_resnet(x, n_dim): in_x = x x = Dense(n_dim, activation="relu")(x) x = Dense(n_dim, activation="relu")(x) return Add()([in_x, x])
def modelv3(nbChannels, stride=2, nbFilters=[64, 64, 128, 128, 256], nbEmbeddingDim=32, nbQuantizationBins=16, nbResidualLayers=1, **kwargs): inputs = Input(shape=(None, nbChannels)) x = inputs def _gated_block(x, kernelSize, nbFilters, noisy=False, name=None): # Define the gated activation unit for the input f = Conv1D(nbFilters, kernelSize, padding='same', activation='linear', name=name)(x) # filter f = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(f) g = Conv1D(nbFilters, kernelSize, padding='same', activation='sigmoid')(x) # gate zi = Multiply()([f, g]) if noisy: # Define the gated activation unit for the noise n = Lambda(lambda x: K.random_normal( shape=K.shape(x), mean=0.0, stddev=1.0))(x) f = Conv1D(nbFilters, kernelSize, padding='same', activation='linear')(n) # filter f = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(f) g = Conv1D(nbFilters, kernelSize, padding='same', activation='sigmoid')(x) # gate zn = Multiply()([f, g]) # Add signal and noise y = Add()([zi, zn]) else: y = zi # Define the parametrized skip connection skip = Conv1D(nbFilters, 1, padding='same')(y) return y, skip kernelSize = 5 with K.name_scope('encoder'): for i, n in enumerate(nbFilters): # Non-linear convolution followed by linear downsampling skips = [] for _ in range(nbResidualLayers): x, skip = _gated_block(x, kernelSize, n, noisy=False, name='codec-conv-ds-%d' % (i + 1)) skips.append(skip) if len(skips) > 1: x = Add()(skips) else: x = skips[0] x = Resample1D(scale=1.0 / stride, method='linear')(x) x = Conv1D(nbEmbeddingDim, kernel_size=kernelSize, strides=1, activation='linear', padding='same', name='codec-conv-embedding')(x) # Quantization if nbQuantizationBins is not None: quantization = SoftmaxQuantization(nbQuantizationBins, name='quantization') x = quantization(x) # Dequantization if nbQuantizationBins is not None: x = SoftmaxDequantization(nbQuantizationBins)(x) kernelSize = 5 with K.name_scope('decoder'): # Upsampling with resize convolutions for i, n in enumerate(reversed(nbFilters)): # Linear upsampling followed by non-linear convolution x = Resample1D(stride, method='linear')(x) skips = [] for _ in range(nbResidualLayers): x, skip = _gated_block(x, kernelSize, n, noisy=False, name='codec-conv-us-%d' % (i + 1)) skips.append(skip) if len(skips) > 1: x = Add()(skips) else: x = skips[0] xr = Conv1D(nbChannels, kernelSize, strides=1, activation='linear', padding='same', name='codec-conv-out')(x) outputs = xr return Model(inputs, outputs, **kwargs)
def add_conv(input_dim, tobj, l): return Add()([ conv131(input_dim, l, tobj=get(tobj, 1), first_layer=True), Conv(input_dim * 2, 1, tobj=get(tobj, 12))(l) ])
def ThreeLevelsMaxPool( input_dims, num_classes, init='glorot_normal', encoder_act_function='elu', decoder_act_function='relu', classification_act_function='softmax', loss_function='categorical_crossentropy', learning_rate=1e-05, min_filters_per_layer=16, use_kernel_reg=False, use_dropout=False, ): if use_kernel_reg == True: lvl2_reg = l2(0.005) lvl3_reg = l2(0.01) else: lvl2_reg = None lvl3_reg = None if use_dropout == True: lvl1_dropout = 0.0 lvl2_dropout = 0.25 lvl3_dropout = 0.5 else: lvl1_dropout = 0.0 lvl2_dropout = 0.0 lvl3_dropout = 0.0 lvl1_filters = (3, 3, 3) lvl2_filters = (3, 3, 3) lvl3_filters = (3, 3, 3) # ----------------------------------- # ENCODER - LEVEL 1 # ----------------------------------- with tf.device('/gpu:0'): inputs = Input(input_dims, name='main_input') enc_lvl1_block1 = myConv3DBlock( input_tensor=inputs, filters=min_filters_per_layer, kernel_size=lvl1_filters, activation=encoder_act_function, kernel_initializer=init, padding='same', block_name='enc_lvl1a', kernel_regularizer=None, use_batch_norm=False, dropout_rate=lvl1_dropout, ) max_pool_1to2 = MaxPooling3D(pool_size=(4, 4, 4), name='max_pool_1to2')(enc_lvl1_block1) # ----------------------------------- # ENCODER - LEVEL 2 # ----------------------------------- enc_lvl2_block1 = myConv3DBlock( input_tensor=max_pool_1to2, filters=2 * min_filters_per_layer, kernel_size=lvl2_filters, activation=encoder_act_function, kernel_initializer=init, padding='same', block_name='enc_lvl2a', kernel_regularizer=lvl2_reg, use_batch_norm=False, dropout_rate=lvl2_dropout, ) with tf.device('/gpu:1'): enc_lvl2_block2 = myConv3DBlock( input_tensor=enc_lvl2_block1, filters=2 * min_filters_per_layer, kernel_size=lvl2_filters, activation=encoder_act_function, kernel_initializer=init, padding='same', block_name='enc_lvl2b', kernel_regularizer=lvl2_reg, use_batch_norm=False, dropout_rate=lvl2_dropout, ) max_pool_2to3 = MaxPooling3D(pool_size=(2, 2, 2), name='max_pool_2to3')(enc_lvl2_block2) # ----------------------------------- # BOTTLENECK LAYER # ----------------------------------- bottleneck_block1 = myConv3DBlock( input_tensor=max_pool_2to3, filters=4 * min_filters_per_layer, kernel_size=lvl3_filters, activation=encoder_act_function, kernel_initializer=init, padding='same', block_name='bottleneck_a', kernel_regularizer=lvl3_reg, use_batch_norm=False, dropout_rate=lvl3_dropout, ) bottleneck_block2 = myConv3DBlock( input_tensor=bottleneck_block1, filters=4 * min_filters_per_layer, kernel_size=lvl3_filters, activation=encoder_act_function, kernel_initializer=init, padding='same', block_name='bottleneck_b', kernel_regularizer=lvl3_reg, use_batch_norm=False, dropout_rate=lvl3_dropout, ) bottleneck_block3 = myConv3DBlock( input_tensor=bottleneck_block2, filters=4 * min_filters_per_layer, kernel_size=lvl3_filters, activation=encoder_act_function, kernel_initializer=init, padding='same', block_name='bottleneck_c', kernel_regularizer=lvl3_reg, use_batch_norm=False, dropout_rate=lvl3_dropout, ) # ----------------------------------- # DECODER - LEVEL 2 # ----------------------------------- up_conv_3to2 = myUpConv3DBlock( input_tensor=bottleneck_block3, filters=2 * min_filters_per_layer, kernel_size=(2, 2, 2), strides=(2, 2, 2), kernel_initializer=init, padding='same', block_name='up_conv_3to2', kernel_regularizer=lvl2_reg, use_batch_norm=False, dropout_rate=lvl2_dropout, ) with tf.device('/gpu:2'): skip_conn_lvl2 = Add(name='lvl2_longskip')( [enc_lvl2_block2, up_conv_3to2]) dec_lvl2_block1 = myConv3DBlock( input_tensor=skip_conn_lvl2, filters=2 * min_filters_per_layer, kernel_size=lvl2_filters, activation=encoder_act_function, kernel_initializer=init, padding='same', block_name='dec_lvl2a', kernel_regularizer=lvl2_reg, use_batch_norm=False, dropout_rate=lvl2_dropout, ) dec_lvl2_block2 = myConv3DBlock( input_tensor=dec_lvl2_block1, filters=2 * min_filters_per_layer, kernel_size=lvl2_filters, activation=encoder_act_function, kernel_initializer=init, padding='same', block_name='dec_lvl2b', kernel_regularizer=lvl2_reg, use_batch_norm=False, dropout_rate=lvl2_dropout, ) # ----------------------------------- # DECODER - LEVEL 1 # ----------------------------------- up_conv_2to1 = myUpConv3DBlock( input_tensor=dec_lvl2_block2, filters=min_filters_per_layer, kernel_size=(4, 4, 4), strides=(4, 4, 4), kernel_initializer=init, padding='same', block_name='up_conv_2to1', kernel_regularizer=None, use_batch_norm=False, dropout_rate=lvl2_dropout, ) skip_conn_lvl1 = Add(name='lvl1_longskip')( [enc_lvl1_block1, up_conv_2to1]) with tf.device('/gpu:3'): dec_lvl1_block1 = myConv3DBlock( input_tensor=skip_conn_lvl1, filters=min_filters_per_layer, kernel_size=lvl1_filters, activation=encoder_act_function, kernel_initializer=init, padding='same', block_name='dec_lvl1a', kernel_regularizer=None, use_batch_norm=False, dropout_rate=lvl1_dropout, ) outputs = Conv3D( filters=num_classes, kernel_size=(1, 1, 1), activation=classification_act_function, kernel_initializer=init, name='main_output', )(dec_lvl1_block1) # define the model object and the optimizer model = Model(inputs=[inputs], outputs=[outputs]) my_adam = keras.optimizers.Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999) # compile the model model.compile(loss=losses_dict[loss_function], optimizer=my_adam, metrics=[ 'categorical_crossentropy', losses_dict['multiclass_dice_coeff'], losses_dict['average_multiclass_dice_coeff'], losses_dict['tanimoto_coefficient'] ]) # return the model object return model
def add_id(input_dim, tobj, l): return Add()( [conv131(input_dim, l, tobj=get(tobj, 1), first_layer=False), l])
# 모델 2 input2 = Input(shape=(3, 1)) model2 = LSTM(6, activation='relu')(input1) dense11 = Dense(4)(model2) dense21 = Dense(3)(dense11) dense31 = Dense(2)(dense21) model2 = Dense(1)(dense31) # 1번째 모델 합치는 방법 #from keras.layers.merge import concatenate #merge1 = concatenate([output1,output2]) # input 모델 합치기 # concatenate는 모델 1,2의 아웃풋 노드수가 달라도 된다. 엮는것이므로 # 2번째 방법 from keras.layers.merge import Add merge1 = Add()([model1, model2]) # 모델1,2 의 아웃풋 노드수가 같아야 한다. middle1 = Dense(4)(merge1) middle2 = Dense(3)(middle1) # 1번째 output output_1 = Dense(2)(middle2) output_1 = Dense(1)(output_1) # y의 벡터가 1개이므로 1로 설정 # 2번째 output output_2 = Dense(3)(middle2) output_2 = Dense(1)(output_2) model = Model(inputs=[input1, input2], outputs=[output_1, output_2]) model.summary()
def encoder3D(x, img_shape, conv_chans=None, n_convs_per_stage=1, min_h=5, min_c=None, prefix='vte', ks=3, return_skips=False, use_residuals=False, use_maxpool=False, max_time_downsample=None): skip_layers = [] concat_skip_sizes = [] if max_time_downsample is None: # do not attempt to downsample beyond 1 max_time_downsample = int(np.floor(np.log2(img_shape[-2]))) - 1 print('Max downsamples in time: {}'.format(max_time_downsample)) if conv_chans is None: n_convs = int(np.floor(np.log2(img_shape[0] / min_h))) conv_chans = [min_c * 2] * (n_convs - 1) + [min_c] elif not type(conv_chans) == list: n_convs = int(np.floor(np.log2(img_shape[0] / min_h))) conv_chans = [conv_chans] * (n_convs - 1) + [min_c] for i in range(len(conv_chans)): if n_convs_per_stage is not None and n_convs_per_stage > 1 or use_maxpool and n_convs_per_stage is not None: for ci in range(n_convs_per_stage): x = Conv3D(conv_chans[i], kernel_size=ks, padding='same', name='{}_enc_conv3D_{}_{}'.format(prefix, i, ci + 1))(x) if ci == 0 and use_residuals: residual_input = x elif ci == n_convs_per_stage - 1 and use_residuals: x = Add(name='{}_enc_{}_add_residual'.format(prefix, i))([residual_input, x]) x = LeakyReLU(0.2, name='{}_enc_leakyrelu_{}_{}'.format(prefix, i, ci + 1))(x) if return_skips: skip_layers.append(x) concat_skip_sizes.append(x.get_shape().as_list()[1:-1]) # only downsample if we are below the max number of downsamples in time if i < max_time_downsample: strides = (2, 2, 2) else: strides = (2, 2, 1) if use_maxpool: x = MaxPooling3D(pool_size=strides, name='{}_enc_maxpool_{}'.format(prefix, i))(x) else: x = Conv3D(conv_chans[i], kernel_size=ks, strides=strides, padding='same', name='{}_enc_conv3D_{}'.format(prefix, i))(x) if i < len(conv_chans) - 1: # no activation on last convolution x = LeakyReLU(0.2, name='{}_enc_leakyrelu_{}'.format(prefix, i))(x) if min_c is not None and min_c > 0: if n_convs_per_stage is not None and n_convs_per_stage > 1: for ci in range(n_convs_per_stage): x = Conv3D(min_c, kernel_size=ks, padding='same', name='{}_enc_conv3D_last_{}'.format(prefix, ci + 1))(x) if ci == 0 and use_residuals: residual_input = x elif ci == n_convs_per_stage - 1 and use_residuals: x = Add(name='{}_enc_{}_add_residual'.format(prefix, 'last'))([residual_input, x]) x = LeakyReLU(0.2, name='{}_enc_leakyrelu_last'.format(prefix))(x) x = Conv3D(min_c, kernel_size=ks, strides=(1, 1, 1), padding='same', name='{}_enc_conv3D_last'.format(prefix))(x) if return_skips: skip_layers.append(x) concat_skip_sizes.append(x.get_shape().as_list()[1:-1]) if return_skips: return x, skip_layers, concat_skip_sizes else: return x