def spatial(self, ind_emb): kernel_size = 3 kernel_initializer1 = tf.contrib.layers.variance_scaling_initializer() # with tf.variable_scope(name): avg_emb = [] max_emb = [] std_emb = [] for emb in ind_emb: avg_pool = Lambda(lambda x: K.mean(x, axis=[1], keepdims=True))( emb) assert avg_pool.get_shape()[1] == 1 avg_emb.append(avg_pool) max_pool = Lambda(lambda x: K.max(x, axis=[1], keepdims=True))(emb) assert max_pool.get_shape()[1] == 1 max_emb.append(max_pool) # std_pool = Lambda(lambda x: K.std(x, axis=[1], keepdims=True))(emb) # assert std_pool.get_shape()[1] == 1 # std_emb.append(std_pool) avg_emb = Concatenate(axis=1)(avg_emb) max_emb = Concatenate(axis=1)(max_emb) # std_emb = Concatenate(axis=1)(std_emb) concat = Concatenate(axis=1)([avg_emb, max_emb]) #assert concat.get_shape()[1] == 6 concat = Lambda(lambda x: K.expand_dims(x, 0))(concat) concat = Lambda(lambda x: K.permute_dimensions(x, (0, 2, 1, 3, 4)))( concat) concat = Conv3D(filters=1, kernel_size=[kernel_size, kernel_size, kernel_size], strides=[1, 1, 1], padding="same", activation=None, kernel_initializer='random_uniform', use_bias=False, name='conv')(concat) #assert concat.get_shape()[-1] == 1 concat = Activation('sigmoid')(concat) concat = Lambda(lambda x: K.squeeze(x, 0))(concat) ind_emb = [ Multiply()([ a, Lambda(lambda x: K.permute_dimensions(x, (1, 0, 2, 3)))(concat) ]) for a in ind_emb ] # concat = Lambda(lambda x: K.permute_dimensions(x,(1,0,2,3)))(concat) # return Lambda(lambda x: x[:,0:4,:,:])(concat) return Lambda(lambda x: ave(x))(ind_emb)
def unet(x_in, pose_in, nf_enc, nf_dec): x0 = my_conv(x_in, nf_enc[0], ks=7) # 256 x1 = my_conv(x0, nf_enc[1], strides=2) # 128 x2 = concatenate([x1, pose_in]) x3 = my_conv(x2, nf_enc[2]) x4 = my_conv(x3, nf_enc[3], strides=2) # 64 x5 = my_conv(x4, nf_enc[4]) x6 = my_conv(x5, nf_enc[5], strides=2) # 32 x7 = my_conv(x6, nf_enc[6]) x8 = my_conv(x7, nf_enc[7], strides=2) # 16 x9 = my_conv(x8, nf_enc[8]) x10 = my_conv(x9, nf_enc[9], strides=2) # 8 x = my_conv(x10, nf_enc[10]) skips = [x9, x7, x5, x3, x0] filters = [nf_enc[10], nf_dec[0], nf_dec[1], nf_dec[2], nf_enc[3]] for i in range(5): out_sz = 8 * (2**(i + 1)) print(x.get_shape()) x = Lambda(interp_upsampling, output_shape=(out_sz, out_sz, filters[i]))(x) print("@@@@@@@@@@@@@@@@@@@@@@@@@@") print(i, x.get_shape(), skips[i].get_shape()) print("@@@@@@@@@@@@@@@@@@@@@@@@@@") x = concatenate([x, skips[i]]) x = my_conv(x, nf_dec[i]) return x
def set_model(self): ''' Encoder ''' # define input with 'channels_first' input_encoder = Input(shape=self.input_shape, name='encoder_input') conv_1 = Conv2D(1, kernel_size=self.kernel_size, padding='same', activation='relu')(input_encoder) conv_2 = Conv2D(self.filters, kernel_size=self.kernel_size, padding='same', activation='relu')(conv_1) conv2_out_shape = tuple(conv_2.get_shape().as_list()) flat = Flatten()(conv_2) hidden = Dense(self.pre_latent_size, activation='relu')(flat) z_mean = Dense(self.latent_size)(hidden) z_log_var = Dense(self.latent_size)(hidden) # sample from normal with z_mean and z_log_var z = Lambda(self.sampling, name='latent_space')([z_mean, z_log_var]) encoder_out_shape = tuple(z.get_shape().as_list()) # we instantiate these layers separately so as to reuse them later input_decoder = Input(shape=encoder_out_shape[1:]) decoder_hid = Dense(self.pre_latent_size, activation='relu') decoder_upsample = Dense(np.prod(conv2_out_shape[1:]), activation='relu') output_shape = (self.filters, conv2_out_shape[2], conv2_out_shape[3]) decoder_reshape = Reshape(output_shape) decoder_deconv_1 = Conv2DTranspose(self.filters, kernel_size=self.kernel_size, padding='same', activation='relu') decoder_mean_squash = Conv2D(1, kernel_size=self.kernel_size, padding='same', activation='sigmoid') hid_decoded = decoder_hid(input_decoder) up_decoded = decoder_upsample(hid_decoded) reshape_decoded = decoder_reshape(up_decoded) deconv_1_decoded = decoder_deconv_1(reshape_decoded) x_decoded_mean = decoder_mean_squash(deconv_1_decoded) # For parent fitting function self.encoder = Model(input_encoder, z) self.decoder = Model(input_decoder, x_decoded_mean) self.model = Model(input_encoder, self.decoder(self.encoder(input_encoder))) # For parent loss function self.z_mean = z_mean self.z_log_var = z_log_var self.z = z
def KerasModel(isCompile=True): In = Input(shape=(None, 1, 1)) x = Lambda(fourierLayer, output_shape=fourierLayerShape)(In) #x = Convolution2D(32, 7, 7, subsample=(3,3), activation='relu', W_regularizer=l2(L2_RATE))(x) x = Conv2D(32, (7, 7), activation='relu', strides=(3, 3), kernel_regularizer=l2(L2_RATE))(x) #x=Conv2D(32, kernel_size=(7, 7), activation='relu')(x) #x = Dropout(DROP_RATE)(x) #x = Convolution2D(64, 7, 5, subsample=(3,3), activation='relu', W_regularizer=l2(L2_RATE))(x) x = Conv2D(64, (7, 5), activation='relu', strides=(3, 3), kernel_regularizer=l2(L2_RATE))(x) #x=Conv2D(64, kernel_size=(7, 5), activation='relu')(x) #x = Dropout(DROP_RATE)(x) #x = Convolution2D(64, 3, 3, subsample=(2,2), activation='relu', W_regularizer=l2(L2_RATE))(x) x = Conv2D(64, (2, 2), activation='relu', strides=(2, 2), kernel_regularizer=l2(L2_RATE))(x) #x=Conv2D(64, kernel_size=(3, 3), activation='relu')(x) #x = Dropout(DROP_RATE)(x) #x = Convolution2D(32, 3, 3, subsample=(2,2), activation='relu', W_regularizer=l2(L2_RATE))(x) x = Conv2D(32, (2, 2), activation='relu', strides=(2, 2), kernel_regularizer=l2(L2_RATE))(x) # x=Conv2D(32, kernel_size=(3, 3), activation='relu')(x) #x = Dropout(DROP_RATE)(x) freq, chan = x.get_shape()[2:4] x = TimeDistributed(Reshape([int(freq) * int(chan)]))(x) x = Bidirectional(LSTM(128, ))(x) #x = Dropout(DROP_RATE)(x) x = Dense(64, activation='relu')(x) #x = Dropout(DROP_RATE)(x) x = Dense(24, activation='softmax')(x) model = Model(inputs=In, outputs=x) if isCompile: opt = Adam(1e-4) model.compile( optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'], ) return model
def f(x, chosen, style): time_steps = int(x.get_shape()[1]) # Shift target one note to the left. shift_chosen = Lambda(lambda x: tf.pad( x[:, :, :-1, :], [[0, 0], [0, 0], [1, 0], [0, 0]]))(chosen) # [batch, time, notes, 1] total = int(shift_chosen.get_shape()[3]) * int( shift_chosen.get_shape()[2]) * int(shift_chosen.get_shape()[1]) dim = int(total / (time_steps * NUM_NOTES)) shift_chosen_rsp = Reshape((time_steps, NUM_NOTES, dim))(shift_chosen) ##shift_chosen = Reshape((time_steps, NUM_NOTES, -1))(shift_chosen) ##shift_chosen_rsp=shift_chosen[:,:,:,-1] ##shift_chosen_rsp=Reshape((time_steps, NUM_NOTES, 1))(shift_chosen_rsp) ##shift_chosen_rsp = Reshape((time_steps, NUM_NOTES, 1))(shift_chosen) # [batch, time, notes, features + 1] x = Concatenate(axis=3)([x, shift_chosen_rsp]) for l in range(NOTE_AXIS_LAYERS): # Integrate style if l not in dense_layer_cache: dense_layer_cache[l] = Dense(int(x.get_shape()[3])) ##a=x.get_shape()[3] ##dense_layer_cache[l] = Dense(int(a)) style_proj = dense_layer_cache[l](style) style_proj = TimeDistributed(RepeatVector(NUM_NOTES))(style_proj) style_proj = Activation('tanh')(style_proj) style_proj = Dropout(dropout)(style_proj) x = Add()([x, style_proj]) if l not in lstm_layer_cache: lstm_layer_cache[l] = LSTM(NOTE_AXIS_UNITS, return_sequences=True) x = TimeDistributed(lstm_layer_cache[l])(x) x = Dropout(dropout)(x) return Concatenate()([note_dense(x), volume_dense(x)])
def set_model(self): ''' Initialisers ''' weight_seed = None kernel_initializer = initializers.glorot_uniform(seed = weight_seed) bias_initializer = initializers.glorot_uniform(seed = weight_seed) ''' Encoder ''' # define input with 'channels_first' input_encoder = Input(shape=self.input_shape, name='encoder_input') x = Conv2D(self.filters, self.kernel_size, strides=(2, 2), activation='relu', kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_conv2D_1')(input_encoder) x = Conv2D(2*self.filters, self.kernel_size, strides=(2, 2), activation='relu', kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_conv2D_3')(x) before_flatten_shape = tuple(x.get_shape().as_list()) x = Flatten()(x) # x = Dense(self.pre_latent_size, activation='relu', name='encoder_dense_1')(x) # separate dense layers for mu and log(sigma), both of size latent_dim z_mean = Dense(self.latent_size, activation=None, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_z_mean')(x) z_log_var = Dense(self.latent_size, activation=None, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_z_log_var')(x) # sample from normal with z_mean and z_log_var z = Lambda(self.sampling, name='encoder_z')([z_mean, z_log_var]) ''' Decoder ''' # take encoder output shape encoder_out_shape = tuple(z.get_shape().as_list()) # define rest of model input_decoder = Input(shape=encoder_out_shape[1:], name='decoder_input') # x = Dense(self.pre_latent_size, activation='relu')(input_decoder) x = Dense(np.prod(before_flatten_shape[1:]), activation='relu')(input_decoder) x = Reshape(before_flatten_shape[1:])(x) x = Conv2DTranspose(self.filters, self.kernel_size, strides=(2, 2), activation='relu', kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_conv2DT_2')(x) decoded_img = Conv2DTranspose(1, self.kernel_size, strides=(2, 2), activation='sigmoid', kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_conv2DT_3')(x) ''' Necessary definitions ''' # For parent fitting function self.encoder = Model(input_encoder, z) self.decoder = Model(input_decoder, decoded_img) self.model = Model(input_encoder, self.decoder(self.encoder(input_encoder))) # For parent loss function self.z_mean = z_mean self.z_log_var = z_log_var self.z = z
def build_nn(): nn_input = Input((num_antennas, num_sub, 2)) dropout_rate = 0.25 num_complex_channels = 4 def k_mean(tensor): return K.mean(tensor, axis=2) mean_input = Lambda(k_mean)(nn_input) print(mean_input.get_shape()) # complex to polar real = Lambda(lambda x: x[:, :, :, 0])(nn_input) imag = Lambda(lambda x: x[:, :, :, 1])(nn_input) # complex_crop = Lambda(lambda x: x[:, :, 0, :], output_shape=(Nb_Antennas, 2, 1))(complex_input) # complex_input = Reshape((Nb_Antennas, 2, 1))(mean_input) real_squared = Multiply()([real, real]) imag_squared = Multiply()([imag, imag]) real_imag_squared_sum = Add()([real_squared, imag_squared]) # amplitude def k_sqrt(tensor): r = K.sqrt(tensor) return r r = Lambda(k_sqrt)(real_imag_squared_sum) r = Reshape((num_antennas, num_sub, 1))(r) print(r.get_shape()) # phase def k_atan(tensor): t = tf.math.atan2(tensor[0], tensor[1]) return t t = Lambda(k_atan)([imag, real]) t = Reshape((num_antennas, num_sub, 1))(t) print(t.get_shape()) polar_input = Concatenate()([r, t]) total_input = Concatenate()([nn_input, polar_input]) print("total", total_input.get_shape()) # reduce dimension of time axis lay_input = Reshape((num_antennas, num_sub, num_complex_channels, 1))(total_input) layD1 = Conv3D(8, (1, 23, num_complex_channels), strides=(1, 5, 1), padding='same')(lay_input) layD1 = LeakyReLU(alpha=0.3)(layD1) layD1 = Dropout(dropout_rate)(layD1) layD2 = Conv3D(8, (1, 23, 1), padding='same')(layD1) layD2 = LeakyReLU(alpha=0.3)(layD2) layD2 = Concatenate()([layD1, layD2]) layD2 = Conv3D(8, (1, 1, num_complex_channels), padding='same')(layD2) layD2 = LeakyReLU(alpha=0.3)(layD2) layD2 = Conv3D(8, (1, 23, 1), strides=(1, 5, 1), padding='same', kernel_regularizer=regularizers.l2(0.01))(layD2) layD2 = LeakyReLU(alpha=0.3)(layD2) layD2 = Dropout(dropout_rate)(layD2) layD3 = Conv3D(8, (1, 23, 1), padding='same')(layD2) layD3 = LeakyReLU(alpha=0.3)(layD3) layD3 = Concatenate()([layD2, layD3]) layD3 = Conv3D(8, (1, 1, num_complex_channels), padding='same')(layD3) layD3 = LeakyReLU(alpha=0.3)(layD3) layD3 = Conv3D(8, (1, 23, 1), strides=(1, 5, 1), padding='same', kernel_regularizer=regularizers.l2(0.01))(layD3) layD3 = LeakyReLU(alpha=0.3)(layD3) layD3 = Dropout(dropout_rate)(layD3) layD4 = Conv3D(8, (1, 23, 1), padding='same')(layD3) layD4 = LeakyReLU(alpha=0.3)(layD4) layD4 = Concatenate()([layD4, layD3]) layD4 = Conv3D(8, (1, 1, num_complex_channels), padding='same')(layD4) layD4 = LeakyReLU(alpha=0.3)(layD4) # layD4 = Conv3D(8, (1, 23, 1), strides=(1, 5, 1), padding='same', # kernel_regularizer=regularizers.l2(0.01))(layD4) # layD4 = LeakyReLU(alpha=0.3)(layD4) # layD4 = Dropout(dropout_rate)(layD4) # conv over antenna layers layV1 = Conv3D(8, (16, 1, 1), padding='same')(layD4) layV1 = LeakyReLU(alpha=0.3)(layV1) layV1 = Dropout(dropout_rate)(layV1) layV1 = Concatenate()([layV1, layD4]) layV2 = Conv3D(8, (16, 1, 1), padding='same', kernel_regularizer=regularizers.l2(0.01))(layV1) layV2 = LeakyReLU(alpha=0.3)(layV2) layV2 = Dropout(dropout_rate)(layV2) layV2 = Concatenate()([layV2, layV1]) layV3 = Conv3D(8, (16, 1, 1), padding='same')(layV2) layV3 = LeakyReLU(alpha=0.3)(layV3) layV3 = Dropout(dropout_rate)(layV3) # layV3 = Dropout(dropout_rate)(layV3) # layV4 = Conv3D(8, (16, 1, 1), padding='valid')(layV3) # layV4 = LeakyReLU(alpha=0.3)(layV4) # layV4 = Dropout(dropout_rate)(layV4) # layV5 = Conv3D(8, (16, 1, 1), padding='valid', # kernel_regularizer=regularizers.l2(0.01))(layV4) # layV5 = LeakyReLU(alpha=0.3)(layV5) # layV5 = Dropout(dropout_rate)(layV5) # layV6 = Conv3D(8, (16, 1, 1), padding='valid')(layV5) # layV6 = LeakyReLU(alpha=0.3)(layV6) # layV6 = Dropout(dropout_rate)(layV6) # conv over complex layers # layH1 = Conv3D(12, (1,1,2), strides=(1,1,2), padding='valid', activation='relu')(layV3) # layH1 = Dropout(dropout_rate)(layH1) # layH2 = Conv3D(12, (1,1,2), padding='same', activation='relu')(layH1) # layH2 = Dropout(dropout_rate)(layH2) # layH2 = Concatenate()([layH1, layH2]) # layH3 = Conv3D(12, (1,1,4), padding='same', activation='relu')(layH2) # layH3 = Dropout(dropout_rate)(layH3) nn_output = Flatten()(layV3) nn_output = Dense(64, activation='relu')(nn_output) nn_output = Dense(32, activation='relu')(nn_output) nn_output = Dense(2, activation='linear')(nn_output) nn = Model(inputs=nn_input, outputs=nn_output) nn.compile(optimizer='Adam', loss='mse', metrics=[dist]) nn.summary() return nn
bias_initializer=bias_initializer, name='encoder_z_mean')(x) z_log_var = Conv2D(4 * filters, kernal_size, activation=None, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_z_log_var')(x) # sample from normal with z_mean and z_log_var z = Lambda(sampling, name='latent_space')([z_mean, z_log_var]) ''' Decoder ''' # define input with 'channels_first' encoder_out_shape = tuple(z.get_shape().as_list()) input_decoder = Input(shape=(encoder_out_shape[1], encoder_out_shape[2], encoder_out_shape[3]), name='decoder_input') # transposed convolution and up sampling x = Conv2DTranspose(2 * filters, kernal_size, activation='relu', kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='decoder_conv2DT_1')(input_decoder) x = UpSampling2D(pool_size, name='decoder_up_sampling_1')(x) # transposed convolution and up sampling x = Conv2DTranspose(filters,
def channel(self, ind_emb): #kernel_initializer = tf.contrib.layers.variance_scaling_initializer() #bias_initializer = tf.constant_initializer(value=0.0) ratio = 4 # with tf.variable_scope(name): scale = [] output = [] channel = ind_emb[0].get_shape()[1] shared_mean_1 = Dense(units=int(channel // ratio), activation='relu', kernel_initializer='random_uniform', bias_initializer='zeros', name='mean_1') shared_mean_2 = Dense(units=int(channel), kernel_initializer='random_uniform', bias_initializer='zeros', name='mean_2') shared_max_1 = Dense(units=int(channel // ratio), activation='relu', kernel_initializer='random_uniform', bias_initializer='zeros', name='max_1') shared_max_2 = Dense(units=int(channel), kernel_initializer='random_uniform', bias_initializer='zeros', name='max_2') shared_std_1 = Dense(units=int(channel // ratio), activation='relu', kernel_initializer='random_uniform', bias_initializer='zeros', name='std_1') shared_std_2 = Dense(units=int(channel), kernel_initializer='random_uniform', bias_initializer='zeros', name='std_2') for i, emb in enumerate(ind_emb): #emb = K.permute_dimensions(emb, (0, 2, 3,1)) avg_pool = Lambda(lambda x: K.mean(x, axis=[0, 2, 3]))(emb) avg_pool = Lambda(lambda x: K.expand_dims(x, axis=0))(avg_pool) assert avg_pool.get_shape()[-1] == channel avg_pool = shared_mean_1(avg_pool) assert avg_pool.get_shape()[-1] == channel // ratio avg_pool = shared_mean_2(avg_pool) assert avg_pool.get_shape()[-1] == channel max_pool = Lambda(lambda x: K.max(x, axis=[0, 2, 3]))(emb) max_pool = Lambda(lambda x: K.expand_dims(x, axis=0))(max_pool) assert max_pool.get_shape()[-1] == channel max_pool = shared_max_1(max_pool) assert max_pool.get_shape()[-1] == channel // ratio max_pool = shared_max_2(max_pool) assert max_pool.get_shape()[-1] == channel std_pool = Lambda(lambda x: K.std(x, axis=[0, 2, 3]))(emb) std_pool = Lambda(lambda x: K.expand_dims(x, axis=0))(std_pool) assert std_pool.get_shape()[-1] == channel std_pool = shared_std_1(std_pool) assert std_pool.get_shape()[-1] == channel // ratio std_pool = shared_std_2(std_pool) assert std_pool.get_shape()[-1] == channel added = Add()([avg_pool, max_pool, std_pool]) activation = Activation('sigmoid')(added) # scale = Lambda(lambda x: K.permute_dimensions(x,(1,0)))(activation) # scale = Concatenate(axis=-1)([scale for j in range(channel)]) if i == 0: # output = K.permute_dimensions(K.dot(K.permute_dimensions(emb, (0, 2, 3,1)),scale),(0,3,2,1)) emb = Lambda(lambda x: K.permute_dimensions(x, (0, 2, 3, 1)))( emb) mul = Multiply()([emb, activation]) output = Lambda( lambda x: K.permute_dimensions(x, (0, 3, 1, 2)))(mul) else: # output = K.concatenate([output,K.permute_dimensions(K.dot(K.permute_dimensions(emb, (0, 2, 3,1)),scale),(0,3,2,1))],axis=1) emb = Lambda(lambda x: K.permute_dimensions(x, (0, 2, 3, 1)))( emb) mul = Multiply()([emb, activation]) mul = Lambda(lambda x: K.permute_dimensions(x, (0, 3, 1, 2)))( mul) output = Concatenate(axis=1)([output, mul]) # emb = Lambda(lambda x: K.permute_dimensions(x,(0, 2, 3,1)))(ind_emb[0]) return output
def get_gru_ctc_model( image_size=image_size, seq_len=4, # 字符最大长 label_count=37): # 标签数量 img_height, img_width = image_size[0], image_size[1] input_tensor = Input((img_height, img_width, 3), name='input') x = input_tensor x = preprocess(x) x = Lambda(cnn_part)(x) # x = cnn_part(x) conv_shape = x.get_shape() print(conv_shape) x = Reshape(target_shape=(int(conv_shape[1]), int(conv_shape[2] * conv_shape[3])))(x) x = Dense(32, activation='relu')(x) gru_1 = GRU(32, return_sequences=True, kernel_initializer='he_normal', name='gru1')(x) gru_1b = GRU(32, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='gru1_b')(x) gru1_merged = Add()([gru_1, gru_1b]) gru_2 = GRU(32, return_sequences=True, kernel_initializer='he_normal', name='gru2')(gru1_merged) gru_2b = GRU(32, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='gru2_b')(gru1_merged) x = Concatenate()([gru_2, gru_2b]) x = Dropout(0.25)(x) x = Dense(label_count, kernel_initializer='he_normal', activation='softmax', name='output')(x) base_model = Model(inputs=input_tensor, outputs=x) labels = Input(name='the_labels', shape=[seq_len], dtype='float32') input_length = Input(name='input_length', shape=[1], dtype='int64') label_length = Input(name='label_length', shape=[1], dtype='int64') loss_out = Lambda(ctc_lambda_func, output_shape=(1, ), name='ctc')([x, labels, input_length, label_length]) ctc_model = Model( inputs=[input_tensor, labels, input_length, label_length], outputs=[loss_out]) sgd = optimizers.Adam(lr=0.005) ctc_model.compile(loss={ 'ctc': lambda y_true, y_pred: y_pred }, optimizer=sgd) ctc_model.summary() return conv_shape, base_model, ctc_model
def set_model(self): ''' Initialisers ''' weight_seed = None kernel_initializer = initializers.glorot_uniform(seed=weight_seed) bias_initializer = initializers.glorot_uniform(seed=weight_seed) ''' Encoder ''' # define input with 'channels_first' input_encoder = Input(shape=input_shape, name='encoder_input') # 'kernel_size' convolution with 'filters' output filters, stride 1x1 and 'valid' border_mode x = Conv2D(self.filters, self.kernel_size, padding='valid', activation='relu', kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_conv2D_1')(input_encoder) x = MaxPooling2D(self.pool_size, name='encoder_max_pooling_1')(x) x = Conv2D(2 * self.filters, self.kernel_size, padding='valid', activation='relu', kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_conv2D_2')(x) # separate dense layers for mu and log(sigma), both of size latent_dim z_mean = Conv2D(self.latent_filters, self.kernel_size, padding='valid', activation=None, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_z_mean')(x) z_log_var = Conv2D(self.latent_filters, self.kernel_size, padding='valid', activation=None, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_z_log_var')(x) # sample from normal with z_mean and z_log_var z = Lambda(self.sampling, name='latent_space')([z_mean, z_log_var]) ''' Decoder ''' # define input with 'channels_first' encoder_out_shape = tuple(z.get_shape().as_list()) input_decoder = Input(shape=(encoder_out_shape[1], encoder_out_shape[2], encoder_out_shape[3]), name='decoder_input') # transposed convolution and up sampling x = Conv2DTranspose(2 * self.filters, self.kernel_size, padding='valid', activation='relu', kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='decoder_conv2DT_1')(input_decoder) x = Conv2DTranspose(self.filters, self.kernel_size, padding='valid', activation='relu', kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='decoder_conv2DT_2')(x) x = UpSampling2D(self.pool_size, name='decoder_up_sampling_1')(x) x = Conv2DTranspose(1, self.kernel_size, padding='valid', activation='sigmoid', kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='decoder_conv2DT_3')(x) # define decoded image to be image in last layer decoded_img = x ''' Necessary definitions ''' # For parent fitting function self.encoder = Model(input_encoder, z) self.decoder = Model(input_decoder, decoded_img) self.model = Model(input_encoder, self.decoder(self.encoder(input_encoder))) # For parent loss function self.z_mean = z_mean self.z_log_var = z_log_var self.z = z
def create_VAE(input_size=16): ''' Returns a VAE model that takes in a image of size (input_size, input_size) Inputs: inputs_size = the size of input images to the encoder Returns: 3-Tuple: VAE model, VAE encoder, VAE decoder ''' encoding_size = 3 epsilon_std = 1.0 conv_filters = 32 kernel_size = (3, 3) pool_size = (2, 2) conv_levels = 3 intermediate_size = 128 x = Input(shape=(input_size, input_size, 1)) h = Conv2D(conv_filters, kernel_size, activation='relu', padding='same')(x) for _ in range(conv_levels - 1): h = MaxPooling2D(pool_size, padding='same')(h) h = Conv2D(conv_filters, kernel_size, activation='relu', padding='same')(h) # h = MaxPooling2D(pool_size, padding='same')(h) # h = Conv2D(cov_filters, kernel_size, activation='relu', padding='same')(h) original_shape = h.get_shape().as_list()[1:] h = Flatten()(h) h = Dense(intermediate_size)(h) z_mean = Dense(encoding_size)(h) z_log_var = Dense(encoding_size)(h) def sampling(args): z_mean, z_log_sigma = args epsilon = K.random_normal(shape=(K.shape(z_mean)[0], encoding_size), mean=0., stddev=epsilon_std) return z_mean + K.exp(z_log_sigma / 2) * epsilon z = Lambda(sampling)([z_mean, z_log_var]) generator_input = Input(shape=z.get_shape().as_list()[1:]) h = Dense(intermediate_size) decode_h = h(z) generator_h = h(generator_input) h = Dense(np.cumprod(original_shape)[-1]) decode_h = h(decode_h) generator_h = h(generator_h) h = Reshape(original_shape) decode_h = h(decode_h) generator_h = h(generator_h) h = Conv2D(conv_filters, kernel_size, activation='relu', padding='same') decode_h = h(decode_h) generator_h = h(generator_h) for _ in range(conv_levels - 1): h = UpSampling2D(pool_size) decode_h = h(decode_h) generator_h = h(generator_h) h = Conv2D(conv_filters, kernel_size, activation='relu', padding='same') decode_h = h(decode_h) generator_h = h(generator_h) h = Dense(1, activation='sigmoid') decode_h = h(decode_h) generator_h = h(generator_h) encoder = Model(x, z) generator = Model(generator_input, generator_h) vae = Model(x, decode_h) def vae_loss(x, x_decoded_mean): xent_loss = K.sum(K.sum(metrics.binary_crossentropy(x, x_decoded_mean), axis=-1), axis=-1) kl_loss = - 0.5 * K.mean(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1) return xent_loss + kl_loss opt = optimizers.Adam() vae.compile(optimizer=opt, loss=vae_loss) generator.compile(optimizer=opt, loss='binary_crossentropy') encoder.compile(optimizer=opt, loss='binary_crossentropy') return vae, encoder, generator
return K.ctc_batch_cost(labels, y_pred, input_length, label_length) rnn_size = 128 input_tensor = Input((width, height, 3)) x = input_tensor x = Lambda(lambda x: (x - 127.5) / 127.5)(x) for i in range(3): for j in range(2): x = Conv2D(32 * 2**i, 3, kernel_initializer='he_uniform')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = MaxPooling2D(pool_size=(2, 2))(x) conv_shape = x.get_shape().as_list() rnn_length = conv_shape[1] rnn_dimen = conv_shape[2] * conv_shape[3] print(conv_shape, rnn_length, rnn_dimen) x = Reshape(target_shape=(rnn_length, rnn_dimen))(x) rnn_length -= 2 x = Dense(rnn_size, kernel_initializer='he_uniform')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Dropout(0.2)(x) gru_1 = GRU(rnn_size, return_sequences=True, kernel_initializer='he_uniform',
def gan_2(input_img, input_channel=3, z_num=64, no_of_pairs=5, hidden_num=128, activation_fn=tf.nn.elu, noise_dim=64): # Encoder encoder_layer_list = [] x = Conv2D(hidden_num, kernel_size=3, strides=1, activation=activation_fn, padding='same')(input_img) prev_channel_num = hidden_num for idx in range(no_of_pairs): # to increase number of filter by (filters)*(index+1) ex: 16, 32, 48 ... channel_num = hidden_num * (idx + 1) res = x x = Conv2D(channel_num, kernel_size=3, strides=1, activation=activation_fn, padding='same')(x) x = Conv2D(channel_num, kernel_size=3, strides=1, activation=activation_fn, padding='same')(x) if idx > 0: encoder_layer_list.append(x) if idx < no_of_pairs - 1: x = Conv2D(channel_num, kernel_size=3, strides=2, activation=activation_fn, padding='same')(x) if noise_dim > 0: x = Lambda(get_noise)(x) for idx in range(no_of_pairs): if idx < no_of_pairs - 1: x = Concatenate(axis=-1)( [x, encoder_layer_list[no_of_pairs - 2 - idx]]) channel_num = x.get_shape().as_list()[-1] x = Conv2D(hidden_num, kernel_size=3, strides=1, activation=activation_fn, padding='same')(x) x = Conv2D(hidden_num, kernel_size=3, strides=1, activation=activation_fn, padding='same')(x) if idx < no_of_pairs - 1: x = UpSampling2D(2)(x) out = Conv2D(input_channel, kernel_size=3, strides=1, activation=None, padding='same')(x) return out
def googleNet_n(x, data_format='channels_last', num_classes=24, num_layers=[1, 1, 2, 1], features=[1, 1, 1, 1, 1]): xft = Lambda(lambda v: tf_fft(v))(x) x = Reshape(in_shp + (1, ), input_shape=in_shp)(x) x = Conv2D(filters=64 * features[0], kernel_size=[2, 7], strides=[2, 2], data_format=data_format, padding='same', activation='relu')(x) x = MaxPooling2D([1, 3], strides=[1, 2], padding='same')(x) for dep in range(num_layers[0]): x = Conv2D(filters=192 * features[1], kernel_size=[1, 3], strides=[1, 1], padding='same', activation='relu')(x) x = MaxPooling2D([1, 3], strides=[1, 2], padding='same')(x) xft = Reshape(in_shp + (1, ), input_shape=in_shp)(xft) xft = Conv2D(filters=64 * features[0], kernel_size=[2, 4], strides=[2, 2], data_format=data_format, padding='same', activation='relu')(xft) xft = MaxPooling2D([1, 3], strides=[1, 2], padding='same')(xft) for dep in range(num_layers[0]): xft = Conv2D(filters=192 * features[1], kernel_size=[1, 3], strides=[1, 1], padding='same', activation='relu')(xft) xft = MaxPooling2D([1, 3], strides=[1, 2], padding='same')(xft) print(x.get_shape(), xft.get_shape()) x = keras.layers.concatenate([x, xft], axis=3) print(x.get_shape()) for dep in range(num_layers[1]): x = inception(x, height=2, fs=np.array([32, 32, 32, 32, 32]) * features[2], tw_tower=True) x = MaxPooling2D([1, 3], strides=2, padding='same')(x) for dep in range(num_layers[2]): x = inception(x, height=2, fs=np.array([48, 96, 48, 96, 96]) * features[3], with_residual=True) #out_mid = out_tower(x, dr=0.3) #for dep in range(num_layers[3]): # x = inception(x, height=2, fs=np.array([48,96,48,96,96])*features[4], with_residual=True) x = MaxPooling2D([2, 3], strides=2, padding='same')(x) for dep in range(num_layers[3]): x = inception(x, height=1, fs=np.array([32, 32, 32, 32, 32]) * features[4]) out = out_tower(x, dr=0.5, reg=args.confireg) #out = Average()([out_mid, out_late]) return out
def prot_encoder(model, seq, seq_size): encoder = model.prot_encoder if model.dataset.name in LIST_AA_DATASETS: if encoder['conv_strides'] != 1 or encoder['name'] == 'conv_biLSTM_att'\ or model.batch_size != 1: print('!! ERROR !!: AA prediction but conv_strides > 1 ' + 'OR tt_mech at the end of encoder OR batch_size > 1 !!') exit(1) kreg, breg = None, None # if model.prot_reg != 0: # kreg, breg = regularizers.l2(model.prot_reg), regularizers.l2(model.prot_reg) # else: # kreg, breg = None, None seq = Lambda(lambda t: expand_last_dim(t), name="expand_last_dim")(seq) if encoder['name'] == 'conv' or encoder['name'] == 'conv_aa': n_steps = encoder['n_steps'] n_filters = encoder['nb_conv_filters'] # seq = Lambda(lambda t: t, name='temp1')(seq) for n in range(n_steps): seq_shape = seq.get_shape().as_list() kernel_size = (seq_shape[1], encoder['filter_size']) strides = (seq_shape[1], encoder['conv_strides']) seq = seq_conv(model, seq, n_filters, kernel_size, strides, kreg, breg, model.prot_reg, model.prot_dropout, model.prot_BN, n) print('seq', seq) seq_size = Lambda(lambda t: update_seq_size(t, float(strides[1])))( seq_size) # seq = Lambda(lambda t: t, name='temp2')(seq) if model.dataset.name not in LIST_AA_DATASETS and 'aa' not in encoder[ 'name']: embedding = Lambda(lambda t: agg_sum(t, 2), name='agg_sum')(seq) print('###', embedding) embedding = Lambda(lambda t: squeeze(t, -1), name='prot_embedding')(embedding) elif 'aa' in encoder['name']: seq = Lambda(lambda t: squeeze(t, -1), name='conv_emb')(seq) seq = Permute((2, 1))(seq) embedding = Lambda(lambda t: t, name='prot_embedding')(seq) else: seq = Permute((2, 1, 3))(seq) embedding = Lambda(lambda t: squeeze(squeeze(t, -1), 0), name='prot_embedding')(seq) elif 'conv_biLSTM' in encoder['name'] or 'conv_biGRU' in encoder['name']: seq_shape = seq.get_shape().as_list() n_filters = encoder['nb_conv_filters'] kernel_size = (seq_shape[1], encoder['filter_size']) strides = (seq_shape[1], encoder['conv_strides']) seq_size = Lambda(lambda t: update_seq_size(t, float(strides[1])))( seq_size) seq = seq_conv(model, seq, n_filters, kernel_size, strides, kreg, breg, model.prot_reg, model.prot_dropout, model.prot_BN, 0) seq = Lambda(lambda t: squeeze(t, -1), name='conv_emb')(seq) seq = Permute((2, 1))(seq) if model.dataset.name not in LIST_AA_DATASETS: seq = Masking(mask_value=0.0)(seq) rseq = True if 'att' not in encoder['name'] and 'aa' not in encoder[ 'name'] else True if 'biLSTM' in encoder['name']: lstm_layer = LSTM(n_filters, return_sequences=rseq, activation='tanh', kernel_regularizer=kreg, recurrent_regularizer=kreg, bias_regularizer=breg, dropout=model.prot_dropout) elif 'biGRU' in encoder['name']: lstm_layer = GRU(n_filters, return_sequences=rseq, activation='tanh', kernel_regularizer=kreg, recurrent_regularizer=kreg, bias_regularizer=breg, dropout=model.prot_dropout) # import pdb; pdb.Pdb().set_trace() seq = Bidirectional(lstm_layer, merge_mode='concat', weights=None, name='biLSTM')(seq) if model.prot_reg != 0: seq = GaussianNoise(model.prot_reg)(seq) if 'att' in encoder['name']: attention_probs = Dense(n_filters * 2, activation='softmax', name='attention_probs')(seq) embedding = Multiply(name='prot_embedding')([seq, attention_probs]) elif model.dataset.name in LIST_AA_DATASETS or 'aa' in encoder['name']: embedding = Lambda(lambda t: t, name='prot_embedding')(seq) else: embedding = Lambda(lambda t: agg_sum(t, 1), name='prot_embedding')(seq) # embedding = Lambda(lambda t: t, name='prot_embedding')(seq) # seq = Permute((2, 1))(seq) # import pdb; pdb.Pdb().set_trace() # embedding = Lambda(lambda t: squeeze(t, 0), name='prot_embedding')(seq) if encoder['hand_crafted_features']: embedding = Concatenate(axis=-1)([embedding, model.features]) # import pdb; pdb.Pdb().set_trace() return embedding, seq_size
def build_ctc_network(args, training=True): global N_COL global N_ROW """ if args.stn == True: from STN.STN import locnet input_tensor = locnet(input, sampling_size = (N_COL, N_ROW)) print("STN") """ #stn_model = None if hasattr(args, 'stn_model'): pass else: args.stn_model = False if args.model == 'resnet18': NOT_CARE = 1 base_model = resnet.ResnetBuilder.build_resnet_18(input_shape=(N_COL, N_ROW, 3), num_outputs=NOT_CARE, include_top=False, args=args) elif args.model == 'resnet18_2222': NOT_CARE = 1 base_model = resnet.ResnetBuilder.build_resnet_18_2222( input_shape=(N_COL, N_ROW, 3), num_outputs=NOT_CARE, include_top=False, args=args) elif args.model == 'resnet18_2222_64': NOT_CARE = 1 base_model = resnet.ResnetBuilder.build_resnet_18_2222_start_from64( input_shape=(N_COL, N_ROW, 3), num_outputs=NOT_CARE, include_top=False, args=args) elif args.model == 'resnet18_2222_48': NOT_CARE = 1 base_model = resnet.ResnetBuilder.build_resnet_18_2222_start_from48( input_shape=(N_COL, N_ROW, 3), num_outputs=NOT_CARE, include_top=False, args=args) elif args.model == 'resnet18_2222_32': NOT_CARE = 1 base_model = resnet.ResnetBuilder.build_resnet_18_2222_start_from32( input_shape=(N_COL, N_ROW, 3), num_outputs=NOT_CARE, include_top=False, args=args) elif args.model == 'resnet18_2222_16': NOT_CARE = 1 base_model = resnet.ResnetBuilder.build_resnet_18_2222_start_from16( input_shape=(N_COL, N_ROW, 3), num_outputs=NOT_CARE, include_top=False, args=args) elif args.model == 'resnet34': NOT_CARE = 1 base_model = resnet.ResnetBuilder.build_resnet_34(input_shape=(N_COL, N_ROW, 3), num_outputs=NOT_CARE, include_top=False, args=args) elif args.model == 'resnet50': NOT_CARE = 1 base_model = resnet.ResnetBuilder.build_resnet_50(input_shape=(N_COL, N_ROW, 3), num_outputs=NOT_CARE, include_top=False, args=args) else: raise TypeError('model should be in the list of the supported model!') #print('Input col: ', N_COL) #print('Input row: ', N_ROW) x = base_model.output #CNN to RNN x = Lambda(lambda x: K.permute_dimensions(x, (0, 2, 1, 3)))( x) # switchaxes from [b,h,w,c] to [b,w,h,c] conv_shape = x.get_shape() # b, h,w,c resnet 18 -> (?, 8, 16, 256) #print('conv_shape', conv_shape) x = Reshape(target_shape=(int(conv_shape[1]), int(conv_shape[2] * conv_shape[3])), name='reshape')(x) #(?, 16, 8, 256) x = Dense(para.dense_size, activation='relu', kernel_initializer='he_normal', name='dense1')(x) #5_exp ->128 #x = BatchNormalization()(x) # GRU RNN GRU_UNIT = CuDNNGRU bi_gru1 = Bidirectional(GRU_UNIT(para.rnn_size, return_sequences=True, kernel_initializer='he_normal', name='bi_gru1'), merge_mode='sum')(x) bi_gru1 = BatchNormalization()(bi_gru1) bi_gru2 = Bidirectional(GRU_UNIT(para.rnn_size, return_sequences=True, kernel_initializer='he_normal', name='bi_gru2'), merge_mode='concat')(bi_gru1) bi_gru2 = BatchNormalization()(bi_gru2) #attention #att = AttentionWithContext()(bi_gru2) inner = Dense(para.num_classes, kernel_initializer='he_normal', name='dense2')(bi_gru2) y_pred = Activation('softmax', name='softmax')(inner) labels = Input(name='the_labels', shape=[para.max_text_len], dtype='float32') # (None ,7) input_length = Input(name='input_length', shape=[1], dtype='int64') # (None, 1) label_length = Input(name='label_length', shape=[1], dtype='int64') # (None, 1) # Keras doesn't currently support loss funcs with extra parameters # so CTC loss is implemented in a lambda layer loss_out = Lambda(ctc_lambda_func, output_shape=(1, ), name='ctc')([y_pred, labels, input_length, label_length]) #(None, 1) if training: return Model( inputs=[base_model.input, labels, input_length, label_length], outputs=loss_out), conv_shape[1] else: return Model(inputs=[base_model.input], outputs=y_pred)
def build_model(width: int, height: int, rnn_size: int, classify_count: int, classify_len: int, learning_rate: float): input_tensor = Input((width, height, 3)) x = input_tensor x = Lambda(lambda a: (a - 127.5) / 127.5)(x) for j in range(3): for i in range(2): x = Conv2D(32 * 2**i, (3, 3), kernel_initializer='he_uniform')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = MaxPooling2D((2, 2))(x) conv_shape = x.get_shape().as_list() rnn_length = conv_shape[1] rnn_dimen = conv_shape[2] * conv_shape[3] x = Reshape((rnn_length, rnn_dimen))(x) rnn_length -= 2 x = Dense(rnn_size, kernel_initializer='he_uniform')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Dropout(0.2)(x) gru_1 = GRU(rnn_size, return_sequences=True, kernel_initializer='he_uniform', name='gru1')(x) gru_1b = GRU(rnn_size, return_sequences=True, kernel_initializer='he_uniform', go_backwards=True, name='gru1_b')(x) x = add([gru_1, gru_1b]) gru_2 = GRU(rnn_size, return_sequences=True, kernel_initializer='he_uniform', name='gru2')(x) gru_2b = GRU(rnn_size, return_sequences=True, kernel_initializer='he_uniform', go_backwards=True, name='gru2_b')(x) x = concatenate([gru_2, gru_2b]) x = Dropout(0.2)(x) x = Dense(classify_count, activation='softmax')(x) base_model = Model(inputs=input_tensor, outputs=x) labels = Input(name='the_labels', shape=[classify_len], dtype='float32') input_length = Input(name='input_length', shape=[1], dtype='int64') label_length = Input(name='label_length', shape=[1], dtype='int64') loss_out = Lambda(ctc_lambda, output_shape=(1, ), name='ctc')([x, labels, input_length, label_length]) opt = Adam(lr=learning_rate) ctc_model = Model( inputs=[input_tensor, labels, input_length, label_length], outputs=[loss_out]) ctc_model.compile(loss={ 'ctc': lambda y_true, y_pred: y_pred }, optimizer=opt) return base_model, ctc_model, rnn_length
def NN_model(args, training=True): global N_COL global N_ROW if args.model == 'densenet121': from keras.applications.densenet import DenseNet121 input_tensor = Input(shape=(N_COL, N_ROW, 3)) base_model = DenseNet121(input_shape=(N_COL, N_ROW, 3), include_top=False, weights='imagenet', input_tensor=input_tensor, pooling=None) elif args.model == 'resnet18': import resnet NOT_CARE = 1 base_model = resnet.ResnetBuilder.build_resnet_18(input_shape=(N_COL, N_ROW, 3), num_outputs=NOT_CARE, include_top=False) elif args.model == 'resnet18_2222': import resnet NOT_CARE = 1 base_model = resnet.ResnetBuilder.build_resnet_18_2222( input_shape=(N_COL, N_ROW, 3), num_outputs=NOT_CARE, include_top=False) elif args.model == 'resnet34': import resnet NOT_CARE = 1 base_model = resnet.ResnetBuilder.build_resnet_34(input_shape=(N_COL, N_ROW, 3), num_outputs=NOT_CARE, include_top=False) elif args.model == 'resnet50': import resnet NOT_CARE = 1 base_model = resnet.ResnetBuilder.build_resnet_50(input_shape=(N_COL, N_ROW, 3), num_outputs=NOT_CARE, include_top=False) elif args.model == 'resnet101': import resnet NOT_CARE = 1 base_model = resnet.ResnetBuilder.build_resnet_101( input_shape=(N_COL, N_ROW, 3), num_outputs=NOT_CARE, include_top=False) else: raise TypeError('model should be in the list of the supported model!') print('Input col: ', N_COL) print('Input row: ', N_ROW) x = base_model.output #CNN to RNN x = Lambda(lambda x: K.permute_dimensions(x, (0, 2, 1, 3)))( x) # switchaxes from [b,h,w,c] to [b,w,h,c] conv_shape = x.get_shape() # b, h,w,c resnet 18 -> (?, 16, 32, 256) print('conv_shape', conv_shape) x = Reshape(target_shape=(int(conv_shape[1]), int(conv_shape[2] * conv_shape[3])), name='reshape')(x) x = Dense(para.dense_size, activation='relu', kernel_initializer='he_normal', name='dense1')(x) #x = BatchNormalization()(x) # GRU RNN gru_1 = GRU(para.rnn_size, return_sequences=True, init='he_normal', name='gru1')(x) gru_1b = GRU(para.rnn_size, return_sequences=True, go_backwards=True, init='he_normal', name='gru1_b')(x) gru1_merged = add([gru_1, gru_1b]) gru1_merged = BatchNormalization()(gru1_merged) gru_2 = GRU(para.rnn_size, return_sequences=True, init='he_normal', name='gru2')(gru1_merged) gru_2b = GRU(para.rnn_size, return_sequences=True, go_backwards=True, init='he_normal', name='gru2_b')(gru1_merged) gru2_merged = concatenate([gru_2, gru_2b]) gru2_merged = BatchNormalization()(gru2_merged) inner = Dense(para.num_classes, kernel_initializer='he_normal', name='dense2')(gru2_merged) y_pred = Activation('softmax', name='softmax')(inner) labels = Input(name='the_labels', shape=[para.max_text_len], dtype='float32') # (None ,7) input_length = Input(name='input_length', shape=[1], dtype='int64') # (None, 1) label_length = Input(name='label_length', shape=[1], dtype='int64') # (None, 1) # Keras doesn't currently support loss funcs with extra parameters # so CTC loss is implemented in a lambda layer loss_out = Lambda(ctc_lambda_func, output_shape=(1, ), name='ctc')([y_pred, labels, input_length, label_length]) #(None, 1) if training: return Model( inputs=[base_model.input, labels, input_length, label_length], outputs=loss_out), conv_shape[1] else: return Model(inputs=[base_model.input], outputs=y_pred)
def build_nn(num_antenna=64): nn_input = Input((num_antenna, num_sub, 2)) dropout_rate = 0.25 num_complex_channels = 6 def k_mean(tensor): return K.mean(tensor, axis=2) mean_input = Lambda(k_mean)(nn_input) print(mean_input.get_shape()) # complex to polar real = Lambda(lambda x: x[:, :, :, 0])(nn_input) imag = Lambda(lambda x: x[:, :, :, 1])(nn_input) # complex_crop = Lambda(lambda x: x[:, :, 0, :], output_shape=(Nb_Antennas, 2, 1))(complex_input) # complex_input = Reshape((Nb_Antennas, 2, 1))(mean_input) real_squared = Multiply()([real, real]) imag_squared = Multiply()([imag, imag]) real_imag_squared_sum = Add()([real_squared, imag_squared]) # amplitude def k_sqrt(tensor): r = K.sqrt(tensor) return r r = Lambda(k_sqrt)(real_imag_squared_sum) r = Reshape((num_antenna, num_sub, 1))(r) print(r.get_shape()) # phase def k_atan(tensor): import tensorflow as tf t = tf.math.atan2(tensor[0], tensor[1]) return t t = Lambda(k_atan)([imag, real]) t = Reshape((num_antenna, num_sub, 1))(t) print(t.get_shape()) def ifft(x): y = tf.complex(x[:, :, :, 0], x[:, :, :, 1]) ifft = tf.spectral.ifft(y) return tf.stack([tf.math.real(ifft), tf.math.imag(ifft)], axis=3) polar_input = Concatenate()([r, t]) time_input = Lambda(ifft)(nn_input) total_input = Concatenate()([nn_input, polar_input, time_input]) # print("total", total_input.get_shape()) # reduce dimension of time axis lay_input = Reshape( (num_antenna, num_sub, num_complex_channels, 1))(total_input) layD1 = Conv3D(8, (1, 23, num_complex_channels), strides=(1, 5, 1), padding='same')(lay_input) layD1 = LeakyReLU(alpha=0.3)(layD1) layD1 = Dropout(dropout_rate)(layD1) layD2 = Conv3D(8, (1, 23, 1), padding='same')(layD1) layD2 = LeakyReLU(alpha=0.3)(layD2) layD2 = Concatenate()([layD1, layD2]) layD2 = Conv3D(8, (1, 1, num_complex_channels), padding='same')(layD2) layD2 = LeakyReLU(alpha=0.3)(layD2) layD2 = Conv3D(8, (1, 23, 1), strides=(1, 5, 1), padding='same', kernel_regularizer=regularizers.l2(0.01))(layD2) layD2 = LeakyReLU(alpha=0.3)(layD2) layD2 = Dropout(dropout_rate)(layD2) layD3 = Conv3D(8, (1, 23, 1), padding='same')(layD2) layD3 = LeakyReLU(alpha=0.3)(layD3) layD3 = Concatenate()([layD2, layD3]) layD3 = Conv3D(8, (1, 1, num_complex_channels), padding='same')(layD3) layD3 = LeakyReLU(alpha=0.3)(layD3) layD3 = Conv3D(8, (1, 23, 1), strides=(1, 5, 1), padding='same', kernel_regularizer=regularizers.l2(0.01))(layD3) layD3 = LeakyReLU(alpha=0.3)(layD3) layD3 = Dropout(dropout_rate)(layD3) layD4 = Conv3D(8, (1, 23, 1), padding='same')(layD3) layD4 = LeakyReLU(alpha=0.3)(layD4) layD4 = Concatenate()([layD4, layD3]) layD4 = Conv3D(8, (1, 1, num_complex_channels), padding='same')(layD4) layD4 = LeakyReLU(alpha=0.3)(layD4) layV1 = Conv3D(8, (8, 1, 1), padding='same')(layD4) layV1 = LeakyReLU(alpha=0.3)(layV1) layV1 = Dropout(dropout_rate)(layV1) layV1 = Concatenate()([layV1, layD4]) layV2 = Conv3D(8, (8, 1, 1), padding='same', kernel_regularizer=regularizers.l2(0.01))(layV1) layV2 = LeakyReLU(alpha=0.3)(layV2) layV2 = Dropout(dropout_rate)(layV2) layV2 = Concatenate()([layV2, layV1]) layV3 = Conv3D(8, (8, 1, 1), padding='same')(layV2) layV3 = LeakyReLU(alpha=0.3)(layV3) layV3 = Dropout(dropout_rate)(layV3) layV3 = Concatenate()([layV3, layV2]) layV4 = Conv3D(8, (8, 1, 1), padding='same')(layV3) layV4 = LeakyReLU(alpha=0.3)(layV4) layV4 = Dropout(dropout_rate)(layV4) layV4 = Concatenate()([layV4, layV3]) layV5 = Conv3D(8, (8, 1, 1), padding='same')(layV4) layV5 = LeakyReLU(alpha=0.3)(layV5) layV5 = Dropout(dropout_rate)(layV5) nn_output = Flatten()(layV5) nn_output = Dense(64, activation='relu')(nn_output) nn_output = Dense(32, activation='relu')(nn_output) nn_output = Dense(2, activation='linear')(nn_output) nn = Model(inputs=nn_input, outputs=nn_output) # nn = multi_gpu_model(nn, gpus=2) nn.compile(optimizer='Adam', loss='mse', metrics=[dist]) nn.summary() return nn
def build_model(train_batch_size, input_shape, smpl_path, output_wh, num_classes, encoder_architecture='resnet50', use_IEF=False, vertex_sampling=None, scaledown=0.005): """ Function to build indirect learning via body-part segmentation model. Builds encoder, decoder and segmenter. :param train_batch_size: training batch size. :param input_shape: (H, W, 3) :param smpl_path: path to SMPL model parameters. :param output_wh: output width and height :param num_classes: number of body-part classes :param encoder_architecture: resnet50 or enet :param use_IEF: use iterative error feedback flag :param vertex_sampling: :param scaledown: how much to scaledown regression module output by :return: segs_model, smpl_model, verts_model, projects_model """ num_camera_params = 4 num_smpl_params = 72 + 10 num_total_params = num_smpl_params + num_camera_params # --- BACKBONE --- if encoder_architecture == 'enet': inp = Input(shape=input_shape) img_features = build_enet(inp) # (N, 32, 32, 128) output size from enet img_features = Conv2D(256, (3, 3), activation='relu', padding='same')(img_features) img_features = BatchNormalization()(img_features) img_features = MaxPooling2D(pool_size=(4, 4))(img_features) img_features = Conv2D(512, (3, 3), activation='relu', padding='same')(img_features) img_features = BatchNormalization()(img_features) img_features = MaxPooling2D()(img_features) img_features = Conv2D(2048, (3, 3), activation='relu', padding='same')(img_features) img_features = BatchNormalization()(img_features) img_features = MaxPooling2D(pool_size=(4, 4))(img_features) img_features = Reshape((2048,))(img_features) elif encoder_architecture == 'resnet50': resnet = resnet50.ResNet50(include_top=False, weights=None, input_shape=input_shape) inp = resnet.input img_features = resnet.output img_features = Reshape((2048,))(img_features) # --- REGRESSION MODULE --- if use_IEF: # --- IEF MODULE --- # Instantiate ief layers IEF_layer_1 = Dense(1024, activation='relu', name='IEF_layer_1') IEF_layer_2 = Dense(1024, activation='relu', name='IEF_layer_2') IEF_layer_3 = Dense(num_total_params, activation='linear', name='IEF_layer_3') state1 = Lambda(concat_mean_param, arguments={'img_wh': output_wh})(img_features) param1 = Lambda(lambda x: x[:, 2048:])(state1) print("State1 shape", state1.get_shape()) print("Param1 shape", param1.get_shape()) # Iteration 1 delta1 = IEF_layer_1(state1) delta1 = IEF_layer_2(delta1) delta1 = IEF_layer_3(delta1) delta1 = Lambda(lambda x, d: x * d, arguments={"d": scaledown})(delta1) param2 = Add()([param1, delta1]) state2 = Concatenate()([img_features, param2]) # Iteration 2 delta2 = IEF_layer_1(state2) delta2 = IEF_layer_2(delta2) delta2 = IEF_layer_3(delta2) delta2 = Lambda(lambda x, d: x * d, arguments={"d": scaledown})(delta2) param3 = Add()([param2, delta2]) state3 = Concatenate()([img_features, param3]) # Iteration 3 delta3 = IEF_layer_1(state3) delta3 = IEF_layer_2(delta3) delta3 = IEF_layer_3(delta3) delta3 = Lambda(lambda x, d: x * d, arguments={"d": scaledown})(delta3) final_param = Add()([param3, delta3]) else: smpl = Dense(2048, activation='relu')(img_features) smpl = Dense(1024, activation='relu')(smpl) smpl = Dense(num_total_params, activation='linear')(smpl) smpl = Lambda(lambda x: x * scaledown, name="scale_down")(smpl) final_param = Lambda(load_mean_set_cam_params, arguments={'img_wh': output_wh})(smpl) # --- DECODER --- verts = SMPLLayer(smpl_path, batch_size=train_batch_size)(final_param) # --- SEGMENTER --- projects_with_depth = Lambda(orthographic_project, arguments={'vertex_sampling': vertex_sampling}, name='project')([verts, final_param]) masks = Lambda(compute_mask, name='compute_mask')(projects_with_depth) segs = Lambda(projects_to_seg, arguments={'img_wh': output_wh, 'vertex_sampling': vertex_sampling}, name='segment')([projects_with_depth, masks]) segs = Reshape((output_wh * output_wh, num_classes), name="final_reshape")(segs) segs = Activation('softmax', name="final_softmax")(segs) segs_model = Model(inputs=inp, outputs=segs) smpl_model = Model(inputs=inp, outputs=final_param) verts_model = Model(inputs=inp, outputs=verts) projects_model = Model(inputs=inp, outputs=projects_with_depth) print(segs_model.summary()) return segs_model, smpl_model, verts_model, projects_model
def create_model(existing='', is_twohundred=False, is_halffeatures=True, weights='imagenet'): if len(existing) == 0: print('Loading base model (DenseNet)..') input = Input(shape=(None, None, None)) image, aux_output, has_locations = Lambda(split_input)(input) # Encoder Layers if is_twohundred: base_model = applications.DenseNet201(input_tensor=image, weights=weights, include_top=False) else: base_model = applications.DenseNet169(input_tensor=image, weights=weights, include_top=False) # Starting point for decoder base_model_output_shape = base_model.layers[-1].output.shape print('Base model loaded [output: %s].' % (base_model_output_shape, )) # Layer freezing? for layer in base_model.layers: layer.trainable = True # Starting number of decoder filters if is_halffeatures: decode_filters = int(int(base_model_output_shape[-1]) / 2) else: decode_filters = int(base_model_output_shape[-1]) # Define upsampling layer def upproject(tensor, filters, name, concat_with): up_i = BilinearUpSampling2D((2, 2), name=name + '_upsampling2d')(tensor) up_i = Concatenate(name=name + '_concat')( [up_i, base_model.get_layer(concat_with).output]) # Skip connection up_i = Conv2D(filters=filters, kernel_size=3, strides=1, padding='same', name=name + '_convA')(up_i) up_i = LeakyReLU(alpha=0.2)(up_i) up_i = Conv2D(filters=filters, kernel_size=3, strides=1, padding='same', name=name + '_convB')(up_i) up_i = LeakyReLU(alpha=0.2)(up_i) return up_i # Decoder Layers decoder = Conv2D(filters=decode_filters, kernel_size=1, padding='same', input_shape=base_model_output_shape, name='conv2')(base_model.output) decoder = upproject(decoder, int(decode_filters / 2), 'up1', concat_with='pool3_pool') decoder = upproject(decoder, int(decode_filters / 4), 'up2', concat_with='pool2_pool') decoder = upproject(decoder, int(decode_filters / 8), 'up3', concat_with='pool1') decoder = upproject(decoder, int(decode_filters / 16), 'up4', concat_with='conv1/relu') if False: decoder = upproject(decoder, int(decode_filters / 32), 'up5', concat_with='input_1') # Extract depths (final layer) conv3 = Conv2D(filters=1, kernel_size=3, strides=1, padding='same', name='conv3')(decoder) output = Lambda(concat_output)([conv3, aux_output, has_locations]) output._keras_shape = output.get_shape().as_list() # Create the model model = Model(inputs=input, outputs=output) else: # Load model from file if not existing.endswith('.h5'): sys.exit( 'Please provide a correct model file when using [existing] argument.' ) custom_objects = { 'BilinearUpSampling2D': BilinearUpSampling2D, 'depth_loss_function': depth_loss_function } model = load_model(existing, custom_objects=custom_objects) print('\nExisting model loaded.\n') print('Model created.') return model
def conductance_model(input_shape=(11, 9, 1), filter_shape=[21, 1], num_filter=2, sum_over_space=True, fit_reversal=False): reg_val = 1 v_leak = 0 v_exc = 60 v_inh = -30 g_leak = 1 filter_shape[1] = 1 pad_x = int((filter_shape[1] - 1))+2 pad_t = int((filter_shape[0] - 1)) # Define the input as a tensor with shape input_shape image_in = Input(input_shape) s1 = Lambda(lambda lam: lam[:, :, 0:-2, :])(image_in) s2 = Lambda(lambda lam: lam[:, :, 1:-1, :])(image_in) s3 = Lambda(lambda lam: lam[:, :, 2:, :])(image_in) g1 = Conv2D(num_filter, filter_shape, strides=(1, 1), name='g1', kernel_initializer=glorot_uniform(seed=None), activation='relu', kernel_regularizer=l1_reg_sqrt)(s1) g2 = Conv2D(num_filter, filter_shape, strides=(1, 1), name='g2', kernel_initializer=glorot_uniform(seed=None), activation='relu', kernel_regularizer=l1_reg_sqrt)(s2) g3 = Conv2D(num_filter, filter_shape, strides=(1, 1), name='g3', kernel_initializer=glorot_uniform(seed=None), activation='relu', kernel_regularizer=l1_reg_sqrt)(s3) if fit_reversal: expand_last = Lambda(lambda lam: K.expand_dims(lam, axis=-1)) squeeze_last = Lambda(lambda lam: K.squeeze(lam, axis=-1)) g1 = expand_last(g1) g2 = expand_last(g2) g3 = expand_last(g3) numerator_in = Lambda(lambda inputs: K.concatenate(inputs, axis=4))([g1, g2, g3]) numerator = Conv3D(1, (1, 1, 1), strides=(1, 1, 1), name='create_numerator', kernel_initializer=glorot_uniform(seed=None), kernel_regularizer=l1_reg_sqrt, use_bias=False)(numerator_in) denominator = Lambda(lambda inputs: g_leak + inputs[0] + inputs[1] + inputs[2])([g1, g2, g3]) vm = Lambda(lambda inputs: inputs[0] / inputs[1])([numerator, denominator]) vm = squeeze_last(vm) else: g1_v_inh = Lambda(lambda lam: lam * v_inh)(g1) g2_v_exc = Lambda(lambda lam: lam * v_exc)(g2) g3_v_inh = Lambda(lambda lam: lam * v_inh)(g3) numerator = Lambda(lambda inputs: inputs[0] + inputs[1] + inputs[2])([g1_v_inh, g2_v_exc, g3_v_inh]) denominator = Lambda(lambda inputs: g_leak + inputs[0] + inputs[1] + inputs[2])([g1, g2, g3]) vm = Lambda(lambda inputs: inputs[0] / inputs[1])([numerator, denominator]) vm_bias = BiasLayer()(vm) vm_rect = Lambda(lambda lam: K.relu(lam))(vm_bias) if sum_over_space: conv_x_size = vm.get_shape().as_list()[2] else: conv_x_size = 1 combine_filters = Conv2D(1, (1, conv_x_size), strides=(1, 1), name='conv2', kernel_initializer=glorot_uniform(seed=None), kernel_regularizer=l1_reg_sqrt, use_bias=False)(vm_rect) # Create model model = Model(inputs=image_in, outputs=combine_filters, name='conductance_model') return model, pad_x, pad_t
def build(self, x_shape, c_shape=None): input_shape = x_shape x_inputs = Input(input_shape, name="%s_x_input" % self.name) y_inputs = Input(input_shape, name="%s_y_input" % self.name) concat_inputs = Concatenate(name="%s_concat_input" % self.name, axis=-1)([x_inputs, y_inputs]) inputs = [x_inputs, y_inputs] if c_shape is not None: c_inputs = Input(c_shape, name="%s_c_input" % self.name) inputs = [x_inputs, c_inputs, y_inputs] wdout = concat_inputs wodout = concat_inputs nfilters = self.nfilters * 2 for il, stride in enumerate(self.strides): conv = Convolution2D(nfilters, kernel_size=self.kernel_size, use_bias=self.use_bias, name="%s_conv%d" % (self.name, il), strides=(stride, stride), padding='valid') if il > 0: bn = BatchNormalization(axis=1, name="%s_bn%d" % (self.name, il)) wdout = bn(wdout) wodout = bn(wodout) do = Dropout(self.dropout_rate, name="%s_do%d" % (self.name, il)) act = LeakyReLU(0.2, name="%s_act%d" % (self.name, il)) wdout = act(do(conv(wdout))) wodout = act(conv(wodout)) nfilters *= 2 if c_shape is not None: nembed = nfilters // 2 c_bn0 = BatchNormalization(axis=-1, name="%s_c_bn0" % self.name)(c_inputs) c_dense = Dense(nembed, use_bias=self.use_bias, name="%s_c_dense" % self.name)(c_bn0) c_bn = BatchNormalization(axis=-1, name="%s_c_bn" % self.name)(c_dense) c_rshp = Reshape((1, 1, nembed), name="%s_c_rshp" % self.name)(c_bn) ups = wdout.get_shape() ups = (int(ups[1]), int(ups[2])) c_up = UpSampling2D(size=ups, name="%s_c_up" % self.name)(c_rshp) concat = Concatenate(name="%s_concat" % self.name, axis=-1) wdout = concat([wdout, c_up]) wodout = concat([wodout, c_up]) if self.obo_layers is not None: nch = self.obo_nch if nch == 0: nch = nfilters // 2 for il in range(self.obo_layers): fc = Convolution2D(nch, kernel_size=(1, 1), use_bias=self.use_bias, strides=(1, 1), name="%s_fc%d" % (self.name, il)) wdout = fc(wdout) wodout = fc(wodout) bn = BatchNormalization(axis=1, name="%s_obo_bn%d" % (self.name, il)) wdout = bn(wdout) wodout = bn(wodout) act = LeakyReLU(0.2, name="%s_obo_act%d" % (self.name, il)) wdout = act(wdout) wodout = act(wodout) if self.obo_nch_red != 0: nch = max(self.obo_nch_min, nch // self.obo_nch_red) flat = Flatten(name="%s_flat" % self.name) wdout = flat(wdout) wodout = flat(wodout) dense = Dense(1, name="%s_dense" % self.name) wdout = dense(wdout) wodout = dense(wodout) out = Activation("sigmoid", name="%s_out" % self.name) wdout = out(wdout) wodout = out(wodout) wdooutputs = [wdout] wodooutputs = [wodout] if self.do_total: sum_x = Lambda( lambda x: K.sum(x, axis=(1, 2)), #output_shape=(1,), name="%s_sum_x" % self.name)(x_inputs) sum_y = Lambda( lambda x: K.sum(x, axis=(1, 2)), #output_shape=(1,), name="%s_sum_y" % self.name)(y_inputs) concat_sum = [sum_x, sum_y] print(sum_x.get_shape(), sum_y.get_shape()) if c_shape is not None: c_flat = Flatten(name="%s_c_flt" % self.name)(c_bn) concat_sum.append(c_flat) concat_sum = Concatenate(name="%s_concat_sum" % self.name, axis=-1)(concat_sum) wdout = concat_sum wodout = concat_sum for il, nc in enumerate(self.total_layers): t_dense = Dense(nembed, use_bias=self.use_bias, name="%s_t_dense%d" % (self.name, il)) t_bn = BatchNormalization(axis=-1, name="%s_t_bn%d" % (self.name, il)) t_do = Dropout(self.dropout_rate, name="%s_t_do%d" % (self.name, il)) t_act = LeakyReLU(0.2, name="%s_t_act%d" % (self.name, il)) wdout = t_act(t_do(t_bn(t_dense(wdout)))) wodout = t_act(t_bn(t_dense(wodout))) dense = Dense(1, name="%s_t_dense" % self.name) wdout = dense(wdout) wodout = dense(wodout) out = Activation("sigmoid", name="%s_t_out" % self.name) wdout = out(wdout) wodout = out(wodout) wdooutputs.append(wdout) wodooutputs.append(wodout) model = (Model(inputs=inputs, outputs=wdooutputs), Model(inputs=inputs, outputs=wodooutputs)) return model
def set_model(self): ''' Initialisers ''' weight_seed = None kernel_initializer = initializers.glorot_uniform(seed = weight_seed) bias_initializer = initializers.glorot_uniform(seed = weight_seed) ''' Encoder ''' # define input with 'channels_first' input_encoder = Input(shape=self.input_shape, name='encoder_input') x = Conv2D(self.filters, self.kernel_size, padding='same', activation=None, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_conv2D_1')(input_encoder) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(2*self.filters, self.kernel_size, padding='same', activation=None, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_conv2D_2')(x) x = BatchNormalization()(x) x = Activation('relu')(x) # take z_mean / z_log_var input shape latent_input_shape = tuple(x.get_shape().as_list()) latent_width, latent_height = latent_input_shape[2:] # separate dense layers for mu and log(sigma), both of size latent_dim z_mean = Conv2D(self.latent_channels, kernel_size=(latent_width, latent_height), activation=None, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_z_mean')(x) z_log_var = Conv2D(self.latent_channels, kernel_size=(latent_width, latent_height), activation=None, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_z_log_var')(x) # sample from normal with z_mean and z_log_var z = Lambda(self.sampling, name='encoder_z')([z_mean, z_log_var]) ''' Decoder ''' # take encoder output shape encoder_out_shape = tuple(z.get_shape().as_list()) # define rest of model input_decoder = Input(shape=encoder_out_shape[1:], name='decoder_input') x = Conv2DTranspose(2*self.filters, kernel_size=(latent_width, latent_height), padding='same', activation=None, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='decoder_conv2DT_1')(input_decoder) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2DTranspose(self.filters, self.kernel_size, activation=None, padding='same', kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='decoder_conv2DT_2')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2DTranspose(self.filters, self.kernel_size, activation=None, padding='same', kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='decoder_conv2DT_3')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2DTranspose(1, self.kernel_size, activation=None, padding='same', kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='decoder_conv2DT_4')(x) x = BatchNormalization()(x) decoded_img = Activation('sigmoid')(x) ''' Necessary definitions ''' # For parent fitting function self.encoder = Model(input_encoder, z) self.decoder = Model(input_decoder, decoded_img) self.model = Model(input_encoder, self.decoder(self.encoder(input_encoder))) # For parent loss function self.z_mean = z_mean self.z_log_var = z_log_var self.z = z