def generator_nw_unet(**kwargs): n_x = kwargs['input_dim'] #first encoder #generator params fe1 = 64 fe2 = 64 fe3 = 32 BottleneckDim = 16 fe5 = 32 fe6 = 64 a = 0.2 #alpha dout = 0.2 #dropout X_in = Input(shape=(n_x, ), name='financial_cond_input') Y_in = Input( shape=(1, ), name='financial_manip') #this is the dimension we are manipulating concat_en = concat([X_in, Y_in]) #image encoder layers h1_en = Dense(fe1)(concat_en) h1_en = Dropout(dout)(h1_en) h2_en = Dense(fe2)(h1_en) h2_en = LeakyReLU(alpha=a)(h2_en) h2_en = Dropout(dout)(h2_en) h3_en = Dense(fe3)(h2_en) h3_en = LeakyReLU(alpha=a)(h3_en) h3_en = Dropout(dout)(h3_en) h4_bot = Dense(BottleneckDim)(h3_en) h4_bot = LeakyReLU(alpha=a)(h4_bot) h4_bot = Dropout(dout)(h4_bot) h5_dec = Dense(fe5)(h4_bot) h5_dec = LeakyReLU(alpha=a)(h5_dec) h5_dec = Dropout(dout)(h5_dec) h5_dec = concat([h5_dec, h2_en]) h6_dec = Dense(fe6)(h5_dec) h6_dec = LeakyReLU(alpha=a)(h6_dec) h6_dec = Dropout(dout)(h6_dec) h6_dec = concat([h6_dec, h1_en]) out_dec = Dense(n_x)(h6_dec) out_dec = concat([out_dec, Y_in]) Generator = Model([X_in, Y_in], out_dec) Generator.summary() Generator.compile(optimizer='adam', loss=recon_loss) return Generator
def CVAE_Model(Optimizer, batch_size=16, lr=0.01, dim_size=32): input3D_shape = (dim_size, dim_size, dim_size, 1) input2D_shape = (dim_size, dim_size, 1) output2D_dim = 32 output3D_dim = 32 latent_dim = 10 decoder3D_dense_shape = (4 * 4 * 4 * 64) decoder3D_reshape_shape = (4, 4, 4, 64) input_3d = Input(input3D_shape, name='3D_Encoder_input') input_2d = Input(input2D_shape, name='2D_Encoder_input') Code_3d = Encoder3D(input_3d, output3D_dim) Code_2d = Encoder2D(input_2d, output2D_dim) inputs = concat([Code_3d, Code_2d], name='input_concat') encoder = Dense(512, activation='relu', name='encoder_dense')(inputs) mu = Dense(latent_dim, activation='linear', name='mu')(encoder) sigma = Dense(latent_dim, activation='linear', name='sigma')(encoder) latent = Lambda(sampling, output_shape=(latent_dim, ), name='latent')([mu, sigma]) latent_concat = concat([latent, Code_2d], name='latent_concat') output = Decoder3D(latent_concat, decoder3D_dense_shape, decoder3D_reshape_shape) cvae = Model([input_3d, input_2d], output) encoder = Model([input_3d, input_2d], mu) d_in = Input(shape=(latent_dim + output2D_dim, )) d_out = Decoder3D(d_in, decoder3D_dense_shape, decoder3D_reshape_shape) decoder = Model(d_in, d_out) Optimizer.lr = lr cvae.compile(optimizer=Optimizer, loss=vae_loss, metrics=[KL_loss, recon_loss]) cvae.summary() #cvae.save('cvae.h5') return cvae
def cond_gan_nw(Generator, Discriminator, x_train): input_dim = x_train.shape[1] #build gan model make_trainable(Discriminator, False) gan_input = Input(shape=(input_dim, )) gan_inputy = Input(shape=(1, )) gan_gen_out = Generator([gan_input, gan_inputy]) gan_discrim_inp = concat([gan_gen_out, gan_input]) gan_output = Discriminator(gan_discrim_inp) GAN = Model([gan_input, gan_inputy], [gan_gen_out, gan_output]) GAN.summary() #gan_loss_weights=[1E2,1] return GAN
encoder_dim1 = 512 # dim of encoder hidden layer #encoder_dim2 = 128 # dim of encoder hidden layer decoder_dim = 512 # dim of decoder hidden layer decoder_out_dim = 784 # dim of decoder output layer activ = 'relu' optim = Adam(lr=0.0005) n_x = X_train.shape[1] n_y = y_train.shape[1] n_epoch = 100 X = Input(shape=(n_x, )) label = Input(shape=(n_y, )) inputs = concat([X, label]) encoder_h = Dense(encoder_dim1, activation=activ, activity_regularizer='l2')(inputs) #encoder_h = Dense(encoder_dim2, activation=activ)(encoder_h) mu = Dense(n_z, activation='linear')(encoder_h) l_sigma = Dense(n_z, activation='linear')(encoder_h) def sample_z(args): mu, l_sigma = args eps = K.random_normal(shape=(m, n_z), mean=0., stddev=1.) return mu + K.exp(l_sigma / 2) * eps # Sampling latent space
input3D_shape = (32, 32, 32, 1) input2D_shape = (32, 32, 1) output2D_dim = 32 output3D_dim = 32 latent_dim = 10 decoder3D_dense_shape = (4 * 4 * 4 * 64) decoder3D_reshape_shape = (4, 4, 4, 64) batch_size = 16 input_3d = Input(input3D_shape, name='3D_Encoder_input') input_2d = Input(input2D_shape, name='2D_Encoder_input') Code_3d = Encoder3D(input_3d, output3D_dim) Code_2d = Encoder2D(input_2d, output2D_dim) inputs = concat([Code_3d, Code_2d], name='input_concat') #encoder = Dense(512, activation='relu',name='encoder_dense')(inputs) mu = Dense(latent_dim, activation='linear', name='mu')(inputs) sigma = Dense(latent_dim, activation='linear', name='sigma')(inputs) latent = Lambda(sampling, output_shape=(latent_dim, ), name='latent')([mu, sigma]) latent_concat = concat([latent, Code_2d], name='latent_concat') output = Decoder3D(latent_concat, decoder3D_dense_shape, decoder3D_reshape_shape) cvae = Model(inputs=[input_3d, input_2d], outputs=output) encoder = Model([input_3d, input_2d], mu) d_in = Input(shape=(latent_dim + output2D_dim, ))
def generator_nw_5_u(x_train, g_noise=0.00, nodes=[64, 32, 16, 32, 64], y=False, prelu_bias=0.1, drop_ra=0.0, ker_init=None, compile=True, output_dim=None): input_dim = x_train.shape[1] if output_dim is None: output_dim = input_dim fe1 = nodes[0] fe2 = nodes[1] Bottleneck_Dim = nodes[2] fe3 = nodes[3] fe4 = nodes[4] #Maybe 1 or 2 inputs: if y: X_in = Input(shape=(input_dim, ), name='financial_cond_input') Y_in = Input( shape=(1, ), name='financial_manip') #this is the dimension we are manipulating #concatenate the two inputs concat_en = concat([X_in, Y_in]) inpu = [X_in, Y_in] else: X_in = Input(shape=(input_dim, ), name='financial_cond_input') concat_en = X_in inpu = X_in #image encoder layers h1_en = Dropout(drop_ra, name='H1_dropout')(concat_en) h1_en = GaussianNoise(g_noise, name='H1_noise')(h1_en) h1_en = Dense(fe1, kernel_initializer=ker_init, name='H1_layer')(h1_en) h1_en = PReLU(name='H1_activation', alpha_initializer=Constant(value=prelu_bias))(h1_en) h1_en = BatchNormalization(name='H1_batch_norm')(h1_en) h2_en = Dropout(drop_ra, name='H2_dropout')(h1_en) h2_en = GaussianNoise(g_noise, name='H12_noise')(h2_en) h2_en = Dense(fe2, kernel_initializer=ker_init, name='H2_layer')(h2_en) h2_en = PReLU(alpha_initializer=Constant(value=prelu_bias), name='H2_activation')(h2_en) h2_en = BatchNormalization(name='H2_batch_norm')(h2_en) h3_en = Dropout(drop_ra, name='H3_dropout')(h2_en) h3_en = GaussianNoise(g_noise, name='H3_noise')(h3_en) h3_en = Dense(Bottleneck_Dim, kernel_initializer=ker_init, name='H3_layer')(h3_en) h3_en = PReLU(alpha_initializer=Constant(value=prelu_bias), name='H3_activation')(h3_en) Latent_space = BatchNormalization(name='H3_batch_norm')(h3_en) h4_dec = concat([Latent_space, h2_en]) h4_dec = Dense(fe3, kernel_initializer=ker_init, name='H4_layer')(Latent_space) h4_dec = PReLU(alpha_initializer=Constant(value=prelu_bias), name='H4_activation')(h4_dec) h5_dec = concat([Latent_space, h1_en]) h5_dec = Dense(fe4, kernel_initializer=ker_init, name='H5_layer')(h4_dec) h5_dec = PReLU(alpha_initializer=Constant(value=prelu_bias), name='H5_activation')(h5_dec) out_dec = Dense(output_dim, name='Output_layer')(h4_dec) if y: out_dec = concat([out_dec, Y_in]) Generator = Model(inpu, out_dec) Generator.summary() #if compile: # gen_compile_dic={'loss':sparse_recon_loss_mse,'metrics':metrics,'optimizer':'adam','early_stop':ES} # Generator.compile(**gen_compile_dic) return Generator
def _configure(self): # build the encoder outputs for the whole powerset encoder_outputs_powerset = [ ] # Holds M=|encoder_powerset|-1 encoder networks for encoder_set in self.encoder_powerset: element_output = [] for encoder_element in encoder_set: element_output.append( encoder_element[0] ) # The first element is always an input layer # Append the next layer to the currently appended element in the set for encoder_layer in encoder_element[1:]: element_output[-1] = encoder_layer(element_output[-1]) # Concat all elements of the current set # Note: It is "just" concat to enforce the parameter layers (z_mean & z_logvar) to learn linear # combinations of the uni-modal cases # TODO: Extension to add a non-linear layer after the concatination if len(element_output) > 1: encoder_outputs_powerset.append(concat(element_output)) ################################################################################################# #encoder_outputs_powerset.append(Dense(int(intermediate_dim/2), activation='relu')(concat(element_output))) ################################################################################################# else: encoder_outputs_powerset.append(element_output[0]) # Add sampling for every permutation self.Z = [] self.Z_mean = [] self.Z_logvar = [] for encoder_output in encoder_outputs_powerset: self.Z_mean.append(Dense(self.z_dim)(encoder_output)) self.Z_logvar.append(Dense(self.z_dim)(encoder_output)) self.Z.append( self.sampling_layer([self.Z_mean[-1], self.Z_logvar[-1]])) # Add a decoder output for every permutation self.decoder_outputs_powerset = [] for z, decoder_set in zip(self.Z, self.decoder_powerset): element_output = [] for decoder_element in decoder_set: element_output.append(decoder_element[0]( z)) # Input the sample z into the first decoder layer # Append the next layer to the currently appended element in the set for decoder_layer in decoder_element[1:]: #print(element_output[-1]) element_output[-1] = decoder_layer(element_output[-1]) self.decoder_outputs_powerset.append(element_output) # collection of loss layers self.loss_layers = [] # Calculate entropy losses for all sets in the powerset reconstruction_loss = [] # Traverse the sets of the powerset for x_set, x_decoded_mean_set, encoder_inputs_dim, reconstruction_loss_metrics in \ zip(self.encoder_inputs_powerset, \ self.decoder_outputs_powerset, \ self.encoder_inputs_dim_powerset, \ self.reconstruction_loss_metrics_powerset): reconstruction_loss_set = [ ] # Holds the loss for the whole powerset # Traverse the elements per set for x, x_decoded_mean, encoder_input_dim, reconstruction_loss_metric in zip(x_set, \ x_decoded_mean_set, \ encoder_inputs_dim, \ reconstruction_loss_metrics): # Choose the proper reconstruction loss metric #print("encoder_input_dim: ", encoder_input_dim) rl = { ReconstructionLoss.MSE: LosslayerReconstructionMSE(weight=encoder_input_dim), ReconstructionLoss.BCE: LosslayerReconstructionBCE(weight=encoder_input_dim), } #print("reconstruction_loss_metric: ", reconstruction_loss_metric) loss_layer = rl.get(reconstruction_loss_metric) self.loss_layers.append( loss_layer) # Backup the layer for callbacks, etc. loss = loss_layer([x, x_decoded_mean]) reconstruction_loss_set.append(loss) reconstruction_loss.extend(reconstruction_loss_set) # Calculate the prior losses for all sets in the powerset kl_prior_loss = [] for z_mean, z_logvar, inputs, encoder_inputs_dim in zip( self.Z_mean, self.Z_logvar, self.encoder_powerset, self.encoder_inputs_dim_powerset): loss_layer = LosslayerDistributionGaussianPrior( weight=self.get_beta(x_dim=sum(encoder_inputs_dim))) self.loss_layers.append( loss_layer) # Backup the layer for callbacks, etc. loss = loss_layer([z_mean, z_logvar]) kl_prior_loss.append(loss) # Calculate the mutual KL divergences for the to sets A and B of the powerset, # where |A|=|B|-1 and A is a proper subset of B (which is always valid for only one pair of sets) kl_mutual_loss = [] subset_idx, superset_idx = setfun.find_proper_subsets( self.encoder_inputs_powerset, cardinality_difference=1, debug=True) for A_idx, B_idx in zip(subset_idx, superset_idx): loss_layer = LosslayerDistributionGaussianMutual( weight=self.beta_mutual) self.loss_layers.append( loss_layer) # Backup the layer for callbacks, etc. loss = loss_layer([ self.Z_mean[B_idx], self.Z_mean[A_idx], self.Z_logvar[B_idx], self.Z_logvar[A_idx] ]) kl_mutual_loss.append(loss) loss_list = reconstruction_loss + kl_prior_loss + kl_mutual_loss #print("\n +++++++++++++++++++++++ \n") #print("loss_list: ", loss_list) #print("reconstruction_loss: ", reconstruction_loss) #print("kl_prior_loss: ", kl_prior_loss) #print("kl_mutual_loss: ", kl_mutual_loss) #print("\n +++++++++++++++++++++++ \n") return loss_list
n_y = ytrain.shape[1] epochs = 100 # okay, let's define our model # okay, so the variational autoeencoder consists of two parts - firstly the encoder which is a mapping/NN representation of the probability distribution p(z|X) i.e. it maps from the input space X to the latent variable space Z. we want to maximise the usefulness ot this thing and also how closely it approximates the standard normal N(0,1) as regularisation # secondly we want to map in the decoder from the latent variable to another X X^ and therefore maximise p(X|z) to match the data which is a stanard log thing, so I don't know # we'll generally want a much more dimensional latent variable, and I'm not sure how I shuold get it tbh # like the mu and sigma are just meant to be a standard normal vector, right? in multiple dimensions # so we should be able to deal with that, but we can't, so what do we do there? X = Input(shape=(n_x,)) label = Input(shape=(n_y,)) #now we concatenate x and y so we can igure this out inputs = concat([X, label]) # not sure why we need this # we need to merge them within the context of the graph encoder_h = Dense(encoder_dim1, activation=activation, activity_regularizer='l2')(inputs) mu = Dense(n_z, activation='linear')(encoder_h) l_sigma = Dense(n_z, activation='linear')(encoder_h) # this is our mapping t omu and sigma via ANNs #now we have a function which add random noise to our sampling process, and use a lambda layer in keras for this - cool! # this is the reparametrization trick. basicaly, we're meant to be sampling from our calculated z normal N(mu, sigma) which are the outputs of the neural network, but this isn't differentiable, so what we do is instead sample from the standard normal, which isn't differentiable, but this doesn't matter as it's not actually dependent on anything in the network or trainable, so it doesn't need to ahve gradients, and then we multiply and add the other bits, which are differentiable, so the reparametrisation works, and the network is differentiable through these things def sample_z(args): mu, l_sigma = args eps = K.random_normal return mu + K.exp(log_sigma/2) * eps # we use a lambda layer to sample our standard thing
def get_mrSabzi_net(): # ------------------------------------------------------------------------------ def vae_loss(y_true, y_pred): recon = K.sum(K.binary_crossentropy(y_true, y_pred), axis=-1) kl = 0.5 * K.sum(K.exp(sigma) + K.square(mu) - 1. - sigma, axis=-1) return recon + kl def KL_loss(y_true, y_pred): return (0.5 * K.sum(K.exp(sigma) + K.square(mu) - 1. - sigma, axis=1)) def recon_loss(y_true, y_pred): return K.sum(K.binary_crossentropy(y_true, y_pred), axis=-1) def sampling(args): mu, l_sigma = args eps = K.random_normal(shape=(batch_size, latent_dim), mean=0., stddev=1.) return mu + K.exp(l_sigma / 2) * eps def Encoder3D(input_3d, ouput_size): conv = Conv3D(8, (3, 3, 3), padding='same', kernel_initializer='he_normal', activation='relu', name='3D_conv1_1')(input_3d) conv = Conv3D(8, (3, 3, 3), padding='same', kernel_initializer='he_normal', activation='relu', name='3D_conv1_2')(conv) pool = MaxPooling3D(name='3d_pool_1')(conv) conv = Conv3D(16, (3, 3, 3), padding='same', kernel_initializer='he_normal', activation='relu', name='3D_conv2_1')(pool) conv = Conv3D(16, (3, 3, 3), padding='same', kernel_initializer='he_normal', activation='relu', name='3D_conv2_2')(conv) pool = MaxPooling3D(name='3d_pool_2')(conv) conv = Conv3D(32, (3, 3, 3), padding='same', kernel_initializer='he_normal', activation='relu', name='3D_conv3_1')(pool) conv = Conv3D(32, (3, 3, 3), padding='same', kernel_initializer='he_normal', activation='relu', name='3D_conv3_2')(conv) pool = MaxPooling3D(name='3d_pool_3')(conv) conv = Conv3D(64, (3, 3, 3), padding='same', kernel_initializer='he_normal', activation='relu', name='3D_conv4_1')(pool) conv = Conv3D(64, (3, 3, 3), padding='same', kernel_initializer='he_normal', activation='relu', name='3D_conv4_2')(conv) flat = Flatten(name='3d_flatten')(conv) dense = Dense(256, activation='relu', name='3d_dense_1')(flat) dense = Dense(ouput_size, activation='relu', name='3d_dense_2')(dense) return dense def Encoder2D(input_2d, ouput_size): conv = Conv2D(8, (3, 3), padding='same', kernel_initializer='he_normal', activation='relu', name='2D_conv1_1')(input_2d) conv = Conv2D(8, (3, 3), padding='same', kernel_initializer='he_normal', activation='relu', name='2D_conv1_2')(conv) pool = MaxPooling2D(name='2d_pool_1')(conv) conv = Conv2D(16, (3, 3), padding='same', kernel_initializer='he_normal', activation='relu', name='2D_conv2_1')(pool) conv = Conv2D(16, (3, 3), padding='same', kernel_initializer='he_normal', activation='relu', name='2D_conv2_2')(conv) pool = MaxPooling2D(name='2d_pool_2')(conv) conv = Conv2D(32, (3, 3), padding='same', kernel_initializer='he_normal', activation='relu', name='2D_conv3_1')(pool) conv = Conv2D(32, (3, 3), padding='same', kernel_initializer='he_normal', activation='relu', name='2D_conv3_2')(conv) pool = MaxPooling2D(name='2d_pool_3')(conv) conv = Conv2D(64, (3, 3), padding='same', kernel_initializer='he_normal', activation='relu', name='2D_conv4_1')(pool) conv = Conv2D(64, (3, 3), padding='same', kernel_initializer='he_normal', activation='relu', name='2D_conv4_2')(conv) flat = Flatten(name='2d_flatten')(conv) dense = Dense(256, activation='relu', name='2d_dense_1')(flat) dense = Dense(ouput_size, activation='relu', name='2d_dense_2')(dense) return dense def Decoder3D(latent_output, decoder3D_dense_shape, decoder3D_reshape_shape): dense = Dense(decoder3D_dense_shape, activation='relu')(latent_output) reshape = Reshape(decoder3D_reshape_shape)(dense) conv = Conv3D(64, (3, 3, 3), kernel_initializer='he_normal', padding='same', activation='relu', name='3D_conv5_1')(reshape) conv = Conv3D(64, (3, 3, 3), kernel_initializer='he_normal', padding='same', activation='relu', name='3D_conv5_2')(conv) up = UpSampling3D(size=(2, 2, 2), name='3D_up1')(conv) conv = Conv3D(32, (3, 3, 3), kernel_initializer='he_normal', padding='same', activation='relu', name='3D_conv6_1')(up) conv = Conv3D(32, (3, 3, 3), kernel_initializer='he_normal', padding='same', activation='relu', name='3D_conv6_2')(conv) up = UpSampling3D(size=(2, 2, 2), name='3D_up2')(conv) conv = Conv3D(16, (3, 3, 3), kernel_initializer='he_normal', padding='same', activation='relu', name='3D_conv7_1')(up) conv = Conv3D(16, (3, 3, 3), kernel_initializer='he_normal', padding='same', activation='relu', name='3D_conv7_2')(conv) up = UpSampling3D(size=(2, 2, 2), name='3D_up3')(conv) conv = Conv3D(8, (3, 3, 3), kernel_initializer='he_normal', padding='same', activation='relu', name='3D_conv8_1')(up) conv = Conv3D(8, (3, 3, 3), kernel_initializer='he_normal', padding='same', activation='relu', name='3D_conv8_2')(conv) conv = Conv3D(2, (3, 3, 3), kernel_initializer='he_normal', padding='same', activation='relu', name='3D_conv8_3')(conv) conv = Conv3D(1, (3, 3, 3), kernel_initializer='he_normal', padding='same', activation='sigmoid', name='3D_conv_output')(conv) return conv input3D_shape = (32, 32, 32, 1) input2D_shape = (32, 32, 1) output2D_dim = 32 output3D_dim = 32 latent_dim = 10 decoder3D_dense_shape = (4 * 4 * 4 * 64) decoder3D_reshape_shape = (4, 4, 4, 64) batch_size = 16 input_3d = Input(input3D_shape, name='3D_Encoder_input') input_2d = Input(input2D_shape, name='2D_Encoder_input') Code_3d = Encoder3D(input_3d, output3D_dim) Code_2d = Encoder2D(input_2d, output2D_dim) inputs = concat([Code_3d, Code_2d], name='input_concat') encoder = Dense(512, activation='relu', name='encoder_dense')(inputs) mu = Dense(latent_dim, activation='linear', name='mu')(encoder) sigma = Dense(latent_dim, activation='linear', name='sigma')(encoder) latent = Lambda(sampling, output_shape=(latent_dim, ), name='latent')([mu, sigma]) latent_concat = concat([latent, Code_2d], name='latent_concat') output = Decoder3D(latent_concat, decoder3D_dense_shape, decoder3D_reshape_shape) cvae = Model([input_3d, input_2d], output) encoder = Model([input_3d, input_2d], mu) d_in = Input(shape=(latent_dim + output2D_dim, )) d_out = Decoder3D(d_in, decoder3D_dense_shape, decoder3D_reshape_shape) decoder = Model(d_in, d_out) cvae.compile(optimizer='Adam', loss=vae_loss, metrics=[KL_loss, recon_loss]) cvae.summary() cvae.save('cvae.h5') return cvae # ------------------------------------------------------------------------------