def StackedAutoencoder(gene_exp_dim, layer_size, dropout_size, encoding_layer): input_exp = Input(shape=(gene_exp_dim, )) # making the encoding Layers encoded = Dense(layer_size, activation='relu')(input_exp) encoded.add(Dropout(dropout_size))(encoded) last_layer = 0 for enc_layer in range(1, encoding_layer): encoded = Dense((layer_size - enc_layer * 2), activation='sigmoid')(encoded) encoded = Dropout(dropout_size)(encoded) last_layer = layer_size - enc_layer * 2 ## making the decoding Layers decoded = Dense(last_layer, activation='sigmoid')(encoded) for dec_layer in range(1, (encoding_layer - 1)): decoded = Dense((dec_layer + dec_layer * 2), activation='sigmoid')(decoded) decoded = Dropout(dropout_size)(decoded) decoded = Dense(shape=(gene_exp_dim, ), activation='sigmoid')(encoded) return decoded
def build(self): input_img = Input(shape=(self.input_size, )) # Encoder encoder = Dense(1000, activation='relu')(input_img) encoder.add(Dense(500, activation='relu')) encoder.add(Dense(250, activation='relu')) encoder.add(Dense(30, activation='relu')) # Decoder decoder = Dense(250, activation='relu')(encoder) decoder = Dense(500, activation='relu')(decoder) decoder = Dense(1000, activation='relu')(decoder) decoder = Dense(784, activation='sigmoid')(decoder) self.autoencoder = Model(input=input_img, output=decoder) self.autoencoder.compile(optimizer='adam', loss='binary_crossentropy') self.encoder = Model(input=input_img, output=encoder) self.encoder.compile(optimizer='adam', loss='binary_crossentropy')
z_log_var_encoded = Activation('relu')(z_log_var_dense_batchnorm) z = Lambda(sampling, output_shape=(z_shape, ))([z_mean_encoded, z_log_var_encoded]) # ~~~~~~~~~~~~~~~~~~~~~~ # DECODER # ~~~~~~~~~~~~~~~~~~~~~~ if depth == 1: decoder_to_reconstruct = Dense(original_dim, kernel_initializer='glorot_uniform', activation='sigmoid') elif depth == 2: decoder_to_reconstruct = Sequential() decoder_to_reconstruct.add(Dense(latent_dim, kernel_initializer='glorot_uniform', activation='relu', input_dim=latent_dim2)) decoder_to_reconstruct.add(Dense(original_dim, kernel_initializer='glorot_uniform', activation='sigmoid')) rnaseq_reconstruct = decoder_to_reconstruct(z) # ~~~~~~~~~~~~~~~~~~~~~~ # CONNECTIONS # ~~~~~~~~~~~~~~~~~~~~~~ adam = optimizers.Adam(lr=learning_rate) vae_layer = CustomVariationalLayer()([rnaseq_input, rnaseq_reconstruct]) vae = Model(rnaseq_input, vae_layer) vae.compile(optimizer=adam, loss=None, loss_weights=[beta]) vae.summary()
def run_vae(rnaseq_file, learning_rate, batch_size, epochs, kappa, depth, first_layer, output_filename, latent_dim, scale, subset_mad_genes, data_basename): # Random seed seed = int(np.random.randint(low=0, high=10000, size=1)) np.random.seed(seed) # Load Data #file = 'train_{}_expression_matrix_processed.tsv.gz'.format(dataset.lower()) #rnaseq_file = os.path.join('..', '0.expression-download', 'data', file) rnaseq_df = pd.read_table(rnaseq_file, index_col=0) # Determine most variably expressed genes and subset if subset_mad_genes is not None: mad_genes = rnaseq_df.mad(axis=0).sort_values(ascending=False) top_mad_genes = mad_genes.iloc[0:int(subset_mad_genes), ].index rnaseq_df = rnaseq_df.loc[:, top_mad_genes] #rnaseq_df.loc[:, top_mad_genes] # Zero One normalize input data if scale: scaler = MinMaxScaler() x = scaler.fit_transform(rnaseq_df) rnaseq_df = pd.DataFrame(x, index=rnaseq_df.index, columns=rnaseq_df.columns) # Set architecture dimensions original_dim = rnaseq_df.shape[1] epsilon_std = 1.0 beta = K.variable(0) if depth == 2: latent_dim2 = int(first_layer) # Random seed seed = int(np.random.randint(low=0, high=10000, size=1)) np.random.seed(seed) # Function for reparameterization trick to make model differentiable def sampling(args): # Function with args required for Keras Lambda function z_mean, z_log_var = args # Draw epsilon of the same shape from a standard normal distribution epsilon = K.random_normal(shape=tf.shape(z_mean), mean=0., stddev=epsilon_std) # The latent vector is non-deterministic and differentiable # in respect to z_mean and z_log_var z = z_mean + K.exp(z_log_var / 2) * epsilon return z class CustomVariationalLayer(Layer): """ Define a custom layer that learns and performs the training """ def __init__(self, **kwargs): # https://keras.io/layers/writing-your-own-keras-layers/ self.is_placeholder = True super(CustomVariationalLayer, self).__init__(**kwargs) def vae_loss(self, x_input, x_decoded): reconstruction_loss = original_dim * \ metrics.binary_crossentropy(x_input, x_decoded) kl_loss = -0.5 * K.sum( 1 + z_log_var_encoded - K.square(z_mean_encoded) - K.exp(z_log_var_encoded), axis=-1) return K.mean(reconstruction_loss + (K.get_value(beta) * kl_loss)) def call(self, inputs): x = inputs[0] x_decoded = inputs[1] loss = self.vae_loss(x, x_decoded) self.add_loss(loss, inputs=inputs) # We won't actually use the output. return x class WarmUpCallback(Callback): def __init__(self, beta, kappa): self.beta = beta self.kappa = kappa # Behavior on each epoch def on_epoch_end(self, epoch, logs={}): if K.get_value(self.beta) <= 1: K.set_value(self.beta, K.get_value(self.beta) + self.kappa) # Process data # Split 10% test set randomly test_set_percent = 0.1 rnaseq_train_df, rnaseq_test_df = train_test_split( rnaseq_df, test_size=test_set_percent, random_state=123) #, shuffle=False # Input place holder for RNAseq data with specific input size rnaseq_input = Input(shape=(original_dim, )) # ~~~~~~~~~~~~~~~~~~~~~~ # ENCODER # ~~~~~~~~~~~~~~~~~~~~~~ # Depending on the depth of the model, the input is eventually compressed into # a mean and log variance vector of prespecified size. Each layer is # initialized with glorot uniform weights and each step (dense connections, # batch norm,and relu activation) are funneled separately # # Each vector of length `latent_dim` are connected to the rnaseq input tensor # In the case of a depth 2 architecture, input_dim -> latent_dim -> latent_dim2 if depth == 1: z_shape = latent_dim z_mean_dense = Dense(latent_dim, kernel_initializer='glorot_uniform')(rnaseq_input) z_log_var_dense = Dense( latent_dim, kernel_initializer='glorot_uniform')(rnaseq_input) elif depth == 2: z_shape = latent_dim2 hidden_dense = Dense(latent_dim, kernel_initializer='glorot_uniform')(rnaseq_input) hidden_dense_batchnorm = BatchNormalization()(hidden_dense) hidden_enc = Activation('relu')(hidden_dense_batchnorm) z_mean_dense = Dense(latent_dim2, kernel_initializer='glorot_uniform')(hidden_enc) z_log_var_dense = Dense( latent_dim2, kernel_initializer='glorot_uniform')(hidden_enc) z_mean_dense_batchnorm = BatchNormalization()(z_mean_dense) z_mean_encoded = Activation('relu')(z_mean_dense_batchnorm) z_log_var_dense_batchnorm = BatchNormalization()(z_log_var_dense) z_log_var_encoded = Activation('relu')(z_log_var_dense_batchnorm) # return the encoded and randomly sampled z vector # Takes two keras layers as input to the custom sampling function layer with a # latent_dim` output z = Lambda(sampling, output_shape=(z_shape, ))([z_mean_encoded, z_log_var_encoded]) # ~~~~~~~~~~~~~~~~~~~~~~ # DECODER # ~~~~~~~~~~~~~~~~~~~~~~ # The layers are different depending on the prespecified depth. # # Single layer: glorot uniform initialized and sigmoid activation. # Double layer: relu activated hidden layer followed by sigmoid reconstruction if depth == 1: decoder_to_reconstruct = Dense(original_dim, kernel_initializer='glorot_uniform', activation='sigmoid') elif depth == 2: decoder_to_reconstruct = Sequential() decoder_to_reconstruct.add( Dense(latent_dim, kernel_initializer='glorot_uniform', activation='relu', input_dim=latent_dim2)) decoder_to_reconstruct.add( Dense(original_dim, kernel_initializer='glorot_uniform', activation='sigmoid')) rnaseq_reconstruct = decoder_to_reconstruct(z) # ~~~~~~~~~~~~~~~~~~~~~~ # CONNECTIONS # ~~~~~~~~~~~~~~~~~~~~~~ adam = optimizers.Adam(lr=learning_rate) vae_layer = CustomVariationalLayer()([rnaseq_input, rnaseq_reconstruct]) vae = Model(rnaseq_input, vae_layer) vae.compile(optimizer=adam, loss=None, loss_weights=[beta]) # fit Model hist = vae.fit(np.array(rnaseq_train_df), shuffle=True, epochs=epochs, batch_size=batch_size, validation_data=(np.array(rnaseq_test_df), None), callbacks=[WarmUpCallback(beta, kappa)]) # Save training performance history_df = pd.DataFrame(hist.history) history_df = history_df.assign(num_components=latent_dim, learning_rate=learning_rate, batch_size=batch_size, epochs=epochs, kappa=kappa, seed=seed, depth=depth, first_layer=first_layer, dataset=data_basename) history_df.to_csv(output_filename, sep='\t')
# 1. DNN # if model_num == 1: input = Input(shape=(28, 5)) model = Dense(64)(input) model = Dense(128)(model) # model = Dense(32)(model) drop_out = Dropout(0.3)(model) flatten = Flatten(name='flatten')(drop_out) output = Dense(1)(flatten) model = Model(inputs=input, outputs=output) # 2. LSTM # elif model_num == 2: model = Sequential() model.add(LSTM(32, activation='relu', input_shape=(input_data_length, 5))) model.add(Dense(64)) model.add(Dense(1)) # 3. DNN ENSEMBLE # elif model_num == 3: input = Input(shape=(28, 5)) model = Dense(64)(input) model = Dense(128)(model) # model = Dense(32)(model) drop_out = Dropout(0.3)(model) flatten = Flatten(name='flatten')(drop_out) output = Dense(1)(flatten) input2 = Input(shape=(28, 5)) model2 = Dense(32)(input2)
def autoencoder(X, layers, mode, act=VAR.ae_act, opt=VAR.ae_opt, \ loss=VAR.ae_loss, dropout=VAR.ae_dropout, \ epochs=VAR.ae_epoch, verbose=VAR.ae_verbose, \ summary_display=VAR.ae_summary_display): from keras.models import Sequential, Model from keras.layers import Dense, Input, Dropout import warnings #ignore warnings warnings.filterwarnings("ignore") #create an input holder num_features = len(X[0]) input_holder = Input(shape=(num_features, )) #input_holder=Input(shape=(num_features,)) #initialise the encoder variables layers_copy = [num_features] for l in layers: layers_copy.append(l) # -------------------------------------------------------------------- # basic autoencoder # -------------------------------------------------------------------- if mode == 0: #initialise the decoder variables n = len(layers_copy) layers_dec = [] for i in range(n - 1, 0, -1): layers_dec.append(layers_copy[i]) layers_dec.append(num_features) #build an encoder encoder = input_holder for i in range(0, n - 1): encoder=Dense(layers_copy[i+1], input_dim=layers_copy[i], \ activation=act)(encoder) # add dropout if dropout != 0: encoder = Dropout(dropout)(encoder) decoder = encoder for i in range(0, len(layers_dec) - 1): decoder=Dense(layers_dec[i+1], input_dim=layers_dec[i], \ activation=act)(decoder) # add dropout if dropout != 0: if i < len(layers_dec) - 2: decoder = Dropout(dropout)(decoder) autoencoder = Model(input=input_holder, output=decoder) autoencoder.compile(optimizer=opt, loss=loss) #train the autoencoder loss_hiss = [autoencoder.fit(X, X, epochs=epochs, verbose=verbose)] # extract the encoder from the trained autoencoder encoder = Model(input=input_holder, output=encoder) if summary_display: print(autoencoder.summary()) return encoder, loss_hiss # -------------------------------------------------------------------- # stacked autoencoder # -------------------------------------------------------------------- elif mode == 1: tmp_holder = input_holder x = X stk_encoders, loss_hist = [], [] #train encoder layers for i in range(len(layers_copy) - 1): print("Training Layer %d/%d ..." % (i + 1, len(layers_copy) - 1)) encoder=Dense(layers_copy[i+1], input_dim=layers_copy[i], \ activation=act)(tmp_holder) if dropout != 0: encoder = Dropout(dropout)(encoder) decoder=Dense(layers_copy[i], input_dim=layers_copy[i+1], \ activation=act)(encoder) autoencoder = Model(input=tmp_holder, output=decoder) autoencoder.compile(optimizer=opt, loss=loss) # train a layer loss_his = autoencoder.fit(x, x, epochs=epochs, verbose=verbose) loss_hist.append(loss_his) # use output of the layer as next training input encoder = Model(input=tmp_holder, output=encoder) x = encoder.predict(x) # update the input_holder tmp_holder = Input(shape=(layers_copy[i + 1], )) # store the trained layer in a list stk_encoders.append(encoder) # connect trained encoder layers as 'encode' encoder = Sequential() for e in stk_encoders: encoder.add(e) if summary_display == 1: print(encoder.summary()) return encoder, loss_hist