def __call__(self, x): if self.n_splines == 0: x = kl.GlobalAvgPool1D()(x) else: # Spline-transformation for the position aggregation # This allows to up-weight positions in the middle x = SplineWeight1D(n_bases=self.n_splines, share_splines=True)(x) x = kl.GlobalAvgPool1D()(x) if self.dropout: x = kl.Dropout(self.dropout)(x) # Hidden units (not used by default) for h in self.hidden: if self.batchnorm: x = kl.BatchNormalization()(x) x = kl.Dense(h, activation='relu')(x) if self.dropout_hidden: x = kl.Dropout(self.dropout_hidden)(x) # Final dense layer if self.batchnorm: x = kl.BatchNormalization()(x) x = kl.Dense(self.n_tasks)(x) return x
def model(reader): assert reader.channeltypes == [ChannelType.SENTENCE] word_mapping = reader.channels[0].vocabulary_map weights = gew.generate_embedding_weights(word_mapping) sentence_len = reader.channels[0].sentence_len word_number = weights.shape[0] word_embedding_size = weights.shape[1] known_in = L.Input(shape=(None, sentence_len), dtype='int32') unknown_in = L.Input(shape=(None, sentence_len), dtype='int32') embedding = L.Embedding(output_dim=word_embedding_size, input_dim=word_number, trainable=False, weights=[weights]) known_emb = embedding(known_in) unknown_emb = embedding(unknown_in) average_sentences = L.Lambda(lambda x: K.sum(x, axis=2) / sentence_len, output_shape=(None, word_embedding_size)) known_sentences_repr = average_sentences(known_emb) unknown_sentences_repr = average_sentences(unknown_emb) feature_extractor1 = L.Bidirectional(L.LSTM(50, return_sequences=True)) feature_extractor2 = L.Bidirectional(L.LSTM(50, return_sequences=True)) pool = L.GlobalAvgPool1D() features_known = pool( feature_extractor2(feature_extractor1(known_sentences_repr))) features_unknown = pool( feature_extractor2(feature_extractor1(unknown_sentences_repr))) abs_diff = L.merge(inputs=[features_known, features_unknown], mode=lambda x: abs(x[0] - x[1]), output_shape=lambda x: x[0], name='absolute_difference') # Dense network. dense1 = L.Dense(100)(abs_diff) pruned = L.Dropout(0.3)(dense1) output = L.Dense(2, activation='softmax', name='output')(pruned) model = Model(inputs=[known_in, unknown_in], outputs=output) optimizer = O.Adam(lr=0.0005) model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy']) return model
def count_prediction(bottleneck, tasks, outputs_per_task, bias_counts_inputs=[], batchnorm=False): x = kl.GlobalAvgPool1D()(bottleneck) if bias_counts_inputs: # add bias as additional features if present x = kl.concatenate([x] + bias_counts_inputs, axis=-1) if batchnorm: x = kl.BatchNormalization()(x) # final layer counts = [ kl.Dense(outputs_per_task[i], name="counts/" + task)(x) for i, task in enumerate(tasks) ] return counts
def model(reader): assert reader.channeltypes == [ChannelType.POS_TAGS] known_in = L.Input(shape=(None, ), dtype='int32') unknown_in = L.Input(shape=(None, ), dtype='int32') embedding = L.Embedding( len(reader.channels[0].vocabulary_above_cutoff) + 2, 1) known_emb = embedding(known_in) unknown_emb = embedding(unknown_in) feature_extractor1 = L.Bidirectional(L.LSTM(100, return_sequences=True)) pool = L.GlobalAvgPool1D() features_known = pool(feature_extractor1(known_emb)) features_unknown = pool(feature_extractor1(unknown_emb)) abs_diff = L.merge(inputs=[features_known, features_unknown], mode=lambda x: abs(x[0] - x[1]), output_shape=lambda x: x[0], name='absolute_difference') # Dense network. dense1 = L.Dense(300)(abs_diff) dense2 = L.Dense(200)(dense1) dense3 = L.Dense(100)(dense2) output1 = L.Dense(2, activation='softmax')(dense1) output2 = L.Dense(2, activation='softmax')(dense2) output3 = L.Dense(2, activation='softmax')(dense3) output = L.Average()([output1, output2, output3]) model = Model(inputs=[known_in, unknown_in], outputs=output) optimizer = O.Adam(lr=0.0005) model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy']) return model
def model(reader): known_in = L.Input(shape=(None, ), dtype='int32') unknown_in = L.Input(shape=(None, ), dtype='int32') embedding = L.Embedding( len(reader.channels[0].vocabulary_above_cutoff) + 2, 150) known_emb = embedding(known_in) unknown_emb = embedding(unknown_in) feature_extractor = L.Bidirectional(L.LSTM(250, return_sequences=True)) features_known = feature_extractor(known_emb) features_unknown = feature_extractor(unknown_emb) avg = L.GlobalAvgPool1D() features_known_avg = avg(features_known) features_unknown_avg = avg(features_unknown) abs_diff = L.merge( inputs=[features_known_avg, features_unknown_avg], mode=lambda x: abs(x[0] - x[1]), output_shape=lambda x: x[0], name='absolute_difference') output = L.Dense(2, activation='softmax', name='output')(abs_diff) model = Model(inputs=[known_in, unknown_in], outputs=output) model.compile( optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) return model
x = kl.add([conv_x, x]) bottleneck = x # heads outputs = [] for task in tasks: # profile shape head px = kl.Reshape((-1, 1, 64))(bottleneck) px = kl.Conv2DTranspose(2, kernel_size=(25, 1), padding='same')(px) outputs.append(kl.Reshape((-1, 2))(px)) cx = kl.GlobalAvgPool1D()(bottleneck) outputs.append(kl.Dense(2)(cx)) def multinomial_nll(true_counts, logits): """Compute the multinomial negative log-likelihood along the sequence (axis=1) and sum the values across all each channels Args: true_counts: observed count values (batch, seqlen, channels) logits: predicted logit values (batch, seqlen, channels) """ print(logits) # swap axes such that the final axis will be the positional axis logits_perm = tf.transpose(logits[0], (0, 2, 1))
def get_keras_model(self): inp, bias_counts_input, bias_profile_input = self.get_inputs() countouttaskname, profileouttaskname = self.get_names() first_conv = kl.Conv1D(self.filters, kernel_size=self.conv1_kernel_size, kernel_initializer=self.kernel_initializer, padding='valid', activation='relu')(inp) curr_layer_size = self.input_seq_len - (self.conv1_kernel_size - 1) prev_layers = first_conv for i in range(1, self.n_dil_layers + 1): dilation_rate = 2**i conv_output = kl.Conv1D(self.filters, kernel_size=self.dil_kernel_size, kernel_initializer=self.kernel_initializer, padding='valid', activation='relu', dilation_rate=dilation_rate)(prev_layers) width_to_trim = dilation_rate * (self.dil_kernel_size - 1) curr_layer_size = (curr_layer_size - width_to_trim) prev_layers = self.trim_flanks_of_conv_layer( conv_layer=prev_layers, output_len=curr_layer_size, width_to_trim=width_to_trim, filters=self.filters) if self.is_add: prev_layers = kl.add([prev_layers, conv_output]) else: prev_layers = kl.average([prev_layers, conv_output]) combined_conv = prev_layers # Counts Prediction gap_combined_conv = kl.GlobalAvgPool1D()(combined_conv) count_out = kl.Dense(2, kernel_initializer=self.kernel_initializer, name=countouttaskname)(kl.concatenate( [gap_combined_conv, bias_counts_input], axis=-1)) # Profile Prediction profile_out_prebias = kl.Conv1D( filters=2, kernel_size=self.outconv_kernel_size, kernel_initializer=self.kernel_initializer, padding='valid')(combined_conv) profile_out = kl.Conv1D(2, kernel_size=1, kernel_initializer=self.kernel_initializer, name=profileouttaskname)(kl.concatenate( [profile_out_prebias, bias_profile_input], axis=-1)) model = keras.models.Model( inputs=[inp, bias_counts_input, bias_profile_input], outputs=[count_out, profile_out]) model.compile(keras.optimizers.Adam(lr=self.lr), loss=['mse', MultichannelMultinomialNLL(2)], loss_weights=[self.c_task_weight, self.p_task_weight]) return model
def get_keras_model(self): inp, bias_counts_input, bias_profile_input = self.get_inputs() rev_inp = kl.Lambda(lambda x: x[:, ::-1, ::-1])(inp) countouttaskname, profileouttaskname = self.get_names() first_conv = kl.Conv1D(self.filters, kernel_size=self.conv1_kernel_size, kernel_initializer=self.kernel_initializer, padding='valid', activation='relu') first_conv_fwd = first_conv(inp) first_conv_rev = first_conv(rev_inp) curr_layer_size = self.input_seq_len - (self.conv1_kernel_size - 1) prev_layers_fwd = first_conv_fwd prev_layers_rev = first_conv_rev for i in range(1, self.n_dil_layers + 1): dilation_rate = 2**i conv_output = kl.Conv1D(self.filters, kernel_size=self.dil_kernel_size, padding='valid', kernel_initializer=self.kernel_initializer, activation='relu', dilation_rate=dilation_rate) conv_output_fwd = conv_output(prev_layers_fwd) conv_output_rev = conv_output(prev_layers_rev) width_to_trim = dilation_rate * (self.dil_kernel_size - 1) curr_layer_size = (curr_layer_size - width_to_trim) prev_layers_fwd = self.trim_flanks_of_conv_layer( conv_layer=prev_layers_fwd, output_len=curr_layer_size, width_to_trim=width_to_trim, filters=self.filters) prev_layers_rev = self.trim_flanks_of_conv_layer_revcomp( conv_layer=prev_layers_rev, output_len=curr_layer_size, width_to_trim=width_to_trim, filters=self.filters) if (self.is_add): prev_layers_fwd = kl.add([prev_layers_fwd, conv_output_fwd]) prev_layers_rev = kl.add([prev_layers_rev, conv_output_rev]) else: prev_layers_fwd = kl.average( [prev_layers_fwd, conv_output_fwd]) prev_layers_rev = kl.average( [prev_layers_rev, conv_output_rev]) combined_conv_fwd = prev_layers_fwd combined_conv_rev = prev_layers_rev # Counts Prediction counts_dense_layer = kl.Dense( 2, kernel_initializer=self.kernel_initializer, ) gap_combined_conv_fwd = kl.GlobalAvgPool1D()(combined_conv_fwd) gap_combined_conv_rev = kl.GlobalAvgPool1D()(combined_conv_rev) main_count_out_fwd = counts_dense_layer( kl.concatenate([gap_combined_conv_fwd, bias_counts_input], axis=-1)) main_count_out_rev = counts_dense_layer( kl.concatenate([bias_counts_input, gap_combined_conv_rev], axis=-1)) rc_rev_count_out = kl.Lambda(lambda x: x[:, ::-1])(main_count_out_rev) avg_count_out = kl.Average(name=countouttaskname)( [main_count_out_fwd, rc_rev_count_out]) # Profile Prediction profile_penultimate_conv = kl.Conv1D( filters=2, kernel_size=self.outconv_kernel_size, kernel_initializer=self.kernel_initializer, padding='valid') profile_final_conv = kl.Conv1D( 2, kernel_size=1, kernel_initializer=self.kernel_initializer, ) profile_out_prebias_fwd = profile_penultimate_conv(combined_conv_fwd) main_profile_out_fwd = profile_final_conv( kl.concatenate([profile_out_prebias_fwd, bias_profile_input], axis=-1)) profile_out_prebias_rev = profile_penultimate_conv(combined_conv_rev) rev_bias_profile_input = kl.Lambda(lambda x: x[:, ::-1, :])( bias_profile_input) main_profile_out_rev = profile_final_conv( kl.concatenate([profile_out_prebias_rev, rev_bias_profile_input], axis=-1)) rc_rev_profile_out = kl.Lambda(lambda x: x[:, ::-1, ::-1])( main_profile_out_rev) avg_profile_out = kl.Average(name=profileouttaskname)( [main_profile_out_fwd, rc_rev_profile_out]) model = keras.models.Model( inputs=[inp, bias_counts_input, bias_profile_input], outputs=[avg_count_out, avg_profile_out]) model.compile(keras.optimizers.Adam(lr=self.lr), loss=['mse', MultichannelMultinomialNLL(2)], loss_weights=[self.c_task_weight, self.p_task_weight]) return model
def get_keras_model(self): from equinet import ToKmerLayer inp, bias_counts_input, bias_profile_input = self.get_inputs() countouttaskname, profileouttaskname = self.get_names() out_pred_len = self.get_output_profile_len() curr_layer_size = self.input_seq_len - (self.conv1_kernel_size - 1) kmer_inp = ToKmerLayer(k=self.kmers)(inp) first_conv = RevCompConv1D(filters=self.filters, kernel_size=self.conv1_kernel_size - self.kmers + 1, kernel_initializer=self.kernel_initializer, padding='valid', activation='relu')(kmer_inp) prev_layers = first_conv for i in range(1, self.n_dil_layers + 1): dilation_rate = 2**i conv_output = RevCompConv1D( filters=self.filters, kernel_size=self.dil_kernel_size, kernel_initializer=self.kernel_initializer, padding='valid', activation='relu', dilation_rate=dilation_rate)(prev_layers) width_to_trim = dilation_rate * (self.dil_kernel_size - 1) curr_layer_size = (curr_layer_size - width_to_trim) prev_layers = self.trim_flanks_of_conv_layer( conv_layer=prev_layers, output_len=curr_layer_size, width_to_trim=width_to_trim, filters=2 * self.filters) if (self.is_add): prev_layers = kl.add([prev_layers, conv_output]) else: prev_layers = kl.average([prev_layers, conv_output]) combined_conv = prev_layers # Counts prediction gap_combined_conv = kl.GlobalAvgPool1D()(combined_conv) count_out = kl.Reshape((-1, ), name=countouttaskname)( RevCompConv1D(filters=1, kernel_size=1, kernel_initializer=self.kernel_initializer) ( kl.Reshape((1, -1))( kl.concatenate( [ # concatenation of the bias layer both before and after # is needed for rc symmetry kl.Lambda(lambda x: x[:, ::-1])(bias_counts_input), gap_combined_conv, bias_counts_input ], axis=-1)))) # Profile prediction profile_out_prebias = RevCompConv1D( filters=1, kernel_size=self.outconv_kernel_size, kernel_initializer=self.kernel_initializer, padding='valid')(combined_conv) profile_out = RevCompConv1D( filters=1, kernel_size=1, name=profileouttaskname, kernel_initializer=self.kernel_initializer )( kl.concatenate( [ # concatenation of the bias layer both before and after # is needed for rc symmetry kl.Lambda(lambda x: x[:, :, ::-1])(bias_profile_input), profile_out_prebias, bias_profile_input ], axis=-1)) model = keras.models.Model( inputs=[inp, bias_counts_input, bias_profile_input], outputs=[count_out, profile_out]) model.compile(keras.optimizers.Adam(lr=self.lr), loss=['mse', MultichannelMultinomialNLL(2)], loss_weights=[self.c_task_weight, self.p_task_weight]) return model
def __call__(self, inp): x = kl.GlobalAvgPool1D()(inp) return kl.Dense(1)(x)
def get_keras_model(self): np.random.seed(self.seed) tf.set_random_seed(self.seed) inp, bias_counts_input, bias_profile_input = self.get_inputs() countouttaskname, profileouttaskname = self.get_names() #define the submodel submodel_inp = keras.layers.Input(shape=(self.input_seq_len, 4)) first_conv = kl.Conv1D(self.filters, kernel_size=self.conv1_kernel_size, kernel_initializer=self.kernel_initializer, padding='valid', activation='relu') first_conv_fwd = first_conv(submodel_inp) curr_layer_size = self.input_seq_len - (self.conv1_kernel_size - 1) prev_layers_fwd = first_conv_fwd for i in range(1, self.n_dil_layers + 1): dilation_rate = 2**i conv_output = kl.Conv1D(self.filters, kernel_size=self.dil_kernel_size, padding='valid', kernel_initializer=self.kernel_initializer, activation='relu', dilation_rate=dilation_rate) conv_output_fwd = conv_output(prev_layers_fwd) width_to_trim = dilation_rate * (self.dil_kernel_size - 1) curr_layer_size = (curr_layer_size - width_to_trim) prev_layers_fwd = self.trim_flanks_of_conv_layer( conv_layer=prev_layers_fwd, output_len=curr_layer_size, width_to_trim=width_to_trim, filters=self.filters) prev_layers_fwd = kl.add([prev_layers_fwd, conv_output_fwd]) combined_conv_submodel = prev_layers_fwd submodel = keras.models.Model(inputs=submodel_inp, outputs=combined_conv_submodel) combined_conv = self.apply_conjoined_rcps(curr_submodel=submodel, input_tensor=inp) #Counts prediction gap_combined_conv = kl.GlobalAvgPool1D()(combined_conv) count_out = kl.Reshape((-1, ), name=countouttaskname)( RevCompConv1D(filters=1, kernel_size=1, kernel_initializer=self.kernel_initializer) ( kl.Reshape((1, -1))( kl.concatenate( [ #concatenation of the bias layer both before and after # is needed for rc symmetry kl.Lambda(lambda x: x[:, ::-1])(bias_counts_input), gap_combined_conv, bias_counts_input ], axis=-1)))) #Profile prediction profile_out_prebias = RevCompConv1D( filters=1, kernel_size=self.outconv_kernel_size, kernel_initializer=self.kernel_initializer, padding='valid')(combined_conv) profile_out = RevCompConv1D( filters=1, kernel_size=1, name=profileouttaskname, kernel_initializer=self.kernel_initializer )( kl.concatenate( [ #concatenation of the bias layer both before and after # is needed for rc symmetry kl.Lambda(lambda x: x[:, :, ::-1])(bias_profile_input), profile_out_prebias, bias_profile_input ], axis=-1)) model = keras.models.Model( inputs=[inp, bias_counts_input, bias_profile_input], outputs=[count_out, profile_out]) model.compile(keras.optimizers.Adam(lr=self.lr), loss=['mse', MultichannelMultinomialNLL(2)], loss_weights=[self.c_task_weight, self.p_task_weight]) return model