def __call__(self, x):
if self.n_splines == 0:
x = kl.GlobalAvgPool1D()(x)
else:
# Spline-transformation for the position aggregation
# This allows to up-weight positions in the middle
x = SplineWeight1D(n_bases=self.n_splines, share_splines=True)(x)
x = kl.GlobalAvgPool1D()(x)
if self.dropout:
x = kl.Dropout(self.dropout)(x)
# Hidden units (not used by default)
for h in self.hidden:
if self.batchnorm:
x = kl.BatchNormalization()(x)
x = kl.Dense(h, activation='relu')(x)
if self.dropout_hidden:
x = kl.Dropout(self.dropout_hidden)(x)
# Final dense layer
if self.batchnorm:
x = kl.BatchNormalization()(x)
x = kl.Dense(self.n_tasks)(x)
return x
def model(reader):
assert reader.channeltypes == [ChannelType.SENTENCE]
word_mapping = reader.channels[0].vocabulary_map
weights = gew.generate_embedding_weights(word_mapping)
sentence_len = reader.channels[0].sentence_len
word_number = weights.shape[0]
word_embedding_size = weights.shape[1]
known_in = L.Input(shape=(None, sentence_len), dtype='int32')
unknown_in = L.Input(shape=(None, sentence_len), dtype='int32')
embedding = L.Embedding(output_dim=word_embedding_size,
input_dim=word_number,
trainable=False,
weights=[weights])
known_emb = embedding(known_in)
unknown_emb = embedding(unknown_in)
average_sentences = L.Lambda(lambda x: K.sum(x, axis=2) / sentence_len,
output_shape=(None, word_embedding_size))
known_sentences_repr = average_sentences(known_emb)
unknown_sentences_repr = average_sentences(unknown_emb)
feature_extractor1 = L.Bidirectional(L.LSTM(50, return_sequences=True))
feature_extractor2 = L.Bidirectional(L.LSTM(50, return_sequences=True))
pool = L.GlobalAvgPool1D()
features_known = pool(
feature_extractor2(feature_extractor1(known_sentences_repr)))
features_unknown = pool(
feature_extractor2(feature_extractor1(unknown_sentences_repr)))
abs_diff = L.merge(inputs=[features_known, features_unknown],
mode=lambda x: abs(x[0] - x[1]),
output_shape=lambda x: x[0],
name='absolute_difference')
# Dense network.
dense1 = L.Dense(100)(abs_diff)
pruned = L.Dropout(0.3)(dense1)
output = L.Dense(2, activation='softmax', name='output')(pruned)
model = Model(inputs=[known_in, unknown_in], outputs=output)
optimizer = O.Adam(lr=0.0005)
model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def count_prediction(bottleneck,
tasks,
outputs_per_task,
bias_counts_inputs=[],
batchnorm=False):
x = kl.GlobalAvgPool1D()(bottleneck)
if bias_counts_inputs:
# add bias as additional features if present
x = kl.concatenate([x] + bias_counts_inputs, axis=-1)
if batchnorm:
x = kl.BatchNormalization()(x)
# final layer
counts = [
kl.Dense(outputs_per_task[i], name="counts/" + task)(x)
for i, task in enumerate(tasks)
]
return counts
def model(reader):
assert reader.channeltypes == [ChannelType.POS_TAGS]
known_in = L.Input(shape=(None, ), dtype='int32')
unknown_in = L.Input(shape=(None, ), dtype='int32')
embedding = L.Embedding(
len(reader.channels[0].vocabulary_above_cutoff) + 2, 1)
known_emb = embedding(known_in)
unknown_emb = embedding(unknown_in)
feature_extractor1 = L.Bidirectional(L.LSTM(100, return_sequences=True))
pool = L.GlobalAvgPool1D()
features_known = pool(feature_extractor1(known_emb))
features_unknown = pool(feature_extractor1(unknown_emb))
abs_diff = L.merge(inputs=[features_known, features_unknown],
mode=lambda x: abs(x[0] - x[1]),
output_shape=lambda x: x[0],
name='absolute_difference')
# Dense network.
dense1 = L.Dense(300)(abs_diff)
dense2 = L.Dense(200)(dense1)
dense3 = L.Dense(100)(dense2)
output1 = L.Dense(2, activation='softmax')(dense1)
output2 = L.Dense(2, activation='softmax')(dense2)
output3 = L.Dense(2, activation='softmax')(dense3)
output = L.Average()([output1, output2, output3])
model = Model(inputs=[known_in, unknown_in], outputs=output)
optimizer = O.Adam(lr=0.0005)
model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def model(reader):
known_in = L.Input(shape=(None, ), dtype='int32')
unknown_in = L.Input(shape=(None, ), dtype='int32')
embedding = L.Embedding(
len(reader.channels[0].vocabulary_above_cutoff) + 2, 150)
known_emb = embedding(known_in)
unknown_emb = embedding(unknown_in)
feature_extractor = L.Bidirectional(L.LSTM(250, return_sequences=True))
features_known = feature_extractor(known_emb)
features_unknown = feature_extractor(unknown_emb)
avg = L.GlobalAvgPool1D()
features_known_avg = avg(features_known)
features_unknown_avg = avg(features_unknown)
abs_diff = L.merge(
inputs=[features_known_avg, features_unknown_avg],
mode=lambda x: abs(x[0] - x[1]),
output_shape=lambda x: x[0],
name='absolute_difference')
output = L.Dense(2, activation='softmax', name='output')(abs_diff)
model = Model(inputs=[known_in, unknown_in], outputs=output)
model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
x = kl.add([conv_x, x])
bottleneck = x
# heads
outputs = []
for task in tasks:
# profile shape head
px = kl.Reshape((-1, 1, 64))(bottleneck)
px = kl.Conv2DTranspose(2, kernel_size=(25, 1), padding='same')(px)
outputs.append(kl.Reshape((-1, 2))(px))
cx = kl.GlobalAvgPool1D()(bottleneck)
outputs.append(kl.Dense(2)(cx))
def multinomial_nll(true_counts, logits):
"""Compute the multinomial negative log-likelihood along the sequence (axis=1)
and sum the values across all each channels
Args:
true_counts: observed count values (batch, seqlen, channels)
logits: predicted logit values (batch, seqlen, channels)
"""
print(logits)
# swap axes such that the final axis will be the positional axis
logits_perm = tf.transpose(logits[0], (0, 2, 1))
def get_keras_model(self):
inp, bias_counts_input, bias_profile_input = self.get_inputs()
countouttaskname, profileouttaskname = self.get_names()
first_conv = kl.Conv1D(self.filters,
kernel_size=self.conv1_kernel_size,
kernel_initializer=self.kernel_initializer,
padding='valid',
activation='relu')(inp)
curr_layer_size = self.input_seq_len - (self.conv1_kernel_size - 1)
prev_layers = first_conv
for i in range(1, self.n_dil_layers + 1):
dilation_rate = 2**i
conv_output = kl.Conv1D(self.filters,
kernel_size=self.dil_kernel_size,
kernel_initializer=self.kernel_initializer,
padding='valid',
activation='relu',
dilation_rate=dilation_rate)(prev_layers)
width_to_trim = dilation_rate * (self.dil_kernel_size - 1)
curr_layer_size = (curr_layer_size - width_to_trim)
prev_layers = self.trim_flanks_of_conv_layer(
conv_layer=prev_layers,
output_len=curr_layer_size,
width_to_trim=width_to_trim,
filters=self.filters)
if self.is_add:
prev_layers = kl.add([prev_layers, conv_output])
else:
prev_layers = kl.average([prev_layers, conv_output])
combined_conv = prev_layers
# Counts Prediction
gap_combined_conv = kl.GlobalAvgPool1D()(combined_conv)
count_out = kl.Dense(2,
kernel_initializer=self.kernel_initializer,
name=countouttaskname)(kl.concatenate(
[gap_combined_conv, bias_counts_input],
axis=-1))
# Profile Prediction
profile_out_prebias = kl.Conv1D(
filters=2,
kernel_size=self.outconv_kernel_size,
kernel_initializer=self.kernel_initializer,
padding='valid')(combined_conv)
profile_out = kl.Conv1D(2,
kernel_size=1,
kernel_initializer=self.kernel_initializer,
name=profileouttaskname)(kl.concatenate(
[profile_out_prebias, bias_profile_input],
axis=-1))
model = keras.models.Model(
inputs=[inp, bias_counts_input, bias_profile_input],
outputs=[count_out, profile_out])
model.compile(keras.optimizers.Adam(lr=self.lr),
loss=['mse', MultichannelMultinomialNLL(2)],
loss_weights=[self.c_task_weight, self.p_task_weight])
return model
def get_keras_model(self):
inp, bias_counts_input, bias_profile_input = self.get_inputs()
rev_inp = kl.Lambda(lambda x: x[:, ::-1, ::-1])(inp)
countouttaskname, profileouttaskname = self.get_names()
first_conv = kl.Conv1D(self.filters,
kernel_size=self.conv1_kernel_size,
kernel_initializer=self.kernel_initializer,
padding='valid',
activation='relu')
first_conv_fwd = first_conv(inp)
first_conv_rev = first_conv(rev_inp)
curr_layer_size = self.input_seq_len - (self.conv1_kernel_size - 1)
prev_layers_fwd = first_conv_fwd
prev_layers_rev = first_conv_rev
for i in range(1, self.n_dil_layers + 1):
dilation_rate = 2**i
conv_output = kl.Conv1D(self.filters,
kernel_size=self.dil_kernel_size,
padding='valid',
kernel_initializer=self.kernel_initializer,
activation='relu',
dilation_rate=dilation_rate)
conv_output_fwd = conv_output(prev_layers_fwd)
conv_output_rev = conv_output(prev_layers_rev)
width_to_trim = dilation_rate * (self.dil_kernel_size - 1)
curr_layer_size = (curr_layer_size - width_to_trim)
prev_layers_fwd = self.trim_flanks_of_conv_layer(
conv_layer=prev_layers_fwd,
output_len=curr_layer_size,
width_to_trim=width_to_trim,
filters=self.filters)
prev_layers_rev = self.trim_flanks_of_conv_layer_revcomp(
conv_layer=prev_layers_rev,
output_len=curr_layer_size,
width_to_trim=width_to_trim,
filters=self.filters)
if (self.is_add):
prev_layers_fwd = kl.add([prev_layers_fwd, conv_output_fwd])
prev_layers_rev = kl.add([prev_layers_rev, conv_output_rev])
else:
prev_layers_fwd = kl.average(
[prev_layers_fwd, conv_output_fwd])
prev_layers_rev = kl.average(
[prev_layers_rev, conv_output_rev])
combined_conv_fwd = prev_layers_fwd
combined_conv_rev = prev_layers_rev
# Counts Prediction
counts_dense_layer = kl.Dense(
2,
kernel_initializer=self.kernel_initializer,
)
gap_combined_conv_fwd = kl.GlobalAvgPool1D()(combined_conv_fwd)
gap_combined_conv_rev = kl.GlobalAvgPool1D()(combined_conv_rev)
main_count_out_fwd = counts_dense_layer(
kl.concatenate([gap_combined_conv_fwd, bias_counts_input],
axis=-1))
main_count_out_rev = counts_dense_layer(
kl.concatenate([bias_counts_input, gap_combined_conv_rev],
axis=-1))
rc_rev_count_out = kl.Lambda(lambda x: x[:, ::-1])(main_count_out_rev)
avg_count_out = kl.Average(name=countouttaskname)(
[main_count_out_fwd, rc_rev_count_out])
# Profile Prediction
profile_penultimate_conv = kl.Conv1D(
filters=2,
kernel_size=self.outconv_kernel_size,
kernel_initializer=self.kernel_initializer,
padding='valid')
profile_final_conv = kl.Conv1D(
2,
kernel_size=1,
kernel_initializer=self.kernel_initializer,
)
profile_out_prebias_fwd = profile_penultimate_conv(combined_conv_fwd)
main_profile_out_fwd = profile_final_conv(
kl.concatenate([profile_out_prebias_fwd, bias_profile_input],
axis=-1))
profile_out_prebias_rev = profile_penultimate_conv(combined_conv_rev)
rev_bias_profile_input = kl.Lambda(lambda x: x[:, ::-1, :])(
bias_profile_input)
main_profile_out_rev = profile_final_conv(
kl.concatenate([profile_out_prebias_rev, rev_bias_profile_input],
axis=-1))
rc_rev_profile_out = kl.Lambda(lambda x: x[:, ::-1, ::-1])(
main_profile_out_rev)
avg_profile_out = kl.Average(name=profileouttaskname)(
[main_profile_out_fwd, rc_rev_profile_out])
model = keras.models.Model(
inputs=[inp, bias_counts_input, bias_profile_input],
outputs=[avg_count_out, avg_profile_out])
model.compile(keras.optimizers.Adam(lr=self.lr),
loss=['mse', MultichannelMultinomialNLL(2)],
loss_weights=[self.c_task_weight, self.p_task_weight])
return model
def get_keras_model(self):
from equinet import ToKmerLayer
inp, bias_counts_input, bias_profile_input = self.get_inputs()
countouttaskname, profileouttaskname = self.get_names()
out_pred_len = self.get_output_profile_len()
curr_layer_size = self.input_seq_len - (self.conv1_kernel_size - 1)
kmer_inp = ToKmerLayer(k=self.kmers)(inp)
first_conv = RevCompConv1D(filters=self.filters,
kernel_size=self.conv1_kernel_size -
self.kmers + 1,
kernel_initializer=self.kernel_initializer,
padding='valid',
activation='relu')(kmer_inp)
prev_layers = first_conv
for i in range(1, self.n_dil_layers + 1):
dilation_rate = 2**i
conv_output = RevCompConv1D(
filters=self.filters,
kernel_size=self.dil_kernel_size,
kernel_initializer=self.kernel_initializer,
padding='valid',
activation='relu',
dilation_rate=dilation_rate)(prev_layers)
width_to_trim = dilation_rate * (self.dil_kernel_size - 1)
curr_layer_size = (curr_layer_size - width_to_trim)
prev_layers = self.trim_flanks_of_conv_layer(
conv_layer=prev_layers,
output_len=curr_layer_size,
width_to_trim=width_to_trim,
filters=2 * self.filters)
if (self.is_add):
prev_layers = kl.add([prev_layers, conv_output])
else:
prev_layers = kl.average([prev_layers, conv_output])
combined_conv = prev_layers
# Counts prediction
gap_combined_conv = kl.GlobalAvgPool1D()(combined_conv)
count_out = kl.Reshape((-1, ), name=countouttaskname)(
RevCompConv1D(filters=1,
kernel_size=1,
kernel_initializer=self.kernel_initializer)
(
kl.Reshape((1, -1))(
kl.concatenate(
[
# concatenation of the bias layer both before and after
# is needed for rc symmetry
kl.Lambda(lambda x: x[:, ::-1])(bias_counts_input),
gap_combined_conv,
bias_counts_input
],
axis=-1))))
# Profile prediction
profile_out_prebias = RevCompConv1D(
filters=1,
kernel_size=self.outconv_kernel_size,
kernel_initializer=self.kernel_initializer,
padding='valid')(combined_conv)
profile_out = RevCompConv1D(
filters=1,
kernel_size=1,
name=profileouttaskname,
kernel_initializer=self.kernel_initializer
)(
kl.concatenate(
[
# concatenation of the bias layer both before and after
# is needed for rc symmetry
kl.Lambda(lambda x: x[:, :, ::-1])(bias_profile_input),
profile_out_prebias,
bias_profile_input
],
axis=-1))
model = keras.models.Model(
inputs=[inp, bias_counts_input, bias_profile_input],
outputs=[count_out, profile_out])
model.compile(keras.optimizers.Adam(lr=self.lr),
loss=['mse', MultichannelMultinomialNLL(2)],
loss_weights=[self.c_task_weight, self.p_task_weight])
return model
def __call__(self, inp):
x = kl.GlobalAvgPool1D()(inp)
return kl.Dense(1)(x)
def get_keras_model(self):
np.random.seed(self.seed)
tf.set_random_seed(self.seed)
inp, bias_counts_input, bias_profile_input = self.get_inputs()
countouttaskname, profileouttaskname = self.get_names()
#define the submodel
submodel_inp = keras.layers.Input(shape=(self.input_seq_len, 4))
first_conv = kl.Conv1D(self.filters,
kernel_size=self.conv1_kernel_size,
kernel_initializer=self.kernel_initializer,
padding='valid',
activation='relu')
first_conv_fwd = first_conv(submodel_inp)
curr_layer_size = self.input_seq_len - (self.conv1_kernel_size - 1)
prev_layers_fwd = first_conv_fwd
for i in range(1, self.n_dil_layers + 1):
dilation_rate = 2**i
conv_output = kl.Conv1D(self.filters,
kernel_size=self.dil_kernel_size,
padding='valid',
kernel_initializer=self.kernel_initializer,
activation='relu',
dilation_rate=dilation_rate)
conv_output_fwd = conv_output(prev_layers_fwd)
width_to_trim = dilation_rate * (self.dil_kernel_size - 1)
curr_layer_size = (curr_layer_size - width_to_trim)
prev_layers_fwd = self.trim_flanks_of_conv_layer(
conv_layer=prev_layers_fwd,
output_len=curr_layer_size,
width_to_trim=width_to_trim,
filters=self.filters)
prev_layers_fwd = kl.add([prev_layers_fwd, conv_output_fwd])
combined_conv_submodel = prev_layers_fwd
submodel = keras.models.Model(inputs=submodel_inp,
outputs=combined_conv_submodel)
combined_conv = self.apply_conjoined_rcps(curr_submodel=submodel,
input_tensor=inp)
#Counts prediction
gap_combined_conv = kl.GlobalAvgPool1D()(combined_conv)
count_out = kl.Reshape((-1, ), name=countouttaskname)(
RevCompConv1D(filters=1,
kernel_size=1,
kernel_initializer=self.kernel_initializer)
(
kl.Reshape((1, -1))(
kl.concatenate(
[
#concatenation of the bias layer both before and after
# is needed for rc symmetry
kl.Lambda(lambda x: x[:, ::-1])(bias_counts_input),
gap_combined_conv,
bias_counts_input
],
axis=-1))))
#Profile prediction
profile_out_prebias = RevCompConv1D(
filters=1,
kernel_size=self.outconv_kernel_size,
kernel_initializer=self.kernel_initializer,
padding='valid')(combined_conv)
profile_out = RevCompConv1D(
filters=1,
kernel_size=1,
name=profileouttaskname,
kernel_initializer=self.kernel_initializer
)(
kl.concatenate(
[
#concatenation of the bias layer both before and after
# is needed for rc symmetry
kl.Lambda(lambda x: x[:, :, ::-1])(bias_profile_input),
profile_out_prebias,
bias_profile_input
],
axis=-1))
model = keras.models.Model(
inputs=[inp, bias_counts_input, bias_profile_input],
outputs=[count_out, profile_out])
model.compile(keras.optimizers.Adam(lr=self.lr),
loss=['mse', MultichannelMultinomialNLL(2)],
loss_weights=[self.c_task_weight, self.p_task_weight])
return model
