def cnn_bidirectional_lstm_model(num_classes, input_shape, embedding_matrix, max_length):
""" Creates Bidirectional LSTM model for classification of emotions with Glove embeddings
:param num_classes: number of classes
:type num_classes: int
:param input_shape: shape of the input
:type input_shape: tuple
:param embedding_matrix: embedding matrix for Keras Embedding layer
:type embedding_matrix: numpy.array
:param max_length: maximum length of the text sequence
:type max_length: int
:return: Bidirectional LSTM model
"""
model = k.Sequential()
model.add(kl.Embedding(input_dim=embedding_matrix.shape[0],
output_dim=embedding_matrix.shape[1],
weights=[embedding_matrix],
input_length=max_length,
trainable=False))
model.add(kl.Convolution1D(32, 3, activation='relu', input_shape=input_shape))
model.add(kl.BatchNormalization())
model.add(kl.MaxPooling1D())
model.add(kl.Convolution1D(64, 3, activation='relu'))
model.add(kl.BatchNormalization())
model.add(kl.MaxPooling1D())
model.add(kl.Dropout(0.1))
model.add(kl.Bidirectional(kl.LSTM(128, dropout=0.2, recurrent_dropout=0.2)))
model.add(kl.Dense(num_classes, activation='sigmoid'))
return model
def cnn_2x(voc_size, max_len, dropout=0.5):
"""One branch embedding a statement. Binary classifier on top.
Args:
voc_size: size of the vocabulary for the input statements.
max_len: maximum length for the input statements.
Returns:
A Keras model instance.
"""
statement_input = layers.Input(shape=(max_len,), dtype='int32')
x = layers.Embedding(
output_dim=256,
input_dim=voc_size,
input_length=max_len)(statement_input)
x = layers.Convolution1D(256, 7, activation='relu')(x)
x = layers.MaxPooling1D(3)(x)
x = layers.Convolution1D(256, 7, activation='relu')(x)
embedded_statement = layers.GlobalMaxPooling1D()(x)
x = layers.Dense(256, activation='relu')(embedded_statement)
x = layers.Dropout(dropout)(x)
prediction = layers.Dense(1, activation='sigmoid')(x)
model = Model(statement_input, prediction)
return model
def combined_model(dense_layers, output_dim):
conv_base = VGG16(weights='imagenet',
include_top=False,
input_shape=(128, 128, 3))
image_model = models.Sequential()
image_model.add(conv_base)
image_model.add(layers.Flatten())
image_model.add(layers.Dense(2048, activation='relu'))
image_model.add(layers.Dropout(0.5))
image_model.add(layers.Dense(1024, activation='relu'))
image_model.add(layers.Dropout(0.5))
image_model.add(layers.Dense(512, activation='relu'))
text_model = models.Sequential()
text_model.add(layers.Embedding(UNK_WORD + 1, 100, input_length=20))
text_model.add(layers.Convolution1D(256, 3, padding='same'))
text_model.add(layers.MaxPool1D(3, 3, padding='same'))
text_model.add(layers.Convolution1D(128, 3, padding='same'))
text_model.add(layers.MaxPool1D(3, 3, padding='same'))
text_model.add(layers.Convolution1D(64, 3, padding='same'))
text_model.add(layers.Flatten())
model = models.Sequential()
model.add(Merge([image_model, text_model], mode='concat'))
model.add(layers.BatchNormalization())
for item in dense_layers:
model.add(layers.Dense(item, activation='relu'))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(output_dim, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def cnn_2x_siamese(voc_size, max_len, dropout=0.5):
"""Two siamese branches, each embedding a statement.
Binary classifier on top.
Args:
voc_size: size of the vocabulary for the input statements.
max_len: maximum length for the input statements.
Returns:
A Keras model instance.
"""
pivot_input = layers.Input(shape=(max_len, ), dtype='int32')
statement_input = layers.Input(shape=(max_len, ), dtype='int32')
x = layers.Embedding(output_dim=256,
input_dim=voc_size,
input_length=max_len)(pivot_input)
x = layers.Convolution1D(256, 7, activation='relu')(x)
x = layers.MaxPooling1D(3)(x)
x = layers.Convolution1D(256, 7, activation='relu')(x)
embedded_pivot = layers.GlobalMaxPooling1D()(x)
encoder_model = Model(pivot_input, embedded_pivot)
embedded_statement = encoder_model(statement_input)
concat = layers.merge([embedded_pivot, embedded_statement], mode='concat')
x = layers.Dense(256, activation='relu')(concat)
x = layers.Dropout(dropout)(x)
prediction = layers.Dense(1, activation='sigmoid')(x)
model = Model([pivot_input, statement_input], prediction)
return model
def cnn_attention_lstm(num_classes, input_shape, embedding_matrix, max_length):
""" Creates LSTM model with attention layer for classification of emotions with Glove embeddings
:param num_classes: number of classes
:type num_classes: int
:param input_shape: shape of the input
:type input_shape: tuple
:param embedding_matrix: embedding matrix for Keras Embedding layer
:type embedding_matrix: numpy.array
:param max_length: maximum length of the text sequence
:type max_length: int
:return: LSTM model
"""
inputs = kl.Input(shape=input_shape)
embeddings = kl.Embedding(input_dim=embedding_matrix.shape[0],
output_dim=embedding_matrix.shape[1],
weights=[embedding_matrix],
input_length=max_length,
trainable=False,
name='embedding_layer')(inputs)
conv1 = kl.Convolution1D(32, 3, activation='relu', input_shape=input_shape)(embeddings)
pool1 = kl.MaxPooling1D()(conv1)
conv2 = kl.Convolution1D(64, 3, activation='relu')(pool1)
pool2 = kl.MaxPooling1D()(conv2)
drop = kl.Dropout(0.2)(pool2)
attention_mul = attention_3d_block(drop, int(drop.shape[1]), True)
attention_mul = kl.LSTM(32, dropout=0.2, recurrent_dropout=0.2)(attention_mul)
output = kl.Dense(num_classes, activation='sigmoid')(attention_mul)
model = k.Model(input=[inputs], output=output)
return model
def cnn_model(num_classes, input_shape, embedding_matrix, max_length):
""" Creates CNN model for classification of emotions with Glove embeddings
:param num_classes: number of classes
:type num_classes: int
:param input_shape: shape of the input
:type input_shape: tuple
:param embedding_matrix: embedding matrix for Keras Embedding layer
:type embedding_matrix: numpy.array
:param max_length: maximum length of the text sequence
:type max_length: int
:return: CNN model
"""
model = k.Sequential()
model.add(kl.Embedding(input_dim=embedding_matrix.shape[0],
output_dim=embedding_matrix.shape[1],
weights=[embedding_matrix],
input_length=max_length,
trainable=False,
name='embedding_layer'))
model.add(kl.Convolution1D(32, 3, activation='relu', input_shape=input_shape))
model.add(kl.MaxPooling1D())
model.add(kl.Convolution1D(64, 3, activation='relu'))
model.add(kl.GlobalMaxPooling1D())
model.add(kl.Dense(128))
model.add(kl.Dropout(0.2))
model.add(kl.Activation('relu'))
model.add(kl.Dense(num_classes))
model.add(kl.Activation('sigmoid'))
return model
def model(reader):
inshape = (reader.max_len, )
known_in = L.Input(shape=inshape, name='known_input')
unknown_in = L.Input(shape=inshape, name='unknown_input')
embedding = L.Embedding(len(reader.vocabulary_above_cutoff) + 2,
5,
input_length=reader.max_len)
known_embed = embedding(known_in)
unknown_embed = embedding(unknown_in)
conv8 = L.Convolution1D(filters=500,
kernel_size=8,
strides=1,
activation='relu',
name='convolutional_8')
conv4 = L.Convolution1D(filters=500,
kernel_size=4,
strides=1,
activation='relu',
name='convolutional_4')
repr_known1 = L.GlobalMaxPooling1D(name='known_repr_8')(conv8(known_embed))
repr_unknown1 = L.GlobalMaxPooling1D(name='unknown_repr_8')(
conv8(unknown_embed))
repr_known2 = L.GlobalMaxPooling1D(name='known_repr_4')(conv4(known_embed))
repr_unknown2 = L.GlobalMaxPooling1D(name='unknown_repr_4')(
conv4(unknown_embed))
repr_known = L.Concatenate()([repr_known1, repr_known2])
repr_unknown = L.Concatenate()([repr_unknown1, repr_unknown2])
abs_diff = L.merge(inputs=[repr_known, repr_unknown],
mode=lambda x: abs(x[0] - x[1]),
output_shape=lambda x: x[0],
name='absolute_difference')
dense1 = L.Dense(500, activation='relu')(abs_diff)
dense2 = L.Dense(500, activation='relu')(dense1)
pruned = L.Dropout(0.3)(dense2)
output = L.Dense(2, activation='softmax', name='output')(pruned)
model = Model(inputs=[known_in, unknown_in], outputs=output)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def create_cnn(inp_size):
# Add an Input Layer
input_layer = layers.Input((inp_size, ))
# embedding layer learnt from above
embedding_layer = layers.Embedding(vocab_size, 200)(input_layer)
# add dropout on this layer
embedding_layer = layers.SpatialDropout1D(0.3)(embedding_layer)
# Add the convolutional Layer
conv_layer = layers.Convolution1D(100, 3,
activation="tanh")(embedding_layer)
# Add the pooling Layer
pooling_layer = layers.GlobalMaxPool1D()(conv_layer)
# Add the output Layers
output_layer1 = layers.Dense(50, activation="relu")(pooling_layer)
output_layer1 = layers.Dropout(0.25)(output_layer1)
output_layer2 = layers.Dense(1, activation="sigmoid")(output_layer1)
# Compile the model
model = models.Model(inputs=input_layer, outputs=output_layer2)
model.compile(optimizer=optimizers.Adam(),
loss='binary_crossentropy',
metrics=['accuracy'])
return model
def create_cnn():
# Adicione uma camada de entrada
input_layer = layers.Input((70, ))
# Adicione a camada de incorporação de palavras
embedding_layer = layers.Embedding(len(word_index) + 1,
300,
weights=[embedding_matrix],
trainable=False)(input_layer)
embedding_layer = layers.SpatialDropout1D(0.3)(embedding_layer)
# Adicione a camada convolucional
conv_layer = layers.Convolution1D(90, 3,
activation="relu")(embedding_layer)
# Adicione a camada de pooling máximo, pega maior valor resltande do mapa de ativação.
pooling_layer = layers.GlobalMaxPool1D()(conv_layer)
# Adicione as camadas de saída
# camada totalmente conectada para normalização dos dados.
output_layer1 = layers.Dropout(0.7)(pooling_layer)
output_layer2 = layers.Dense(1, activation="sigmoid")(output_layer1)
# Compile o modelo
model = models.Model(inputs=input_layer, outputs=output_layer2)
model.compile(optimizer=optimizers.Adamax(),
loss='binary_crossentropy',
metrics=['accuracy'])
return model
def createRCNN(word_index, embedding_matrix):
input_layer = layers.Input((1000, ))
embedding_layer = layers.Embedding(len(word_index) + 1,
300,
weights=[embedding_matrix],
trainable=False)(input_layer)
embedding_layer = layers.SpatialDropout1D(0.3)(embedding_layer)
rnn_layer = layers.Bidirectional(layers.GRU(
50, return_sequences=True))(embedding_layer)
conv_layer = layers.Convolution1D(100, 3, activation="relu")(rnn_layer)
pooling_layer = layers.GlobalMaxPool1D()(conv_layer)
output_layer1 = layers.Dense(50, activation="relu")(pooling_layer)
output_layer1 = layers.Dropout(0.25)(output_layer1)
output_layer2 = layers.Dense(1, activation="sigmoid")(output_layer1)
model = models.Model(inputs=input_layer, outputs=output_layer2)
model.compile(optimizer=optimizers.Adam(), loss='binary_crossentropy')
return model
def create_cnn():
# Add an Input Layer
input_layer = layers.Input((70, ))
# Add the word embedding Layer
embedding_layer = layers.Embedding(len(word_index) + 1,
300,
weights=[embedding_matrix],
trainable=False)(input_layer)
embedding_layer = layers.SpatialDropout1D(0.3)(embedding_layer)
# Add the convolutional Layer
conv_layer = layers.Convolution1D(100, 3,
activation="relu")(embedding_layer)
# Add the pooling Layer
pooling_layer = layers.GlobalMaxPool1D()(conv_layer)
# Add the output Layers
output_layer1 = layers.Dense(50, activation="relu")(pooling_layer)
output_layer1 = layers.Dropout(0.25)(output_layer1)
output_layer2 = layers.Dense(1, activation="sigmoid")(output_layer1)
# Compile the model
model = models.Model(inputs=input_layer, outputs=output_layer2)
model.compile(optimizer=optimizers.Adam(), loss='binary_crossentropy')
return model
def create_rcnn(self):
# add an input layer
input_layer = layers.Input((70, ))
# add the word embedding layer
embedding_layer = layers.Embedding(len(self.word_index) + 1,
300,
weights=[self.embedding_matrix],
trainable=False)(input_layer)
embedding_layer = layers.SpatialDropout1D(0.3)(embedding_layer)
# add the recurrent layer
rnn_layer = layers.Bidirectional(layers.GRU(
50, return_sequences=True))(embedding_layer)
# add the convolutional layer
conv_layer = layers.Convolution1D(100, 3, activation="relu")(rnn_layer)
# add the pooling layer
pooling_layer = layers.GlobalMaxPool1D()(conv_layer)
# Add the output Layers
output_layer1 = layers.Dense(50, activation="relu")(pooling_layer)
output_layer1 = layers.Dropout(0.25)(output_layer1)
output_layer2 = layers.Dense(1, activation="sigmoid")(output_layer1)
# Compile the model
model = models.Model(inputs=input_layer, outputs=output_layer2)
model.compile(optimizer=optimizers.Adam(), loss='binary_crossentropy')
return model
def create_rcnn():
# Add an Input Layer
input_layer = layers.Input((70, ))
# Add the word embedding Layer
embedding_layer = layers.Embedding(len(word_index) + 1, 300, weights=[embedding_matrix], trainable=False)(input_layer)
embedding_layer = layers.SpatialDropout1D(0.3)(embedding_layer)
# Add the recurrent layer
layers.Bidirectional(layers.GRU(50, return_sequences=True))(embedding_layer)
# Add the convolutional Layer
conv_layer = layers.Convolution1D(100, 3, activation="relu")(embedding_layer)
# Add the pooling Layer
pooling_layer = layers.GlobalMaxPool1D()(conv_layer)
# Add the output Layers
output_layer1 = layers.Dense(50, activation="relu")(pooling_layer)
output_layer1 = layers.Dropout(0.25)(output_layer1)
output_layer2 = layers.Dense(units=max(training_label_encoded) + 1, activation="softmax", name="ouput_layer")(output_layer1)
# Compile the model
model = models.Model(inputs=input_layer, outputs=output_layer2)
model.compile(optimizer=optimizers.Adam(), loss='sparse_categorical_crossentropy', metrics=["sparse_categorical_accuracy"])
return model
def cnn(xtrain, ytrain, xvalid, yvalid, epochs=3):
# Add an Input Layer
input_layer = layers.Input((70, ))
# Add the word embedding Layer
embedding_layer = layers.Embedding(len(word_index) + 1,
300,
weights=[embedding_matrix],
trainable=False)(input_layer)
embedding_layer = layers.SpatialDropout1D(0.3)(embedding_layer)
# Add the convolutional Layer
conv_layer = layers.Convolution1D(100, 4,
activation="relu")(embedding_layer)
# Add the pooling Layer
pooling_layer = layers.GlobalMaxPool1D()(conv_layer)
# Add the output Layers
output_layer1 = layers.Dense(50, activation="relu")(pooling_layer)
output_layer1 = layers.Dropout(0.25)(output_layer1)
output_layer2 = layers.Dense(4, activation="softmax")(output_layer1)
# Compile the model
model = models.Model(inputs=input_layer, outputs=output_layer2)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(xtrain, ytrain, batch_size=256, epochs=epochs)
predictions = model.predict(xvalid)
predictions = predictions.argmax(axis=-1)
accuracy = model.evaluate(xvalid, yvalid, verbose=0)
f1score = metrics.f1_score(valid_y, predictions, average='weighted')
return accuracy, f1score
def model(reader):
inshape = (reader.max_len, )
known_in = L.Input(shape=inshape, name='known_input')
unknown_in = L.Input(shape=inshape, name='unknown_input')
embedding = L.Embedding(
len(reader.vocabulary_above_cutoff) + 2,
5,
input_length=reader.max_len)
conv = L.Convolution1D(
filters=1000,
kernel_size=10,
strides=1,
activation='relu',
name='convolution_10')
repr_known = L.GlobalMaxPooling1D(name='repr_known')(conv(
embedding(known_in)))
repr_unknown = L.GlobalMaxPooling1D(name='repr_unknown')(conv(
embedding(unknown_in)))
full_input = L.Concatenate()([repr_known, repr_unknown])
dense = L.Dense(500, activation='relu')(full_input)
output = L.Dense(2, activation='softmax', name='output')(dense)
model = Model(inputs=[known_in, unknown_in], outputs=output)
model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def residual_block(x):
original_x = x
# TODO: initalization, regularization?
# Note: The AtrousConvolution1D with the 'causal' flag is implemented in github.com/basveeling/keras#@wavenet.
tanh_out = CausalAtrousConvolution1D(nb_filters,
2,
atrous_rate=2**i,
border_mode='valid',
causal=True,
bias=use_bias,
name='dilated_conv_%d_tanh_s%d' %
(2**i, s),
activation='tanh')(x)
x = layers.Dropout(0.2)(x)
sigm_out = CausalAtrousConvolution1D(nb_filters,
2,
atrous_rate=2**i,
border_mode='valid',
causal=True,
bias=use_bias,
name='dilated_conv_%d_sigm_s%d' %
(2**i, s),
activation='sigmoid')(x)
x = layers.Merge(mode='mul', name='gated_activation_%d_s%d' %
(i, s))([tanh_out, sigm_out])
res_x = layers.Convolution1D(nb_filters,
1,
border_mode='same',
bias=use_bias)(x)
res_x = layers.Merge(mode='sum')([original_x, res_x])
return res_x
def build_model(fragment_length, nb_filters, nb_output_bins, dilation_depth, nb_stacks, use_skip_connections,
learn_all_outputs, _log, desired_sample_rate, use_bias):
def residual_block(x):
original_x = x
# TODO: initalization, regularization?
# Note: The AtrousConvolution1D with the 'causal' flag is implemented in github.com/basveeling/keras#@wavenet.
tanh_out = layers.AtrousConvolution1D(nb_filters, 2, atrous_rate=2 ** i, border_mode='valid', causal=True,
bias=use_bias,
name='dilated_conv_%d_tanh_s%d' % (2 ** i, s), activation='tanh')(x)
sigm_out = layers.AtrousConvolution1D(nb_filters, 2, atrous_rate=2 ** i, border_mode='valid', causal=True,
bias=use_bias,
name='dilated_conv_%d_sigm_s%d' % (2 ** i, s), activation='sigmoid')(x)
x = layers.Merge(mode='mul', name='gated_activation_%d_s%d' % (i, s))([tanh_out, sigm_out])
x = layers.Convolution1D(nb_filters, 1, border_mode='same', bias=use_bias)(x)
skip_out = x
x = layers.Merge(mode='sum')([original_x, x])
return x, skip_out
input = Input(shape=(fragment_length, nb_output_bins), name='input_part')
out = input
skip_connections = []
out = layers.AtrousConvolution1D(nb_filters, 2, atrous_rate=1, border_mode='valid', causal=True,
name='initial_causal_conv')(out)
for s in xrange(nb_stacks):
for i in xrange(0, dilation_depth + 1):
out, skip_out = residual_block(out)
skip_connections.append(skip_out)
if use_skip_connections:
out = layers.Merge(mode='sum')(skip_connections)
out = layers.Activation('relu')(out)
out = layers.Convolution1D(nb_output_bins, 1, border_mode='same')(out)
out = layers.Activation('relu')(out)
out = layers.Convolution1D(nb_output_bins, 1, border_mode='same')(out)
if not learn_all_outputs:
raise DeprecationWarning('Learning on just all outputs is wasteful, now learning only inside receptive field.')
out = layers.Lambda(lambda x: x[:, -1, :], output_shape=(out._keras_shape[-1],))(
out) # Based on gif in deepmind blog: take last output?
out = layers.Activation('softmax', name="output_softmax")(out)
model = Model(input, out)
receptive_field, receptive_field_ms = compute_receptive_field()
_log.info('Receptive Field: %d (%dms)' % (receptive_field, int(receptive_field_ms)))
return model
def cnn_2x_rnn(voc_size, max_len, dropout=0.5):
statement_input = layers.Input(shape=(max_len, ), dtype='int32')
x = layers.Embedding(output_dim=256,
input_dim=voc_size,
input_length=max_len)(statement_input)
x = layers.Convolution1D(256, 7, activation='relu')(x)
x = layers.MaxPooling1D(3)(x)
x = layers.Convolution1D(256, 7, activation='relu')(x)
x = layers.MaxPooling1D(5)(x)
embedded_statement = layers.SimpleRNN(256)(x)
x = layers.Dense(256, activation='relu')(embedded_statement)
x = layers.Dropout(dropout)(x)
prediction = layers.Dense(1, activation='sigmoid')(x)
model = Model(statement_input, prediction)
return model
def stack_conv(prev, param: Tuple[str, int, int, float, str]):
name, nfilt, kern_size, drop_p, pad = param
l = layers.Convolution1D(nfilt,
kern_size,
activation="relu",
name=name,
padding=pad)(prev)
return layers.Dropout(drop_p)(l) if drop_p else l
def stack_conv(prev: layers.Layer, param: Tuple[str, int, int]):
name, nfilt, kern_size = param
return layers.Convolution1D(
nfilt,
kern_size,
activation="relu",
name=name,
)(prev)
def make_embedding(self, x):
with self.graph.as_default():
# 3 convolutional layers, with max pooling in the last layer
x = layers.Convolution1D(1024,
5,
border_mode='valid',
subsample_length=1,
activation='relu')(x)
x = layers.Convolution1D(1024,
5,
border_mode='valid',
subsample_length=1,
activation='relu')(x)
x = layers.Convolution1D(self.embedding_size,
5,
border_mode='valid',
subsample_length=1,
activation='relu')(x)
x = layers.MaxPooling1D(2, stride=2)(x)
# Use an LSTM to finish off
initializer = tf.truncated_normal_initializer(stddev=0.01,
seed=1337)
cell = tf.contrib.rnn.LSTMCell(self.embedding_size,
use_peepholes=False,
initializer=initializer,
num_proj=None,
num_unit_shards=1,
num_proj_shards=1,
forget_bias=1.0,
state_is_tuple=False)
x, _ = tf.nn.dynamic_rnn(cell,
x,
sequence_length=None,
initial_state=None,
dtype='float32',
parallel_iterations=32,
swap_memory=False)
last_timestep = tf.shape(x)[1]
indices = tf.stack([0, last_timestep - 1, 0])
indices = tf.cast(indices, 'int32')
embedded = tf.slice(x, indices, [-1, 1, -1])
embedded = tf.squeeze(embedded, [1])
embedded.set_shape((None, self.embedding_size))
return embedded
def custom_neural(self):
first = layers.Input(shape=(None, 257))
lyr = first
lyr = layers.Dense(512, activation='sigmoid')(lyr)
lyr = layers.Convolution1D(kernel_size=5,
filters=512,
activation='relu')(lyr)
lyr = layers.Convolution1D(kernel_size=5,
filters=512,
activation='relu')(lyr)
lyr = layers.LSTM(512, return_sequences=True)(lyr)
lyr = layers.LSTM(512, return_sequences=True,
activation='sigmoid')(lyr)
lyr = layers.Lambda(
lambda x: K.stack([x[:, :, :256], x[:, :, 256:]], axis=-1))(lyr)
lyr = layers.Lambda(lambda x: tf.pad(x, ((0, 0), (0, 0), (1, 0),
(0, 0))))(lyr)
lyr = layers.Lambda(lambda x: x[0] * K.stack([x[1], x[1]], axis=-1))(
[lyr, first])
return keras.models.Model(first, lyr)
def cnn_2x_encdec(voc_size, max_len, dropout=0.5):
statement_input = layers.Input(shape=(max_len, ), dtype='int32')
x = layers.Embedding(output_dim=256,
input_dim=voc_size,
input_length=max_len)(statement_input)
x = layers.Convolution1D(256, 7, activation='relu')(x)
x = layers.MaxPooling1D(3)(x)
x = layers.Convolution1D(256, 7, activation='relu')(x)
x = layers.MaxPooling1D(5)(x)
x = layers.GRU(256)(x) #this acts as the encoder
x = layers.RepeatVector(256)(
x) #take the last output of GRU and feed it to the decoder
embedded_statement = layers.GRU(256)(x) #this acts as the decoder
x = layers.Dense(256, activation='tanh')(embedded_statement)
x = layers.Dropout(dropout)(x)
prediction = layers.Dense(1, activation='sigmoid')(x)
model = Model(statement_input, prediction)
return model
def build_model(embed_len, word_len, use_rnn=False, use_cosine=True):
"""Builds the Keras model.
Args:
embed_len: int, the length of the embeddings vector.
word_len: int, the maximum word length.
use_rnn: bool, if set, use the RNN model, else use the CNN model.
use_cosine: bool, if set, use cosine distance, else use MSE.
Returns:
the compiled model for training.
"""
model = models.Sequential()
model.add(
layers.Embedding(
1 + ord('z'),
1 + ord('z'),
init='orthogonal',
trainable=False,
# mask_zero=use_rnn,
input_length=word_len))
if use_rnn:
model.add(layers.Bidirectional(layers.GRU(512)))
model.add(layers.Dense(embed_len, init='glorot_normal'))
else:
model.add(layers.Convolution1D(1000, 5))
model.add(layers.BatchNormalization())
model.add(layers.Activation('tanh'))
model.add(layers.Convolution1D(100, 1))
model.add(layers.BatchNormalization())
model.add(layers.Activation('tanh'))
model.add(layers.Flatten())
model.add(
layers.Dense(embed_len, W_regularizer='l2', init='glorot_normal'))
if use_cosine:
model.add(layers.Activation('tanh'))
model.compile(optimizer='nadam', loss='cosine' if use_cosine else 'mse')
return model
def create_cnn():
input_layer = layers.Input((70,))
embedding_layer = layers.Embedding(len(word_index)+1, 300, weights=[embedding_matrix], trainable=False)(input_layer)
embedding_layer = layers.SpatialDropout1D(0.3)(embedding_layer)
conv_layer = layers.Convolution1D(100, 3, activation='relu')(embedding_layer)
pooling_layer = layers.GlobalMaxPool1D()(conv_layer)
output_layer1 = layers.Dense(50, activation='relu')(pooling_layer)
output_layer1 = layers.Dropout(0.25)(output_layer1)
output_layer2 = layers.Dense(1, activation='sigmoid')(output_layer1)
model = models.Model(inputs=input_layer, outputs=output_layer2)
model.compile(optimizer=optimizers.Adam(), loss='binary_crossentropy')
return model
def residual_block(x):
original_x = x
# TODO: initalization, regularization?
# Note: The AtrousConvolution1D with the 'causal' flag is implemented in github.com/basveeling/keras#@wavenet.
tanh_out = CausalAtrousConvolution1D(nb_filters,
2,
dilation_rate=2**i,
padding='valid',
causal=True,
use_bias=use_bias,
name='dilated_conv_%d_tanh_s%d' %
(2**i, s),
activation='tanh',
kernel_regularizer=l2(res_l2))(x)
sigm_out = CausalAtrousConvolution1D(nb_filters,
2,
dilation_rate=2**i,
padding='valid',
causal=True,
use_bias=use_bias,
name='dilated_conv_%d_sigm_s%d' %
(2**i, s),
activation='sigmoid',
kernel_regularizer=l2(res_l2))(x)
x = layers.Multiply(name='gated_activation_%d_s%d' %
(i, s))([tanh_out, sigm_out])
res_x = layers.Convolution1D(nb_filters,
1,
padding='same',
use_bias=use_bias,
kernel_regularizer=l2(res_l2))(x)
skip_x = layers.Convolution1D(nb_filters,
1,
padding='same',
use_bias=use_bias,
kernel_regularizer=l2(res_l2))(x)
res_x = layers.Add()([original_x, res_x])
return res_x, skip_x
def make_embedding(self, x):
with self.graph.as_default():
x = layers.Convolution1D(256,
1,
border_mode='valid',
subsample_length=1,
activation='relu')(x)
x = layers.Convolution1D(256,
5,
border_mode='valid',
subsample_length=2,
activation='relu')(x)
x = layers.Convolution1D(256,
5,
border_mode='valid',
subsample_length=2,
activation='relu')(x)
x = layers.Convolution1D(self.embedding_size,
5,
border_mode='valid',
activation='relu')(x)
x = tf.reduce_max(x, reduction_indices=1)
return x
def model(reader):
inshape = (None, )
known_in = L.Input(shape=inshape, name='known_input', dtype='int32')
unknown_in = L.Input(shape=inshape, name='unknown_input', dtype='int32')
embedding = L.Embedding(
len(reader.channels[0].vocabulary_above_cutoff) + 2, 5)
known_embed = embedding(known_in)
unknown_embed = embedding(unknown_in)
conv8 = L.Convolution1D(filters=5000,
kernel_size=8,
strides=1,
activation='relu',
name='convolutional_8')
repr_known = L.GlobalMaxPooling1D()(conv8(known_embed))
repr_unknown = L.GlobalMaxPooling1D()(conv8(unknown_embed))
abs_diff = L.merge(inputs=[repr_known, repr_unknown],
mode=lambda x: abs(x[0] - x[1]),
output_shape=lambda x: x[0],
name='absolute_difference')
dense1 = L.Dense(500, activation='relu')(abs_diff)
dense2 = L.Dense(500, activation='relu')(dense1)
dense3 = L.Dense(500, activation='relu')(dense2)
dense4 = L.Dense(500, activation='relu')(dense3)
dense5 = L.Dense(500, activation='relu')(dense4)
pruned = L.Dropout(0.3)(dense5)
output = L.Dense(2, activation='softmax', name='output')(pruned)
model = Model(inputs=[known_in, unknown_in], outputs=output)
optimizer = O.Adam(lr=0.0005)
model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def model(reader):
inshape = (None, )
known_in = L.Input(shape=inshape, dtype='int32')
unknown_in = L.Input(shape=inshape, dtype='int32')
embedding = L.Embedding(len(reader.vocabulary_above_cutoff) + 2, 5)
known_emb = embedding(known_in)
unknown_emb = embedding(unknown_in)
conv8 = L.Convolution1D(filters=500,
kernel_size=8,
strides=1,
activation='relu',
padding='same')
pool = L.MaxPooling1D(pool_size=8)
if 'device:GPU' in str('str(device_lib.list_local_devices())'):
gru = L.CuDNNGRU(100)
else:
gru = L.GRU(200)
features_known = gru(pool(conv8(known_emb)))
features_unknown = gru(pool(conv8(unknown_emb)))
abs_diff = L.merge(inputs=[features_known, features_unknown],
mode=lambda x: abs(x[0] - x[1]),
output_shape=lambda x: x[0],
name='absolute_difference')
dense1 = L.Dense(500, activation='relu')(abs_diff)
pruned = L.Dropout(0.3)(dense1)
output = L.Dense(2, activation='softmax', name='output')(pruned)
model = Model(inputs=[known_in, unknown_in], outputs=output)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def multi_input_model(vocab_size, k_reg=0):
filter_sizes = (3, 8)
k_regularizer = keras.regularizers.l2(k_reg)
text_input = layers.Input(shape=(16,), name='text_input')
embedding = layers.Embedding(input_dim=vocab_size,
output_dim=128,
input_length=16)(text_input)
#x = keras.layers.Flatten()(embedding)
conv_blocks = []
for sz in filter_sizes:
conv = layers.Convolution1D(filters=64,
kernel_size=sz,
padding="valid",
activation="relu",
strides=1)(embedding)
conv = layers.MaxPooling1D(pool_size=2)(conv)
conv = layers.Flatten()(conv)
conv_blocks.append(conv)
text_out = layers.Concatenate()(conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]
text_output = layers.Dense(N_CLASSES, activation='softmax', name='text_output', kernel_regularizer=k_regularizer)(text_out)
#img_input = layers.Input(shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3))
image_model = keras.models.load_model(IMAGE_MODEL_PATH)
image_model.layers.pop() # remove fully connected layers
image_model.layers.pop() # remove fully connected layers
image_model.layers.pop() # remove pooling
flatten = layers.Flatten()(image_model.layers[-3].output)
new_image_model = keras.models.Model(inputs=image_model.input, outputs=flatten)
print(new_image_model.summary)
for layer in new_image_model.layers[int(len(new_image_model.layers) * 2/3):]:
layer.trainable = False
# test with global max pooling as well
concat = keras.layers.concatenate([text_out, new_image_model.output])
x = layers.Dropout(0.2)(concat)
x = layers.Dense(128, activation=None, kernel_initializer='he_uniform',
kernel_regularizer=k_regularizer)(x)
final_output = layers.Dense(N_CLASSES, activation='softmax',
name='final_output', kernel_regularizer=k_regularizer)(x)
mul_inp_model = keras.models.Model(inputs=[new_image_model.input, text_input],
outputs=[final_output, text_output])
return mul_inp_model
