def multi_task_CNN_only_weights():
# Text input
input_for_dimensions = Input(shape=(MAX_SEQUENCE_LENGTH, ),
name='input_dim')
input_for_emotions = Input(shape=(MAX_SEQUENCE_LENGTH, ),
name='input_demo')
embedding1 = Embedding(vocab_size,
embedding_size,
embeddings_initializer=Constant(embedding_matrix),
name='embedding1',
trainable=False)(input_for_dimensions)
# embedding2 = Embedding(vocab_size, embedding_size, embeddings_initializer=Constant(embedding_matrix), name='embedding2', trainable=False)(input_for_emotions)
expend_shape = [
embedding1.get_shape().as_list()[1],
embedding1.get_shape().as_list()[2], 1
]
embedding_chars1 = Reshape(expend_shape)(embedding1)
# expend_shape = [embedding2.get_shape().as_list()[1], embedding2.get_shape().as_list()[2], 1]
# embedding_chars2 = Reshape(expend_shape)(embedding2)
# Shared layers
pooled_outputs = []
for i, filter_size in enumerate(filter_sizes):
conv = Conv2D(filters=num_filters,
kernel_size=[filter_size, embedding_size],
activation='relu',
name=('conv_%d' % filter_size))(embedding_chars1)
# encoded_a = conv(embedding_chars1)
# encoded_b = conv(embedding_chars1)
# merged_conv = keras.layers.concatenate([encoded_b, encoded_a], axis=-1)
max_pool = MaxPool2D(pool_size=[sequence_length - filter_size + 1, 1],
strides=(2, 2),
padding='valid',
name=('max_pool_%d' % filter_size))(conv)
# max_pool = Dropout (0.5)(max_pool)
pooled_outputs.append(max_pool)
num_filters_total = num_filters * len(filter_sizes)
h_pool = Concatenate(axis=3)(pooled_outputs)
merged = Reshape([num_filters_total])(h_pool)
merged1 = Dropout(0.5)(merged)
emo_output = Dense(len(LABELS), activation='softmax', name='emo')(merged1)
seq_output = Dense(len(DIMENSIONS), activation='sigmoid',
name='dim')(merged1)
multitask_model = Model(inputs=[input_for_emotions, input_for_dimensions],
outputs=[emo_output, seq_output])
multitask_model.compile(loss='categorical_crossentropy',
optimizer=OPTIMIZER,
metrics=['accuracy'],
loss_weights=[1, 0.25])
return multitask_model
def text_cnn_model_baseline(sequence_length, vocab_size, embedding_size,
filter_sizes, num_filters, embedding_matrix,
drop_out, LABELS, activation):
input = Input(shape=(sequence_length, ), name='input_text')
embedding = Embedding(vocab_size,
embedding_size,
embeddings_initializer=Constant(embedding_matrix),
name='glove300_embedding',
trainable=False)(input)
expend_shape = [
embedding.get_shape().as_list()[1],
embedding.get_shape().as_list()[2], 1
]
embedding = Reshape(expend_shape)(embedding)
# conv->max pool
pooled_outputs = []
for i, filter_size in enumerate(filter_sizes):
# embedding = Dropout(0.2)(embedding)
conv = Conv2D(filters=num_filters,
kernel_size=[filter_size, embedding_size],
activation='relu',
name=('conv_filtersize%d' % filter_size))(embedding)
# conv = Dropout(0.5)(conv)
max_pool = MaxPool2D(pool_size=[sequence_length - filter_size + 1, 1],
strides=(2, 2),
padding='valid',
name=('max_pool_%d' % filter_size))(conv)
# max_pool = Dropout(drop_out)(max_pool)
pooled_outputs.append(max_pool)
# combine all the pooled features
num_filters_total = num_filters * len(filter_sizes)
h_pool = Concatenate(axis=3)(pooled_outputs)
# h_pool = Dropout(0.5, name='0.5')(h_pool)
h_pool_flat = Reshape([num_filters_total])(h_pool)
fullyconnected = Dense(128, name='fullyconnected2',
activation='relu')(h_pool_flat)
dropout = Dropout(drop_out, name='0.5-')(fullyconnected)
# output layer
output = Dense(len(LABELS),
kernel_initializer='glorot_normal',
bias_initializer=keras.initializers.constant(0.1),
activation=activation,
name='output_emotion')(dropout)
model = Model(input, output)
return model
def text_cnn_model_appraisals(sequence_length, vocab_size, embedding_size,
filter_sizes, num_filters, embedding_matrix,
drop_out, output_count, activation):
input = Input(shape=(sequence_length, ), name='input_text')
try:
embedding = Embedding(
vocab_size,
embedding_size,
embeddings_initializer=Constant(embedding_matrix),
name='glove300_embedding',
trainable=False)(input)
except TypeError:
print('\nERROR: Embedding does not fit dataset:')
print(
' Datasets contains %d terms and embedding contains %d terms...'
% (vocab_size, embedding_matrix.shape[0]))
print('\nExiting.')
sys.exit(1)
except:
print('\nUnexpected error:', sys.exc_info()[1])
sys.exit(1)
expend_shape = [
embedding.get_shape().as_list()[1],
embedding.get_shape().as_list()[2], 1
]
embedding = Reshape(expend_shape)(embedding)
# conv->max pool
pooled_outputs = []
for i, filter_size in enumerate(filter_sizes):
# embedding = Dropout(0.2)(embedding)
conv = Conv2D(filters=num_filters,
kernel_size=[filter_size, embedding_size],
activation='relu',
name=('conv_filtersize%d' % filter_size))(embedding)
# conv = Dropout(0.5)(conv)
max_pool = MaxPool2D(pool_size=[sequence_length - filter_size + 1, 1],
strides=(2, 2),
padding='valid',
name=('max_pool_%d' % filter_size))(conv)
# max_pool = Dropout(drop_out)(max_pool)
pooled_outputs.append(max_pool)
# combine all the pooled features
num_filters_total = num_filters * len(filter_sizes)
h_pool = Concatenate(axis=3)(pooled_outputs)
# h_pool = Dropout(drop_out, name='0.5')(h_pool)
h_pool_flat = Reshape([num_filters_total])(h_pool)
fullyconnected = Dense(128, name='fullyconnected2',
activation='relu')(h_pool_flat)
dropout = Dropout(drop_out, name='d_out')(fullyconnected)
# output layer
output = Dense(output_count,
kernel_initializer='glorot_normal',
bias_initializer=keras.initializers.constant(0.1),
activation=activation,
name='output_emotion')(dropout)
model = Model(input, output)
return model
def draw_cnn_model(hyper_param, embedding_matrix=None, verbose=True):
"""
Input: hyper_parameters dictionary
Construct:
input layers : x , x_pos(o), x_captialization(o)
embedding matrix : use_glove or randomly initialize
conv1 : first convolution layer
conv2 : second convolution layer
conv3 : third convolution layer
max pooling
flatten : concant maxpooled univariate vectors into one long vector
ff1, ff2: two feed forward layers
out_pred: softmax over all ner classes
Returns: keras.models.Model object
"""
# input layer(s)
x = Input(shape=(hyper_param['maxlen'], ), name='x')
if hyper_param['use_pos_tags']:
x_pos = Input(shape=(hyper_param['maxlen'], hyper_param['poslen']),
name='x_pos')
if hyper_param['use_capitalization_info']:
x_capital = Input(shape=(hyper_param['maxlen'],
hyper_param['capitallen']),
name='x_capital')
# embedding matrix
if hyper_param['use_glove']:
embed = Embedding(hyper_param['max_features'], hyper_param['embed_dim'], weights=[embedding_matrix],\
input_length=hyper_param['maxlen'], trainable=hyper_param['allow_glove_retrain'])(x)
else:
embed = Embedding(hyper_param['max_features'], hyper_param['embed_dim'], input_length=hyper_param['maxlen'],\
embeddings_initializer="random_uniform" )(x)
# concat embeddings with additional features
if hyper_param['use_pos_tags'] and hyper_param['use_capitalization_info']:
embed = Concatenate(axis=-1)([embed, x_pos, x_capital])
elif hyper_param['use_pos_tags'] and (
not hyper_param['use_capitalization_info']):
embed = Concatenate(axis=-1)([embed, x_pos])
elif (not hyper_param['use_pos_tags']
) and hyper_param['use_capitalization_info']:
embed = Concatenate(axis=-1)([embed, x_capital])
else:
embed = embed
# feed embeddings into conv1
conv1 = Conv1D( filters=hyper_param['conv1_filters'], \
kernel_size=hyper_param['conv1_kernel_size'],\
strides=hyper_param['conv1_strides'], \
padding=hyper_param['conv1_padding'],\
activation='relu', name='conv1')(embed)
# update this
# make primary capsules
conv2 = Conv1D( filters=hyper_param['conv2_filters'], \
kernel_size=hyper_param['conv2_kernel_size'],\
strides=hyper_param['conv2_strides'], \
padding=hyper_param['conv2_padding'],\
activation='relu', name='conv2')(conv1)
# update this
# make primary capsules
conv3 = Conv1D( filters=hyper_param['conv3_filters'], \
kernel_size=hyper_param['conv3_kernel_size'],\
strides=hyper_param['conv3_strides'], \
padding=hyper_param['conv3_padding'],\
activation='relu', name='conv3')(conv2)
# max pooling layer
max_pooled = MaxPooling1D(pool_size=hyper_param['max_pooling_size'], \
strides=hyper_param['max_pooling_strides'], \
padding=hyper_param['max_pooling_padding'])(conv3)
# dropout
maxpooled_dropout = Dropout(hyper_param['maxpool_dropout'])(max_pooled)
# flatten many univariate vectos into 1 long vector
flattened = Flatten()(maxpooled_dropout)
# to feed-forward layers
ff1 = Dense(hyper_param['feed_forward_1'], activation='relu')(flattened)
ff1_dropout = Dropout(hyper_param['ff1_dropout'])(ff1)
ff2 = Dense(hyper_param['feed_forward_2'], activation='relu')(ff1_dropout)
ff2_dropout = Dropout(hyper_param['ff2_dropout'])(ff2)
out_pred = Dense(hyper_param['ner_classes'],
activation='softmax',
name='out_pred')(ff2) #!
if verbose:
print("x", x.get_shape())
if hyper_param['use_pos_tags']: print("x_pos", x_pos.get_shape())
if hyper_param['use_capitalization_info']:
print("x_capital", x_capital.get_shape())
print("embed", embed.get_shape())
print("embed", embed.get_shape())
print("conv1", conv1.get_shape())
print("conv2", conv2.get_shape())
print("conv3", conv3.get_shape())
print("max_pooled", max_pooled.get_shape())
print("flattened", flattened.get_shape())
print("ff1", ff1.get_shape())
print("ff2", ff2.get_shape())
print("out_pred", out_pred.get_shape())
# return final model
if hyper_param['use_pos_tags'] and hyper_param['use_capitalization_info']:
cnnmodel = Model(inputs=[x, x_pos, x_capital], outputs=[out_pred])
elif hyper_param['use_pos_tags'] and (
not hyper_param['use_capitalization_info']):
cnnmodel = Model(inputs=[x, x_pos], outputs=[out_pred])
elif (not hyper_param['use_pos_tags']
) and hyper_param['use_capitalization_info']:
cnnmodel = Model(inputs=[x, x_capital], outputs=[out_pred])
else:
cnnmodel = Model(inputs=[x], outputs=[out_pred])
return cnnmodel
'''
lx = len(embedding_matrix)
inp = Input(shape=(maxlen, ))
x = Embedding(lx, embed_size, weights=[embedding_matrix])(inp)
x = Conv1D(filters=32, kernel_size=3, padding='same', activation='relu')(x)
x = MaxPooling1D(pool_size=2)(x)
x = LSTM(300, return_sequences=True)(x)
#x = AttentionDecoder(300,maxlen)(x)
#x = GlobalMaxPool1D()(x)
#x = Dense(300, activation="relu")(x)
#x = Dense(100, activation="relu")(x)
#x = Dropout(0.1)(x)
#ylayer=numpy.asarray(ylayer)
print(x.get_shape())
xa = TimeDistributed(Dense(3, activation="tanh"))(x)
xa = Flatten()(xa)
xa = Activation('softmax')(xa)
xa = RepeatVector(300)(xa)
print(xa.get_shape())
xa = Permute([2, 1])(xa)
print(xa.get_shape())
ot = Multiply()([x, xa])
ot = Lambda(lambda xin: K.sum(xin, axis=-1))(ot)
ol = TimeDistributed(Dense(3, activation="sigmoid"))(ot)
model = Model(inputs=inp, outputs=ol)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
def draw_capsnet_model(hyper_param, embedding_matrix=None, verbose=True):
"""
Input: hyper parameters dictionary
Construct:
input layers : x , x_pos(o), x_captialization(o)
embedding matrix : use_glove or randomly initialize
conv1 : first convolution layer
primarycaps : conv2 and squash function applied
ner_caps : make 8 ner capsules of specified dim
out_pred : calc length of 8 ner capsules as 8 prob. predictions over 8 ner classes
Returns:
if decoding/reconstruction disabled --> a single keras.models.Model object
if decoding/reconstruction enabled --> three keras.models.Model objects
"""
# input layer(s)
x = Input(shape=(hyper_param['maxlen'], ), name='x')
if hyper_param['use_pos_tags']:
x_pos = Input(shape=(hyper_param['maxlen'], hyper_param['poslen']),
name='x_pos')
if hyper_param['use_capitalization_info']:
x_capital = Input(shape=(hyper_param['maxlen'],
hyper_param['capitallen']),
name='x_capital')
# embedding matrix
if hyper_param['use_glove']:
embed = Embedding(hyper_param['max_features'], hyper_param['embed_dim'], weights=[embedding_matrix],\
input_length=hyper_param['maxlen'], trainable=hyper_param['allow_glove_retrain'])(x)
else:
embed = Embedding(hyper_param['max_features'], hyper_param['embed_dim'], input_length=hyper_param['maxlen'],\
embeddings_initializer="random_uniform" )(x)
# concat embeddings with additional features
if hyper_param['use_pos_tags'] and hyper_param['use_capitalization_info']:
embed = Concatenate(axis=-1)([embed, x_pos, x_capital])
elif hyper_param['use_pos_tags'] and (
not hyper_param['use_capitalization_info']):
embed = Concatenate(axis=-1)([embed, x_pos])
elif (not hyper_param['use_pos_tags']
) and hyper_param['use_capitalization_info']:
embed = Concatenate(axis=-1)([embed, x_capital])
else:
embed = embed
# add dropout here
if hyper_param['embed_dropout'] > 0.0:
embed = SpatialDropout1D(hyper_param['embed_dropout'])(embed)
# feed embeddings into conv1
conv1 = Conv1D( filters=hyper_param['conv1_filters'], \
kernel_size=hyper_param['conv1_kernel_size'],\
strides=hyper_param['conv1_strides'], \
padding=hyper_param['conv1_padding'],\
activation='relu', name='conv1')(embed)
# make primary capsules
if hyper_param['use_2D_primarycaps']:
convShape = conv1.get_shape().as_list()
conv1 = Reshape((convShape[1], convShape[2], 1))(conv1)
primaryCapLayer = PrimaryCap
else:
primaryCapLayer = PrimaryCap1D
# make primary capsules
primarycaps = primaryCapLayer(conv1, \
dim_capsule=hyper_param['primarycaps_dim_capsule'],\
n_channels=hyper_param['primarycaps_n_channels'],\
kernel_size=hyper_param['primarycaps_kernel_size'], \
strides=hyper_param['primarycaps_strides'], \
padding=hyper_param['primarycaps_padding'])
# make ner capsules
ner_caps = CapsuleLayer(num_capsule=hyper_param['ner_classes'], \
dim_capsule=hyper_param['ner_capsule_dim'], \
routings=hyper_param['num_dynamic_routing_passes'], \
name='nercaps')(primarycaps)
# replace each ner capsuel with its length
out_pred = Length(name='out_pred')(ner_caps)
if verbose:
print("x", x.get_shape())
if hyper_param['use_pos_tags']: print("x_pos", x_pos.get_shape())
if hyper_param['use_capitalization_info']:
print("x_capital", x_capital.get_shape())
print("embed", embed.get_shape())
print("conv1", conv1.get_shape())
print("primarycaps", primarycaps.get_shape())
print("ner_caps", ner_caps.get_shape())
print("out_pred", out_pred.get_shape())
if hyper_param['use_decoder']:
decoder_y_cat = Input(shape=(hyper_param['ner_classes'], ),
name='decoder_y_cat')
masked_by_y = Mask(name='masked_by_y')(
[ner_caps,
decoder_y_cat]) # true label is used to mask during training
masked = Mask()(
ner_caps) # mask using capsule with maximal length for predicion
# decoder for training
train_decoder_dense1 = Dense(hyper_param['decoder_feed_forward_1'], activation='relu',\
input_dim=hyper_param['ner_capsule_dim']*hyper_param['ner_classes'],\
name='train_decoder_dense1')(masked_by_y)
train_decoder_dense1_dropout = Dropout(
hyper_param['decoder_dropout'])(train_decoder_dense1)
train_decoder_dense2 = Dense(hyper_param['decoder_feed_forward_2'], activation='relu',\
name='train_decoder_dense2')(train_decoder_dense1_dropout)
train_decoder_dense2_dropout = Dropout(
hyper_param['decoder_dropout'])(train_decoder_dense2)
train_decoder_output = Dense(hyper_param['embed_dim'], activation=None,\
name='train_decoder_output')(train_decoder_dense2_dropout)
# decoder for evaluation (prediction)
eval_decoder_dense1 = Dense(hyper_param['decoder_feed_forward_1'], activation='relu',\
input_dim=hyper_param['ner_capsule_dim']*hyper_param['ner_classes'],\
name='eval_decoder_dense1')(masked)
eval_decoder_dense2 = Dense(hyper_param['decoder_feed_forward_2'], activation='relu',\
name='eval_decoder_dense2')(eval_decoder_dense1)
eval_decoder_output = Dense(hyper_param['embed_dim'], activation=None,\
name='eval_decoder_output')(eval_decoder_dense2)
if verbose:
print("Decoder model enabled for GloVe vector deconstruction...")
print("decoder_y_cat", decoder_y_cat.get_shape())
print("masked_by_y", masked_by_y.get_shape())
print("train_decoder_dense1", train_decoder_dense1.get_shape())
print("train_decoder_dense1_dropout",
train_decoder_dense1_dropout.get_shape())
print("train_decoder_dense2", train_decoder_dense2.get_shape())
print("train_decoder_dense2_dropout",
train_decoder_dense2_dropout.get_shape())
print("train_decoder_output", train_decoder_output.get_shape())
print("masked", masked.get_shape())
print("eval_decoder_dense1", eval_decoder_dense1.get_shape())
print("eval_decoder_dense2", eval_decoder_dense2.get_shape())
print("eval_decoder_output", eval_decoder_output.get_shape())
# construct input list
if hyper_param['use_pos_tags'] and hyper_param['use_capitalization_info']:
input_list = [x, x_pos, x_capital]
elif hyper_param['use_pos_tags'] and (
not hyper_param['use_capitalization_info']):
input_list = [x, x_pos]
elif (not hyper_param['use_pos_tags']
) and hyper_param['use_capitalization_info']:
input_list = [x, x_capital]
else:
input_list = [x]
if hyper_param['use_decoder'] == False:
print("decoder/reconstruction DISabled")
print("returning 1 model")
return Model(inputs=input_list, outputs=[out_pred])
else:
train_model = Model(inputs=input_list + [decoder_y_cat],
outputs=[out_pred, train_decoder_output])
eval_model = Model(inputs=input_list,
outputs=[out_pred, eval_decoder_output])
print("decoder/reconstruction enabled")
print("returning a list of 2 models: train_model, eval_model")
return train_model, eval_model
def create_CNN_Model():
filter_sizes = [2, 3, 4]
embedding_size = 300
num_filters = 128
sequence_length = MAX_SEQUENCE_LENGTH
# Text input
input_for_dimensions = Input(shape=(MAX_SEQUENCE_LENGTH, ),
name='input_text1')
input_for_emotions = Input(shape=(MAX_SEQUENCE_LENGTH, ),
name='input_text2')
embedding1 = Embedding(vocab_size,
embedding_size,
embeddings_initializer=Constant(embedding_matrix),
name='glove300_embedding1',
trainable=False)(input_for_dimensions)
# embedding2 = Embedding(vocab_size, embedding_size, embeddings_initializer=Constant(embedding_matrix), name='glove300_embedding2', trainable=False)(input_for_emotions)
expend_shape = [
embedding1.get_shape().as_list()[1],
embedding1.get_shape().as_list()[2], 1
]
embedding_chars1 = Reshape(expend_shape)(embedding1)
# expend_shape = [embedding2.get_shape().as_list()[1], embedding2.get_shape().as_list()[2], 1]
# embedding_chars2 = Reshape(expend_shape)(embedding2)
# Shared layers
pooled_outputs = []
for i, filter_size in enumerate(filter_sizes):
conv = Conv2D(filters=num_filters,
kernel_size=[filter_size, embedding_size],
activation='relu',
name=('conv_filtersize%d' % filter_size))
encoded_a = conv(embedding_chars1)
encoded_b = conv(embedding_chars1)
merged_conv = keras.layers.concatenate([encoded_b, encoded_a], axis=-1)
max_pool = MaxPool2D(pool_size=[sequence_length - filter_size + 1, 1],
strides=(2, 2),
padding='valid',
name=('max_pool_%d' % filter_size))(merged_conv)
max_pool = Dropout(0.2)(max_pool)
pooled_outputs.append(max_pool)
num_filters_total = num_filters * len(filter_sizes) * 2
h_pool = Concatenate(axis=3)(pooled_outputs)
merged = Reshape([num_filters_total])(h_pool)
dropout = Dropout(0.5, name='0.5')(merged)
fullyconnected = Dense(128, name='fullyconnected',
activation='relu')(dropout)
emo_output = Dense(
len(LABELS),
activation='softmax',
name='emo_out',
kernel_initializer='glorot_normal',
bias_initializer=keras.initializers.constant(0.1))(fullyconnected)
seq_output = Dense(
len(DIMENSIONS),
activation='sigmoid',
name='dim_out',
kernel_initializer='glorot_normal',
bias_initializer=keras.initializers.constant(0.1))(fullyconnected)
multitask_model = Model(inputs=[input_for_emotions, input_for_dimensions],
outputs=[emo_output, seq_output])
multitask_model.compile(loss='categorical_crossentropy',
optimizer=OPTIMIZER,
metrics=['accuracy'],
loss_weights=[0.5, 1])
return multitask_model
