def multi_task_CNN_only_weights(): # Text input input_for_dimensions = Input(shape=(MAX_SEQUENCE_LENGTH, ), name='input_dim') input_for_emotions = Input(shape=(MAX_SEQUENCE_LENGTH, ), name='input_demo') embedding1 = Embedding(vocab_size, embedding_size, embeddings_initializer=Constant(embedding_matrix), name='embedding1', trainable=False)(input_for_dimensions) # embedding2 = Embedding(vocab_size, embedding_size, embeddings_initializer=Constant(embedding_matrix), name='embedding2', trainable=False)(input_for_emotions) expend_shape = [ embedding1.get_shape().as_list()[1], embedding1.get_shape().as_list()[2], 1 ] embedding_chars1 = Reshape(expend_shape)(embedding1) # expend_shape = [embedding2.get_shape().as_list()[1], embedding2.get_shape().as_list()[2], 1] # embedding_chars2 = Reshape(expend_shape)(embedding2) # Shared layers pooled_outputs = [] for i, filter_size in enumerate(filter_sizes): conv = Conv2D(filters=num_filters, kernel_size=[filter_size, embedding_size], activation='relu', name=('conv_%d' % filter_size))(embedding_chars1) # encoded_a = conv(embedding_chars1) # encoded_b = conv(embedding_chars1) # merged_conv = keras.layers.concatenate([encoded_b, encoded_a], axis=-1) max_pool = MaxPool2D(pool_size=[sequence_length - filter_size + 1, 1], strides=(2, 2), padding='valid', name=('max_pool_%d' % filter_size))(conv) # max_pool = Dropout (0.5)(max_pool) pooled_outputs.append(max_pool) num_filters_total = num_filters * len(filter_sizes) h_pool = Concatenate(axis=3)(pooled_outputs) merged = Reshape([num_filters_total])(h_pool) merged1 = Dropout(0.5)(merged) emo_output = Dense(len(LABELS), activation='softmax', name='emo')(merged1) seq_output = Dense(len(DIMENSIONS), activation='sigmoid', name='dim')(merged1) multitask_model = Model(inputs=[input_for_emotions, input_for_dimensions], outputs=[emo_output, seq_output]) multitask_model.compile(loss='categorical_crossentropy', optimizer=OPTIMIZER, metrics=['accuracy'], loss_weights=[1, 0.25]) return multitask_model
def text_cnn_model_baseline(sequence_length, vocab_size, embedding_size, filter_sizes, num_filters, embedding_matrix, drop_out, LABELS, activation): input = Input(shape=(sequence_length, ), name='input_text') embedding = Embedding(vocab_size, embedding_size, embeddings_initializer=Constant(embedding_matrix), name='glove300_embedding', trainable=False)(input) expend_shape = [ embedding.get_shape().as_list()[1], embedding.get_shape().as_list()[2], 1 ] embedding = Reshape(expend_shape)(embedding) # conv->max pool pooled_outputs = [] for i, filter_size in enumerate(filter_sizes): # embedding = Dropout(0.2)(embedding) conv = Conv2D(filters=num_filters, kernel_size=[filter_size, embedding_size], activation='relu', name=('conv_filtersize%d' % filter_size))(embedding) # conv = Dropout(0.5)(conv) max_pool = MaxPool2D(pool_size=[sequence_length - filter_size + 1, 1], strides=(2, 2), padding='valid', name=('max_pool_%d' % filter_size))(conv) # max_pool = Dropout(drop_out)(max_pool) pooled_outputs.append(max_pool) # combine all the pooled features num_filters_total = num_filters * len(filter_sizes) h_pool = Concatenate(axis=3)(pooled_outputs) # h_pool = Dropout(0.5, name='0.5')(h_pool) h_pool_flat = Reshape([num_filters_total])(h_pool) fullyconnected = Dense(128, name='fullyconnected2', activation='relu')(h_pool_flat) dropout = Dropout(drop_out, name='0.5-')(fullyconnected) # output layer output = Dense(len(LABELS), kernel_initializer='glorot_normal', bias_initializer=keras.initializers.constant(0.1), activation=activation, name='output_emotion')(dropout) model = Model(input, output) return model
def text_cnn_model_appraisals(sequence_length, vocab_size, embedding_size, filter_sizes, num_filters, embedding_matrix, drop_out, output_count, activation): input = Input(shape=(sequence_length, ), name='input_text') try: embedding = Embedding( vocab_size, embedding_size, embeddings_initializer=Constant(embedding_matrix), name='glove300_embedding', trainable=False)(input) except TypeError: print('\nERROR: Embedding does not fit dataset:') print( ' Datasets contains %d terms and embedding contains %d terms...' % (vocab_size, embedding_matrix.shape[0])) print('\nExiting.') sys.exit(1) except: print('\nUnexpected error:', sys.exc_info()[1]) sys.exit(1) expend_shape = [ embedding.get_shape().as_list()[1], embedding.get_shape().as_list()[2], 1 ] embedding = Reshape(expend_shape)(embedding) # conv->max pool pooled_outputs = [] for i, filter_size in enumerate(filter_sizes): # embedding = Dropout(0.2)(embedding) conv = Conv2D(filters=num_filters, kernel_size=[filter_size, embedding_size], activation='relu', name=('conv_filtersize%d' % filter_size))(embedding) # conv = Dropout(0.5)(conv) max_pool = MaxPool2D(pool_size=[sequence_length - filter_size + 1, 1], strides=(2, 2), padding='valid', name=('max_pool_%d' % filter_size))(conv) # max_pool = Dropout(drop_out)(max_pool) pooled_outputs.append(max_pool) # combine all the pooled features num_filters_total = num_filters * len(filter_sizes) h_pool = Concatenate(axis=3)(pooled_outputs) # h_pool = Dropout(drop_out, name='0.5')(h_pool) h_pool_flat = Reshape([num_filters_total])(h_pool) fullyconnected = Dense(128, name='fullyconnected2', activation='relu')(h_pool_flat) dropout = Dropout(drop_out, name='d_out')(fullyconnected) # output layer output = Dense(output_count, kernel_initializer='glorot_normal', bias_initializer=keras.initializers.constant(0.1), activation=activation, name='output_emotion')(dropout) model = Model(input, output) return model
def draw_cnn_model(hyper_param, embedding_matrix=None, verbose=True): """ Input: hyper_parameters dictionary Construct: input layers : x , x_pos(o), x_captialization(o) embedding matrix : use_glove or randomly initialize conv1 : first convolution layer conv2 : second convolution layer conv3 : third convolution layer max pooling flatten : concant maxpooled univariate vectors into one long vector ff1, ff2: two feed forward layers out_pred: softmax over all ner classes Returns: keras.models.Model object """ # input layer(s) x = Input(shape=(hyper_param['maxlen'], ), name='x') if hyper_param['use_pos_tags']: x_pos = Input(shape=(hyper_param['maxlen'], hyper_param['poslen']), name='x_pos') if hyper_param['use_capitalization_info']: x_capital = Input(shape=(hyper_param['maxlen'], hyper_param['capitallen']), name='x_capital') # embedding matrix if hyper_param['use_glove']: embed = Embedding(hyper_param['max_features'], hyper_param['embed_dim'], weights=[embedding_matrix],\ input_length=hyper_param['maxlen'], trainable=hyper_param['allow_glove_retrain'])(x) else: embed = Embedding(hyper_param['max_features'], hyper_param['embed_dim'], input_length=hyper_param['maxlen'],\ embeddings_initializer="random_uniform" )(x) # concat embeddings with additional features if hyper_param['use_pos_tags'] and hyper_param['use_capitalization_info']: embed = Concatenate(axis=-1)([embed, x_pos, x_capital]) elif hyper_param['use_pos_tags'] and ( not hyper_param['use_capitalization_info']): embed = Concatenate(axis=-1)([embed, x_pos]) elif (not hyper_param['use_pos_tags'] ) and hyper_param['use_capitalization_info']: embed = Concatenate(axis=-1)([embed, x_capital]) else: embed = embed # feed embeddings into conv1 conv1 = Conv1D( filters=hyper_param['conv1_filters'], \ kernel_size=hyper_param['conv1_kernel_size'],\ strides=hyper_param['conv1_strides'], \ padding=hyper_param['conv1_padding'],\ activation='relu', name='conv1')(embed) # update this # make primary capsules conv2 = Conv1D( filters=hyper_param['conv2_filters'], \ kernel_size=hyper_param['conv2_kernel_size'],\ strides=hyper_param['conv2_strides'], \ padding=hyper_param['conv2_padding'],\ activation='relu', name='conv2')(conv1) # update this # make primary capsules conv3 = Conv1D( filters=hyper_param['conv3_filters'], \ kernel_size=hyper_param['conv3_kernel_size'],\ strides=hyper_param['conv3_strides'], \ padding=hyper_param['conv3_padding'],\ activation='relu', name='conv3')(conv2) # max pooling layer max_pooled = MaxPooling1D(pool_size=hyper_param['max_pooling_size'], \ strides=hyper_param['max_pooling_strides'], \ padding=hyper_param['max_pooling_padding'])(conv3) # dropout maxpooled_dropout = Dropout(hyper_param['maxpool_dropout'])(max_pooled) # flatten many univariate vectos into 1 long vector flattened = Flatten()(maxpooled_dropout) # to feed-forward layers ff1 = Dense(hyper_param['feed_forward_1'], activation='relu')(flattened) ff1_dropout = Dropout(hyper_param['ff1_dropout'])(ff1) ff2 = Dense(hyper_param['feed_forward_2'], activation='relu')(ff1_dropout) ff2_dropout = Dropout(hyper_param['ff2_dropout'])(ff2) out_pred = Dense(hyper_param['ner_classes'], activation='softmax', name='out_pred')(ff2) #! if verbose: print("x", x.get_shape()) if hyper_param['use_pos_tags']: print("x_pos", x_pos.get_shape()) if hyper_param['use_capitalization_info']: print("x_capital", x_capital.get_shape()) print("embed", embed.get_shape()) print("embed", embed.get_shape()) print("conv1", conv1.get_shape()) print("conv2", conv2.get_shape()) print("conv3", conv3.get_shape()) print("max_pooled", max_pooled.get_shape()) print("flattened", flattened.get_shape()) print("ff1", ff1.get_shape()) print("ff2", ff2.get_shape()) print("out_pred", out_pred.get_shape()) # return final model if hyper_param['use_pos_tags'] and hyper_param['use_capitalization_info']: cnnmodel = Model(inputs=[x, x_pos, x_capital], outputs=[out_pred]) elif hyper_param['use_pos_tags'] and ( not hyper_param['use_capitalization_info']): cnnmodel = Model(inputs=[x, x_pos], outputs=[out_pred]) elif (not hyper_param['use_pos_tags'] ) and hyper_param['use_capitalization_info']: cnnmodel = Model(inputs=[x, x_capital], outputs=[out_pred]) else: cnnmodel = Model(inputs=[x], outputs=[out_pred]) return cnnmodel
''' lx = len(embedding_matrix) inp = Input(shape=(maxlen, )) x = Embedding(lx, embed_size, weights=[embedding_matrix])(inp) x = Conv1D(filters=32, kernel_size=3, padding='same', activation='relu')(x) x = MaxPooling1D(pool_size=2)(x) x = LSTM(300, return_sequences=True)(x) #x = AttentionDecoder(300,maxlen)(x) #x = GlobalMaxPool1D()(x) #x = Dense(300, activation="relu")(x) #x = Dense(100, activation="relu")(x) #x = Dropout(0.1)(x) #ylayer=numpy.asarray(ylayer) print(x.get_shape()) xa = TimeDistributed(Dense(3, activation="tanh"))(x) xa = Flatten()(xa) xa = Activation('softmax')(xa) xa = RepeatVector(300)(xa) print(xa.get_shape()) xa = Permute([2, 1])(xa) print(xa.get_shape()) ot = Multiply()([x, xa]) ot = Lambda(lambda xin: K.sum(xin, axis=-1))(ot) ol = TimeDistributed(Dense(3, activation="sigmoid"))(ot) model = Model(inputs=inp, outputs=ol) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
def draw_capsnet_model(hyper_param, embedding_matrix=None, verbose=True): """ Input: hyper parameters dictionary Construct: input layers : x , x_pos(o), x_captialization(o) embedding matrix : use_glove or randomly initialize conv1 : first convolution layer primarycaps : conv2 and squash function applied ner_caps : make 8 ner capsules of specified dim out_pred : calc length of 8 ner capsules as 8 prob. predictions over 8 ner classes Returns: if decoding/reconstruction disabled --> a single keras.models.Model object if decoding/reconstruction enabled --> three keras.models.Model objects """ # input layer(s) x = Input(shape=(hyper_param['maxlen'], ), name='x') if hyper_param['use_pos_tags']: x_pos = Input(shape=(hyper_param['maxlen'], hyper_param['poslen']), name='x_pos') if hyper_param['use_capitalization_info']: x_capital = Input(shape=(hyper_param['maxlen'], hyper_param['capitallen']), name='x_capital') # embedding matrix if hyper_param['use_glove']: embed = Embedding(hyper_param['max_features'], hyper_param['embed_dim'], weights=[embedding_matrix],\ input_length=hyper_param['maxlen'], trainable=hyper_param['allow_glove_retrain'])(x) else: embed = Embedding(hyper_param['max_features'], hyper_param['embed_dim'], input_length=hyper_param['maxlen'],\ embeddings_initializer="random_uniform" )(x) # concat embeddings with additional features if hyper_param['use_pos_tags'] and hyper_param['use_capitalization_info']: embed = Concatenate(axis=-1)([embed, x_pos, x_capital]) elif hyper_param['use_pos_tags'] and ( not hyper_param['use_capitalization_info']): embed = Concatenate(axis=-1)([embed, x_pos]) elif (not hyper_param['use_pos_tags'] ) and hyper_param['use_capitalization_info']: embed = Concatenate(axis=-1)([embed, x_capital]) else: embed = embed # add dropout here if hyper_param['embed_dropout'] > 0.0: embed = SpatialDropout1D(hyper_param['embed_dropout'])(embed) # feed embeddings into conv1 conv1 = Conv1D( filters=hyper_param['conv1_filters'], \ kernel_size=hyper_param['conv1_kernel_size'],\ strides=hyper_param['conv1_strides'], \ padding=hyper_param['conv1_padding'],\ activation='relu', name='conv1')(embed) # make primary capsules if hyper_param['use_2D_primarycaps']: convShape = conv1.get_shape().as_list() conv1 = Reshape((convShape[1], convShape[2], 1))(conv1) primaryCapLayer = PrimaryCap else: primaryCapLayer = PrimaryCap1D # make primary capsules primarycaps = primaryCapLayer(conv1, \ dim_capsule=hyper_param['primarycaps_dim_capsule'],\ n_channels=hyper_param['primarycaps_n_channels'],\ kernel_size=hyper_param['primarycaps_kernel_size'], \ strides=hyper_param['primarycaps_strides'], \ padding=hyper_param['primarycaps_padding']) # make ner capsules ner_caps = CapsuleLayer(num_capsule=hyper_param['ner_classes'], \ dim_capsule=hyper_param['ner_capsule_dim'], \ routings=hyper_param['num_dynamic_routing_passes'], \ name='nercaps')(primarycaps) # replace each ner capsuel with its length out_pred = Length(name='out_pred')(ner_caps) if verbose: print("x", x.get_shape()) if hyper_param['use_pos_tags']: print("x_pos", x_pos.get_shape()) if hyper_param['use_capitalization_info']: print("x_capital", x_capital.get_shape()) print("embed", embed.get_shape()) print("conv1", conv1.get_shape()) print("primarycaps", primarycaps.get_shape()) print("ner_caps", ner_caps.get_shape()) print("out_pred", out_pred.get_shape()) if hyper_param['use_decoder']: decoder_y_cat = Input(shape=(hyper_param['ner_classes'], ), name='decoder_y_cat') masked_by_y = Mask(name='masked_by_y')( [ner_caps, decoder_y_cat]) # true label is used to mask during training masked = Mask()( ner_caps) # mask using capsule with maximal length for predicion # decoder for training train_decoder_dense1 = Dense(hyper_param['decoder_feed_forward_1'], activation='relu',\ input_dim=hyper_param['ner_capsule_dim']*hyper_param['ner_classes'],\ name='train_decoder_dense1')(masked_by_y) train_decoder_dense1_dropout = Dropout( hyper_param['decoder_dropout'])(train_decoder_dense1) train_decoder_dense2 = Dense(hyper_param['decoder_feed_forward_2'], activation='relu',\ name='train_decoder_dense2')(train_decoder_dense1_dropout) train_decoder_dense2_dropout = Dropout( hyper_param['decoder_dropout'])(train_decoder_dense2) train_decoder_output = Dense(hyper_param['embed_dim'], activation=None,\ name='train_decoder_output')(train_decoder_dense2_dropout) # decoder for evaluation (prediction) eval_decoder_dense1 = Dense(hyper_param['decoder_feed_forward_1'], activation='relu',\ input_dim=hyper_param['ner_capsule_dim']*hyper_param['ner_classes'],\ name='eval_decoder_dense1')(masked) eval_decoder_dense2 = Dense(hyper_param['decoder_feed_forward_2'], activation='relu',\ name='eval_decoder_dense2')(eval_decoder_dense1) eval_decoder_output = Dense(hyper_param['embed_dim'], activation=None,\ name='eval_decoder_output')(eval_decoder_dense2) if verbose: print("Decoder model enabled for GloVe vector deconstruction...") print("decoder_y_cat", decoder_y_cat.get_shape()) print("masked_by_y", masked_by_y.get_shape()) print("train_decoder_dense1", train_decoder_dense1.get_shape()) print("train_decoder_dense1_dropout", train_decoder_dense1_dropout.get_shape()) print("train_decoder_dense2", train_decoder_dense2.get_shape()) print("train_decoder_dense2_dropout", train_decoder_dense2_dropout.get_shape()) print("train_decoder_output", train_decoder_output.get_shape()) print("masked", masked.get_shape()) print("eval_decoder_dense1", eval_decoder_dense1.get_shape()) print("eval_decoder_dense2", eval_decoder_dense2.get_shape()) print("eval_decoder_output", eval_decoder_output.get_shape()) # construct input list if hyper_param['use_pos_tags'] and hyper_param['use_capitalization_info']: input_list = [x, x_pos, x_capital] elif hyper_param['use_pos_tags'] and ( not hyper_param['use_capitalization_info']): input_list = [x, x_pos] elif (not hyper_param['use_pos_tags'] ) and hyper_param['use_capitalization_info']: input_list = [x, x_capital] else: input_list = [x] if hyper_param['use_decoder'] == False: print("decoder/reconstruction DISabled") print("returning 1 model") return Model(inputs=input_list, outputs=[out_pred]) else: train_model = Model(inputs=input_list + [decoder_y_cat], outputs=[out_pred, train_decoder_output]) eval_model = Model(inputs=input_list, outputs=[out_pred, eval_decoder_output]) print("decoder/reconstruction enabled") print("returning a list of 2 models: train_model, eval_model") return train_model, eval_model
def create_CNN_Model(): filter_sizes = [2, 3, 4] embedding_size = 300 num_filters = 128 sequence_length = MAX_SEQUENCE_LENGTH # Text input input_for_dimensions = Input(shape=(MAX_SEQUENCE_LENGTH, ), name='input_text1') input_for_emotions = Input(shape=(MAX_SEQUENCE_LENGTH, ), name='input_text2') embedding1 = Embedding(vocab_size, embedding_size, embeddings_initializer=Constant(embedding_matrix), name='glove300_embedding1', trainable=False)(input_for_dimensions) # embedding2 = Embedding(vocab_size, embedding_size, embeddings_initializer=Constant(embedding_matrix), name='glove300_embedding2', trainable=False)(input_for_emotions) expend_shape = [ embedding1.get_shape().as_list()[1], embedding1.get_shape().as_list()[2], 1 ] embedding_chars1 = Reshape(expend_shape)(embedding1) # expend_shape = [embedding2.get_shape().as_list()[1], embedding2.get_shape().as_list()[2], 1] # embedding_chars2 = Reshape(expend_shape)(embedding2) # Shared layers pooled_outputs = [] for i, filter_size in enumerate(filter_sizes): conv = Conv2D(filters=num_filters, kernel_size=[filter_size, embedding_size], activation='relu', name=('conv_filtersize%d' % filter_size)) encoded_a = conv(embedding_chars1) encoded_b = conv(embedding_chars1) merged_conv = keras.layers.concatenate([encoded_b, encoded_a], axis=-1) max_pool = MaxPool2D(pool_size=[sequence_length - filter_size + 1, 1], strides=(2, 2), padding='valid', name=('max_pool_%d' % filter_size))(merged_conv) max_pool = Dropout(0.2)(max_pool) pooled_outputs.append(max_pool) num_filters_total = num_filters * len(filter_sizes) * 2 h_pool = Concatenate(axis=3)(pooled_outputs) merged = Reshape([num_filters_total])(h_pool) dropout = Dropout(0.5, name='0.5')(merged) fullyconnected = Dense(128, name='fullyconnected', activation='relu')(dropout) emo_output = Dense( len(LABELS), activation='softmax', name='emo_out', kernel_initializer='glorot_normal', bias_initializer=keras.initializers.constant(0.1))(fullyconnected) seq_output = Dense( len(DIMENSIONS), activation='sigmoid', name='dim_out', kernel_initializer='glorot_normal', bias_initializer=keras.initializers.constant(0.1))(fullyconnected) multitask_model = Model(inputs=[input_for_emotions, input_for_dimensions], outputs=[emo_output, seq_output]) multitask_model.compile(loss='categorical_crossentropy', optimizer=OPTIMIZER, metrics=['accuracy'], loss_weights=[0.5, 1]) return multitask_model