def cabasc(self):
def sequence_mask(sequence):
return K.sign(K.max(K.abs(sequence), 2))
def sequence_length(sequence):
return K.cast(K.sum(sequence_mask(sequence), 1), tf.int32)
input_text = Input(shape=(self.max_len, ))
input_text_l = Input(shape=(self.max_len, ))
input_text_r = Input(shape=(self.max_len, ))
input_aspect = Input(shape=(1, ))
input_mask = Input(shape=(self.max_len, ))
if self.use_elmo:
text_elmo_embedding = ELMoEmbedding(
output_mode=self.config.elmo_output_mode,
idx2word=self.config.idx2token,
mask_zero=True,
hub_url=self.config.elmo_hub_url,
elmo_trainable=self.config.elmo_trainable)
l_elmo_embedding = ELMoEmbedding(
output_mode=self.config.elmo_output_mode,
idx2word=self.config.idx2token,
mask_zero=True,
hub_url=self.config.elmo_hub_url,
elmo_trainable=self.config.elmo_trainable)
r_elmo_embedding = ELMoEmbedding(
output_mode=self.config.elmo_output_mode,
idx2word=self.config.idx2token,
mask_zero=True,
hub_url=self.config.elmo_hub_url,
elmo_trainable=self.config.elmo_trainable)
if self.config.use_elmo_alone:
text_embed = SpatialDropout1D(0.2)(
text_elmo_embedding(input_text))
text_l_embed = SpatialDropout1D(0.2)(
l_elmo_embedding(input_text_l))
text_r_embed = SpatialDropout1D(0.2)(
r_elmo_embedding(input_text_r))
else:
word_embedding = Embedding(
input_dim=self.text_embeddings.shape[0],
output_dim=self.config.word_embed_dim,
weights=[self.text_embeddings],
trainable=self.config.word_embed_trainable,
mask_zero=True)
text_embed = SpatialDropout1D(0.2)(concatenate([
word_embedding(input_text),
text_elmo_embedding(input_text)
]))
text_l_embed = SpatialDropout1D(0.2)(concatenate([
word_embedding(input_text_l),
l_elmo_embedding(input_text_l)
]))
text_r_embed = SpatialDropout1D(0.2)(concatenate([
word_embedding(input_text_r),
r_elmo_embedding(input_text_r)
]))
else:
word_embedding = Embedding(
input_dim=self.text_embeddings.shape[0],
output_dim=self.config.word_embed_dim,
weights=[self.text_embeddings],
trainable=self.config.word_embed_trainable,
mask_zero=True)
text_embed = SpatialDropout1D(0.2)(word_embedding(input_text))
text_l_embed = SpatialDropout1D(0.2)(word_embedding(input_text_l))
text_r_embed = SpatialDropout1D(0.2)(word_embedding(input_text_r))
if self.config.aspect_embed_type == 'random':
asp_embedding = Embedding(input_dim=self.n_aspect,
output_dim=self.config.aspect_embed_dim)
else:
asp_embedding = Embedding(
input_dim=self.aspect_embeddings.shape[0],
output_dim=self.config.aspect_embed_dim,
trainable=self.config.aspect_embed_trainable)
aspect_embed = asp_embedding(input_aspect)
aspect_embed = Flatten()(aspect_embed) # reshape to 2d
# regarding aspect string as the first unit
hidden_l = GRU(self.config.lstm_units,
go_backwards=True,
return_sequences=True)(text_l_embed)
hidden_r = GRU(self.config.lstm_units,
return_sequences=True)(text_r_embed)
# left context attention
context_attend_l = TimeDistributed(Dense(
1, activation='sigmoid'))(hidden_l)
# Note: I couldn't find `reverse_sequence` in keras
context_attend_l = Lambda(lambda x: tf.reverse_sequence(
x, sequence_length(x), 1, 0))(context_attend_l)
context_attend_l = Lambda(lambda x: K.squeeze(x, -1))(context_attend_l)
# right context attention
context_attend_r = TimeDistributed(Dense(
1, activation='sigmoid'))(hidden_r)
context_attend_r = Lambda(lambda x: K.squeeze(x, -1))(context_attend_r)
# combine context attention
# aspect_text_embed = subtract([add([text_l_embed, text_r_embed]), text_embed])
# aspect_text_mask = Lambda(lambda x: sequence_mask(x))(aspect_text_embed)
# text_mask = Lambda(lambda x: sequence_mask(x))(text_embed)
# context_mask = subtract([text_mask, aspect_text_mask])
# aspect_text_mask_half = Lambda(lambda x: x*0.5)(aspect_text_mask)
# combine_mask = add([context_mask, aspect_text_mask_half]) # 1 for context, 0.5 for aspect
context_attend = multiply(
[add([context_attend_l, context_attend_r]), input_mask])
# apply context attention
context_attend_expand = Lambda(lambda x: K.expand_dims(x))(
context_attend)
memory = multiply([text_embed, context_attend_expand])
# sentence-level content attention
sentence = Lambda(lambda x: K.mean(x, axis=1))(memory)
final_output = ContentAttention()([memory, aspect_embed, sentence])
return Model(
[input_text, input_text_l, input_text_r, input_aspect, input_mask],
final_output)
def cabasc(self):
def sequence_mask(sequence):
return K.sign(K.max(K.abs(sequence), 2))
def sequence_length(sequence):
return K.cast(K.sum(sequence_mask(sequence), 1), tf.int32)
input_text = Input(shape=(self.max_len, ))
input_text_l = Input(shape=(self.max_len, ))
input_text_r = Input(shape=(self.max_len, ))
input_aspect = Input(shape=(1, ))
input_mask = Input(shape=(self.max_len, ))
word_embedding = Embedding(input_dim=self.max_content_vocab_size,
output_dim=self.content_embed_dim)
text_embed = SpatialDropout1D(0.2)(word_embedding(input_text))
text_l_embed = SpatialDropout1D(0.2)(word_embedding(input_text_l))
text_r_embed = SpatialDropout1D(0.2)(word_embedding(input_text_r))
asp_embedding = Embedding(input_dim=self.max_aspect_vocab_size,
output_dim=self.aspect_embed_dim)
aspect_embed = asp_embedding(input_aspect)
aspect_embed = Flatten()(aspect_embed) # reshape to 2d
# regarding aspect string as the first unit
hidden_l = GRU(self.lstm_units,
go_backwards=True,
return_sequences=True)(text_l_embed)
hidden_r = GRU(self.lstm_units, return_sequences=True)(text_r_embed)
# left context attention
context_attend_l = TimeDistributed(Dense(
1, activation='sigmoid'))(hidden_l)
# Note: I couldn't find `reverse_sequence` in keras
context_attend_l = Lambda(lambda x: tf.reverse_sequence(
x, sequence_length(x), 1, 0))(context_attend_l)
context_attend_l = Lambda(lambda x: K.squeeze(x, -1))(context_attend_l)
# right context attention
context_attend_r = TimeDistributed(Dense(
1, activation='sigmoid'))(hidden_r)
context_attend_r = Lambda(lambda x: K.squeeze(x, -1))(context_attend_r)
# combine context attention
# aspect_text_embed = subtract([add([text_l_embed, text_r_embed]), text_embed])
# aspect_text_mask = Lambda(lambda x: sequence_mask(x))(aspect_text_embed)
# text_mask = Lambda(lambda x: sequence_mask(x))(text_embed)
# context_mask = subtract([text_mask, aspect_text_mask])
# aspect_text_mask_half = Lambda(lambda x: x*0.5)(aspect_text_mask)
# combine_mask = add([context_mask, aspect_text_mask_half]) # 1 for context, 0.5 for aspect
context_attend = multiply(
[add([context_attend_l, context_attend_r]), input_mask])
# apply context attention
context_attend_expand = Lambda(lambda x: K.expand_dims(x))(
context_attend)
memory = multiply([text_embed, context_attend_expand])
# sentence-level content attention
sentence = Lambda(lambda x: K.mean(x, axis=1))(memory)
final_output = ContentAttention()([memory, aspect_embed, sentence])
dense_layer = Dense(self.dense_units, activation='relu')(final_output)
output_layer = Dense(self.n_classes, activation='softmax')(dense_layer)
return Model(
[input_text, input_text_l, input_text_r, input_aspect, input_mask],
output_layer)