def td_lstm(self):
input_l = Input(shape=(self.left_max_len, ))
input_r = Input(shape=(self.right_max_len, ))
if self.use_elmo:
l_elmo_embedding = ELMoEmbedding(
output_mode=self.config.elmo_output_mode,
idx2word=self.config.idx2token,
mask_zero=True,
hub_url=self.config.elmo_hub_url,
elmo_trainable=self.config.elmo_trainable)
r_elmo_embedding = ELMoEmbedding(
output_mode=self.config.elmo_output_mode,
idx2word=self.config.idx2token,
mask_zero=True,
hub_url=self.config.elmo_hub_url,
elmo_trainable=self.config.elmo_trainable)
if self.config.use_elmo_alone:
input_l_embed = SpatialDropout1D(0.2)(
l_elmo_embedding(input_l))
input_r_embed = SpatialDropout1D(0.2)(
r_elmo_embedding(input_r))
else:
word_embedding = Embedding(
input_dim=self.text_embeddings.shape[0],
output_dim=self.config.word_embed_dim,
weights=[self.text_embeddings],
trainable=self.config.word_embed_trainable,
mask_zero=True)
input_l_embed = SpatialDropout1D(0.2)(concatenate(
[word_embedding(input_l),
l_elmo_embedding(input_l)]))
input_r_embed = SpatialDropout1D(0.2)(concatenate(
[word_embedding(input_r),
r_elmo_embedding(input_r)]))
else:
word_embedding = Embedding(
input_dim=self.text_embeddings.shape[0],
output_dim=self.config.word_embed_dim,
weights=[self.text_embeddings],
trainable=self.config.word_embed_trainable,
mask_zero=True)
input_l_embed = SpatialDropout1D(0.2)(word_embedding(input_l))
input_r_embed = SpatialDropout1D(0.2)(word_embedding(input_r))
# regarding aspect string as the last unit
if (self.is_cudnn):
hidden_l = CuDNNLSTM(self.config.lstm_units)(input_l_embed)
else:
hidden_l = LSTM(self.config.lstm_units)(input_l_embed)
if (self.is_cudnn):
hidden_r = CuDNNLSTM(self.config.lstm_units,
go_backwards=True)(input_r_embed)
else:
hidden_r = LSTM(self.config.lstm_units,
go_backwards=True)(input_r_embed)
hidden_concat = concatenate([hidden_l, hidden_r], axis=-1)
return Model([input_l, input_r], hidden_concat)
def ian(self):
input_text = Input(shape=(self.max_len,))
input_aspect_text = Input(shape=(self.asp_max_len,), )
if self.use_elmo:
elmo_embedding = ELMoEmbedding(output_mode=self.config.elmo_output_mode, idx2word=self.config.idx2token,
mask_zero=True, hub_url=self.config.elmo_hub_url,
elmo_trainable=self.config.elmo_trainable)
if self.config.use_elmo_alone:
text_embed = SpatialDropout1D(0.2)(elmo_embedding(input_text))
else:
word_embedding = Embedding(input_dim=self.text_embeddings.shape[0],
output_dim=self.config.word_embed_dim,
weights=[self.text_embeddings], trainable=self.config.word_embed_trainable,
mask_zero=True)
text_embed = SpatialDropout1D(0.2)(concatenate([word_embedding(input_text), elmo_embedding(input_text)]))
else:
word_embedding = Embedding(input_dim=self.text_embeddings.shape[0], output_dim=self.config.word_embed_dim,
weights=[self.text_embeddings], trainable=self.config.word_embed_trainable,
mask_zero=True)
text_embed = SpatialDropout1D(0.2)(word_embedding(input_text))
if self.use_elmo:
asp_elmo_embedding = ELMoEmbedding(output_mode=self.config.elmo_output_mode,
idx2word=self.config.idx2aspect_token,
mask_zero=True, hub_url=self.config.elmo_hub_url,
elmo_trainable=self.config.elmo_trainable)
if self.config.use_elmo_alone:
asp_text_embed = SpatialDropout1D(0.2)(asp_elmo_embedding(input_aspect_text))
else:
asp_text_embedding = Embedding(input_dim=self.aspect_text_embeddings.shape[0],
output_dim=self.config.word_embed_dim,
weights=[self.aspect_text_embeddings],
trainable=self.config.word_embed_trainable,
mask_zero=True)
asp_text_embed = SpatialDropout1D(0.2)(concatenate([asp_text_embedding(input_aspect_text),
asp_elmo_embedding(input_aspect_text)]))
else:
asp_text_embedding = Embedding(input_dim=self.aspect_text_embeddings.shape[0],
output_dim=self.config.word_embed_dim,
weights=[self.aspect_text_embeddings],
trainable=self.config.word_embed_trainable,
mask_zero=True)
asp_text_embed = SpatialDropout1D(0.2)(asp_text_embedding(input_aspect_text))
hidden_text = LSTM(self.config.lstm_units, return_sequences=True)(text_embed)
hidden_asp_text = LSTM(self.config.lstm_units, return_sequences=True)(asp_text_embed)
attend_concat = InteractiveAttention()([hidden_text, hidden_asp_text])
return Model([input_text, input_aspect_text], attend_concat)
def tc_lstm(self):
input_l = Input(shape=(self.left_max_len,))
input_r = Input(shape=(self.right_max_len,))
input_aspect = Input(shape=(1,))
if self.use_elmo:
l_elmo_embedding = ELMoEmbedding(output_mode=self.config.elmo_output_mode, idx2word=self.config.idx2token,
mask_zero=True, hub_url=self.config.elmo_hub_url,
elmo_trainable=self.config.elmo_trainable)
r_elmo_embedding = ELMoEmbedding(output_mode=self.config.elmo_output_mode, idx2word=self.config.idx2token,
mask_zero=True, hub_url=self.config.elmo_hub_url,
elmo_trainable=self.config.elmo_trainable)
if self.config.use_elmo_alone:
input_l_embed = SpatialDropout1D(0.2)(l_elmo_embedding(input_l))
input_r_embed = SpatialDropout1D(0.2)(r_elmo_embedding(input_r))
else:
word_embedding = Embedding(input_dim=self.text_embeddings.shape[0],
output_dim=self.config.word_embed_dim,
weights=[self.text_embeddings], trainable=self.config.word_embed_trainable,
mask_zero=True)
input_l_embed = SpatialDropout1D(0.2)(concatenate([word_embedding(input_l), l_elmo_embedding(input_l)]))
input_r_embed = SpatialDropout1D(0.2)(concatenate([word_embedding(input_r), r_elmo_embedding(input_r)]))
else:
word_embedding = Embedding(input_dim=self.text_embeddings.shape[0], output_dim=self.config.word_embed_dim,
weights=[self.text_embeddings], trainable=self.config.word_embed_trainable,
mask_zero=True)
input_l_embed = SpatialDropout1D(0.2)(word_embedding(input_l))
input_r_embed = SpatialDropout1D(0.2)(word_embedding(input_r))
if self.config.aspect_embed_type == 'random':
asp_embedding = Embedding(input_dim=self.n_aspect, output_dim=self.config.aspect_embed_dim)
else:
asp_embedding = Embedding(input_dim=self.aspect_embeddings.shape[0],
output_dim=self.config.aspect_embed_dim,
trainable=self.config.aspect_embed_trainable)
aspect_embed = asp_embedding(input_aspect)
aspect_embed = Flatten()(aspect_embed)
aspect_repeat_l = RepeatVector(self.left_max_len)(aspect_embed)
input_l_concat = concatenate([input_l_embed, aspect_repeat_l], axis=-1)
aspect_repeat_r = RepeatVector(self.right_max_len)(aspect_embed)
input_r_concat = concatenate([input_r_embed, aspect_repeat_r], axis=-1)
# regarding aspect string as the last unit
hidden_l = LSTM(self.config.lstm_units)(input_l_concat)
hidden_r = LSTM(self.config.lstm_units, go_backwards=True)(input_r_concat)
hidden_concat = concatenate([hidden_l, hidden_r], axis=-1)
return Model([input_l, input_r, input_aspect], hidden_concat)
def at_lstm(self):
input_text = Input(shape=(self.max_len, ))
input_aspect = Input(shape=(1, ), )
if self.use_elmo:
elmo_embedding = ELMoEmbedding(
output_mode=self.config.elmo_output_mode,
idx2word=self.config.idx2token,
mask_zero=True,
hub_url=self.config.elmo_hub_url,
elmo_trainable=self.config.elmo_trainable)
if self.config.use_elmo_alone:
text_embed = SpatialDropout1D(0.2)(elmo_embedding(input_text))
else:
word_embedding = Embedding(
input_dim=self.text_embeddings.shape[0],
output_dim=self.config.word_embed_dim,
weights=[self.text_embeddings],
trainable=self.config.word_embed_trainable,
mask_zero=True)
text_embed = SpatialDropout1D(0.2)(concatenate(
[word_embedding(input_text),
elmo_embedding(input_text)]))
else:
word_embedding = Embedding(
input_dim=self.text_embeddings.shape[0],
output_dim=self.config.word_embed_dim,
weights=[self.text_embeddings],
trainable=self.config.word_embed_trainable,
mask_zero=True)
text_embed = SpatialDropout1D(0.2)(word_embedding(input_text))
if self.config.aspect_embed_type == 'random':
asp_embedding = Embedding(input_dim=self.n_aspect,
output_dim=self.config.aspect_embed_dim)
else:
asp_embedding = Embedding(
input_dim=self.aspect_embeddings.shape[0],
output_dim=self.config.aspect_embed_dim,
trainable=self.config.aspect_embed_trainable)
aspect_embed = asp_embedding(input_aspect)
aspect_embed = Flatten()(aspect_embed) # reshape to 2d
repeat_aspect = RepeatVector(self.max_len)(
aspect_embed) # repeat aspect for every word in sequence
hidden_vecs = LSTM(self.config.lstm_units, return_sequences=True)(
text_embed) # hidden vectors output by lstm
concat = concatenate([
hidden_vecs, repeat_aspect
], axis=-1) # mask after concatenate will be same as hidden_out's mask
# apply attention mechanism
attend_weight = Attention()(concat)
attend_weight_expand = Lambda(lambda x: K.expand_dims(x))(
attend_weight)
attend_hidden = multiply([hidden_vecs, attend_weight_expand])
attend_hidden = Lambda(lambda x: K.sum(x, axis=1))(attend_hidden)
return Model([input_text, input_aspect], attend_hidden)
def ram(self):
n_hop = 3
# input module
input_text = Input(shape=(self.max_len,))
input_aspect = Input(shape=(1,))
inputs = [input_text, input_aspect]
if self.use_elmo:
elmo_embedding = ELMoEmbedding(output_mode=self.config.elmo_output_mode, idx2word=self.config.idx2token,
mask_zero=True, hub_url=self.config.elmo_hub_url,
elmo_trainable=self.config.elmo_trainable)
if self.config.use_elmo_alone:
text_embed = SpatialDropout1D(0.2)(elmo_embedding(input_text))
else:
word_embedding = Embedding(input_dim=self.text_embeddings.shape[0],
output_dim=self.config.word_embed_dim,
weights=[self.text_embeddings], trainable=self.config.word_embed_trainable,
mask_zero=True)
text_embed = SpatialDropout1D(0.2)(concatenate([word_embedding(input_text), elmo_embedding(input_text)]))
else:
word_embedding = Embedding(input_dim=self.text_embeddings.shape[0], output_dim=self.config.word_embed_dim,
weights=[self.text_embeddings], trainable=self.config.word_embed_trainable,
mask_zero=True)
text_embed = SpatialDropout1D(0.2)(word_embedding(input_text))
if self.config.aspect_embed_type == 'random':
asp_embedding = Embedding(input_dim=self.n_aspect, output_dim=self.config.aspect_embed_dim)
else:
asp_embedding = Embedding(input_dim=self.aspect_embeddings.shape[0],
output_dim=self.config.aspect_embed_dim,
trainable=self.config.aspect_embed_trainable)
aspect_embed = asp_embedding(input_aspect)
aspect_embed = Flatten()(aspect_embed) # reshape to 2d
# memory module
hidden_out_1 = Bidirectional(LSTM(self.config.lstm_units, return_sequences=True))(text_embed)
memory = Bidirectional(LSTM(self.config.lstm_units, return_sequences=True))(hidden_out_1)
# position-weighted memory module
if self.config.use_loc_input:
input_loc = Input(shape=(self.max_len,))
inputs.append(input_loc)
input_loc_expand = Lambda(lambda x: K.expand_dims(x))(input_loc)
memory = multiply([memory, input_loc_expand])
if self.config.use_offset_input:
input_offset = Input(shape=(self.max_len,))
inputs.append(input_offset)
input_offset_expand = Lambda(lambda x: K.expand_dims(x))(input_offset)
memory = concatenate([memory, input_offset_expand])
# recurrent attention module
final_attend = RecurrentAttention(units=self.config.lstm_units, n_hop=n_hop)([memory, aspect_embed])
return Model(inputs, final_attend)
def ae_lstm(self):
input_text = Input(shape=(self.max_len, ))
input_aspect = Input(shape=(1, ), )
if self.use_elmo:
elmo_embedding = ELMoEmbedding(
output_mode=self.config.elmo_output_mode,
idx2word=self.config.idx2token,
mask_zero=True,
hub_url=self.config.elmo_hub_url,
elmo_trainable=self.config.elmo_trainable)
if self.config.use_elmo_alone:
text_embed = SpatialDropout1D(0.2)(elmo_embedding(input_text))
else:
word_embedding = Embedding(
input_dim=self.text_embeddings.shape[0],
output_dim=self.config.word_embed_dim,
weights=[self.text_embeddings],
trainable=self.config.word_embed_trainable,
mask_zero=True)
text_embed = SpatialDropout1D(0.2)(concatenate(
[word_embedding(input_text),
elmo_embedding(input_text)]))
else:
word_embedding = Embedding(
input_dim=self.text_embeddings.shape[0],
output_dim=self.config.word_embed_dim,
weights=[self.text_embeddings],
trainable=self.config.word_embed_trainable,
mask_zero=True)
text_embed = SpatialDropout1D(0.2)(word_embedding(input_text))
if self.config.aspect_embed_type == 'random':
asp_embedding = Embedding(input_dim=self.n_aspect,
output_dim=self.config.aspect_embed_dim)
else:
asp_embedding = Embedding(
input_dim=self.aspect_embeddings.shape[0],
output_dim=self.config.aspect_embed_dim,
trainable=self.config.aspect_embed_trainable)
aspect_embed = asp_embedding(input_aspect)
aspect_embed = Flatten()(aspect_embed) # reshape to 2d
repeat_aspect = RepeatVector(self.max_len)(
aspect_embed) # repeat aspect for every word in sequence
input_concat = concatenate([text_embed, repeat_aspect], axis=-1)
if (self.is_cudnn):
hidden = CuDNNLSTM(self.config.lstm_units)(input_concat)
else:
hidden = LSTM(self.config.lstm_units)(input_concat)
return Model([input_text, input_aspect], hidden)
def atae_lstm(self):
input_text = Input(shape=(self.max_len,))
input_aspect = Input(shape=(1,), )
if self.use_elmo:
elmo_embedding = ELMoEmbedding(output_mode=self.config.elmo_output_mode, idx2word=self.config.idx2token,
mask_zero=True, hub_url=self.config.elmo_hub_url,
elmo_trainable=self.config.elmo_trainable)
if self.config.use_elmo_alone:
text_embed = SpatialDropout1D(0.2)(elmo_embedding(input_text))
else:
word_embedding = Embedding(input_dim=self.text_embeddings.shape[0],
output_dim=self.config.word_embed_dim,
weights=[self.text_embeddings], trainable=self.config.word_embed_trainable,
mask_zero=True)
text_embed = SpatialDropout1D(0.2)(concatenate([word_embedding(input_text), elmo_embedding(input_text)]))
else:
word_embedding = Embedding(input_dim=self.text_embeddings.shape[0], output_dim=self.config.word_embed_dim,
weights=[self.text_embeddings], trainable=self.config.word_embed_trainable,
mask_zero=True)
text_embed = SpatialDropout1D(0.2)(word_embedding(input_text))
if self.config.aspect_embed_type == 'random':
asp_embedding = Embedding(input_dim=self.n_aspect, output_dim=self.config.aspect_embed_dim)
else:
asp_embedding = Embedding(input_dim=self.aspect_embeddings.shape[0],
output_dim=self.config.aspect_embed_dim,
trainable=self.config.aspect_embed_trainable)
aspect_embed = asp_embedding(input_aspect)
aspect_embed = Flatten()(aspect_embed) # reshape to 2d
repeat_aspect = RepeatVector(self.max_len)(aspect_embed) # repeat aspect for every word in sequence
input_concat = concatenate([text_embed, repeat_aspect], axis=-1)
hidden_vecs, state_h, _ = LSTM(self.config.lstm_units, return_sequences=True, return_state=True)(input_concat)
concat = concatenate([hidden_vecs, repeat_aspect], axis=-1)
# apply attention mechanism
attend_weight = Attention()(concat)
attend_weight_expand = Lambda(lambda x: K.expand_dims(x))(attend_weight)
attend_hidden = multiply([hidden_vecs, attend_weight_expand])
attend_hidden = Lambda(lambda x: K.sum(x, axis=1))(attend_hidden)
attend_hidden_dense = Dense(self.config.lstm_units)(attend_hidden)
last_hidden_dense = Dense(self.config.lstm_units)(state_h)
final_output = Activation('tanh')(add([attend_hidden_dense, last_hidden_dense]))
return Model([input_text, input_aspect], final_output)
def cabasc(self):
def sequence_mask(sequence):
return K.sign(K.max(K.abs(sequence), 2))
def sequence_length(sequence):
return K.cast(K.sum(sequence_mask(sequence), 1), tf.int32)
input_text = Input(shape=(self.max_len, ))
input_text_l = Input(shape=(self.max_len, ))
input_text_r = Input(shape=(self.max_len, ))
input_aspect = Input(shape=(1, ))
input_mask = Input(shape=(self.max_len, ))
if self.use_elmo:
text_elmo_embedding = ELMoEmbedding(
output_mode=self.config.elmo_output_mode,
idx2word=self.config.idx2token,
mask_zero=True,
hub_url=self.config.elmo_hub_url,
elmo_trainable=self.config.elmo_trainable)
l_elmo_embedding = ELMoEmbedding(
output_mode=self.config.elmo_output_mode,
idx2word=self.config.idx2token,
mask_zero=True,
hub_url=self.config.elmo_hub_url,
elmo_trainable=self.config.elmo_trainable)
r_elmo_embedding = ELMoEmbedding(
output_mode=self.config.elmo_output_mode,
idx2word=self.config.idx2token,
mask_zero=True,
hub_url=self.config.elmo_hub_url,
elmo_trainable=self.config.elmo_trainable)
if self.config.use_elmo_alone:
text_embed = SpatialDropout1D(0.2)(
text_elmo_embedding(input_text))
text_l_embed = SpatialDropout1D(0.2)(
l_elmo_embedding(input_text_l))
text_r_embed = SpatialDropout1D(0.2)(
r_elmo_embedding(input_text_r))
else:
word_embedding = Embedding(
input_dim=self.text_embeddings.shape[0],
output_dim=self.config.word_embed_dim,
weights=[self.text_embeddings],
trainable=self.config.word_embed_trainable,
mask_zero=True)
text_embed = SpatialDropout1D(0.2)(concatenate([
word_embedding(input_text),
text_elmo_embedding(input_text)
]))
text_l_embed = SpatialDropout1D(0.2)(concatenate([
word_embedding(input_text_l),
l_elmo_embedding(input_text_l)
]))
text_r_embed = SpatialDropout1D(0.2)(concatenate([
word_embedding(input_text_r),
r_elmo_embedding(input_text_r)
]))
else:
word_embedding = Embedding(
input_dim=self.text_embeddings.shape[0],
output_dim=self.config.word_embed_dim,
weights=[self.text_embeddings],
trainable=self.config.word_embed_trainable,
mask_zero=True)
text_embed = SpatialDropout1D(0.2)(word_embedding(input_text))
text_l_embed = SpatialDropout1D(0.2)(word_embedding(input_text_l))
text_r_embed = SpatialDropout1D(0.2)(word_embedding(input_text_r))
if self.config.aspect_embed_type == 'random':
asp_embedding = Embedding(input_dim=self.n_aspect,
output_dim=self.config.aspect_embed_dim)
else:
asp_embedding = Embedding(
input_dim=self.aspect_embeddings.shape[0],
output_dim=self.config.aspect_embed_dim,
trainable=self.config.aspect_embed_trainable)
aspect_embed = asp_embedding(input_aspect)
aspect_embed = Flatten()(aspect_embed) # reshape to 2d
# regarding aspect string as the first unit
hidden_l = GRU(self.config.lstm_units,
go_backwards=True,
return_sequences=True)(text_l_embed)
hidden_r = GRU(self.config.lstm_units,
return_sequences=True)(text_r_embed)
# left context attention
context_attend_l = TimeDistributed(Dense(
1, activation='sigmoid'))(hidden_l)
# Note: I couldn't find `reverse_sequence` in keras
context_attend_l = Lambda(lambda x: tf.reverse_sequence(
x, sequence_length(x), 1, 0))(context_attend_l)
context_attend_l = Lambda(lambda x: K.squeeze(x, -1))(context_attend_l)
# right context attention
context_attend_r = TimeDistributed(Dense(
1, activation='sigmoid'))(hidden_r)
context_attend_r = Lambda(lambda x: K.squeeze(x, -1))(context_attend_r)
# combine context attention
# aspect_text_embed = subtract([add([text_l_embed, text_r_embed]), text_embed])
# aspect_text_mask = Lambda(lambda x: sequence_mask(x))(aspect_text_embed)
# text_mask = Lambda(lambda x: sequence_mask(x))(text_embed)
# context_mask = subtract([text_mask, aspect_text_mask])
# aspect_text_mask_half = Lambda(lambda x: x*0.5)(aspect_text_mask)
# combine_mask = add([context_mask, aspect_text_mask_half]) # 1 for context, 0.5 for aspect
context_attend = multiply(
[add([context_attend_l, context_attend_r]), input_mask])
# apply context attention
context_attend_expand = Lambda(lambda x: K.expand_dims(x))(
context_attend)
memory = multiply([text_embed, context_attend_expand])
# sentence-level content attention
sentence = Lambda(lambda x: K.mean(x, axis=1))(memory)
final_output = ContentAttention()([memory, aspect_embed, sentence])
return Model(
[input_text, input_text_l, input_text_r, input_aspect, input_mask],
final_output)
