def create_model(args, maxlen, vocab, wordsref):
def ortho_reg(weight_matrix):
### orthogonal regularization for aspect embedding matrix ###
wn_ = weight_matrix / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(WeightedSum), axis=-1, keepdims=True)), K.floatx())
reg = K.sum(K.square(K.dot(w_n, K.transpose(w_n))-K.eye(w_n.shape[0])))
return args.ortho_reg*reg
vocab_size = len(vocab)
################## Inputs #################
sentence_input = Input(shape=(maxlen,), dtype='int32', name='sentence_input')
neg_input = Input(shape=(args.neg_size, maxlen), dtype='int32', name='neg_input')
word_emb = Embedding(vocab_size, args.emb_dim, mask_zero=True, name='word_emb')
######### Compute sentence representation ########
e_w = word_emb(sentence_input)
e_wordsref = word_emb(wordsref)
y_s = Average()(e_w)
att_words_ref = WordsRef(name='att_words_ref')([e_wordsref, e_w, y_s])
z_ss = WeightedSum()([e_w, att_words_ref])
######## Compute representations of negative instances #######
e_neg = word_emb(neg_input)
z_n = Average()(e_neg)
def create_model(args, maxlen, vocab):
def ortho_reg(weight_matrix):
### orthogonal regularization for aspect embedding matrix ###
w_n = weight_matrix / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(weight_matrix), axis=-1, keepdims=True)), K.floatx())
reg = K.sum(K.square(K.dot(w_n, K.transpose(w_n)) - K.eye((w_n.shape[0]).eval())))
return args.ortho_reg*reg
# 词汇表大小
vocab_size = len(vocab)
##### Inputs #####
# 正例的形状：batch_size * dim, 每个元素是在词汇表中的索引值, 每个句子有多少个词就有多少索引值
# 负例的形状：batch_size * args.neg_size * dim, ditto
# 得到w
sentence_input = Input(batch_shape=(None, maxlen), dtype='int32', name='sentence_input')
neg_input = Input(batch_shape=(None, args.neg_size, maxlen), dtype='int32', name='neg_input')
##### Construct word embedding layer #####
# 嵌入层将正整数（下标）转换为具有固定大小的向量，如[[4],[20]]->[[0.25,0.1],[0.6,-0.2]]
# keras.layers.embeddings.Embedding(input_dim, output_dim, embeddings_initializer='uniform', embeddings_regularizer=None, activity_regularizer=None, embeddings_constraint=None, mask_zero=False, input_length=None)
word_emb = Embedding(vocab_size, args.emb_dim, mask_zero=True, name='word_emb')
##### Compute sentence representation #####
# 计算句子嵌入，这里设计到keras的很多细节，日后还需要深入学习
e_w = word_emb(sentence_input)
y_s = Average()(e_w)
att_weights = Attention(name='att_weights')([e_w, y_s])
z_s = WeightedSum()([e_w, att_weights])
##### Compute representations of negative instances #####
# 计算负例的z_n
e_neg = word_emb(neg_input)
z_n = Average()(e_neg)
##### Reconstruction #####
# 重构过程
p_t = Dense(args.aspect_size)(z_s)
p_t = Activation('softmax', name='p_t')(p_t)
r_s = WeightedAspectEmb(args.aspect_size, args.emb_dim, name='aspect_emb',
W_regularizer=ortho_reg)(p_t)
##### Loss #####
# 损失函数
loss = MaxMargin(name='max_margin')([z_s, z_n, r_s])
model = Model(input=[sentence_input, neg_input], output=loss)
### Word embedding and aspect embedding initialization ######
# 如果定义了emb_path, 就用文件中的数值初始化E矩阵, T使用K-means初始化
if args.emb_path:
from w2vEmbReader import W2VEmbReader as EmbReader
emb_reader = EmbReader(args.emb_path, emb_dim=args.emb_dim)
logger.info('Initializing word embedding matrix')
model.get_layer('word_emb').set_weights(emb_reader.get_emb_matrix_given_vocab(vocab, model.get_layer('word_emb').get_weights()))
logger.info('Initializing aspect embedding matrix as centroid of kmean clusters')
model.get_layer('aspect_emb').W.set_value(emb_reader.get_aspect_matrix(args.aspect_size))
return model
def create_model(args, maxlen, vocab):
def ortho_reg(weight_matrix):
### orthogonal regularization for aspect embedding matrix ###
w_n = weight_matrix / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(weight_matrix), axis=-1, keepdims=True)), K.floatx())
reg = K.sum(K.square(K.dot(w_n, K.transpose(w_n)) - K.eye(w_n.shape[0].value)))
return args.ortho_reg*reg
vocab_size = len(vocab)
##### Inputs #####
sentence_input = Input(shape=(maxlen,), dtype='int32', name='sentence_input')
neg_input = Input(shape=(args.neg_size, maxlen), dtype='int32', name='neg_input')
##### Construct word embedding layer #####
word_emb = Embedding(vocab_size, args.emb_dim, mask_zero=True, name='word_emb')
##### Compute sentence representation #####
e_w = word_emb(sentence_input)
#y_s = LSTM(args.emb_dim, return_sequences=False)(e_w)
y_s = Average()(e_w)
att_weights = Attention(name='att_weights')([e_w, y_s])
z_s = WeightedSum()([e_w, att_weights])
##### Compute representations of negative instances #####
e_neg = word_emb(neg_input)
#z_n = TimeDistributed(LSTM(args.emb_dim, return_sequences=False))(e_neg)
z_n = Average()(e_neg)
##### Reconstruction #####
p_t = Dense(args.aspect_size)(z_s)
p_t = Activation('softmax', name='p_t')(p_t)
r_s = WeightedAspectEmb(args.aspect_size, args.emb_dim, name='aspect_emb',
W_regularizer=ortho_reg)(p_t)
##### Loss #####
loss = MaxMargin(name='max_margin')([z_s, z_n, r_s])
model = Model(inputs=[sentence_input, neg_input], outputs=loss)
### Word embedding and aspect embedding initialization ######
if args.emb_path:
from w2vEmbReader import W2VEmbReader as EmbReader
emb_reader = EmbReader(args.emb_path, emb_dim=args.emb_dim)
logger.info('Initializing word embedding matrix')
# model.get_layer('word_emb').set_weights(emb_reader.get_emb_matrix_given_vocab(vocab, model.get_layer('word_emb').get_weights()[0]))
K.set_value(
model.get_layer('word_emb').embeddings,
emb_reader.get_emb_matrix_given_vocab(vocab, K.get_value(model.get_layer('word_emb').embeddings)))
logger.info('Initializing aspect embedding matrix as centroid of kmean clusters')
K.set_value(
model.get_layer('aspect_emb').W,
emb_reader.get_aspect_matrix(args.aspect_size))
return model
X, Y = read_node_label(args.label_file)
node_size = len(X)
###############################################################################################################################
## Building model
sentence_input = Input(shape=(args.kstep, node_size),
dtype='float32',
name='sentence_inputt')
neg_input = Input(shape=(args.neg_size, args.kstep, node_size),
dtype='float32',
name='neg_inputt')
predict = Input(shape=(17, ), dtype='int32', name='predictt')
e_w = sentence_input
y_s = Average()(sentence_input)
att_weights = Attention(name='att_weights')([e_w, y_s])
z_s = WeightedSum()([e_w, att_weights])
##### Compute representations of negative instances #####
e_neg = neg_input
z_n = Average()(e_neg)
##### Reconstruction2 #####
dense1 = Dense(512, activation='relu')(z_s)
#dense1 = noise.GaussianNoise(0.09)(dense1)
p_t = Dense(128, activation='relu')(dense1)
dense2 = Dense(512, activation='relu')(p_t)
new_p_t = Activation('softmax', name='p_t')(dense2)
r_s = Dense(node_size, init='uniform')(new_p_t)
def create_model(args, maxlen, vocab):
def ortho_reg(weight_matrix):
### orthogonal regularization for aspect embedding matrix ###
w_n = K.l2_normalize(weight_matrix, axis=-1)
reg = K.sum(
K.square(
K.dot(w_n, K.transpose(w_n)) -
K.eye(w_n.get_shape().as_list()[0])))
return args.ortho_reg * reg
vocab_size = len(vocab)
if args.emb_name:
from w2vEmbReader import W2VEmbReader as EmbReader
emb_reader = EmbReader(
os.path.join("..", "preprocessed_data", args.domain),
args.emb_name)
aspect_matrix = emb_reader.get_aspect_matrix(args.aspect_size)
args.aspect_size = emb_reader.aspect_size
args.emb_dim = emb_reader.emb_dim
##### Inputs #####
sentence_input = Input(shape=(maxlen, ),
dtype='int32',
name='sentence_input')
neg_input = Input(shape=(args.neg_size, maxlen),
dtype='int32',
name='neg_input')
##### Construct word embedding layer #####
word_emb = Embedding(vocab_size,
args.emb_dim,
mask_zero=True,
name='word_emb')
#, embeddings_constraint=MaxNorm(10))
##### Compute sentence representation #####
e_w = word_emb(sentence_input)
y_s = Average()(e_w)
att_weights = Attention(name='att_weights')([e_w, y_s])
#W_constraint=MaxNorm(10),
#b_constraint=MaxNorm(10))([e_w, y_s])
z_s = WeightedSum()([e_w, att_weights])
##### Compute representations of negative instances #####
e_neg = word_emb(neg_input)
z_n = Average()(e_neg)
##### Reconstruction #####
p_t = Dense(args.aspect_size)(z_s)
p_t = Activation('softmax', name='p_t')(p_t)
r_s = WeightedAspectEmb(
args.aspect_size,
args.emb_dim,
name='aspect_emb',
#W_constraint=MaxNorm(10),
W_regularizer=ortho_reg)(p_t)
##### Loss #####
loss = MaxMargin(name='max_margin')([z_s, z_n, r_s])
model = Model(inputs=[sentence_input, neg_input], outputs=[loss])
### Word embedding and aspect embedding initialization ######
if args.emb_name:
from w2vEmbReader import W2VEmbReader as EmbReader
logger.info('Initializing word embedding matrix')
embs = model.get_layer('word_emb').embeddings
K.set_value(
embs,
emb_reader.get_emb_matrix_given_vocab(vocab, K.get_value(embs)))
logger.info(
'Initializing aspect embedding matrix as centroid of kmean clusters'
)
K.set_value(model.get_layer('aspect_emb').W, aspect_matrix)
return model
def create_model(args, kstep, node_size):
def ortho_reg(weight_matrix):
### orthogonal regularization for aspect embedding matrix ###
w_n = weight_matrix / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(weight_matrix), axis=-1, keepdims=True)), K.floatx())
reg = K.sum(K.square(K.dot(w_n, K.transpose(w_n)) - K.eye(w_n.shape[0].eval())))
return args.ortho_reg*reg
##### Inputs #####
sentence_input = Input(shape=(kstep, node_size), dtype='float32', name='sentence_input')
neg_input = Input(shape=(args.neg_size, kstep, node_size), dtype='float32', name='neg_input')
print("sentence_input.ndim", sentence_input.ndim)
print("neg_input.ndim", neg_input.ndim)
e_w = sentence_input
y_s = Average()(sentence_input)
print(y_s.ndim)
print(e_w.ndim)
print(K.int_shape(e_w), K.int_shape(y_s))
att_weights = Attention(name='att_weights')([e_w, y_s])
z_s = WeightedSum()([e_w, att_weights])
print("z_s----------- %d", (z_s.ndim))
##### Compute representations of negative instances #####
#e_neg = word_emb(neg_input)
e_neg = neg_input
z_n = Average()(e_neg)
print("e_neg.ndim")
print(e_neg.ndim)
print("z_n.ndim")
print(z_n.ndim)
##### Reconstruction #####
p_t = Dense(args.aspect_size)(z_s)
p_t = Activation('softmax', name='p_t')(p_t)
r_s = WeightedAspectEmb(args.aspect_size, 2405, name='aspect_emb',
W_regularizer=ortho_reg)(p_t)
##### Loss #####
print("losssssssssssssssssssssssssssssssssssssssssssssssssssssssssss")
print(K.int_shape(z_s), K.int_shape(r_s))
loss = MaxMargin(name='max_margin')([z_s, z_n, r_s])
model = Model(input=[sentence_input, neg_input], output=loss)
### Word embedding and aspect embedding initialization ######
if args.emb_path:
from w2vEmbReader import W2VEmbReader as EmbReader
emb_reader = EmbReader(args.emb_path, emb_dim=args.emb_dim)
logger.info('Initializing word embedding matrix')
model.get_layer('word_emb').W.set_value(emb_reader.get_emb_matrix_given_vocab(vocab, model.get_layer('word_emb').W.get_value()))
logger.info('Initializing aspect embedding matrix as centroid of kmean clusters')
model.get_layer('aspect_emb').W.set_value(emb_reader.get_aspect_matrix(args.aspect_size))
return model
def create_model(args, maxlen, vocab):
def ortho_reg(weight_matrix):
### orthogonal regularization for aspect embedding matrix ###
w_n = K.l2_normalize(weight_matrix, axis=-1) # K表示调用该函数的当前layer
reg = K.sum(
K.square(
K.dot(w_n, K.transpose(w_n)) -
K.eye(w_n.shape[0].value))) # 自身矩阵的内积的平方根-自身特征值 = 越小, 越说明分量为0
return args.ortho_reg * reg # 这东西越小越好, 因为能保证各个特征分的越开
vocab_size = len(vocab)
if args.emb_name: # 获取已经保存的embedding???
from w2vEmbReader import W2VEmbReader as EmbReader
emb_reader = EmbReader(
os.path.join(
"/content/drive/My Drive/Attention-Based-Aspect-Extraction-master",
"preprocessed_data", args.domain), args.emb_name)
aspect_matrix = emb_reader.get_aspect_matrix(args.aspect_size)
args.aspect_size = emb_reader.aspect_size
args.emb_dim = emb_reader.emb_dim
##### Inputs #####
sentence_input = Input(shape=(maxlen, ),
dtype='int32',
name='sentence_input')
neg_input = Input(shape=(args.neg_size, maxlen),
dtype='int32',
name='neg_input')
##### Construct word embedding layer #####
word_emb = Embedding(vocab_size,
args.emb_dim,
mask_zero=True,
name='word_emb',
embeddings_constraint=MaxNorm(10))
##### Compute sentence representation ##### pre-processing 根据attention组合句子
e_w = word_emb(sentence_input) # 将input转换为embedding
y_s = Average()(e_w) # 默认求平均 layer
att_weights = Attention(name='att_weights',
W_constraint=MaxNorm(10),
b_constraint=MaxNorm(10))([e_w,
y_s]) # attention layer
z_s = WeightedSum()([e_w, att_weights]) # encoding layer
##### Compute representations of negative instances ##### 增加准确性的tricks
e_neg = word_emb(neg_input)
z_n = Average()(e_neg)
##### Reconstruction ##### 构建dense层, 希望能够decoding attention sentences的特征
p_t = Dense(args.aspect_size)(z_s)
p_t = Activation('softmax',
name='p_t')(p_t) # softmax一下, nodes数量不改变, 数值被soft了一下
r_s = WeightedAspectEmb(args.aspect_size,
args.emb_dim,
name='aspect_emb',
W_constraint=MaxNorm(10),
W_regularizer=ortho_reg)(
p_t) # 标准化0-10区间, 且正则项为自定义的ortho_reg
##### Loss #####
loss = MaxMargin(name='max_margin')([z_s, z_n,
r_s]) # 自定义loss function??? 这是在做啥???
model = Model(inputs=[sentence_input, neg_input],
outputs=[loss]) # negative input是需要自己分开数据集的吗??
### Word embedding and aspect embedding initialization ######
if args.emb_name:
from w2vEmbReader import W2VEmbReader as EmbReader
logger.info('Initializing word embedding matrix')
embs = model.get_layer('word_emb').embeddings
K.set_value(
embs,
emb_reader.get_emb_matrix_given_vocab(vocab, K.get_value(embs)))
logger.info(
'Initializing aspect embedding matrix as centroid of kmean clusters'
) # 为何初始化要用到kmeans
K.set_value(model.get_layer('aspect_emb').W, aspect_matrix) # r-s
return model
def create_model(args, maxlen, vocab):
def ortho_reg(weight_matrix):
### orthogonal regularization for aspect embedding matrix ###
w_n = K.l2_normalize(weight_matrix, axis=-1)
reg = K.sum(
K.square(K.dot(w_n, K.transpose(w_n)) - K.eye(w_n.shape[0].value)))
return args.ortho_reg * reg
vocab_size = len(vocab)
if args.emb_name:
if args.emb_technique == "w2v":
logger.info("Load {} glove embedding for {}".format(
args.lang, config.word_emb_training_type))
if args.lang == 'en':
emb_reader = EmbReader(
config.emb_dir_en["w2v"].format(
config.word_emb_training_type), args.emb_name)
elif args.lang == 'de':
#emb_reader = EmbReader(config.emb_dir_de["w2v"].format(config.word_emb_training_type),
#args.emb_name)
emb_reader = FineTuneEmbed_cca(
'../preprocessed_data/german/w2v/fine_tuned', 'w2v_emb',
'../preprocessed_data/german/w2v/full_trained',
'w2v_embedding_300')
elif args.emb_technique == 'fasttext':
if args.lang == 'en':
#emb_reader = FastTextEmbReader(config.emb_dir_en["fasttext"].format(config.word_emb_training_type),
#args.emb_name, config.fine_tuned_enabled)
emb_reader = FineTuneEmbed_ortho_procrustes(
'../preprocessed_data/fasttext/fine_tuned',
'fasttext_pre_trained',
'../preprocessed_data/fasttext/full_trained',
'w2v_embedding_skipgram_300')
elif args.lang == 'de':
emb_reader = FastTextEmbReader(
config.emb_dir_de["fasttext"].format(
config.word_emb_training_type), args.emb_name,
config.fine_tuned_enabled)
#emb_reader = FineTuneEmbed_ortho_procrustes('../preprocessed_data/fasttext/fine_tuned','fasttext_pre_trained','../preprocessed_data/fasttext/full_trained', 'w2v_embedding_skipgram_300')
elif args.emb_technique == "glove":
if args.lang == 'de':
logger.info('Load german glove embedding')
emb_reader = GloveEmbedding(config.emb_dir_de["glove"],
args.emb_name)
else:
logger.info("Load en glove embedding for {}".format(
config.word_emb_training_type))
emb_reader = GloveEmbedding(
config.emb_dir_en["glove"].format(
config.word_emb_training_type), args.emb_name)
elif args.emb_technique == "MUSE_supervised":
emb_reader = MUSEEmbedding(config.emb_dir_biling['supervised'],
args.emb_name)
elif args.emb_technique == "MUSE_unsupervised":
emb_reader = MUSEEmbedding(config.emb_dir_biling['unsupervised'],
args.emb_name)
aspect_matrix = emb_reader.get_aspect_matrix(args.aspect_size)
args.aspect_size = emb_reader.aspect_size
args.emb_dim = emb_reader.emb_dim
##### Inputs #####
sentence_input = Input(shape=(maxlen, ),
dtype='int32',
name='sentence_input')
neg_input = Input(shape=(args.neg_size, maxlen),
dtype='int32',
name='neg_input')
##### Construct word embedding layer #####
word_emb = Embedding(vocab_size,
args.emb_dim,
mask_zero=True,
name='word_emb',
embeddings_constraint=MaxNorm(10))
##### Compute sentence representation #####
e_w = word_emb(sentence_input)
y_s = Average()(e_w)
att_weights = Attention(name='att_weights',
W_constraint=MaxNorm(10),
b_constraint=MaxNorm(10))([e_w, y_s])
z_s = WeightedSum()([e_w, att_weights])
##### Compute representations of negative instances #####
e_neg = word_emb(neg_input)
z_n = Average()(e_neg)
##### Reconstruction #####
p_t = Dense(args.aspect_size)(z_s)
p_t = Activation('softmax', name='p_t')(p_t)
r_s = WeightedAspectEmb(args.aspect_size,
args.emb_dim,
name='aspect_emb',
W_constraint=MaxNorm(10),
W_regularizer=ortho_reg)(p_t)
##### Loss #####
loss = MaxMargin(name='max_margin')([z_s, z_n, r_s])
model = Model(inputs=[sentence_input, neg_input], outputs=[loss])
### Word embedding and aspect embedding initialization ######
if args.emb_name:
logger.info('Initializing word embedding matrix')
embs = model.get_layer('word_emb').embeddings
K.set_value(
embs,
emb_reader.get_emb_matrix_given_vocab(vocab, K.get_value(embs)))
logger.info(
'Initializing aspect embedding matrix as centroid of kmean clusters'
)
K.set_value(model.get_layer('aspect_emb').W, aspect_matrix)
return model
def create_model(args, vocab, num_outputs, overal_maxlen, maxlen_aspect):
###############################################################################################################################
## Recurrence unit type
#
if args.recurrent_unit == 'lstm':
from keras.layers.recurrent import LSTM as RNN
elif args.recurrent_unit == 'gru':
from keras.layers.recurrent import GRU as RNN
elif args.recurrent_unit == 'simple':
from keras.layers.recurrent import SimpleRNN as RNN
###############################################################################################################################
## Create Model
#
dropout = args.dropout_W
recurrent_dropout = args.dropout_U
vocab_size = len(vocab)
logger.info('Building a LSTM attention model to predict term/aspect sentiment')
print '\n\n'
##### Inputs #####
sentence_input = Input(shape=(overal_maxlen,), dtype='int32', name='sentence_input')
aspect_input = Input(shape=(maxlen_aspect,), dtype='int32', name='aspect_input')
pretrain_input = Input(shape=(None,), dtype='int32', name='pretrain_input')
##### construct word embedding layer #####
word_emb = Embedding(vocab_size, args.emb_dim, mask_zero=True, name='word_emb')
### represent aspect as averaged word embedding ###
print 'use average term embs as aspect embedding'
aspect_term_embs = word_emb(aspect_input)
aspect_embs = Average(mask_zero=True, name='aspect_emb')(aspect_term_embs)
### sentence representation ###
sentence_output = word_emb(sentence_input)
pretrain_output = word_emb(pretrain_input)
print 'use a rnn layer'
rnn = RNN(args.rnn_dim, return_sequences=True, dropout=dropout, recurrent_dropout=recurrent_dropout, name='lstm')
sentence_output = rnn(sentence_output)
pretrain_output = rnn(pretrain_output)
print 'use content attention to get term weights'
att_weights = Attention(name='att_weights')([sentence_output, aspect_embs])
sentence_output = WeightedSum()([sentence_output, att_weights])
pretrain_output = Average(mask_zero=True)(pretrain_output)
if args.dropout_prob > 0:
print 'use dropout layer'
sentence_output = Dropout(args.dropout_prob)(sentence_output)
pretrain_output = Dropout(args.dropout_prob)(pretrain_output)
sentence_output = Dense(num_outputs, name='dense_1')(sentence_output)
pretrain_output = Dense(num_outputs, name='dense_2')(pretrain_output)
aspect_probs = Activation('softmax', name='aspect_model')(sentence_output)
doc_probs = Activation('softmax', name='pretrain_model')(pretrain_output)
model = Model(inputs=[sentence_input, aspect_input, pretrain_input], outputs=[aspect_probs, doc_probs])
logger.info(' Done')
###############################################################################################################################
## Initialize embeddings if requested
#
if args.is_pretrain:
import pickle
print 'Set embedding, lstm, and dense weights from pre-trained models'
if args.domain == 'lt':
f_1 = open('../pretrained_weights/lstm_weights_lt%.1f.pkl'%(args.percetage), 'rb')
f_2 = open('../pretrained_weights/dense_weights_lt%.1f.pkl'%(args.percetage), 'rb')
else:
f_1 = open('../pretrained_weights/lstm_weights_res%.1f.pkl'%(args.percetage), 'rb')
f_2 = open('../pretrained_weights/dense_weights_res%.1f.pkl'%(args.percetage), 'rb')
lstm_weights = pickle.load(f_1)
dense_weights = pickle.load(f_2)
model.get_layer('lstm').set_weights(lstm_weights)
model.get_layer('dense_1').set_weights(dense_weights)
model.get_layer('dense_2').set_weights(dense_weights)
from w2vEmbReader import W2VEmbReader as EmbReader
logger.info('Initializing lookup table')
emb_path = '../glove/%s.txt'%(args.domain)
emb_reader = EmbReader(args, emb_path)
model.get_layer('word_emb').set_weights(emb_reader.get_emb_matrix_given_vocab(vocab, model.get_layer('word_emb').get_weights()))
logger.info(' Done')
return model
