def create_model(args, maxlen, vocab):
def ortho_reg(weight_matrix):
### orthogonal regularization for aspect embedding matrix ###
w_n = weight_matrix / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(weight_matrix), axis=-1, keepdims=True)), K.floatx())
reg = K.sum(K.square(K.dot(w_n, K.transpose(w_n)) - K.eye((w_n.shape[0]).eval())))
return args.ortho_reg*reg
# 词汇表大小
vocab_size = len(vocab)
##### Inputs #####
# 正例的形状：batch_size * dim, 每个元素是在词汇表中的索引值, 每个句子有多少个词就有多少索引值
# 负例的形状：batch_size * args.neg_size * dim, ditto
# 得到w
sentence_input = Input(batch_shape=(None, maxlen), dtype='int32', name='sentence_input')
neg_input = Input(batch_shape=(None, args.neg_size, maxlen), dtype='int32', name='neg_input')
##### Construct word embedding layer #####
# 嵌入层将正整数（下标）转换为具有固定大小的向量，如[[4],[20]]->[[0.25,0.1],[0.6,-0.2]]
# keras.layers.embeddings.Embedding(input_dim, output_dim, embeddings_initializer='uniform', embeddings_regularizer=None, activity_regularizer=None, embeddings_constraint=None, mask_zero=False, input_length=None)
word_emb = Embedding(vocab_size, args.emb_dim, mask_zero=True, name='word_emb')
##### Compute sentence representation #####
# 计算句子嵌入，这里设计到keras的很多细节，日后还需要深入学习
e_w = word_emb(sentence_input)
y_s = Average()(e_w)
att_weights = Attention(name='att_weights')([e_w, y_s])
z_s = WeightedSum()([e_w, att_weights])
##### Compute representations of negative instances #####
# 计算负例的z_n
e_neg = word_emb(neg_input)
z_n = Average()(e_neg)
##### Reconstruction #####
# 重构过程
p_t = Dense(args.aspect_size)(z_s)
p_t = Activation('softmax', name='p_t')(p_t)
r_s = WeightedAspectEmb(args.aspect_size, args.emb_dim, name='aspect_emb',
W_regularizer=ortho_reg)(p_t)
##### Loss #####
# 损失函数
loss = MaxMargin(name='max_margin')([z_s, z_n, r_s])
model = Model(input=[sentence_input, neg_input], output=loss)
### Word embedding and aspect embedding initialization ######
# 如果定义了emb_path, 就用文件中的数值初始化E矩阵, T使用K-means初始化
if args.emb_path:
from w2vEmbReader import W2VEmbReader as EmbReader
emb_reader = EmbReader(args.emb_path, emb_dim=args.emb_dim)
logger.info('Initializing word embedding matrix')
model.get_layer('word_emb').set_weights(emb_reader.get_emb_matrix_given_vocab(vocab, model.get_layer('word_emb').get_weights()))
logger.info('Initializing aspect embedding matrix as centroid of kmean clusters')
model.get_layer('aspect_emb').W.set_value(emb_reader.get_aspect_matrix(args.aspect_size))
return model
def create_model(args, maxlen, vocab):
def ortho_reg(weight_matrix):
### orthogonal regularization for aspect embedding matrix ###
w_n = weight_matrix / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(weight_matrix), axis=-1, keepdims=True)), K.floatx())
reg = K.sum(K.square(K.dot(w_n, K.transpose(w_n)) - K.eye(w_n.shape[0].value)))
return args.ortho_reg*reg
vocab_size = len(vocab)
##### Inputs #####
sentence_input = Input(shape=(maxlen,), dtype='int32', name='sentence_input')
neg_input = Input(shape=(args.neg_size, maxlen), dtype='int32', name='neg_input')
##### Construct word embedding layer #####
word_emb = Embedding(vocab_size, args.emb_dim, mask_zero=True, name='word_emb')
##### Compute sentence representation #####
e_w = word_emb(sentence_input)
#y_s = LSTM(args.emb_dim, return_sequences=False)(e_w)
y_s = Average()(e_w)
att_weights = Attention(name='att_weights')([e_w, y_s])
z_s = WeightedSum()([e_w, att_weights])
##### Compute representations of negative instances #####
e_neg = word_emb(neg_input)
#z_n = TimeDistributed(LSTM(args.emb_dim, return_sequences=False))(e_neg)
z_n = Average()(e_neg)
##### Reconstruction #####
p_t = Dense(args.aspect_size)(z_s)
p_t = Activation('softmax', name='p_t')(p_t)
r_s = WeightedAspectEmb(args.aspect_size, args.emb_dim, name='aspect_emb',
W_regularizer=ortho_reg)(p_t)
##### Loss #####
loss = MaxMargin(name='max_margin')([z_s, z_n, r_s])
model = Model(inputs=[sentence_input, neg_input], outputs=loss)
### Word embedding and aspect embedding initialization ######
if args.emb_path:
from w2vEmbReader import W2VEmbReader as EmbReader
emb_reader = EmbReader(args.emb_path, emb_dim=args.emb_dim)
logger.info('Initializing word embedding matrix')
# model.get_layer('word_emb').set_weights(emb_reader.get_emb_matrix_given_vocab(vocab, model.get_layer('word_emb').get_weights()[0]))
K.set_value(
model.get_layer('word_emb').embeddings,
emb_reader.get_emb_matrix_given_vocab(vocab, K.get_value(model.get_layer('word_emb').embeddings)))
logger.info('Initializing aspect embedding matrix as centroid of kmean clusters')
K.set_value(
model.get_layer('aspect_emb').W,
emb_reader.get_aspect_matrix(args.aspect_size))
return model
dense1 = Dense(512, activation='relu')(z_s)
#dense1 = noise.GaussianNoise(0.09)(dense1)
p_t = Dense(128, activation='relu')(dense1)
dense2 = Dense(512, activation='relu')(p_t)
new_p_t = Activation('softmax', name='p_t')(dense2)
r_s = Dense(node_size, init='uniform')(new_p_t)
##### Loss1 #####
lable_ls1 = Dense(22, activation='relu')(p_t)
lable_ls = Dense(17, activation='softmax',
name='lable_classification')(lable_ls1)
#loss2 = Square_loss(name='square_loss')([predict, lable_ls])
##### Loss2 #####
loss1 = MaxMargin(name='max_margin')([z_s, z_n, r_s])
##### Model #####
# model_auto = Model(input=[sentence_input, neg_input], output=[lable_ls, loss1])
#model_auto = Model(input=sentence_input, output=lable_ls)
model_auto = Model(input=[sentence_input, neg_input], output=loss1)
model_lable = Model(input=sentence_input, output=lable_ls)
low_encoder = Model(sentence_input, p_t)
###############################################################################################################################
## Training1 GRArep
t1 = time.time()
g = Graph()
print "Reading data..."
if args.graph_format == 'adjlist':
g.read_adjlist(filename=args.input)
def create_model(args, maxlen, vocab):
def ortho_reg(weight_matrix):
### orthogonal regularization for aspect embedding matrix ###
w_n = K.l2_normalize(weight_matrix, axis=-1)
reg = K.sum(
K.square(
K.dot(w_n, K.transpose(w_n)) -
K.eye(w_n.get_shape().as_list()[0])))
return args.ortho_reg * reg
vocab_size = len(vocab)
if args.emb_name:
from w2vEmbReader import W2VEmbReader as EmbReader
emb_reader = EmbReader(
os.path.join("..", "preprocessed_data", args.domain),
args.emb_name)
aspect_matrix = emb_reader.get_aspect_matrix(args.aspect_size)
args.aspect_size = emb_reader.aspect_size
args.emb_dim = emb_reader.emb_dim
##### Inputs #####
sentence_input = Input(shape=(maxlen, ),
dtype='int32',
name='sentence_input')
neg_input = Input(shape=(args.neg_size, maxlen),
dtype='int32',
name='neg_input')
##### Construct word embedding layer #####
word_emb = Embedding(vocab_size,
args.emb_dim,
mask_zero=True,
name='word_emb')
#, embeddings_constraint=MaxNorm(10))
##### Compute sentence representation #####
e_w = word_emb(sentence_input)
y_s = Average()(e_w)
att_weights = Attention(name='att_weights')([e_w, y_s])
#W_constraint=MaxNorm(10),
#b_constraint=MaxNorm(10))([e_w, y_s])
z_s = WeightedSum()([e_w, att_weights])
##### Compute representations of negative instances #####
e_neg = word_emb(neg_input)
z_n = Average()(e_neg)
##### Reconstruction #####
p_t = Dense(args.aspect_size)(z_s)
p_t = Activation('softmax', name='p_t')(p_t)
r_s = WeightedAspectEmb(
args.aspect_size,
args.emb_dim,
name='aspect_emb',
#W_constraint=MaxNorm(10),
W_regularizer=ortho_reg)(p_t)
##### Loss #####
loss = MaxMargin(name='max_margin')([z_s, z_n, r_s])
model = Model(inputs=[sentence_input, neg_input], outputs=[loss])
### Word embedding and aspect embedding initialization ######
if args.emb_name:
from w2vEmbReader import W2VEmbReader as EmbReader
logger.info('Initializing word embedding matrix')
embs = model.get_layer('word_emb').embeddings
K.set_value(
embs,
emb_reader.get_emb_matrix_given_vocab(vocab, K.get_value(embs)))
logger.info(
'Initializing aspect embedding matrix as centroid of kmean clusters'
)
K.set_value(model.get_layer('aspect_emb').W, aspect_matrix)
return model
def create_model(args, kstep, node_size):
def ortho_reg(weight_matrix):
### orthogonal regularization for aspect embedding matrix ###
w_n = weight_matrix / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(weight_matrix), axis=-1, keepdims=True)), K.floatx())
reg = K.sum(K.square(K.dot(w_n, K.transpose(w_n)) - K.eye(w_n.shape[0].eval())))
return args.ortho_reg*reg
##### Inputs #####
sentence_input = Input(shape=(kstep, node_size), dtype='float32', name='sentence_input')
neg_input = Input(shape=(args.neg_size, kstep, node_size), dtype='float32', name='neg_input')
print("sentence_input.ndim", sentence_input.ndim)
print("neg_input.ndim", neg_input.ndim)
e_w = sentence_input
y_s = Average()(sentence_input)
print(y_s.ndim)
print(e_w.ndim)
print(K.int_shape(e_w), K.int_shape(y_s))
att_weights = Attention(name='att_weights')([e_w, y_s])
z_s = WeightedSum()([e_w, att_weights])
print("z_s----------- %d", (z_s.ndim))
##### Compute representations of negative instances #####
#e_neg = word_emb(neg_input)
e_neg = neg_input
z_n = Average()(e_neg)
print("e_neg.ndim")
print(e_neg.ndim)
print("z_n.ndim")
print(z_n.ndim)
##### Reconstruction #####
p_t = Dense(args.aspect_size)(z_s)
p_t = Activation('softmax', name='p_t')(p_t)
r_s = WeightedAspectEmb(args.aspect_size, 2405, name='aspect_emb',
W_regularizer=ortho_reg)(p_t)
##### Loss #####
print("losssssssssssssssssssssssssssssssssssssssssssssssssssssssssss")
print(K.int_shape(z_s), K.int_shape(r_s))
loss = MaxMargin(name='max_margin')([z_s, z_n, r_s])
model = Model(input=[sentence_input, neg_input], output=loss)
### Word embedding and aspect embedding initialization ######
if args.emb_path:
from w2vEmbReader import W2VEmbReader as EmbReader
emb_reader = EmbReader(args.emb_path, emb_dim=args.emb_dim)
logger.info('Initializing word embedding matrix')
model.get_layer('word_emb').W.set_value(emb_reader.get_emb_matrix_given_vocab(vocab, model.get_layer('word_emb').W.get_value()))
logger.info('Initializing aspect embedding matrix as centroid of kmean clusters')
model.get_layer('aspect_emb').W.set_value(emb_reader.get_aspect_matrix(args.aspect_size))
return model
def create_model(args, maxlen, vocab):
def ortho_reg(weight_matrix):
### orthogonal regularization for aspect embedding matrix ###
w_n = K.l2_normalize(weight_matrix, axis=-1) # K表示调用该函数的当前layer
reg = K.sum(
K.square(
K.dot(w_n, K.transpose(w_n)) -
K.eye(w_n.shape[0].value))) # 自身矩阵的内积的平方根-自身特征值 = 越小, 越说明分量为0
return args.ortho_reg * reg # 这东西越小越好, 因为能保证各个特征分的越开
vocab_size = len(vocab)
if args.emb_name: # 获取已经保存的embedding???
from w2vEmbReader import W2VEmbReader as EmbReader
emb_reader = EmbReader(
os.path.join(
"/content/drive/My Drive/Attention-Based-Aspect-Extraction-master",
"preprocessed_data", args.domain), args.emb_name)
aspect_matrix = emb_reader.get_aspect_matrix(args.aspect_size)
args.aspect_size = emb_reader.aspect_size
args.emb_dim = emb_reader.emb_dim
##### Inputs #####
sentence_input = Input(shape=(maxlen, ),
dtype='int32',
name='sentence_input')
neg_input = Input(shape=(args.neg_size, maxlen),
dtype='int32',
name='neg_input')
##### Construct word embedding layer #####
word_emb = Embedding(vocab_size,
args.emb_dim,
mask_zero=True,
name='word_emb',
embeddings_constraint=MaxNorm(10))
##### Compute sentence representation ##### pre-processing 根据attention组合句子
e_w = word_emb(sentence_input) # 将input转换为embedding
y_s = Average()(e_w) # 默认求平均 layer
att_weights = Attention(name='att_weights',
W_constraint=MaxNorm(10),
b_constraint=MaxNorm(10))([e_w,
y_s]) # attention layer
z_s = WeightedSum()([e_w, att_weights]) # encoding layer
##### Compute representations of negative instances ##### 增加准确性的tricks
e_neg = word_emb(neg_input)
z_n = Average()(e_neg)
##### Reconstruction ##### 构建dense层, 希望能够decoding attention sentences的特征
p_t = Dense(args.aspect_size)(z_s)
p_t = Activation('softmax',
name='p_t')(p_t) # softmax一下, nodes数量不改变, 数值被soft了一下
r_s = WeightedAspectEmb(args.aspect_size,
args.emb_dim,
name='aspect_emb',
W_constraint=MaxNorm(10),
W_regularizer=ortho_reg)(
p_t) # 标准化0-10区间, 且正则项为自定义的ortho_reg
##### Loss #####
loss = MaxMargin(name='max_margin')([z_s, z_n,
r_s]) # 自定义loss function??? 这是在做啥???
model = Model(inputs=[sentence_input, neg_input],
outputs=[loss]) # negative input是需要自己分开数据集的吗??
### Word embedding and aspect embedding initialization ######
if args.emb_name:
from w2vEmbReader import W2VEmbReader as EmbReader
logger.info('Initializing word embedding matrix')
embs = model.get_layer('word_emb').embeddings
K.set_value(
embs,
emb_reader.get_emb_matrix_given_vocab(vocab, K.get_value(embs)))
logger.info(
'Initializing aspect embedding matrix as centroid of kmean clusters'
) # 为何初始化要用到kmeans
K.set_value(model.get_layer('aspect_emb').W, aspect_matrix) # r-s
return model
def create_model(args, maxlen, vocab):
def ortho_reg(weight_matrix):
### orthogonal regularization for aspect embedding matrix ###
w_n = K.l2_normalize(weight_matrix, axis=-1)
reg = K.sum(
K.square(K.dot(w_n, K.transpose(w_n)) - K.eye(w_n.shape[0].value)))
return args.ortho_reg * reg
vocab_size = len(vocab)
if args.emb_name:
if args.emb_technique == "w2v":
logger.info("Load {} glove embedding for {}".format(
args.lang, config.word_emb_training_type))
if args.lang == 'en':
emb_reader = EmbReader(
config.emb_dir_en["w2v"].format(
config.word_emb_training_type), args.emb_name)
elif args.lang == 'de':
#emb_reader = EmbReader(config.emb_dir_de["w2v"].format(config.word_emb_training_type),
#args.emb_name)
emb_reader = FineTuneEmbed_cca(
'../preprocessed_data/german/w2v/fine_tuned', 'w2v_emb',
'../preprocessed_data/german/w2v/full_trained',
'w2v_embedding_300')
elif args.emb_technique == 'fasttext':
if args.lang == 'en':
#emb_reader = FastTextEmbReader(config.emb_dir_en["fasttext"].format(config.word_emb_training_type),
#args.emb_name, config.fine_tuned_enabled)
emb_reader = FineTuneEmbed_ortho_procrustes(
'../preprocessed_data/fasttext/fine_tuned',
'fasttext_pre_trained',
'../preprocessed_data/fasttext/full_trained',
'w2v_embedding_skipgram_300')
elif args.lang == 'de':
emb_reader = FastTextEmbReader(
config.emb_dir_de["fasttext"].format(
config.word_emb_training_type), args.emb_name,
config.fine_tuned_enabled)
#emb_reader = FineTuneEmbed_ortho_procrustes('../preprocessed_data/fasttext/fine_tuned','fasttext_pre_trained','../preprocessed_data/fasttext/full_trained', 'w2v_embedding_skipgram_300')
elif args.emb_technique == "glove":
if args.lang == 'de':
logger.info('Load german glove embedding')
emb_reader = GloveEmbedding(config.emb_dir_de["glove"],
args.emb_name)
else:
logger.info("Load en glove embedding for {}".format(
config.word_emb_training_type))
emb_reader = GloveEmbedding(
config.emb_dir_en["glove"].format(
config.word_emb_training_type), args.emb_name)
elif args.emb_technique == "MUSE_supervised":
emb_reader = MUSEEmbedding(config.emb_dir_biling['supervised'],
args.emb_name)
elif args.emb_technique == "MUSE_unsupervised":
emb_reader = MUSEEmbedding(config.emb_dir_biling['unsupervised'],
args.emb_name)
aspect_matrix = emb_reader.get_aspect_matrix(args.aspect_size)
args.aspect_size = emb_reader.aspect_size
args.emb_dim = emb_reader.emb_dim
##### Inputs #####
sentence_input = Input(shape=(maxlen, ),
dtype='int32',
name='sentence_input')
neg_input = Input(shape=(args.neg_size, maxlen),
dtype='int32',
name='neg_input')
##### Construct word embedding layer #####
word_emb = Embedding(vocab_size,
args.emb_dim,
mask_zero=True,
name='word_emb',
embeddings_constraint=MaxNorm(10))
##### Compute sentence representation #####
e_w = word_emb(sentence_input)
y_s = Average()(e_w)
att_weights = Attention(name='att_weights',
W_constraint=MaxNorm(10),
b_constraint=MaxNorm(10))([e_w, y_s])
z_s = WeightedSum()([e_w, att_weights])
##### Compute representations of negative instances #####
e_neg = word_emb(neg_input)
z_n = Average()(e_neg)
##### Reconstruction #####
p_t = Dense(args.aspect_size)(z_s)
p_t = Activation('softmax', name='p_t')(p_t)
r_s = WeightedAspectEmb(args.aspect_size,
args.emb_dim,
name='aspect_emb',
W_constraint=MaxNorm(10),
W_regularizer=ortho_reg)(p_t)
##### Loss #####
loss = MaxMargin(name='max_margin')([z_s, z_n, r_s])
model = Model(inputs=[sentence_input, neg_input], outputs=[loss])
### Word embedding and aspect embedding initialization ######
if args.emb_name:
logger.info('Initializing word embedding matrix')
embs = model.get_layer('word_emb').embeddings
K.set_value(
embs,
emb_reader.get_emb_matrix_given_vocab(vocab, K.get_value(embs)))
logger.info(
'Initializing aspect embedding matrix as centroid of kmean clusters'
)
K.set_value(model.get_layer('aspect_emb').W, aspect_matrix)
return model