def call(self, inputs):
if not hasattr(self, 'kernel'):
embedding_layer = inputs._keras_history[0]
if embedding_layer.name != self.embedding_name:
def recursive_search(layer):
"""递归向上搜索，根据名字找Embedding层
"""
last_layer = layer._inbound_nodes[0].inbound_layers
if isinstance(last_layer, list):
if len(last_layer) == 0:
return None
else:
last_layer = last_layer[0]
if last_layer.name == self.embedding_name:
return last_layer
else:
return recursive_search(last_layer)
embedding_layer = recursive_search(embedding_layer)
if embedding_layer is None:
raise Exception('Embedding layer not found')
self.kernel = K.transpose(embedding_layer.embeddings)
self.units = K.int_shape(self.kernel)[1]
self.bias = self.add_weight(name='bias',
shape=(self.units, ),
initializer='zeros')
outputs = K.dot(inputs, self.kernel)
outputs = K.bias_add(outputs, self.bias)
outputs = self.activation(outputs)
return outputs
def call(self, inputs, mode='embedding'):
"""新增mode参数，可以为embedding或dense。如果为embedding，
则等价于普通Embedding层；如果为dense，则等价于无bias的Dense层。
"""
if mode == 'embedding':
return super(Embedding, self).call(inputs)
else:
kernel = K.transpose(self.embeddings)
return K.dot(inputs, kernel)
def call(self, x):
x, mask = x
mask = K.squeeze(mask, axis=2)
# linear
key = K.bias_add(K.dot(x, self.weight), self.bias)
# compute attention
outputs = K.squeeze(K.dot(key, self.query), axis=2)
outputs -= 1e32 * (1 - mask)
attn_scores = K.softmax(outputs)
attn_scores *= mask
attn_scores = K.reshape(attn_scores,
shape=(-1, 1, attn_scores.shape[-1]))
outputs = K.squeeze(K.batch_dot(attn_scores, key), axis=1)
return outputs
def compute_loss_of_similarity(self, inputs, mask=None):
_, _, y_pred, _ = inputs
y_true = self.get_labels_of_similarity(y_pred) # 构建标签
y_pred = K.l2_normalize(y_pred, axis=1) # 句向量归一化
similarities = K.dot(y_pred, K.transpose(y_pred)) # 相似度矩阵
similarities = similarities - K.eye(K.shape(y_pred)[0]) * 1e12 # 排除对角线
similarities = similarities * 30 # scale
loss = K.categorical_crossentropy(y_true,
similarities,
from_logits=True)
return loss
def build(self, input_shape):
super(FactorizedEmbedding, self).build(input_shape)
self._embeddings = self.add_weight(name='embeddings',
shape=(self.input_dim,
self.hidden_dim),
initializer='uniform')
self._project_kernel = self.add_weight(name='project_kernel',
shape=(self.hidden_dim,
self.output_dim),
initializer='glorot_uniform')
self.embeddings = K.dot(self._embeddings, self._project_kernel)
def call(self, x):
x, mask = x
# 因为 self.weight只有两个维度，所以这里要进行维度处理
mask = K.squeeze(mask, axis=2) # 维度压缩 去掉一个维度 axis=2,但是数据还是不变的
# linear 线性变化
# K.dot()进行 点乘，然后加了self.bias
key = K.bias_add(K.dot(x, self.weight), self.bias)
# compute attention
outputs = K.squeeze(K.dot(key, self.query), axis=2) # 计算注意力
outputs -= 1e32 * (1 - mask)
attn_scores = K.softmax(outputs) # 使用 softmax 计算得分
attn_scores *= mask
attn_scores = K.reshape(attn_scores,
shape=(-1, 1, attn_scores.shape[-1]))
outputs = K.squeeze(K.batch_dot(attn_scores, key), axis=1)
return outputs
def compute_loss_of_similarity(self, inputs, mask=None):
_, _, y_pred, _ = inputs
y_true = self.get_labels_of_similarity(
y_pred) # 构建标签 (btz,btz) 左右两个btz互为true
y_pred = K.l2_normalize(y_pred, axis=1) # 句向量归一化 (?, 768)
similarities = K.dot(y_pred, K.transpose(y_pred)) # 相似度矩阵 (btz,btz)
similarities = similarities - K.eye(K.shape(
y_pred)[0]) * 1e12 # 排除对角线，因为对角线的是自己跟自己比 (btz,btz) 对角线上的值会变得无穷小
similarities = similarities * 30 # scale (btz,btz)
loss = K.categorical_crossentropy(
y_true, similarities, from_logits=True) # (?,) 由此可以计算一个btz内的句子相似度
return loss
def call(self, inputs):
if not hasattr(self, 'kernel'):
embedding_layer = search_layer(inputs, self.embedding_name)
if embedding_layer is None:
raise Exception('Embedding layer not found')
self.kernel = K.transpose(embedding_layer.embeddings)
self.units = K.int_shape(self.kernel)[1]
self.bias = self.add_weight(name='bias',
shape=(self.units, ),
initializer='zeros')
outputs = K.dot(inputs, self.kernel)
outputs = K.bias_add(outputs, self.bias)
outputs = self.activation(outputs)
return outputs
def call(self, inputs):
if K.dtype(inputs) != 'int32':
inputs = K.cast(inputs, 'int32')
outputs = K.gather(self._embeddings, inputs)
outputs = K.dot(outputs, self._project_kernel)
return outputs
