def call(self, inputs): if not hasattr(self, 'kernel'): embedding_layer = inputs._keras_history[0] if embedding_layer.name != self.embedding_name: def recursive_search(layer): """递归向上搜索,根据名字找Embedding层 """ last_layer = layer._inbound_nodes[0].inbound_layers if isinstance(last_layer, list): if len(last_layer) == 0: return None else: last_layer = last_layer[0] if last_layer.name == self.embedding_name: return last_layer else: return recursive_search(last_layer) embedding_layer = recursive_search(embedding_layer) if embedding_layer is None: raise Exception('Embedding layer not found') self.kernel = K.transpose(embedding_layer.embeddings) self.units = K.int_shape(self.kernel)[1] self.bias = self.add_weight(name='bias', shape=(self.units, ), initializer='zeros') outputs = K.dot(inputs, self.kernel) outputs = K.bias_add(outputs, self.bias) outputs = self.activation(outputs) return outputs
def call(self, inputs, mode='embedding'): """新增mode参数,可以为embedding或dense。如果为embedding, 则等价于普通Embedding层;如果为dense,则等价于无bias的Dense层。 """ if mode == 'embedding': return super(Embedding, self).call(inputs) else: kernel = K.transpose(self.embeddings) return K.dot(inputs, kernel)
def call(self, x): x, mask = x mask = K.squeeze(mask, axis=2) # linear key = K.bias_add(K.dot(x, self.weight), self.bias) # compute attention outputs = K.squeeze(K.dot(key, self.query), axis=2) outputs -= 1e32 * (1 - mask) attn_scores = K.softmax(outputs) attn_scores *= mask attn_scores = K.reshape(attn_scores, shape=(-1, 1, attn_scores.shape[-1])) outputs = K.squeeze(K.batch_dot(attn_scores, key), axis=1) return outputs
def compute_loss_of_similarity(self, inputs, mask=None): _, _, y_pred, _ = inputs y_true = self.get_labels_of_similarity(y_pred) # 构建标签 y_pred = K.l2_normalize(y_pred, axis=1) # 句向量归一化 similarities = K.dot(y_pred, K.transpose(y_pred)) # 相似度矩阵 similarities = similarities - K.eye(K.shape(y_pred)[0]) * 1e12 # 排除对角线 similarities = similarities * 30 # scale loss = K.categorical_crossentropy(y_true, similarities, from_logits=True) return loss
def build(self, input_shape): super(FactorizedEmbedding, self).build(input_shape) self._embeddings = self.add_weight(name='embeddings', shape=(self.input_dim, self.hidden_dim), initializer='uniform') self._project_kernel = self.add_weight(name='project_kernel', shape=(self.hidden_dim, self.output_dim), initializer='glorot_uniform') self.embeddings = K.dot(self._embeddings, self._project_kernel)
def call(self, x): x, mask = x # 因为 self.weight只有两个维度,所以这里要进行维度处理 mask = K.squeeze(mask, axis=2) # 维度压缩 去掉一个维度 axis=2,但是数据还是不变的 # linear 线性变化 # K.dot()进行 点乘,然后加了self.bias key = K.bias_add(K.dot(x, self.weight), self.bias) # compute attention outputs = K.squeeze(K.dot(key, self.query), axis=2) # 计算注意力 outputs -= 1e32 * (1 - mask) attn_scores = K.softmax(outputs) # 使用 softmax 计算得分 attn_scores *= mask attn_scores = K.reshape(attn_scores, shape=(-1, 1, attn_scores.shape[-1])) outputs = K.squeeze(K.batch_dot(attn_scores, key), axis=1) return outputs
def compute_loss_of_similarity(self, inputs, mask=None): _, _, y_pred, _ = inputs y_true = self.get_labels_of_similarity( y_pred) # 构建标签 (btz,btz) 左右两个btz互为true y_pred = K.l2_normalize(y_pred, axis=1) # 句向量归一化 (?, 768) similarities = K.dot(y_pred, K.transpose(y_pred)) # 相似度矩阵 (btz,btz) similarities = similarities - K.eye(K.shape( y_pred)[0]) * 1e12 # 排除对角线,因为对角线的是自己跟自己比 (btz,btz) 对角线上的值会变得无穷小 similarities = similarities * 30 # scale (btz,btz) loss = K.categorical_crossentropy( y_true, similarities, from_logits=True) # (?,) 由此可以计算一个btz内的句子相似度 return loss
def call(self, inputs): if not hasattr(self, 'kernel'): embedding_layer = search_layer(inputs, self.embedding_name) if embedding_layer is None: raise Exception('Embedding layer not found') self.kernel = K.transpose(embedding_layer.embeddings) self.units = K.int_shape(self.kernel)[1] self.bias = self.add_weight(name='bias', shape=(self.units, ), initializer='zeros') outputs = K.dot(inputs, self.kernel) outputs = K.bias_add(outputs, self.bias) outputs = self.activation(outputs) return outputs
def call(self, inputs): if K.dtype(inputs) != 'int32': inputs = K.cast(inputs, 'int32') outputs = K.gather(self._embeddings, inputs) outputs = K.dot(outputs, self._project_kernel) return outputs