def forward(self, span_representation, position_embedding) -> torch.Tensor:
h = self.layer_norm1(
gelu(
self.dense1(
torch.cat([span_representation, position_embedding],
dim=-1))))
token_representation = self.layer_norm2(gelu(self.dense2(h)))
scores = self.prediction(token_representation)
return scores
def forward(self, x):
for i, layer in enumerate(self.layers):
x = layer(self.drop_out(x))
if i < len(self.layers) - 1:
x = gelu(x)
if len(self.norm_layers):
x = self.norm_layers[i](x)
return x
def forward(self, A, x):
layer1_diffusion = A.t().mm(gelu(self.diffusion(x)))
x = gelu(self.retained(x) + layer1_diffusion)
layer2_diffusion = A.t().mm(gelu(self.diffusion(x)))
x = gelu(self.retained(x) + layer2_diffusion)
return self.predict(x).squeeze(-1)