def forward(self, x):
#import pdb
#pdb.set_trace()
X_ = self.attn(x) # (n, w) -> (n,num_hidden)
# should be dot(X_, W)
E = self.attn(X_) # (n, hidden) -> (n, hidden)
attn_weights = F.softmax(E, axis=1) # (n, hidden)
attn_applied = F.elemwise_mul(attn_weights, X_) #(n,hidden)
output = self.c*(F.elemwise_mul(X_, attn_weights)) + (1-self.c)*X_
output = self.out(output) #(n,hidden) -> (n,output_size)
return output
def forward(self, x):
#x: 'nwc'
#import pdb
#pdb.set_trace()
x = F.transpose(x, axes=(0, 2, 1)) # (nwc) -> (ncw)
X_ = F.batch_dot(self.w1.data(ctx), x) # (n,c,w) -> (n,c,w)
# E = dot(X_, W)
E = F.batch_dot(X_, self.w.data(ctx)) # (n,c,w) -> (n,c,w)
attn_weights = F.softmax(E, axis=2) # (n, c, w)
attn_applied = F.elemwise_mul(attn_weights, X_) #(n,c,w)
output = self.c.data(ctx) * (attn_applied) + (
1 - self.c.data(ctx)) * X_ # (n,c,w)
output = F.batch_dot(output, self.w2.data(ctx)) + self.b.data(
ctx) # (n, c,w)
output = F.transpose(output, axes=(0, 2, 1)) # (ncw) -> (nwc)
return output