def wnorm(self):
print("Weight normalization applied to PosFF")
self.ff1_net, self.ff1_fn = weight_norm(module=self.ff1_net,
names=['weight'],
dim=0)
self.ff2_net, self.ff2_fn = weight_norm(module=self.ff2_net,
names=['weight'],
dim=0)
def wnorm(self):
print("Weight normalization applied to SA")
self.qkv_net, self.qkv_fn = weight_norm(module=self.qkv_net,
names=['weight'],
dim=0)
self.r_net, self.r_fn = weight_norm(module=self.r_net,
names=['weight'],
dim=0)
self.o_net, self.o_fn = weight_norm(module=self.o_net,
names=['weight'],
dim=0)