def wnorm(self): print("Weight normalization applied to PosFF") self.ff1_net, self.ff1_fn = weight_norm(module=self.ff1_net, names=['weight'], dim=0) self.ff2_net, self.ff2_fn = weight_norm(module=self.ff2_net, names=['weight'], dim=0)
def wnorm(self): print("Weight normalization applied to SA") self.qkv_net, self.qkv_fn = weight_norm(module=self.qkv_net, names=['weight'], dim=0) self.r_net, self.r_fn = weight_norm(module=self.r_net, names=['weight'], dim=0) self.o_net, self.o_fn = weight_norm(module=self.o_net, names=['weight'], dim=0)