def __init__(self, size: int, eps: float = 1e-5, momentum: float = 0.1) -> None: super().__init__() self.size = size self.eps = Tensor(eps) self.momentum = Tensor(momentum) # To make the final output affine self.gamma = Tensor.ones(self.size, requires_grad=True, is_parameter=True, name="bn_gamma") self.beta = Tensor.zeros(self.size, requires_grad=True, is_parameter=True, name="bn_beta") # Running mean and var self.running_mean = Tensor.zeros(self.size, requires_grad=False, is_parameter=False, name="bn_running_mean") self.running_var = Tensor.ones(self.size, requires_grad=False, is_parameter=False, name="bn_running_var")
def __init__(self, in_channel: int, out_channel: int, kernel_size: int, stride: int = 1, padding: int = 0, weight_initialization: str = "kaiming_normal") -> None: super().__init__() self.in_channel, self.out_channel = in_channel, out_channel self.kernel_size = kernel_size if isinstance( kernel_size, tuple) else (kernel_size, kernel_size) self.stride, self.padding = stride, padding self.weight_initialization = weight_initialization shape = (self.out_channel, self.in_channel, *self.kernel_size) self.weight = init_weights(shape, self.weight_initialization, requires_grad=True, is_parameter=True, name="conv_weight_2d") self.bias = Tensor.zeros(self.out_channel, requires_grad=True, is_parameter=True, name="conv_bias_2d")
def __init__(self, params:list, lr:float=1e-3, momentum:float=0.9) -> None: r""" Args: params (list of Tensors): parameters to be updated lr (float): learning rate momentum (float): Nesterov momentum """ super().__init__(params) self.lr, self.momentum = lr, momentum self.momentums = [Tensor.zeros(p.shape) for p in self.params] assert self.momentum >= 0.0, "Momentum can't be negative"
def __init__(self, in_features: int, out_features: int, weight_initialization: str = "kaiming_normal", fan_mode: str = "fan_in") -> None: super().__init__() self.in_features = in_features self.out_features = out_features self.weight = init_weights((self.out_features, self.in_features), weight_initialization, fan_mode, requires_grad=True, is_parameter=True, name="lin_weight") self.bias = Tensor.zeros(self.out_features, requires_grad=True, is_parameter=True, name="lin_bias")
def reset(self) -> None: self.momentums = [Tensor.zeros(p.shape) for p in self.params]
def _init_parameters(self) -> None: self.moments1 = [Tensor.zeros(p.shape) for p in self.params] self.moments2 = [Tensor.zeros(p.shape) for p in self.params] self.t = 0