def __init__(self,
size: int,
eps: float = 1e-5,
momentum: float = 0.1) -> None:
super().__init__()
self.size = size
self.eps = Tensor(eps)
self.momentum = Tensor(momentum)
# To make the final output affine
self.gamma = Tensor.ones(self.size,
requires_grad=True,
is_parameter=True,
name="bn_gamma")
self.beta = Tensor.zeros(self.size,
requires_grad=True,
is_parameter=True,
name="bn_beta")
# Running mean and var
self.running_mean = Tensor.zeros(self.size,
requires_grad=False,
is_parameter=False,
name="bn_running_mean")
self.running_var = Tensor.ones(self.size,
requires_grad=False,
is_parameter=False,
name="bn_running_var")
def __init__(self,
in_channel: int,
out_channel: int,
kernel_size: int,
stride: int = 1,
padding: int = 0,
weight_initialization: str = "kaiming_normal") -> None:
super().__init__()
self.in_channel, self.out_channel = in_channel, out_channel
self.kernel_size = kernel_size if isinstance(
kernel_size, tuple) else (kernel_size, kernel_size)
self.stride, self.padding = stride, padding
self.weight_initialization = weight_initialization
shape = (self.out_channel, self.in_channel, *self.kernel_size)
self.weight = init_weights(shape,
self.weight_initialization,
requires_grad=True,
is_parameter=True,
name="conv_weight_2d")
self.bias = Tensor.zeros(self.out_channel,
requires_grad=True,
is_parameter=True,
name="conv_bias_2d")
def __init__(self, params:list, lr:float=1e-3, momentum:float=0.9) -> None:
r"""
Args:
params (list of Tensors): parameters to be updated
lr (float): learning rate
momentum (float): Nesterov momentum
"""
super().__init__(params)
self.lr, self.momentum = lr, momentum
self.momentums = [Tensor.zeros(p.shape) for p in self.params]
assert self.momentum >= 0.0, "Momentum can't be negative"
def __init__(self,
in_features: int,
out_features: int,
weight_initialization: str = "kaiming_normal",
fan_mode: str = "fan_in") -> None:
super().__init__()
self.in_features = in_features
self.out_features = out_features
self.weight = init_weights((self.out_features, self.in_features),
weight_initialization,
fan_mode,
requires_grad=True,
is_parameter=True,
name="lin_weight")
self.bias = Tensor.zeros(self.out_features,
requires_grad=True,
is_parameter=True,
name="lin_bias")
def reset(self) -> None:
self.momentums = [Tensor.zeros(p.shape) for p in self.params]
def _init_parameters(self) -> None:
self.moments1 = [Tensor.zeros(p.shape) for p in self.params]
self.moments2 = [Tensor.zeros(p.shape) for p in self.params]
self.t = 0
