def mini_nn_forward(): # MiniNN relu = ReLU() input = Tensor(x_init) weights = Tensor(weight_init) bias = Tensor(bias_init) out = relu(input.dot(weights) + bias) return out.value
def __init__(self, in_feats, out_feats, bias=True): self.in_feats = in_feats self.out_feats = out_feats self.weights = Tensor.random_uniform(self.in_feats, self.out_feats) self.bias_flag = bias if self.bias_flag: self.bias = Tensor.random_uniform(1, self.out_feats)
def __call__(input): val = np.maximum(input.value, 0) output = Tensor(val, parents=(input, ), fun='ReLUBackward') def _backward(): input.grad += output.grad * ((val > 0).astype(np.float32)) output._backward = _backward return output
def __call__(input): with np.warnings.catch_warnings(): np.warnings.filterwarnings('ignore') val = np.tanh(input.value) output = Tensor(val, parents=(input, ), fun='TanhBackard') def _backward(): input.grad += output.grad * (1 - (val**2)) output._backward = _backward return output
def __call__(input, alpha=0.01): val = np.maximum(input.value, alpha * input.value) output = Tensor(val, parents=(input, ), fun='LeakyReLUBackward') grad = np.zeros_like(val) grad[(val > 0)] = 1 grad[(val <= 0)] = alpha def _backward(): input.grad += output.grad * (grad) output._backward = _backward return output
def __call__(input, dim): exp = np.exp(input.value) sum = np.sum(np.exp(input.value), 1) sum = np.expand_dims(sum, 1) val = exp / sum output = Tensor(val, parents=(input, ), fun='SoftmaxBackward') def _backward(): R_bar = -np.sum(output.grad * exp, axis=1, keepdims=True) / (sum** 2) input.grad += (output.grad) * (exp / sum) + R_bar * exp output._backward = _backward return output
def __next__(self): if self.iter < self.num_batches: self.iter += 1 # Could be some overlap with this sample, could use np.argpartition or something samp = np.random.randint(0, self.data.shape[0], size=(self.batch_size)) X = self.data[samp].reshape((-1, 28 * 28)).astype(np.float32) Y = self.labels[samp] y = np.zeros((len(samp), 10), np.float32) y[range(y.shape[0]), Y] = 1 self.raw_labels = Y # return Tensor(X, requires_grad=False), Tensor(y, requires_grad=False) return Tensor(X), Tensor(y) else: raise StopIteration
def __call__(input): # Disable overflow warnings with np.warnings.catch_warnings(): np.warnings.filterwarnings('ignore') # These cases are needed, overflow warning else val = np.where(input.value >= 0, 1 / (1 + np.exp(-input.value)), np.exp(input.value) / (1 + np.exp(input.value))) output = Tensor(val, parents=(input, ), fun='SigmoidBackard') def _backward(): input.grad += output.grad * (val * (1 - val)) output._backward = _backward return output
def __call__(input, dim): exp = np.exp(input.value) sum = np.sum(np.exp(input.value), 1) sum = np.expand_dims(sum, 1) val = np.log(exp / sum) output = Tensor(val, parents=(input, ), fun='LogSoftmaxBackward') # Complete credit to TinyGrad for this gradient... def _backward(): input.grad += (output.grad) - ( np.exp(val) * (np.sum(output.grad, axis=1).reshape((-1, 1)))) output._backward = _backward return output
def __call__(self, input): if self.p > 0: dropout_vals = np.random.binomial([np.ones(input.shape)], 1 - self.p)[0] val = input.value * dropout_vals * (1 / (1 - self.p)) output = Tensor(val, parents=(input, ), fun='DropoutBackward') def _backward(): # Same issue as ReLU input.grad += output.grad * ((val != 0) * (1 / (1 - self.p))) output._backward = _backward return output else: return input
def mini_nn_backward(): relu = ReLU() criterion = MSELoss() input = Tensor(x_init) weights = Tensor(weight_regression_init) bias = Tensor(bias_regression_init) y = Tensor(y_init) out = relu((input.dot(weights)) + bias) loss = criterion(out, y) loss.backward() return out.value, weights.grad, bias.grad
def mini_nn_backward(): relu = ReLU() logsoftmax = LogSoftmax() criterion = NLLLoss() input = Tensor(x_init) weights = Tensor(weight_init) bias = Tensor(bias_init) # Making target tensor - construction should be done within NLLLoss targets = np.zeros((len(targets_init), 500), np.float32) targets[range(targets.shape[0]), targets_init] = 1 targets = Tensor(targets) out = relu((input.dot(weights)) + bias) out = logsoftmax(out, 1) loss = criterion(out, targets) loss.backward() return out.value, weights.grad, bias.grad