class TwoLayerNet: def __init__(self, n_input, n_output, hidden_layer_size, reg): """ Initializes the neural network :param n_input: int - dimension of the model input :param n_output: int - number of classes to predict :param hidden_layer_size: int - number of neurons in the hidden layer :param reg: float - L2 regularization strength """ self.reg = reg self.layer_one = FullyConnectedLayer(n_input, hidden_layer_size) self.layer_ReLU = ReLULayer() self.layer_two = FullyConnectedLayer(hidden_layer_size, n_output) def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples :param X: np array (batch_size, input_features) - input data :param y: np array of int (batch_size) - classes :return: loss: float - single value, cross entropy loss """ for param in self.params().values(): param.grad = np.zeros_like(param.value) loss, grad = softmax_with_cross_entropy( self.layer_two.forward( self.layer_ReLU.forward(self.layer_one.forward(X))), y) self.layer_one.backward( self.layer_ReLU.backward(self.layer_two.backward(grad))) for param in self.params().values(): reg_loss, reg_grad = l2_regularization(param.value, self.reg) loss += reg_loss param.grad += reg_grad return loss def predict(self, X): """ Produces classifier predictions on the set :param X: np array (test_samples, num_features) :return: y_pred: np.array of int (test_samples) """ preds = self.layer_two.forward( self.layer_ReLU.forward(self.layer_one.forward(X))) y_pred = np.argmax(preds, axis=1) return y_pred def params(self): result = { 'layer_one_W': self.layer_one.W, 'layer_one_B': self.layer_one.B, 'layer_two_W': self.layer_two.W, 'layer_two_B': self.layer_two.B } return result
class TwoLayerNet: """ Neural network with two fully connected layers """ def __init__(self, n_input, n_output, hidden_layer_size, reg): """ Initializes the neural network Arguments: n_input, int - dimension of the model input n_output, int - number of classes to predict hidden_layer_size, int - number of neurons in the hidden layer reg, float - L2 regularization strength """ self.reg = reg self.l1 = FullyConnectedLayer(n_input, hidden_layer_size) self.relu = ReLULayer() self.l2 = FullyConnectedLayer(hidden_layer_size, n_output) def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, input_features) - input data y, np array of int (batch_size) - classes """ for k in self.params().keys(): self.params()[k].grad = np.zeros_like(self.params()[k].grad) out = self.l2.forward(self.relu.forward(self.l1.forward(X))) loss, grad = softmax_with_cross_entropy(out, y) self.l1.backward(self.relu.backward(self.l2.backward(grad))) for k in self.params().keys(): l2_loss, l2_grad = l2_regularization(self.params()[k].value, self.reg) loss += l2_loss self.params()[k].grad += l2_grad return loss def predict(self, X): """ Produces classifier predictions on the set Arguments: X, np array (test_samples, num_features) Returns: y_pred, np.array of int (test_samples) """ return self.l2.forward(self.relu.forward(self.l1.forward(X))).argmax(axis=1) def params(self): return {"W1": self.l1.W, "B1": self.l1.B, "W2": self.l2.W, "B2": self.l2.B}
class ConvNet: """ Implements a very simple conv net Input -> Conv[3x3] -> Relu -> Maxpool[4x4] -> Conv[3x3] -> Relu -> MaxPool[4x4] -> Flatten -> FC -> Softmax """ def __init__(self, input_shape, n_output_classes, conv1_channels, conv2_channels): """ Initializes the neural network Arguments: input_shape, tuple of 3 ints - image_width, image_height, n_channels Will be equal to (32, 32, 3) n_output_classes, int - number of classes to predict conv1_channels, int - number of filters in the 1st conv layer conv2_channels, int - number of filters in the 2nd conv layer """ # TODO Create necessary layers input_width, input_height, input_channels = input_shape self.conv1 = ConvolutionalLayer(input_channels, conv1_channels, filter_size=3, padding=1) self.relu1 = ReLULayer() self.maxpool1 = MaxPoolingLayer(pool_size=4, stride=4) self.conv2 = ConvolutionalLayer(conv1_channels, conv2_channels, filter_size=3, padding=1) self.relu2 = ReLULayer() self.maxpool2 = MaxPoolingLayer(pool_size=4, stride=4) self.flattener = Flattener() self.fc = FullyConnectedLayer( input_width * input_height * conv2_channels // (4**4), n_output_classes) def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, height, width, input_features) - input data y, np array of int (batch_size) - classes """ # Before running forward and backward pass through the model, # clear parameter gradients aggregated from the previous pass for layer in (self.conv1, self.conv2, self.fc): for param in layer.params().values(): param.zero_grad() # TODO Compute loss and fill param gradients # Don't worry about implementing L2 regularization, we will not # need it in this assignment conv1_fwd = self.conv1.forward(X) relu1_fwd = self.relu1.forward(conv1_fwd) maxpool1_fwd = self.maxpool1.forward(relu1_fwd) conv2_fwd = self.conv2.forward(maxpool1_fwd) relu2_fwd = self.relu2.forward(conv2_fwd) maxpool2_fwd = self.maxpool2.forward(relu2_fwd) flattener_fwd = self.flattener.forward(maxpool2_fwd) fc_fwd = self.fc.forward(flattener_fwd) loss, dprediction = softmax_with_cross_entropy(fc_fwd, y) fc_bwd = self.fc.backward(dprediction) flattener_bwd = self.flattener.backward(fc_bwd) maxpool2_bwd = self.maxpool2.backward(flattener_bwd) relu2_bwd = self.relu2.backward(maxpool2_bwd) conv2_bwd = self.conv2.backward(relu2_bwd) maxpool1_bwd = self.maxpool1.backward(conv2_bwd) relu1_bwd = self.relu1.backward(maxpool1_bwd) conv1_bwd = self.conv1.backward(relu1_bwd) return loss def predict(self, X): # You can probably copy the code from previous assignment pred = np.zeros(X.shape[0], np.int) conv1_fwd = self.conv1.forward(X) relu1_fwd = self.relu1.forward(conv1_fwd) maxpool1_fwd = self.maxpool1.forward(relu1_fwd) conv2_fwd = self.conv2.forward(maxpool1_fwd) relu2_fwd = self.relu2.forward(conv2_fwd) maxpool2_fwd = self.maxpool2.forward(relu2_fwd) flattener_fwd = self.flattener.forward(maxpool2_fwd) fc_fwd = self.fc.forward(flattener_fwd) pred = np.argmax(fc_fwd, axis=1) return pred def params(self): result = {} # TODO: Aggregate all the params from all the layers # which have parameters for layer_name, layer in (('conv1', self.conv1), ('conv2', self.conv2), ('fc', self.fc)): params = layer.params() for param_name in params: result[f'{layer_name}.{param_name}'] = params[param_name] return result
class TwoLayerNet: """ Neural network with two fully connected layers """ def __init__(self, n_input, n_output, hidden_layer_size, reg): """ Initializes the neural network Arguments: n_input, int - dimension of the model input n_output, int - number of classes to predict hidden_layer_size, int - number of neurons in the hidden layer reg, float - L2 regularization strength """ self.reg = reg # TODO Create necessary layers self.layer1 = FullyConnectedLayer(n_input, n_output) self.layer2 = ReLULayer() self.layer3 = FullyConnectedLayer(n_output, n_output) #raise Exception("Not implemented!") def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, input_features) - input data y, np array of int (batch_size) - classes """ # Before running forward and backward pass through the model, # clear parameter gradients aggregated from the previous pass # TODO Set parameter gradient to zeros # Hint: using self.params() might be useful! #raise Exception("Not implemented!") self.layer1.W.grad = np.zeros_like(self.layer1.W.grad) self.layer3.W.grad = np.zeros_like(self.layer3.W.grad) self.layer1.B.grad = np.zeros_like(self.layer1.B.grad) self.layer3.B.grad = np.zeros_like(self.layer3.B.grad) # TODO Compute loss and fill param gradients # by running forward and backward passes through the model out1 = self.layer1.forward(X) out2 = self.layer2.forward(out1) out3 = self.layer3.forward(out2) # After that, implement l2 regularization on all params # Hint: self.params() is useful again! #raise Exception("Not implemented!") loss, grad = softmax_with_cross_entropy( out3, y) #+ l2_regularization(W, reg) back3 = self.layer3.backward(grad) # бэкпропагатион reg_loss3, dw_dr3 = l2_regularization( self.layer3.W.value, self.reg) # считаем регуляризационные члены self.layer3.W.grad += dw_dr3 back2 = self.layer2.backward(back3) back1 = self.layer1.backward(back2) reg_loss1, dw_dr1 = l2_regularization(self.layer1.W.value, self.reg) self.layer1.W.grad += dw_dr1 return loss + reg_loss1 + reg_loss3 def predict(self, X): """ Produces classifier predictions on the set Arguments: X, np array (test_samples, num_features) Returns: y_pred, np.array of int (test_samples) """ # TODO: Implement predict # Hint: some of the code of the compute_loss_and_gradients # can be reused pred = np.zeros(X.shape[0], np.int) out1 = self.layer1.forward(X) out2 = self.layer2.forward(out1) out3 = self.layer3.forward(out2) pred = np.argmax(out3, axis=1) #raise Exception("Not implemented!") return pred def params(self): result = { 'layer1.W': self.layer1.W, 'layer1.B': self.layer1.B, 'layer3.W': self.layer3.W, 'layer1.B': self.layer3.B } # TODO Implement aggregating all of the params #raise Exception("Not implemented!") return result
class ConvNet: """ Implements a very simple conv net Input -> Conv[3x3] -> Relu -> Maxpool[4x4] -> Conv[3x3] -> Relu -> MaxPool[4x4] -> Flatten -> FC -> Softmax """ def __init__(self, input_shape, n_output_classes, conv1_channels, conv2_channels): """ Initializes the neural network Arguments: input_shape, tuple of 3 ints - image_width, image_height, n_channels Will be equal to (32, 32, 3) n_output_classes, int - number of classes to predict conv1_channels, int - number of filters in the 1st conv layer conv2_channels, int - number of filters in the 2nd conv layer """ # TODO Create necessary layers self.out_classes = n_output_classes image_width, image_height, in_channels = input_shape self.Conv1 = ConvolutionalLayer(in_channels, conv1_channels, 3, 1) self.ReLU1 = ReLULayer() self.MaxPool1 = MaxPoolingLayer(4, 4) self.Conv2 = ConvolutionalLayer(conv1_channels, conv2_channels, 3, 1) self.ReLU2 = ReLULayer() self.MaxPool2 = MaxPoolingLayer(4, 4) self.Flatten = Flattener() self.FC = FullyConnectedLayer(4 * conv2_channels, n_output_classes) def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, height, width, input_features) - input data y, np array of int (batch_size) - classes """ # Before running forward and backward pass through the model, # clear parameter gradients aggregated from the previous pass # TODO Compute loss and fill param gradients # Don't worry about implementing L2 regularization, we will not # need it in this assignment for param in self.params().values(): param.grad = np.zeros_like(param.value) preds = self.Conv1.forward(X) preds = self.ReLU1.forward(preds) preds = self.MaxPool1.forward(preds) preds = self.Conv2.forward(preds) preds = self.ReLU2.forward(preds) preds = self.MaxPool2.forward(preds) preds = self.Flatten.forward(preds) preds = self.FC.forward(preds) loss, grad = softmax_with_cross_entropy(preds, y) grad = self.FC.backward(grad) grad = self.Flatten.backward(grad) grad = self.MaxPool2.backward(grad) grad = self.ReLU2.backward(grad) grad = self.Conv2.backward(grad) grad = self.MaxPool1.backward(grad) grad = self.ReLU1.backward(grad) grad = self.Conv1.backward(grad) return loss def predict(self, X): # You can probably copy the code from previous assignment preds = self.Conv1.forward(X) preds = self.ReLU1.forward(preds) preds = self.MaxPool1.forward(preds) preds = self.Conv2.forward(preds) preds = self.ReLU2.forward(preds) preds = self.MaxPool2.forward(preds) preds = self.Flatten.forward(preds) preds = self.FC.forward(preds) probs = softmax(preds) return np.argmax(probs, axis=1) def params(self): # TODO: Aggregate all the params from all the layers # which have parameters return { 'Conv1.W': self.Conv1.W, 'Conv1.B': self.Conv1.B, 'Conv2.W': self.Conv2.W, 'Conv2.B': self.Conv2.B, 'FC.W': self.FC.W, 'FC.B': self.FC.B }
class ConvNet: """ Implements a very simple conv net Input -> Conv[3x3] -> Relu -> Maxpool[4x4] -> Conv[3x3] -> Relu -> MaxPool[4x4] -> Flatten -> FC -> Softmax """ def __init__(self, input_shape, n_output_classes, conv1_channels, conv2_channels): """ Initializes the neural network Arguments: input_shape, tuple of 3 ints - image_width, image_height, n_channels Will be equal to (32, 32, 3) n_output_classes, int - number of classes to predict conv1_channels, int - number of filters in the 1st conv layer conv2_channels, int - number of filters in the 2nd conv layer """ self.layer_1 = ConvolutionalLayer(input_shape[2], conv1_channels, 3, 1) self.layer_2 = ReLULayer() self.layer_3 = MaxPoolingLayer(4, 4) self.layer_4 = ConvolutionalLayer(conv1_channels, conv2_channels, 3, 1) self.layer_5 = ReLULayer() self.layer_6 = MaxPoolingLayer(4, 4) self.layer_7 = Flattener() self.layer_8 = FullyConnectedLayer( (input_shape[0] * input_shape[1] * conv2_channels) // (16**2), n_output_classes) def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, height, width, input_features) - input data y, np array of int (batch_size) - classes """ params = self.params() for param_key in params: param = params[param_key] param.grad = np.zeros_like(param.grad) step_1f = self.layer_1.forward(X) step_2f = self.layer_2.forward(step_1f) step_3f = self.layer_3.forward(step_2f) step_4f = self.layer_4.forward(step_3f) step_5f = self.layer_5.forward(step_4f) step_6f = self.layer_6.forward(step_5f) step_7f = self.layer_7.forward(step_6f) step_8f = self.layer_8.forward(step_7f) loss, dpred = softmax_with_cross_entropy(step_8f, y) step_8b = self.layer_8.backward(dpred) step_7b = self.layer_7.backward(step_8b) step_6b = self.layer_6.backward(step_7b) step_5b = self.layer_5.backward(step_6b) step_4b = self.layer_4.backward(step_5b) step_3b = self.layer_3.backward(step_4b) step_2b = self.layer_2.backward(step_3b) step_1b = self.layer_1.backward(step_2b) return loss def predict(self, X): step_1f = self.layer_1.forward(X) step_2f = self.layer_2.forward(step_1f) step_3f = self.layer_3.forward(step_2f) step_4f = self.layer_4.forward(step_3f) step_5f = self.layer_5.forward(step_4f) step_6f = self.layer_6.forward(step_5f) step_7f = self.layer_7.forward(step_6f) step_8f = self.layer_8.forward(step_7f) probs = softmax(step_8f) pred = np.array(list(map(lambda x: x.argsort()[-1], probs))) return pred def params(self): result = {} result['W1'] = self.layer_1.W result['W2'] = self.layer_4.W result['W3'] = self.layer_8.W result['B1'] = self.layer_1.B result['B2'] = self.layer_4.B result['B3'] = self.layer_8.B return result
class TwoLayerNet: """ Neural network with two fully connected layers """ def __init__(self, n_input, n_output, hidden_layer_size, reg): """ Initializes the neural network Arguments: n_input, int - dimension of the model input n_output, int - number of classes to predict hidden_layer_size, int - number of neurons in the hidden layer reg, float - L2 regularization strength """ self.reg = reg self.fulllayer1 = FullyConnectedLayer(n_input, hidden_layer_size) self.reglayer1 = ReLULayer() self.fulllayer2 = FullyConnectedLayer(hidden_layer_size, n_output) def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, input_features) - input data y, np array of int (batch_size) - classes """ # Before running forward and backward pass through the model, # clear parameter gradients aggregated from the previous pass # TODO Set parameter gradient to zeros # Hint: using self.params() might be useful! self.fulllayer1.W.grad = np.zeros_like(self.fulllayer1.W.grad) self.fulllayer1.B.grad = np.zeros_like(self.fulllayer1.B.grad) self.fulllayer2.W.grad = np.zeros_like(self.fulllayer2.W.grad) self.fulllayer2.B.grad = np.zeros_like(self.fulllayer2.B.grad) # TODO Compute loss and fill param gradients # by running forward and backward passes through the model res = self.fulllayer1.forward(X) res2 = self.reglayer1.forward(res) res3 = self.fulllayer2.forward(res2) loss, grad = softmax_with_cross_entropy(res3, y) back3 = self.fulllayer2.backward(grad) back2 = self.reglayer1.backward(back3) back = self.fulllayer1.backward(back2) # After that, implement l2 regularization on all params # Hint: self.params() is useful again! for params in self.params().keys(): # print(params) # print(self.params()[params].value) loc_loss, loc_grad = l2_regularization(self.params()[params].value, self.reg) loss += loc_loss self.params()[params].grad += loc_grad return loss def predict(self, X): """ Produces classifier predictions on the set Arguments: X, np array (test_samples, num_features) Returns: y_pred, np.array of int (test_samples) """ # TODO: Implement predict # Hint: some of the code of the compute_loss_and_gradients # can be reused pred = np.zeros(X.shape[0], np.int) res = self.fulllayer1.forward(X) res2 = self.reglayer1.forward(res) # pred = np.argmax(self.fulllayer2.forward(res2), axis=0) pred = np.argmax(self.fulllayer2.forward(res2), axis=1) return pred def params(self): # TODO Implement aggregating all of the params result = { 'W1': self.fulllayer1.W, 'B1': self.fulllayer1.B, 'W2': self.fulllayer2.W, 'B2': self.fulllayer2.B } return result
class ConvNet: """ Implements a very simple conv net Input -> Conv[3x3] -> Relu -> Maxpool[4x4] -> Conv[3x3] -> Relu -> MaxPool[4x4] -> Flatten -> FC -> Softmax """ def __init__(self, input_shape, n_output_classes, conv1_channels, conv2_channels): """ Initializes the neural network Arguments: input_shape, tuple of 3 ints - image_width, image_height, n_channels Will be equal to (32, 32, 3) n_output_classes, int - number of classes to predict conv1_channels, int - number of filters in the 1st conv layer conv2_channels, int - number of filters in the 2nd conv layer """ _, _, input_channels = input_shape self.conv1 = ConvolutionalLayer(input_channels, conv1_channels, 3, 1) self.relu1 = ReLULayer() self.pool1 = MaxPoolingLayer(4, 4) self.conv2 = ConvolutionalLayer(conv1_channels, conv2_channels, 3, 1) self.relu2 = ReLULayer() self.pool2 = MaxPoolingLayer(4, 4) self.flattener = Flattener() self.fc = FullyConnectedLayer(4 * conv2_channels, n_output_classes) def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, height, width, input_features) - input data y, np array of int (batch_size) - classes """ for param_ix in self.params(): self.params()[param_ix].grad = np.zeros_like( self.params()[param_ix].value) out = self.pool1.forward(self.relu1.forward(self.conv1.forward(X))) out = self.pool2.forward(self.relu2.forward(self.conv2.forward(out))) out = self.fc.forward(self.flattener.forward(out)) loss, grad = softmax_with_cross_entropy(out, y) grad = self.flattener.backward(self.fc.backward(grad)) grad = self.conv2.backward( self.relu2.backward(self.pool2.backward(grad))) grad = self.conv1.backward( self.relu1.backward(self.pool1.backward(grad))) return loss def predict(self, X): out = self.pool1.forward(self.relu1.forward(self.conv1.forward(X))) out = self.pool2.forward(self.relu2.forward(self.conv2.forward(out))) out = self.fc.forward(self.flattener.forward(out)) predictions = softmax(out) return np.argmax(predictions, axis=1) def params(self): result = { 'conv1.W': self.conv1.W, 'conv1.B': self.conv1.B, 'conv2.W': self.conv1.W, 'conv2.B': self.conv1.B, 'fc.W': self.fc.W, 'fc.B': self.fc.B } return result
class ConvNet: """ Implements a very simple conv net Input -> Conv[3x3] -> Relu -> Maxpool[4x4] -> Conv[3x3] -> Relu -> MaxPool[4x4] -> Flatten -> FC -> Softmax """ def __init__(self, input_shape, n_output_classes, conv1_channels, conv2_channels): """ Initializes the neural network Arguments: input_shape, tuple of 3 ints - image_width, image_height, n_channels Will be equal to (32, 32, 3) n_output_classes, int - number of classes to predict conv1_channels, int - number of filters in the 1st conv layer conv2_channels, int - number of filters in the 2nd conv layer """ # TODO Create necessary layers width, height, n_channels = input_shape self.conv1 = ConvolutionalLayer(n_channels, conv1_channels, 3, 1) self.relu1 = ReLULayer() self.maxpool1 = MaxPoolingLayer(4, 4) self.conv2 = ConvolutionalLayer(conv1_channels, conv2_channels, 3, 1) self.relu2 = ReLULayer() self.maxpool2 = MaxPoolingLayer(4, 4) self.flatten = Flattener() self.fc = FullyConnectedLayer( (height // 4 // 4) * (width // 4 // 4) * conv2_channels, n_output_classes) self.conv1_params = self.conv1.params() self.conv2_params = self.conv2.params() self.fc_params = self.fc.params() def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, height, width, input_features) - input data y, np array of int (batch_size) - classes """ # Before running forward and backward pass through the model, # clear parameter gradients aggregated from the previous pass for key, value in self.params().items(): value.grad.fill(0) # TODO Compute loss and fill param gradients # Don't worry about implementing L2 regularization, we will not # need it in this assignment conv1 = self.conv1.forward(X) relu1 = self.relu1.forward(conv1) maxpool1 = self.maxpool1.forward(relu1) conv2 = self.conv2.forward(maxpool1) relu2 = self.relu2.forward(conv2) maxpool2 = self.maxpool2.forward(relu2) flatten = self.flatten.forward(maxpool2) fc = self.fc.forward(flatten) loss, d_preds = softmax_with_cross_entropy(fc, y) fc = self.fc.backward(d_preds) flatten = self.flatten.backward(fc) maxpool2 = self.maxpool2.backward(flatten) relu2 = self.relu2.backward(maxpool2) conv2 = self.conv2.backward(relu2) maxpool1 = self.maxpool1.backward(conv2) relu1 = self.relu1.backward(maxpool1) conv1 = self.conv1.backward(relu1) return loss def predict(self, X): # You can probably copy the code from previous assignment conv1 = self.conv1.forward(X) relu1 = self.relu1.forward(conv1) maxpool1 = self.maxpool1.forward(relu1) conv2 = self.conv2.forward(maxpool1) relu2 = self.relu2.forward(conv2) maxpool2 = self.maxpool2.forward(relu2) flatten = self.flatten.forward(maxpool2) fc = self.fc.forward(flatten) return np.argmax(fc, axis=1) def params(self): result = {} # TODO: Aggregate all the params from all the layers # which have parameters d1 = {k + '1': v for k, v in self.conv1_params.items()} d2 = {k + '2': v for k, v in self.conv2_params.items()} d3 = {k + '3': v for k, v in self.fc_params.items()} result = {**d1, **d2, **d3} return result
class TwoLayerNet: """ Neural network with two fully connected layers """ def __init__(self, n_input, n_output, hidden_layer_size, reg): """ Initializes the neural network Arguments: n_input, int - dimension of the model input n_output, int - number of classes to predict hidden_layer_size, int - number of neurons in the hidden layer reg, float - L2 regularization strength """ self.reg = reg # TODO Create necessary layers self.input_layer = FullyConnectedLayer(n_input, hidden_layer_size) self.relu = ReLULayer() self.output_layer = FullyConnectedLayer(hidden_layer_size, n_output) self.W_in = self.input_layer.params()['W'] self.W_out = self.output_layer.params()['W'] self.B_in = self.input_layer.params()['B'] self.B_out = self.output_layer.params()['B'] def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, input_features) - input data y, np array of int (batch_size) - classes """ # Before running forward and backward pass through the model, # clear parameter gradients aggregated from the previous pass # TODO Set parameter gradient to zeros # Hint: using self.params() might be useful! for _, param in self.params().items(): param.grad.fill(0) # TODO Compute loss and fill param gradients # by running forward and backward passes through the model to_relu = self.input_layer.forward(X) to_output_layer = self.relu.forward(to_relu) pred = self.output_layer.forward(to_output_layer) loss, dprediction = softmax_with_cross_entropy(pred, y) grad_output_layer = self.output_layer.backward(dprediction) grad_relu_layer = self.relu.backward(grad_output_layer) grad_input_layer = self.input_layer.backward(grad_relu_layer) # After that, implement l2 regularization on all params # Hint: self.params() is useful again! for key, param in self.params().items(): loss_l2, grad_l2 = l2_regularization(param.value, self.reg) loss += loss_l2 param.grad += grad_l2 return loss def predict(self, X): """ Produces classifier predictions on the set Arguments: X, np array (test_samples, num_features) Returns: y_pred, np.array of int (test_samples) """ to_relu = self.input_layer.forward(X) to_output_layer = self.relu.forward(to_relu) weights = self.output_layer.forward(to_output_layer) pred = np.argmax(weights, axis=-1) return pred def params(self): # TODO Implement aggregating all of the params result = { 'W_out': self.W_out, 'W_in': self.W_in, 'B_out': self.B_out, 'B_in': self.B_in } return result
class TwoLayerNet: """ Neural network with two fully connected layers """ def __init__(self, n_input, n_output, hidden_layer_size, reg): """ Initializes the neural network Arguments: n_input, int - dimension of the model input n_output, int - number of classes to predict hidden_layer_size, int - number of neurons in the hidden layer reg, float - L2 regularization strength """ self.reg = reg self.layer_1 = FullyConnectedLayer(n_input, hidden_layer_size) self.non_linier = ReLULayer() self.layer_2 = FullyConnectedLayer(hidden_layer_size, n_output) def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, input_features) - input data y, np array of int (batch_size) - classes """ params = self.params() W1 = params["W1"] B1 = params["B1"] W2 = params["W2"] B2 = params["B2"] W1.grad = np.zeros_like(W1.value) B1.grad = np.zeros_like(B1.value) W2.grad = np.zeros_like(W2.value) B2.grad = np.zeros_like(B2.value) f1 = self.layer_1.forward(X) f2 = self.non_linier.forward(f1) f3 = self.layer_2.forward(f2) loss, grad = softmax_with_cross_entropy(f3, y) d2 = self.layer_2.backward(grad) d_nl = self.non_linier.backward(d2) d1 = self.layer_1.backward(d_nl) l2_W1_loss, l2_W1_grad = l2_regularization(W1.value, self.reg) l2_B1_loss, l2_B1_grad = l2_regularization(B1.value, self.reg) l2_W2_loss, l2_W2_grad = l2_regularization(W2.value, self.reg) l2_B2_loss, l2_B2_grad = l2_regularization(B2.value, self.reg) l2_reg = l2_W1_loss + l2_B1_loss + l2_W2_loss + l2_B2_loss loss += l2_reg W1.grad += l2_W1_grad B1.grad += l2_B1_grad W2.grad += l2_W2_grad B2.grad += l2_B2_grad return loss def predict(self, X): """ Produces classifier predictions on the set Arguments: X, np array (test_samples, num_features) Returns: y_pred, np.array of int (test_samples) """ y_pred = np.zeros(X.shape[0], np.int) out1 = self.layer_1.forward(X) out_relu = self.non_linier.forward(out1) predictions = self.layer_2.forward(out_relu) probs = softmax(predictions) y_pred = np.argmax(probs, axis=1) return y_pred def params(self): result = { 'W1': self.layer_1.W, 'B1': self.layer_1.B, 'W2': self.layer_2.W, 'B2': self.layer_2.B } return result
class TwoLayerNet: """ Neural network with two fully connected layers """ def __init__(self, n_input, n_output, hidden_layer_size, reg): """ Initializes the neural network Arguments: n_input, int - dimension of the model input n_output, int - number of classes to predict hidden_layer_size, int - number of neurons in the hidden layer reg, float - L2 regularization strength """ self.reg = reg # TODO Create necessary layers self.fc1 = FullyConnectedLayer(n_input, hidden_layer_size) self.ReLU = ReLULayer() self.fc2 = FullyConnectedLayer(hidden_layer_size, n_output) # self.n_output = n_output def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, input_features) - input data y, np array of int (batch_size) - classes """ # Before running forward and backward pass through the model, # clear parameter gradients aggregated from the previous pass # TODO Set parameter gradient to zeros # Hint: using self.params() might be useful! for param in self.params().values(): param.grad = np.zeros_like(param.value) # TODO Compute loss and fill param gradients # by running forward and backward passes through the model fc1_forw = self.fc1.forward(X) relu_forw = self.ReLU.forward(fc1_forw) fc2_forw = self.fc2.forward(relu_forw) loss, grad = softmax_with_cross_entropy(fc2_forw, y + 1) fc2_back = self.fc2.backward(grad) relu_back = self.ReLU.backward(fc2_back) self.fc1.backward(relu_back) # After that, implement l2 regularization on all params # Hint: self.params() is useful again! loss_l2_fc1, grad_l2_fc1 = l2_regularization( self.params()['fc1_W'].value, self.reg) loss_l2_fc2, grad_l2_fc2 = l2_regularization( self.params()['fc2_W'].value, self.reg) self.params()['fc1_W'].grad += grad_l2_fc1 self.params()['fc2_W'].grad += grad_l2_fc2 loss += loss_l2_fc1 + loss_l2_fc2 return loss def predict(self, X): """ Produces classifier predictions on the set Arguments: X, np array (test_samples, num_features) Returns: y_pred, np.array of int (test_samples) """ # TODO: Implement predict # Hint: some of the code of the compute_loss_and_gradients # can be reused predictions = self.fc2.forward(self.ReLU.forward(self.fc1.forward(X))) pred = np.argmax(predictions, axis=1) return pred def params(self): result = { 'fc1_W': self.fc1.params()['W'], 'fc2_W': self.fc2.params()['W'], 'fc1_B': self.fc1.params()['B'], 'fc2_B': self.fc2.params()['B'] } # TODO Implement aggregating all of the params return result
class TwoLayerNet: """ Neural network with two fully connected layers """ def __init__(self, n_input, n_output, hidden_layer_size, reg): """ Initializes the neural network Arguments: n_input, int - dimension of the model input n_output, int - number of classes to predict hidden_layer_size, int - number of neurons in the hidden layer reg, float - L2 regularization strength """ self.reg = reg self.fc1 = FullyConnectedLayer(n_input, hidden_layer_size) self.act1 = ReLULayer() self.fc2 = FullyConnectedLayer(hidden_layer_size, n_output) self.act2 = ReLULayer() def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, input_features) - input data y, np array of int (batch_size) - classes """ # Clear gradients params = self.params() for p in params: params[p].grad = 0 X = self.fc1.forward(X) X = self.act1.forward(X) X = self.fc2.forward(X) loss, d_pred = softmax_with_cross_entropy(X, y) # X = self.act2.forward(X) # d_act2 = self.act2.backward(d_pred) d_fc2 = self.fc2.backward(d_pred) d_act1 = self.act1.backward(d_fc2) d_fc1 = self.fc1.backward(d_act1) for p in params: regular_loss, regular_grad = l2_regularization( params[p].value, self.reg) loss += regular_loss params[p].grad += regular_grad return loss def predict(self, X): """ Produces classifier predictions on the set Arguments: X, np array (test_samples, num_features) Returns: y_pred, np.array of int (test_samples) """ pred = np.zeros(X.shape[0], np.int) X = self.fc1.forward(X) X = self.act1.forward(X) X = self.fc2.forward(X) X = self.act2.forward(X) pred = np.argmax(X, axis=1) return pred def params(self): result = {} for param in self.fc1.params(): result[param + '_fc1'] = self.fc1.params()[param] for param in self.fc2.params(): result[param + '_fc2'] = self.fc2.params()[param] return result
class TwoLayerNet: """ Neural network with two fully connected layers """ def __init__(self, n_input, n_output, hidden_layer_size, reg): """ Initializes the neural network Arguments: n_input, int - dimension of the model input n_output, int - number of classes to predict hidden_layer_size, int - number of neurons in the hidden layer reg, float - L2 regularization strength """ self.reg = reg self.hidden_layer = FullyConnectedLayer(n_input, hidden_layer_size) self.relu_layer = ReLULayer() self.output_layer = FullyConnectedLayer(hidden_layer_size, n_output) self.n_input = n_input self.n_output = n_output self.hidden_layer_size = hidden_layer_size # TODO Create necessary layers # raise Exception("Not implemented!") def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, input_features) - input data y, np array of int (batch_size) - classes """ # Before running forward and backward pass through the model, # clear parameter gradients aggregated from the previous pass # TODO Set parameter gradient to zeros # Hint: using self.params() might be useful! hidden_layer_params = self.hidden_layer.params() output_layer_params = self.output_layer.params() hidden_layer_params['W'].grad = np.zeros_like( hidden_layer_params['W'].grad) hidden_layer_params['B'].grad = np.zeros_like( hidden_layer_params['B'].grad) output_layer_params['W'].grad = np.zeros_like( output_layer_params['W'].grad) output_layer_params['B'].grad = np.zeros_like( output_layer_params['B'].grad) # TODO Compute loss and fill param gradients # by running forward and backward passes through the model hidden_l_out = self.hidden_layer.forward(X) relu_l_out = self.relu_layer.forward(hidden_l_out) output_l_out = self.output_layer.forward(relu_l_out) ce_loss, d_pred = softmax_with_cross_entropy(output_l_out, y) reg_loss_first, d_R_first = l2_regularization( hidden_layer_params['W'].value, self.reg) reg_loss_second, d_R_second = l2_regularization( output_layer_params['W'].value, self.reg) loss = ce_loss + reg_loss_first + reg_loss_second d_input_out_layer = self.output_layer.backward(d_pred) output_layer_params['W'].grad += d_R_second d_input_relu_layer = self.relu_layer.backward(d_input_out_layer) d_input_hidden_layer = self.hidden_layer.backward(d_input_relu_layer) hidden_layer_params['W'].grad += d_R_first # After that, implement l2 regularization on all params # Hint: self.params() is useful again return loss def predict(self, X): """ Produces classifier predictions on the set Arguments: X, np array (test_samples, num_features) Returns: y_pred, np.array of int (test_samples) """ # TODO: Implement predict # Hint: some of the code of the compute_loss_and_gradients # can be reused hidden_l_output = self.hidden_layer.forward(X) relu_output = self.relu_layer.forward(hidden_l_output) output_l_output = self.output_layer.forward(relu_output) pred = np.argmax(output_l_output, axis=1) return pred def params(self): result = {} # TODO Implement aggregating all of the params hidden_layer_params = self.hidden_layer.params() result['W1'] = hidden_layer_params['W'] result['B1'] = hidden_layer_params['B'] output_layer_params = self.output_layer.params() result['W2'] = output_layer_params['W'] result['B2'] = output_layer_params['B'] return result
class TwoLayerNet: """ Neural network with two fully connected layers """ def __init__(self, n_input, n_output, hidden_layer_size, reg): """ Initializes the neural network Arguments: n_input, int - dimension of the model input n_output, int - number of classes to predict hidden_layer_size, int - number of neurons in the hidden layer reg, float - L2 regularization strength """ self.reg = reg self.layer1 = FullyConnectedLayer(n_input, hidden_layer_size) self.relu = ReLULayer() self.layer2 = FullyConnectedLayer(hidden_layer_size, n_output) def zero_grads(self): for param in self.params().values(): param.grad[:] = 0.0 def forward(self, X): out1 = self.layer1.forward(X) out2 = self.relu.forward(out1) preds = self.layer2.forward(out2) return preds def backward(self, d_preds): d_layer2 = self.layer2.backward(d_preds) d_relu = self.relu.backward(d_layer2) d_layer1 = self.layer1.backward(d_relu) return d_layer1 def l2_regularization(self): for param in self.params().values(): l2_loss, l2_grad = l2_regularization(param.value, self.reg) param.grad += l2_grad return l2_loss def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, input_features) - input data y, np array of int (batch_size) - classes """ self.zero_grads() preds = self.forward(X) loss, d_preds = softmax_with_cross_entropy(preds, y) self.backward(d_preds) loss += self.l2_regularization() return loss def predict(self, X): """ Produces classifier predictions on the set Arguments: X, np array (test_samples, num_features) Returns: y_pred, np.array of int (test_samples) """ preds = self.forward(X) return preds.argmax(axis=1) def params(self): result = {} result['W1'] = self.layer1.W result['B1'] = self.layer1.B result['W2'] = self.layer2.W result['B2'] = self.layer2.B return result
class TwoLayerNet: """ Neural network with two fully connected layers """ def __init__(self, n_input, n_output, hidden_layer_size, reg): """ Initializes the neural network Arguments: n_input, int - dimension of the model input n_output, int - number of classes to predict hidden_layer_size, int - number of neurons in the hidden layer reg, float - L2 regularization strength """ self.reg = reg self.hidden_layer = FullyConnectedLayer(n_input, hidden_layer_size) self.non_linearity = ReLULayer() self.output_layer = FullyConnectedLayer(hidden_layer_size, n_output) def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, input_features) - input data y, np array of int (batch_size) - classes """ # Before running forward and backward pass through the model, # clear parameter gradients aggregated from the previous pass # TODO Set parameter gradient to zeros # Hint: using self.params() might be useful! params_ = self.params() for param in params_: params_[param].clear_grad() # TODO Compute loss and fill param gradients # by running forward and backward passes through the model temp_res = self.hidden_layer.forward(X) temp_res = self.non_linearity.forward(temp_res) temp_res = self.output_layer.forward(temp_res) loss, dpred = softmax_with_cross_entropy(temp_res, y) temp_grad = self.output_layer.backward(dpred) temp_grad = self.non_linearity.backward(temp_grad) temp_grad = self.hidden_layer.backward(temp_grad) # After that, implement l2 regularization on all params # Hint: self.params() is useful again! params_ = self.params() for param in params_: loss_l2, grad_l2 = l2_regularization(params_[param].value, self.reg) loss += loss_l2 params_[param].grad += grad_l2 return loss def predict(self, X): """ Produces classifier predictions on the set Arguments: X, np array (test_samples, num_features) Returns: y_pred, np.array of int (test_samples) """ # TODO: Implement predict # Hint: some of the code of the compute_loss_and_gradients # can be reused pred = np.zeros(X.shape[0], np.int) temp_res = self.hidden_layer.forward(X) temp_res = self.non_linearity.forward(temp_res) pred = self.output_layer.forward(temp_res) pred = np.argmax(softmax(pred), axis=1) return pred def params(self): result = {} # TODO Implement aggregating all of the params result['W_h'] = self.hidden_layer.W result['B_h'] = self.hidden_layer.B result['W_o'] = self.output_layer.W result['B_o'] = self.output_layer.B return result
class TwoLayerNet: """ Neural network with two fully connected layers """ def __init__(self, n_input, n_output, hidden_layer_size, reg): """ Initializes the neural network Arguments: n_input, int - dimension of the model input n_output, int - number of classes to predict hidden_layer_size, int - number of neurons in the hidden layer reg, float - L2 regularization strength """ self.reg = reg # TODO Create necessary layers self.FCL1 = FullyConnectedLayer(n_input, hidden_layer_size) self.ReLu = ReLULayer() self.FCL2 = FullyConnectedLayer(hidden_layer_size, n_output) def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, input_features) - input data y, np array of int (batch_size) - classes """ # Before running forward and backward pass through the model, # clear parameter gradients aggregated from the previous pass # TODO Set parameter gradient to zeros # Hint: using self.params() might be useful! prms = self.params() for p in prms: prms[p].grad = np.zeros_like(prms[p].value) # TODO Compute loss and fill param gradients # by running forward and backward passes through the model preds = self.FCL2.forward( self.ReLu.forward( self.FCL1.forward(X))) (loss, dprediction) = softmax_with_cross_entropy(preds, y) d_fcl2 = self.FCL2.backward(dprediction) d_relu = self.ReLu.backward(d_fcl2) d_fcl1 = self.FCL1.backward(d_relu) # After that, implement l2 regularization on all params # Hint: self.params() is useful again! for p in prms: (loss_l2, grad_l2) = l2_regularization(prms[p].value, self.reg) loss += loss_l2 prms[p].grad += grad_l2 return loss def predict(self, X): """ Produces classifier predictions on the set Arguments: X, np array (test_samples, num_features) Returns: y_pred, np.array of int (test_samples) """ # TODO: Implement predict # Hint: some of the code of the compute_loss_and_gradients # can be reused probs = softmax(self.FCL2.forward( self.ReLu.forward( self.FCL1.forward(X)))) pred = np.argmax(probs, axis=1).astype(np.int) return pred def params(self): result = {} # TODO Implement aggregating all of the params result['W1'] = self.FCL1.W result['B1'] = self.FCL1.B result['W2'] = self.FCL2.W result['B2'] = self.FCL2.B return result
class TwoLayerNet: """ Neural network with two fully connected layers """ def __init__(self, n_input, n_output, hidden_layer_size, reg): """ Initializes the neural network Arguments: n_input, int - dimension of the model input n_output, int - number of classes to predict hidden_layer_size, int - number of neurons in the hidden layer reg, float - L2 regularization strength """ self.reg = reg self.input_layer = FullyConnectedLayer(n_input, hidden_layer_size) self.relu = ReLULayer() self.output_layer = FullyConnectedLayer(hidden_layer_size, n_output) def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, input_features) - input data y, np array of int (batch_size) - classes """ params = self.params() for param in params: params[param].clear_grad() res = self.input_layer.forward(X) res = self.relu.forward(res) res = self.output_layer.forward(res) loss, dpred = softmax_with_cross_entropy(res, y) grad = self.output_layer.backward(dpred) grad = self.relu.backward(grad) grad = self.input_layer.backward(grad) for param in params: loss_l2, grad_l2 = l2_regularization(params[param].value, self.reg) loss += loss_l2 params[param].grad += grad_l2 return loss def predict(self, X): """ Produces classifier predictions on the set Arguments: X, np array (test_samples, num_features) Returns: y_pred, np.array of int (test_samples) """ pred = self.input_layer.forward(X) pred = self.relu.forward(pred) pred = self.output_layer.forward(pred) pred = np.argmax(softmax(pred), axis=1) return pred def params(self): result = {'W_h': self.input_layer.W, 'B_h': self.input_layer.B, 'W_o': self.output_layer.W, 'B_o': self.output_layer.B} return result
class TwoLayerNet: """ Neural network with two fully connected layers """ def __init__(self, n_input, n_output, hidden_layer_size, reg): """ Initializes the neural network Arguments: n_input, int - dimension of the model input n_output, int - number of classes to predict hidden_layer_size, int - number of neurons in the hidden layer reg, float - L2 regularization strength """ self.reg = reg self.fc1 = FullyConnectedLayer(n_input, hidden_layer_size) self.ReLU1 = ReLULayer() self.fc2 = FullyConnectedLayer(hidden_layer_size, n_output) def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, input_features) - input data y, np array of int (batch_size) - classes """ # Before running forward and backward pass through the model, # clear parameter gradients aggregated from the previous pass # TODO Set parameter gradient to zeros # Hint: using self.params() might be useful! params = self.params() for key in params.keys(): params[key].grad = np.zeros_like(params[key].value) # TODO Compute loss and fill param gradients # by running forward and backward passes through the model # After that, implement l2 regularization on all params # Hint: self.params() is useful again! out1 = self.ReLU1.forward(self.fc1.forward(X)) out2 = self.fc2.forward(out1) loss, grad = softmax_with_cross_entropy(out2, y) for key in params.keys(): l2_loss, l2_grad = l2_regularization(params[key].value, self.reg) loss += l2_loss params[key].grad += l2_grad d_out2 = self.fc2.backward(grad) self.fc1.backward(self.ReLU1.backward(d_out2)) return loss def predict(self, X): """ Produces classifier predictions on the set Arguments: X, np array (test_samples, num_features) Returns: y_pred, np.array of int (test_samples) """ # TODO: Implement predict # Hint: some of the code of the compute_loss_and_gradients # can be reused out1 = self.ReLU1.forward(self.fc1.forward(X)) out2 = self.fc2.forward(out1) pred = out2.argmax(axis=1) return pred def params(self): result = {} # TODO Implement aggregating all of the params result['W1'] = self.fc1.W result['B1'] = self.fc1.B result['W2'] = self.fc2.W result['B2'] = self.fc2.B return result
class ConvNet: """ Implements a very simple conv net Input -> Conv[3x3] -> Relu -> Maxpool[4x4] -> Conv[3x3] -> Relu -> MaxPool[4x4] -> Flatten -> FC -> Softmax """ def __init__(self, input_shape, n_output_classes, conv1_channels, conv2_channels): """ Initializes the neural network Arguments: input_shape, tuple of 3 ints - image_width, image_height, n_channels Will be equal to (32, 32, 3) n_output_classes, int - number of classes to predict conv1_channels, int - number of filters in the 1st conv layer conv2_channels, int - number of filters in the 2nd conv layer """ filter_size = 3 pool_size = 4 self.conv1 = ConvolutionalLayer(input_shape[2], conv1_channels, filter_size, padding=1) self.relu1 = ReLULayer() self.max_pool1 = MaxPoolingLayer(pool_size, stride=pool_size) self.conv2 = ConvolutionalLayer(conv1_channels, conv2_channels, filter_size, padding=1) self.relu2 = ReLULayer() self.max_pool2 = MaxPoolingLayer(pool_size, stride=pool_size) self.flatten = Flattener() self.fc = FullyConnectedLayer(n_input=4 * conv2_channels, n_output=n_output_classes) def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, height, width, input_features) - input data y, np array of int (batch_size) - classes """ # Before running forward and backward pass through the model, # clear parameter gradients aggregated from the previous pass params = self.params() for param_key, param_value in params.items(): param_value.grad = np.zeros_like(param_value.value) # Compute loss and fill param gradients # Don't worry about implementing L2 regularization, we will not # need it in this assignment X = self.conv1.forward(X) X = self.relu1.forward(X) X = self.max_pool1.forward(X) X = self.conv2.forward(X) X = self.relu2.forward(X) X = self.max_pool2.forward(X) X = self.flatten.forward(X) X = self.fc.forward(X) loss, grad = softmax_with_cross_entropy(X, y) grad = self.fc.backward(grad) grad = self.flatten.backward(grad) grad = self.max_pool2.backward(grad) grad = self.relu2.backward(grad) grad = self.conv2.backward(grad) grad = self.max_pool1.backward(grad) grad = self.relu1.backward(grad) grad = self.conv1.backward(grad) return loss def predict(self, X): X = self.conv1.forward(X) X = self.relu1.forward(X) X = self.max_pool1.forward(X) X = self.conv2.forward(X) X = self.relu2.forward(X) X = self.max_pool2.forward(X) X = self.flatten.forward(X) X = self.fc.forward(X) X = softmax(X) return np.argmax(X, axis=1) def params(self): # Aggregate all the params from all the layers # which have parameters result = {} layers_with_params = [self.conv1, self.conv2, self.fc] for i in range(len(layers_with_params)): layer = layers_with_params[i] layer_number = str(i) for param_key, param_value in layer.params().items(): result[param_key + str(layer_number)] = param_value return result
class TwoLayerNet: """ Neural network with two fully connected layers """ def __init__(self, n_input, n_output, hidden_layer_size, reg): """ Initializes the neural network Arguments: n_input, int - dimension of the model input n_output, int - number of classes to predict hidden_layer_size, int - number of neurons in the hidden layer reg, float - L2 regularization strength """ self.reg = reg #self.pred=[] self.n_output=n_output self.fc1 = FullyConnectedLayer(n_input,hidden_layer_size) self.relu1=ReLULayer() self.fc2=FullyConnectedLayer(hidden_layer_size,n_output) self.relu2=ReLULayer() # TODO Create necessary layers #raise Exception("Not implemented!") def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, input_features) - input data y, np array of int (batch_size) - classes """ # Before running forward and backward pass through the model, # clear parameter gradients aggregated from the previous pass # TODO Set parameter gradient to zeros # Hint: using self.params() might be useful! #raise Exception("Not implemented!") #--------forward way --------- #print('self.fc1.W.value', self.fc1.W.value) #print('self.fc1.W.grad', self.fc1.W.grad) fc1_out=self.fc1.forward(X) #print('self.fc1.W.value after forward', self.fc1.W.value) #print('fc1_out',fc1_out) relu1_out=self.relu1.forward(fc1_out) #print('relu1_out',relu1_out) fc2_out=self.fc2.forward(relu1_out) #print('fc2_out',fc1_out) #relu2_out=self.relu2.forward(fc2_out) loss,d_preds=softmax_with_cross_entropy(fc2_out, y) #self.pred=fc2_out #print('output.shape', output.shape) #print('loss',loss) loss_fc1_l2,grad_fc1_l2=l2_regularization(self.fc1.W.value,self.reg) #print('loss fc1 l2',loss_fc1_l2) loss_fc2_l2,grad_fc2_l2=l2_regularization(self.fc2.W.value,self.reg) #print('loss fc2 l2',loss_fc2_l2) loss_total=loss+loss_fc1_l2+loss_fc2_l2 #loss_total=loss #-------backward way ----------- #d_out = np.ones_like(output) #print('d_preds.shape',d_preds.shape) d_out=d_preds #print('self.fc1.W.grad befor backward', self.fc1.W.grad) #print('d_out befor backward',d_out) #relu1_dX=self.relu1.backward(d_out) #relu2_dX=self.relu2.backward(d_out) fc2_dX=self.fc2.backward(d_out) #print('self.fc2.W.grad',self.fc2.W.grad) relu1_dX=self.relu1.backward(fc2_dX) fc1_dX=self.fc1.backward(relu1_dX) self.fc1.W.grad+=grad_fc1_l2 self.fc2.W.grad+=grad_fc2_l2 return loss_total def predict(self, X): """ Produces classifier predictions on the set Arguments: X, np array (test_samples, num_features) Returns: y_pred, np.array of int (test_samples) """ # TODO: Implement predict # Hint: some of the code of the compute_loss_and_gradients # can be reused fc1_out=self.fc1.forward(X) relu1_out=self.relu1.forward(fc1_out) fc2_out=self.fc2.forward(relu1_out) relu2_out=self.relu2.forward(fc2_out) output = fc2_out pred=np.argmax(output,axis=1) #pred= np.argmax(self.pred,axis=1) #print('pred_1', pred_1) #raise Exception("Not implemented!") return pred def params(self): #result{} result = {'W2':self.fc2.W, 'B2':self.fc2.B, 'B1':self.fc1.B, 'W1':self.fc1.W} # TODO Implement aggregating all of the params #raise Exception("Not implemented!") return result
class TwoLayerNet: """ Neural network with two fully connected layers """ def __init__(self, n_input, n_output, hidden_layer_size, reg): """ Initializes the neural network Arguments: n_input, int - dimension of the model input n_output, int - number of classes to predict hidden_layer_size, int - number of neurons in the hidden layer reg, float - L2 regularization strength """ self.reg = reg # TODO Create necessary layers self.layer_1 = FullyConnectedLayer(n_input, hidden_layer_size) self.relu = ReLULayer() self.layer_2 = FullyConnectedLayer(hidden_layer_size, n_output) self.hidden_layer_size = hidden_layer_size self.n_input = n_input self.n_output = n_output #raise Exception("Not implemented!") def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, input_features) - input data y, np array of int (batch_size) - classes """ # Before running forward and backward pass through the model, # clear parameter gradients aggregated from the previous pass # TODO Set parameter gradient to zeros for v in self.params().values(): v.grad.fill(0) # Hint: using self.params() might be useful! #raise Exception("Not implemented!") # TODO Compute loss and fill param gradients # by running forward and backward passes through the model #Compute FullyConnectedLayer_1 l_1 = self.layer_1.forward(X) #Compute ReLuLayer l_relu = self.relu.forward(l_1) #Compute FullyConnectedLayer_2 l_2 = self.layer_2.forward(l_relu) #compute loss and grad of F loss, grad_pred = softmax_with_cross_entropy(l_2, y) for v in self.params().values(): l2_loss, l2_grad = l2_regularization(v.value, self.reg) loss += l2_loss v.grad += l2_grad grad_l_2 = self.layer_2.backward(grad_pred) grad_relu = self.relu.backward(grad_l_2) grad_l_1 = self.layer_1.backward(grad_relu) # After that, implement l2 regularization on all params # Hint: self.params() is useful again! #raise Exception("Not implemented!") return loss def predict(self, X): """ Produces classifier predictions on the set Arguments: X, np array (test_samples, num_features) Returns: y_pred, np.array of int (test_samples) """ # TODO: Implement predict # Hint: some of the code of the compute_loss_and_gradients # can be reused pred = np.zeros(X.shape[0], np.int) #Compute FullyConnectedLayer_1 l_1 = self.layer_1.forward(X) #Compute ReLuLayer l_relu = self.relu.forward(l_1) #Compute FullyConnectedLayer_2 l_2 = self.layer_2.forward(l_relu) #Compute pred pred = np.argmax(l_2, axis=1) #raise Exception("Not implemented!") return pred def params(self): p1 = self.layer_1.params() p2 = self.layer_2.params() result = {"W1": p1["W"], "B1": p1["B"], "W2": p2["W"], "B2": p2["B"]} #result = {'W1': self.w1, 'W2': self.w2, 'B1': self.b1, 'B2': self.b2} # TODO Implement aggregating all of the params #raise Exception("Not implemented!") return result
class TwoLayerNet: """ Neural network with two fully connected layers """ def __init__(self, n_input, n_output, hidden_layer_size, reg): """ Initializes the neural network Arguments: n_input, int - dimension of the model input n_output, int - number of classes to predict hidden_layer_size, int - number of neurons in the hidden layer reg, float - L2 regularization strength """ self.reg = reg # TODO Create necessary layers self.first_layer = FullyConnectedLayer(n_input, hidden_layer_size) self.first_relu = ReLULayer() self.second_layer = FullyConnectedLayer(hidden_layer_size, n_output) def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, input_features) - input data y, np array of int (batch_size) - classes """ # Before running forward and backward pass through the model, # clear parameter gradients aggregated from the previous pass # TODO Set parameter gradient to zeros # Hint: using self.params() might be useful! for i, w in self.params().items(): w.grad = np.zeros(w.grad.shape) # TODO Compute loss and fill param gradients # by running forward and backward passes through the model value = self.first_layer.forward(X) value = self.first_relu.forward(value) value = self.second_layer.forward(value) loss, grads = softmax_with_cross_entropy(value, y) value = self.second_layer.backward(grads) value = self.first_relu.backward(value) value = self.first_layer.backward(value) # After that, implement l2 regularization on all params # Hint: self.params() is useful again! for i, w in self.params().items(): loss_delta, grad_delta = l2_regularization(w.value, self.reg) w.grad += grad_delta loss += loss_delta return loss def predict(self, X): """ Produces classifier predictions on the set Arguments: X, np array (test_samples, num_features) Returns: y_pred, np.array of int (test_samples) """ # TODO: Implement predict # Hint: some of the code of the compute_loss_and_gradients # can be reused pred = np.zeros(X.shape[0], np.int) pred = self.first_layer.forward(X) pred = self.first_relu.forward(pred) pred = self.second_layer.forward(pred) pred = np.argmax(pred, axis=1) return pred def params(self): result = {} # TODO Implement aggregating all of the params result['first_layer_W'] = self.first_layer.params()['W'] result['second_layer_W'] = self.second_layer.params()['W'] result['first_layer_B'] = self.first_layer.params()['B'] result['second_layer_B'] = self.second_layer.params()['B'] return result
class TwoLayerNet: """ Neural network with two fully connected layers """ def __init__(self, n_input, n_output, hidden_layer_size, reg): """ Initializes the neural network Arguments: n_input, int - dimension of the model input n_output, int - number of classes to predict hidden_layer_size, int - number of neurons in the hidden layer reg, float - L2 regularization strength """ self.reg = reg self.layer1 = FullyConnectedLayer(n_input, hidden_layer_size) self.ReLULayer1 = ReLULayer() self.layer2 = FullyConnectedLayer(hidden_layer_size, n_output) def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, input_features) - input data y, np array of int (batch_size) - classes """ params = self.params() for key in params.keys(): params[key].grad = np.zeros_like(params[key].value) out1 = self.ReLULayer1.forward(self.layer1.forward(X)) out2 = self.layer2.forward(out1) loss, grad = softmax_with_cross_entropy(out2, y) for key in params.keys(): l2_loss, l2_grad = l2_regularization(params[key].value, self.reg) loss += l2_loss params[key].grad += l2_grad d_out2 = self.layer2.backward(grad) self.layer1.backward(self.ReLULayer1.backward(d_out2)) return loss def predict(self, X): """ Produces classifier predictions on the set Arguments: X, np array (test_samples, num_features) Returns: y_pred, np.array of int (test_samples) """ out1 = self.ReLULayer1.forward(self.layer1.forward(X)) out2 = self.layer2.forward(out1) pred = out2.argmax(axis=1) return pred def params(self): result = {} result['W1'] = self.layer1.W result['B1'] = self.layer1.B result['W2'] = self.layer2.W result['B2'] = self.layer2.B return result
class TwoLayerNet: """ Neural network with two fully connected layers """ def __init__(self, n_input, n_output, hidden_layer_size, reg=0): """ Initializes the neural network Arguments: n_input, int - dimension of the model input n_output, int - number of classes to predict hidden_layer_size, int - number of neurons in the hidden layer reg, float - L2 regularization strength """ self.reg = reg self.n_input = n_input self.n_output = n_output self.h_size = hidden_layer_size # TODO Create necessary layers self.RL = ReLULayer() self.FC1 = FullyConnectedLayer(n_input=self.n_input, n_output=self.h_size) self.FC2 = FullyConnectedLayer(self.h_size, self.n_output) def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, input_features) - input data y, np array of int (batch_size) - classes """ # Before running forward and backward pass through the model, # clear parameter gradients aggregated from the previous pass # TODO Set parameter gradient to zeros # Hint: using self.params() might be useful! self.FC1.W.grad = np.zeros_like(self.FC1.W.grad) self.FC1.B.grad = np.zeros_like(self.FC1.B.grad) self.FC2.W.grad = np.zeros_like(self.FC2.W.grad) self.FC2.B.grad = np.zeros_like(self.FC2.B.grad) # raise Exception("Not implemented!") # TODO Compute loss and fill param gradients # by running forward and backward passes through the model # After that, implement l2 regularization on all params # Hint: self.params() is useful again! # raise Exception("Not implemented!") params = self.params() x = X.copy() x = self.FC1.forward(x) x = self.RL.forward(x) pred = self.FC2.forward(x) # print(f'SHAPE fc1: \n {np.sum(self.FC1.W.grad)}') # print(f'SHAPE b2: \n {np.sum(self.FC1.B.grad)}') loss, dpred = softmax_with_cross_entropy(pred, target_index=y) d_out = self.FC2.backward(dpred) d_out = self.RL.backward(d_out) grad = self.FC1.backward(d_out) # print(f'SHAPE fc1: \n {np.sum(self.FC1.W.grad)}') # print(f'SHAPE fc2: \n {np.sum(self.FC2.W.grad)}') if self.reg > 0: rloss_fc1, dW_rfc1 = l2_regularization(self.FC1.W.value, self.reg) rloss_fc2, dW_rfc2 = l2_regularization(self.FC2.W.value, self.reg) rloss_fc1B, dB_rfc1 = l2_regularization(self.FC1.B.value, self.reg) rloss_fc2B, dB_rfc2 = l2_regularization(self.FC2.B.value, self.reg) loss = loss + rloss_fc1 + rloss_fc2 + rloss_fc1B + rloss_fc2B self.FC1.W.grad += dW_rfc1 self.FC2.W.grad += dW_rfc2 self.FC1.B.grad += dB_rfc1 self.FC2.B.grad += dB_rfc2 # result = {'fc1_w': self.FC1.W.grad, # 'fc1_b': self.FC1.B.grad, # 'fc2_w': self.FC2.W.grad, # 'fc2_b': self.FC2.B.grad} return loss, grad def predict(self, X): """ Produces classifier predictions on the set Arguments: X, np array (test_samples, num_features) Returns: y_pred, np.array of int (test_samples) """ # TODO: Implement predict # Hint: some of the code of the compute_loss_and_gradients # can be reused x = X.copy() x = self.FC1.forward(x) x = self.RL.forward(x) x = self.FC2.forward(x) y_hat = softmax(predictions=x) y_hat = np.argmax(y_hat, axis=1) return y_hat def params(self): result = { 'FC1.W': self.FC1.W, 'FC1.B': self.FC1.B, 'FC2.W': self.FC2.W, 'FC2.B': self.FC2.B } return result
class TwoLayerNet: """ Neural network with two fully connected layers """ def __init__(self, n_input, n_output, hidden_layer_size, reg): """ Initializes the neural network Arguments: n_input, int - dimension of the model input n_output, int - number of classes to predict hidden_layer_size, int - number of neurons in the hidden layer reg, float - L2 regularization strength """ self.reg = reg self.layer_1 = FullyConnectedLayer(n_input, hidden_layer_size) self.layer_2 = ReLULayer() self.layer_3 = FullyConnectedLayer(hidden_layer_size, n_output) def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, input_features) - input data y, np array of int (batch_size) - classes """ params = self.params() for p in params: param = params[p] param.grad = np.zeros_like(param.grad) step_1f = self.layer_1.forward(X) step_2f = self.layer_2.forward(step_1f) step_3f = self.layer_3.forward(step_2f) loss, dpred = softmax_with_cross_entropy(step_3f, y) step_3b = self.layer_3.backward(dpred) step_2b = self.layer_2.backward(step_3b) step_1b = self.layer_1.backward(step_2b) for p in params: param = params[p] loss_l2, grad_l2 = l2_regularization(param.value, self.reg) param.grad += grad_l2 loss += loss_l2 return loss def predict(self, X): """ Produces classifier predictions on the set Arguments: X, np array (test_samples, num_features) Returns: y_pred, np.array of int (test_samples) """ step_1f = self.layer_1.forward(X) step_2f = self.layer_2.forward(step_1f) step_3f = self.layer_3.forward(step_2f) probs = softmax(step_3f) pred = np.array(list(map(lambda x: x.argsort()[-1], probs))) return pred def params(self): result = {} result['W1'] = self.layer_1.W result['W2'] = self.layer_3.W result['B1'] = self.layer_1.B result['B2'] = self.layer_3.B return result
class TwoLayerNet: """ Neural network with two fully connected layers """ def __init__(self, n_input, n_output, hidden_layer_size, reg): """ Initializes the neural network Arguments: n_input, int - dimension of the model input n_output, int - number of classes to predict hidden_layer_size, int - number of neurons in the hidden layer reg, float - L2 regularization strength """ self.reg = reg # TODO Create necessary layers self.first = FullyConnectedLayer(n_input, hidden_layer_size) self.relu = ReLULayer() self.second = FullyConnectedLayer(hidden_layer_size, n_output) def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, input_features) - input data y, np array of int (batch_size) - classes """ # Before running forward and backward pass through the model, # clear parameter gradients aggregated from the previous pass # TODO Set parameter gradient to zeros # Hint: using self.params() might be useful! for param in self.params().values(): param.grad = np.zeros_like(param.value) # TODO Compute loss and fill param gradients # by running forward and backward passes through the model preds = self.first.forward(X) preds = self.relu.forward(preds) preds = self.second.forward(preds) loss, grad = softmax_with_cross_entropy(preds, y) grad = self.second.backward(grad) grad = self.relu.backward(grad) grad = self.first.backward(grad) # After that, implement l2 regularization on all params # Hint: self.params() is useful again! for param in self.params().values(): loss_l2, grad_l2 = l2_regularization(param.value, self.reg) loss += loss_l2 param.grad += grad_l2 return loss def predict(self, X): """ Produces classifier predictions on the set Arguments: X, np array (test_samples, num_features) Returns: y_pred, np.array of int (test_samples) """ # TODO: Implement predict # Hint: some of the code of the compute_loss_and_gradients # can be reused preds = self.first.forward(X) preds = self.relu.forward(preds) preds = self.second.forward(preds) probs = softmax(preds) return np.argmax(probs, axis=1) def params(self): # TODO Implement aggregating all of the params return {'first.W': self.first.W, 'first.B': self.first.B, 'second.W': self.second.W, 'second.B': self.second.B}
class ConvNet: """ Implements a very simple conv net Input -> Conv[3x3] -> Relu -> Maxpool[4x4] -> Conv[3x3] -> Relu -> MaxPool[4x4] -> Flatten -> FC -> Softmax """ def __init__(self, input_shape, n_output_classes, conv1_channels, conv2_channels): """ Initializes the neural network Arguments: input_shape, tuple of 3 ints - image_width, image_height, n_channels Will be equal to (32, 32, 3) n_output_classes, int - number of classes to predict conv1_channels, int - number of filters in the 1st conv layer conv2_channels, int - number of filters in the 2nd conv layer """ # TODO Create necessary layers self.input_shape = input_shape self.n_output_classes = n_output_classes self.layer1 = ConvolutionalLayer(3, conv1_channels, 3, 1) #32x32x3xconv1_channels self.layer2 = ReLULayer() self.layer3 = MaxPoolingLayer(4, 4) #8x8x3xconv1_channels self.layer4 = ConvolutionalLayer( conv1_channels, conv2_channels, 3, 1) #8x8x3x conv1_channels x conv2_channels self.layer5 = ReLULayer() self.layer6 = MaxPoolingLayer( 4, 4) #2x2x3 conv1_channels x conv2_channels self.layer7 = Flattener() self.layer8 = FullyConnectedLayer(conv1_channels * conv2_channels * 2, n_output_classes) def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, height, width, input_features) - input data y, np array of int (batch_size) - classes """ # Before running forward and backward pass through the model, # clear parameter gradients aggregated from the previous pass # TODO Compute loss and fill param gradients # Don't worry about implementing L2 regularization, we will not # need it in this assignment self.layer1.W.grad = np.zeros_like(self.layer1.W.grad) self.layer4.W.grad = np.zeros_like(self.layer4.W.grad) self.layer1.B.grad = np.zeros_like(self.layer1.B.grad) self.layer4.B.grad = np.zeros_like(self.layer4.B.grad) self.layer8.W.grad = np.zeros_like(self.layer8.W.grad) self.layer8.B.grad = np.zeros_like(self.layer8.B.grad) out1 = self.layer1.forward(X) out2 = self.layer2.forward(out1) out3 = self.layer3.forward(out2) out4 = self.layer4.forward(out3) out5 = self.layer5.forward(out4) out6 = self.layer6.forward(out5) out7 = self.layer7.forward(out6) out8 = self.layer8.forward(out7) loss, grad = softmax_with_cross_entropy(out8, y) back8 = self.layer8.backward(grad) back7 = self.layer7.backward(back8) back6 = self.layer6.backward(back7) back5 = self.layer5.backward(back6) back4 = self.layer4.backward(back5) back3 = self.layer3.backward(back4) back2 = self.layer2.backward(back3) back1 = self.layer1.backward(back2) return loss def predict(self, X): # You can probably copy the code from previous assignment out1 = self.layer1.forward(X) out2 = self.layer2.forward(out1) out3 = self.layer3.forward(out2) out4 = self.layer4.forward(out3) out5 = self.layer5.forward(out4) out6 = self.layer6.forward(out5) out7 = self.layer7.forward(out6) out8 = self.layer8.forward(out7) pred = np.argmax(out8, axis=1) return pred def params(self): result = { 'layer1.W': self.layer1.W, 'layer1.B': self.layer1.B, 'layer4.W': self.layer4.W, 'layer1.B': self.layer4.B, 'layer8.W': self.layer8.W, 'layer8.B': self.layer8.B, } # TODO: Aggregate all the params from all the layers # which have parameters #raise Exception("Not implemented!") return result
class ConvNet: """ Implements a very simple conv net Input -> Conv[3x3] -> Relu -> Maxpool[4x4] -> Conv[3x3] -> Relu -> MaxPool[4x4] -> Flatten -> FC -> Softmax """ def __init__(self, input_shape, n_output_classes, conv1_channels, conv2_channels): """ Initializes the neural network Arguments: input_shape, tuple of 3 ints - image_width, image_height, n_channels Will be equal to (32, 32, 3) n_output_classes, int - number of classes to predict conv1_channels, int - number of filters in the 1st conv layer conv2_channels, int - number of filters in the 2nd conv layer """ # TODO Create necessary layers self.layer1 = ConvolutionalLayer(input_shape[2], conv1_channels, 3, 1) self.layer2 = ReLULayer() self.layer3 = MaxPoolingLayer(4, 4) self.layer4 = ConvolutionalLayer(conv1_channels, conv2_channels, 3, 1) self.layer5 = ReLULayer() self.layer6 = MaxPoolingLayer(4, 4) self.layer7 = Flattener() self.layer8 = FullyConnectedLayer( input_shape[0] * input_shape[1] * conv2_channels // (16 * 16), n_output_classes) def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, height, width, input_features) - input data y, np array of int (batch_size) - classes """ # Before running forward and backward pass through the model, # clear parameter gradients aggregated from the previous pass # TODO Compute loss and fill param gradients # Don't worry about implementing L2 regularization, we will not # need it in this assignment params = self.params() for param_key in params: param = params[param_key] param.grad = np.zeros_like(param.grad) step1 = self.layer1.forward(X) step2 = self.layer2.forward(step1) step3 = self.layer3.forward(step2) step4 = self.layer4.forward(step3) step5 = self.layer5.forward(step4) step6 = self.layer6.forward(step5) step7 = self.layer7.forward(step6) step8 = self.layer8.forward(step7) loss, dL = softmax_with_cross_entropy(step8, y) dstep8 = self.layer8.backward(dL) dstep7 = self.layer7.backward(dstep8) dstep6 = self.layer6.backward(dstep7) dstep5 = self.layer5.backward(dstep6) dstep4 = self.layer4.backward(dstep5) dstep3 = self.layer3.backward(dstep4) dstep2 = self.layer2.backward(dstep3) dstep1 = self.layer1.backward(dstep2) return loss def predict(self, X): # You can probably copy the code from previous assignment step1 = self.layer1.forward(X) step2 = self.layer2.forward(step1) step3 = self.layer3.forward(step2) step4 = self.layer4.forward(step3) step5 = self.layer5.forward(step4) step6 = self.layer6.forward(step5) step7 = self.layer7.forward(step6) step8 = self.layer8.forward(step7) probs = softmax(step8) pred = np.array(list(map(lambda x: x.argsort()[-1], probs))) return pred def params(self): # TODO: Aggregate all the params from all the layers # which have parameters return { 'W1': self.layer1.W, 'B1': self.layer1.B, 'W2': self.layer4.W, 'B2': self.layer4.B, 'W3': self.layer8.W, 'B3': self.layer8.B }
class ConvNet: """ Implements a very simple conv net Input -> Conv[3x3] -> Relu -> Maxpool[4x4] -> Conv[3x3] -> Relu -> MaxPool[4x4] -> Flatten -> FC -> Softmax """ def __init__(self, input_shape, n_output_classes, conv1_channels, conv2_channels): """ Initializes the neural network Arguments: input_shape, tuple of 3 ints - image_width, image_height, n_channels Will be equal to (32, 32, 3) n_output_classes, int - number of classes to predict conv1_channels, int - number of filters in the 1st conv layer conv2_channels, int - number of filters in the 2nd conv layer """ width, height, channels = input_shape filter_size = 3 padding = 1 pool_size = 4 pool_stride = 4 self.Conv1 = ConvolutionalLayer(channels, conv1_channels, filter_size, padding) self.ReLU1 = ReLULayer() self.MaxPool1 = MaxPoolingLayer(pool_size, pool_stride) self.Conv2 = ConvolutionalLayer(conv1_channels, conv2_channels, filter_size, padding) self.ReLU2 = ReLULayer() self.MaxPool2 = MaxPoolingLayer(pool_size, pool_stride) left_width = width // pool_stride // pool_stride left_height = height // pool_stride // pool_stride self.Flat = Flattener() self.FullyConnected = FullyConnectedLayer( left_width * left_height * conv2_channels, n_output_classes) def compute_loss_and_gradients(self, X, y): """ Computes total loss and updates parameter gradients on a batch of training examples Arguments: X, np array (batch_size, height, width, input_features) - input data y, np array of int (batch_size) - classes """ # Before running forward and backward pass through the model, # clear parameter gradients aggregated from the previous pass # TODO Compute loss and fill param gradients # Don't worry about implementing L2 regularization, we will not # need it in this assignment for _, v in self.params().items(): v.grad = np.zeros(v.grad.shape) out = self.Conv1.forward(X) out = self.ReLU1.forward(out) out = self.MaxPool1.forward(out) out = self.Conv2.forward(out) out = self.ReLU2.forward(out) out = self.MaxPool2.forward(out) out = self.Flat.forward(out) out = self.FullyConnected.forward(out) loss, d_out = softmax_with_cross_entropy(out, y) d_out = self.FullyConnected.backward(d_out) d_out = self.Flat.backward(d_out) d_out = self.MaxPool2.backward(d_out) d_out = self.ReLU2.backward(d_out) d_out = self.Conv2.backward(d_out) d_out = self.MaxPool1.backward(d_out) d_out = self.ReLU1.backward(d_out) d_out = self.Conv1.backward(d_out) return loss def predict(self, X): # You can probably copy the code from previous assignment out = self.Conv1.forward(X) out = self.ReLU1.forward(out) out = self.MaxPool1.forward(out) out = self.Conv2.forward(out) out = self.ReLU2.forward(out) out = self.MaxPool2.forward(out) out = self.Flat.forward(out) out = self.FullyConnected.forward(out) pred = np.argmax(out, axis=1) return pred def params(self): result = {} # TODO: Aggregate all the params from all the layers # which have parameters name2layer = { "Conv1": self.Conv1, "Conv2": self.Conv2, "Fully": self.FullyConnected } for name, layer in name2layer.items(): for k, v in layer.params().items(): result['{}_{}'.format(name, k)] = v return result