def backward(self, out_grad, input, weights, bias): """ # Arguments out_grad: gradient to the forward output of conv layer, with shape (batch, out_channel, out_height, out_width) input: numpy array with shape (batch, in_channel, in_height, in_width) weights: numpy array with shape (out_channel, in_channel, kernel_h, kernel_w) bias: numpy array with shape (out_channel) # Returns in_grad: gradient to the forward input of conv layer, with same shape as input w_grad: gradient to weights, with same shape as weights b_bias: gradient to bias, with same shape as bias """ kernel_h = self.conv_params['kernel_h'] # height of kernel kernel_w = self.conv_params['kernel_w'] # width of kernel pad = self.conv_params['pad'] stride = self.conv_params['stride'] in_channel = self.conv_params['in_channel'] out_channel = self.conv_params['out_channel'] batch, in_channel, in_height, in_width = input.shape ################################################################################# # code here opt_h = int(((in_height - kernel_h + pad) / stride) + 1) opt_w = int(((in_width - kernel_w + pad) / stride) + 1) in_grad = np.zeros((batch, in_channel, in_height + 2 * pad, in_width + 2 * pad)) # add padding to input pad_scheme = (pad//2, pad - pad//2) input_pad = np.pad(input, pad_width=((0,0), (0,0), pad_scheme, pad_scheme), mode='constant', constant_values=0) recep_fields_h = [stride*i for i in range(opt_h)] recep_fields_w = [stride*i for i in range(opt_w)] input_pool = img2col(input_pad, recep_fields_h, recep_fields_w, kernel_h, kernel_w) input_pool_grad = np.stack(map(lambda x: np.matmul(weights.reshape(out_channel, -1).T, x), out_grad.reshape(batch, out_channel, -1)), axis=0) flag = 0 for h in recep_fields_h: for w in recep_fields_w: grad = input_pool_grad[:,:,flag].reshape(batch, in_channel, kernel_h, kernel_w) in_grad[:,:,h:h+kernel_h, w:w+kernel_w] += grad flag += 1 in_grad = in_grad[:, :, pad:pad+in_height, pad:pad+in_width] w_grad = sum(list(map(lambda x: np.matmul(x[0], x[1].T), zip(out_grad.reshape(batch, out_channel, -1), input_pool)))) \ .reshape(weights.shape) b_grad = out_grad.sum(axis=(0, 2, 3)) ################################################################################# return in_grad, w_grad, b_grad
def backward(self, out_grad, input): """ # Arguments out_grad: gradient to the forward output of conv layer, with shape (batch, in_channel, out_height, out_width) input: numpy array with shape (batch, in_channel, in_height, in_width) # Returns in_grad: gradient to the forward input of pool layer, with same shape as input """ pool_type = self.pool_params['pool_type'] pool_height = self.pool_params['pool_height'] pool_width = self.pool_params['pool_width'] stride = self.pool_params['stride'] pad = self.pool_params['pad'] batch, in_channel, in_height, in_width = input.shape out_height = 1 + (in_height - pool_height + pad) // stride out_width = 1 + (in_width - pool_width + pad) // stride pad_scheme = (pad // 2, pad - pad // 2) input_pad = np.pad(input, pad_width=((0, 0), (0, 0), pad_scheme, pad_scheme), mode='constant', constant_values=0) recep_fields_h = [stride * i for i in range(out_height)] recep_fields_w = [stride * i for i in range(out_width)] input_pool = img2col(input_pad, recep_fields_h, recep_fields_w, pool_height, pool_width) input_pool = input_pool.reshape(batch, in_channel, -1, out_height, out_width) if pool_type == 'max': input_pool_grad = (input_pool == np.max(input_pool, axis=2, keepdims=True)) * \ out_grad[:, :, np.newaxis, :, :] elif pool_type == 'avg': scale = 1 / (pool_height * pool_width) input_pool_grad = scale * \ np.repeat(out_grad[:, :, np.newaxis, :, :], pool_height*pool_width, axis=2) input_pool_grad = input_pool_grad.reshape(batch, in_channel, -1, out_height * out_width) input_pad_grad = np.zeros(input_pad.shape) idx = 0 for i in recep_fields_h: for j in recep_fields_w: input_pad_grad[:, :, i:i+pool_height, j:j+pool_width] += \ input_pool_grad[:, :, :, idx].reshape( batch, in_channel, pool_height, pool_width) idx += 1 in_grad = input_pad_grad[:, :, pad:pad + in_height, pad:pad + in_width] return in_grad
def forward(self, input, weights, bias): """ # Arguments input: numpy array with shape (batch, in_channel, in_height, in_width) weights: numpy array with shape (out_channel, in_channel, kernel_h, kernel_w) bias: numpy array with shape (out_channel) # Returns output: numpy array with shape (batch, out_channel, out_height, out_width) """ kernel_h = self.conv_params['kernel_h'] # height of kernel kernel_w = self.conv_params['kernel_w'] # width of kernel pad = self.conv_params['pad'] stride = self.conv_params['stride'] in_channel = self.conv_params['in_channel'] out_channel = self.conv_params['out_channel'] batch, in_channel, in_height, in_width = input.shape ##################################################################################### # code here opt_h = int(((in_height - kernel_h + pad) / stride) + 1) opt_w = int(((in_width - kernel_w + pad) / stride) + 1) # add padding to input pad_scheme = (pad//2, pad - pad//2) input_pad = np.pad(input, pad_width=((0,0), (0,0), pad_scheme, pad_scheme), mode='constant', constant_values=0) recep_fields_h = [stride*i for i in range(opt_h)] recep_fields_w = [stride*i for i in range(opt_w)] input_pool = img2col(input_pad, recep_fields_h, recep_fields_w, kernel_h, kernel_w) weight_h = np.reshape(weights, (out_channel, in_channel * kernel_h * kernel_w)) # initialize outtput output = np.zeros((batch, out_channel, opt_h * opt_w)) bias_h = np.zeros((out_channel, opt_h * opt_w)) for i in range(out_channel): for j in range(opt_h * opt_w): bias_h[i][j] = bias[i] for i in range(batch): output[i] = np.matmul(weight_h, input_pool[i]) + bias_h # print(bias.shape) # print(input_pool.shape) # print(weight_h.shape) output = output.reshape(batch, out_channel, opt_h, opt_w) ##################################################################################### return output
def forward(self, input): """ # Arguments input: numpy array with shape (batch, in_channel, in_height, in_width) # Returns output: numpy array with shape (batch, in_channel, out_height, out_width) """ pool_type = self.pool_params['pool_type'] pool_height = self.pool_params['pool_height'] pool_width = self.pool_params['pool_width'] stride = self.pool_params['stride'] pad = self.pool_params['pad'] batch, in_channel, in_height, in_width = input.shape # get output shape out_height = 1 + (in_height - pool_height + pad) // stride out_width = 1 + (in_width - pool_width + pad) // stride # do padding pad_scheme = (pad // 2, pad - pad // 2) input_pad = np.pad(input, pad_width=((0, 0), (0, 0), pad_scheme, pad_scheme), mode='constant', constant_values=0) # get initial nodes of receptive fields in height and width direction recep_fields_h = [stride * i for i in range(out_height)] recep_fields_w = [stride * i for i in range(out_width)] # do img2col input_pool = img2col(input_pad, recep_fields_h, recep_fields_w, pool_height, pool_width) #print(input_pool.shape) # (10, 48, 225) # re-arrage to separate the channels, make every recep_field into a column input_pool = input_pool.reshape(batch, in_channel, -1, out_height, out_width) #print(input_pool.shape) #(10, 3, 16, 15, 15) if pool_type == 'max': after_pool = np.max(input_pool, axis=2, keepdims=True) elif pool_type == 'avg': after_pool = np.mean(input_pool, axis=2, keepdims=True) else: raise ValueError("Only supports max or avg pooling.") output = after_pool.reshape((batch, in_channel, out_height, out_width)) return output
def forward(self, input, weights, bias): """ # Arguments input: numpy array with shape (batch, in_channel, in_height, in_width) weights: numpy array with shape (out_channel, in_channel, kernel_h, kernel_w) bias: numpy array with shape (out_channel) # Returns output: numpy array with shape (batch, out_channel, out_height, out_width) """ kernel_h = self.conv_params['kernel_h'] # height of kernel kernel_w = self.conv_params['kernel_w'] # width of kernel pad = self.conv_params['pad'] stride = self.conv_params['stride'] in_channel = self.conv_params['in_channel'] out_channel = self.conv_params['out_channel'] batch, in_channel, in_height, in_width = input.shape ##################################################################################### # code here out_height = 1 + (in_height - kernel_h + pad) // stride out_width = 1 + (in_width - kernel_w + pad) // stride pad_scheme = (pad // 2, pad - pad // 2) input_pad = np.pad(input, pad_width=((0, 0), (0, 0), pad_scheme, pad_scheme), mode='constant', constant_values=0) recep_fields_h = [stride * i for i in range(out_height)] recep_fields_w = [stride * i for i in range(out_width)] input_conv = img2col(input_pad, recep_fields_h, recep_fields_w, kernel_h, kernel_w) input_conv = input_conv.reshape(batch, in_channel * kernel_h * kernel_w, out_height * out_width) weights_conv = np.repeat(weights.reshape( 1, out_channel, in_channel * kernel_h * kernel_w), batch, axis=0) out_conv = np.matmul(weights_conv, input_conv) out_conv += bias.reshape(out_channel, -1) output = out_conv.reshape(batch, out_channel, out_height, out_width) ##################################################################################### return output
def forward(self, input, weights, bias): """ # Arguments input: numpy array with shape (batch, in_channel, in_height, in_width) weights: numpy array with shape (out_channel, in_channel, kernel_h, kernel_w) bias: numpy array with shape (out_channel) # Returns output: numpy array with shape (batch, out_channel, out_height, out_width) """ kernel_h = self.conv_params['kernel_h'] # height of kernel kernel_w = self.conv_params['kernel_w'] # width of kernel pad = self.conv_params['pad'] stride = self.conv_params['stride'] in_channel = self.conv_params['in_channel'] out_channel = self.conv_params['out_channel'] batch, in_channel, in_height, in_width = input.shape ##################################################################################### out_height = int((in_height + pad - kernel_h) / stride) + 1 out_width = int((in_width + pad - kernel_w) / stride) + 1 pad_scheme = (pad // 2, pad - pad // 2) input_pad = np.pad(input, pad_width=((0, 0), (0, 0), pad_scheme, pad_scheme), mode='constant', constant_values=0) output = np.zeros((batch, out_channel, out_height, out_width)) recep_fields_h = [stride * i for i in range(out_height)] recep_fields_w = [stride * i for i in range(out_width)] X_col = img2col(input_pad, recep_fields_h, recep_fields_w, kernel_h, kernel_w) weights_reshaped = weights.reshape(out_channel, -1) X_col_conv = np.stack(map( lambda x: np.matmul(weights_reshaped, x) + bias.reshape(-1, 1), X_col), axis=0) output = X_col_conv.reshape(batch, out_channel, out_height, out_width) ##################################################################################### return output
def forward(self, input, weights, bias): """ # Arguments input: numpy array with shape (batch, in_channel, in_height, in_width) weights: numpy array with shape (out_channel, in_channel, kernel_h, kernel_w) bias: numpy array with shape (out_channel) # Returns output: numpy array with shape (batch, out_channel, out_height, out_width) """ kernel_h = self.conv_params['kernel_h'] # height of kernel kernel_w = self.conv_params['kernel_w'] # width of kernel pad = self.conv_params['pad'] stride = self.conv_params['stride'] in_channel = self.conv_params['in_channel'] out_channel = self.conv_params['out_channel'] batch, in_channel, in_height, in_width = input.shape # reshape kernels into row format kernel_rows = weights.reshape( (out_channel, in_channel * kernel_h * kernel_w)) # get output shape out_height = 1 + (in_height - kernel_h + pad) // stride out_width = 1 + (in_width - kernel_w + pad) // stride # do padding pad_scheme = (pad // 2, pad - pad // 2) input_pad = np.pad(input, pad_width=((0, 0), (0, 0), pad_scheme, pad_scheme), mode='constant', constant_values=0) # get initial nodes of receptive fields in height and width direction recep_fields_h = [stride * i for i in range(out_height)] recep_fields_w = [stride * i for i in range(out_width)] # do img2col input_conv = img2col(input_pad, recep_fields_h, recep_fields_w, kernel_h, kernel_w) # matrix multiplication and add bias for every tensor in the batch output = np.array([(np.dot(kernel_rows, i).T + bias).T.reshape( (out_channel, out_height, out_width)) for i in input_conv]) return output
def forward(self, input): """ # Arguments input: numpy array with shape (batch, in_channel, in_height, in_width) # Returns output: numpy array with shape (batch, in_channel, out_height, out_width) """ pool_type = self.pool_params['pool_type'] pool_height = self.pool_params['pool_height'] pool_width = self.pool_params['pool_width'] stride = self.pool_params['stride'] pad = self.pool_params['pad'] batch, in_channel, in_height, in_width = input.shape ##################################################################################### # code here out_height = 1 + (in_height - pool_height + pad) // stride out_width = 1 + (in_width - pool_width + pad) // stride pad_scheme = (pad // 2, pad - pad // 2) input_pad = np.pad(input, pad_width=((0, 0), (0, 0), pad_scheme, pad_scheme), mode='constant', constant_values=0) recep_fields_h = [stride * i for i in range(out_height)] recep_fields_w = [stride * i for i in range(out_width)] input_pool = img2col(input_pad, recep_fields_h, recep_fields_w, pool_height, pool_width) input_pool = input_pool.reshape(batch, in_channel, -1, out_height, out_width) if pool_type == 'max': output = np.max(input_pool, axis=2, keepdims=True) elif pool_type == 'avg': scale = 1 / (pool_height * pool_width) output = scale * \ np.sum(input_pool, axis=2) output = output.reshape(batch, in_channel, out_height, out_width) # output = None ##################################################################################### return output
def forward(self, input, weights, bias): """ # Arguments input: numpy array with shape (batch, in_channel, in_height, in_width) weights: numpy array with shape (out_channel, in_channel, kernel_h, kernel_w) bias: numpy array with shape (out_channel) # Returns output: numpy array with shape (batch, out_channel, out_height, out_width) """ kernel_h = self.conv_params['kernel_h'] # height of kernel kernel_w = self.conv_params['kernel_w'] # width of kernel pad = self.conv_params['pad'] stride = self.conv_params['stride'] in_channel = self.conv_params['in_channel'] out_channel = self.conv_params['out_channel'] batch, in_channel, in_height, in_width = input.shape out_height = 1 + (in_height - kernel_h + pad) // stride out_width = 1 + (in_width - kernel_w + pad) // stride pad_scheme = (pad // 2, pad - pad // 2) input_pad = np.pad(input, pad_width=((0, 0), (0, 0), pad_scheme, pad_scheme), mode='constant', constant_values=0) # get initial nodes of receptive fields in height and width direction recep_fields_h = [stride * i for i in range(out_height)] recep_fields_w = [stride * i for i in range(out_width)] x = img2col(input_pad, recep_fields_h, recep_fields_w, kernel_h, kernel_w) output = np.matmul(weights.reshape(out_channel, -1), x).reshape( batch, out_channel, out_height, out_width) + bias[np.newaxis, :, np.newaxis, np.newaxis] return output
def backward(self, out_grad, input, weights, bias): """ # Arguments out_grad: gradient to the forward output of conv layer, with shape (batch, out_channel, out_height, out_width) input: numpy array with shape (batch, in_channel, in_height, in_width) weights: numpy array with shape (out_channel, in_channel, kernel_h, kernel_w) bias: numpy array with shape (out_channel) # Returns in_grad: gradient to the forward input of conv layer, with same shape as input w_grad: gradient to weights, with same shape as weights b_grad: gradient to bias, with same shape as bias """ kernel_h = self.conv_params['kernel_h'] # height of kernel kernel_w = self.conv_params['kernel_w'] # width of kernel pad = self.conv_params['pad'] stride = self.conv_params['stride'] in_channel = self.conv_params['in_channel'] out_channel = self.conv_params['out_channel'] batch, in_channel, in_height, in_width = input.shape ################################################################################# in_grad = np.zeros_like(input) w_grad = np.zeros_like(weights) b_grad = np.zeros_like(bias) out_height = int((in_height + pad - kernel_h) / stride) + 1 out_width = int((in_width + pad - kernel_w) / stride) + 1 b_grad = np.sum(out_grad, axis=(0, 2, 3)) pad_scheme = (pad // 2, pad - pad // 2) input_pad = np.pad(input, pad_width=((0, 0), (0, 0), pad_scheme, pad_scheme), mode='constant', constant_values=0) recep_fields_h = [stride * i for i in range(out_height)] recep_fields_w = [stride * i for i in range(out_width)] X_col = img2col(input_pad, recep_fields_h, recep_fields_w, kernel_h, kernel_w) weights_reshaped = weights.reshape(out_channel, -1) out_grad_reshaped = out_grad.reshape(batch, out_channel, -1) #check out map lambda x #https://book.pythontips.com/en/latest/map_filter.html in_grad_col = np.stack(map(lambda x: np.matmul(weights_reshaped.T, x), out_grad_reshaped), axis=0) #w_grad = out_grad_reshaped.dot(X_col.T).reshape(weights.shape) input_pad_grad = np.zeros( (batch, in_channel, in_height + 2 * pad, in_width + 2 * pad)) comb = zip(out_grad_reshaped, X_col) grad_list = list(map(lambda x: np.matmul(x[0], x[1].T), comb)) w_grad = sum(grad_list) #similar to backward pool idx = 0 for i in recep_fields_h: for j in recep_fields_w: input_pad_grad[:, :, i:i + kernel_h, j:j + kernel_w] += \ in_grad_col[:, :, idx].reshape(batch, in_channel, kernel_h, kernel_w) idx += 1 in_grad = input_pad_grad[:, :, pad:pad + in_height, pad:pad + in_width] w_grad = w_grad.reshape(weights.shape) ################################################################################# return in_grad, w_grad, b_grad
def backward(self, out_grad, input, weights, bias): """ # Arguments out_grad: gradient to the forward output of conv layer, with shape (batch, out_channel, out_height, out_width) input: numpy array with shape (batch, in_channel, in_height, in_width) weights: numpy array with shape (out_channel, in_channel, kernel_h, kernel_w) bias: numpy array with shape (out_channel) # Returns in_grad: gradient to the forward input of conv layer, with same shape as input w_grad: gradient to weights, with same shape as weights b_bias: gradient to bias, with same shape as bias """ kernel_h = self.conv_params['kernel_h'] # height of kernel kernel_w = self.conv_params['kernel_w'] # width of kernel pad = self.conv_params['pad'] stride = self.conv_params['stride'] in_channel = self.conv_params['in_channel'] out_channel = self.conv_params['out_channel'] batch, in_channel, in_height, in_width = input.shape out_height = 1 + (in_height - kernel_h + pad) // stride out_width = 1 + (in_width - kernel_w + pad) // stride pad_scheme = (pad // 2, pad - pad // 2) input_pad = np.pad(input, pad_width=((0, 0), (0, 0), pad_scheme, pad_scheme), mode='constant', constant_values=0) # get initial nodes of receptive fields in height and width direction recep_fields_h = [stride * i for i in range(out_height)] recep_fields_w = [stride * i for i in range(out_width)] input_conv = img2col(input_pad, recep_fields_h, recep_fields_w, kernel_h, kernel_w) input_conv_grad = np.stack(map( lambda x: np.matmul(weights.reshape(out_channel, -1).T, x), out_grad.reshape(batch, out_channel, -1)), axis=0) input_pad_grad = np.zeros( (batch, in_channel, in_height + 2 * pad, in_width + 2 * pad)) idx = 0 for i in recep_fields_h: for j in recep_fields_w: input_pad_grad[:, :, i:i+kernel_h, j:j+kernel_w] += \ input_conv_grad[:, :, idx].reshape( batch, in_channel, kernel_h, kernel_w) idx += 1 in_grad = input_pad_grad[:, :, pad:pad + in_height, pad:pad + in_width] w_grad = sum( list( map(lambda x: np.matmul(x[0], x[1].T), zip(out_grad.reshape(batch, out_channel, -1), input_conv)))) w_grad = w_grad.reshape(weights.shape) b_grad = out_grad.sum(axis=(0, 2, 3)) return in_grad, w_grad, b_grad
def backward(self, out_grad, input, weights, bias): """ # Arguments out_grad: gradient to the forward output of conv layer, with shape (batch, out_channel, out_height, out_width) input: numpy array with shape (batch, in_channel, in_height, in_width) weights: numpy array with shape (out_channel, in_channel, kernel_h, kernel_w) bias: numpy array with shape (out_channel) # Returns in_grad: gradient to the forward input of conv layer, with same shape as input w_grad: gradient to weights, with same shape as weights b_bias: gradient to bias, with same shape as bias """ kernel_h = self.conv_params['kernel_h'] # height of kernel kernel_w = self.conv_params['kernel_w'] # width of kernel pad = self.conv_params['pad'] stride = self.conv_params['stride'] in_channel = self.conv_params['in_channel'] out_channel = self.conv_params['out_channel'] batch, in_channel, in_height, in_width = input.shape ################################################################################# # code here # in_grad = none # w_grad = none # b_grad = none out_height = 1 + (in_height - kernel_h + pad) // stride out_width = 1 + (in_width - kernel_w + pad) // stride pad_scheme = (pad // 2, pad - pad // 2) input_pad = np.pad(input, pad_width=((0, 0), (0, 0), pad_scheme, pad_scheme), mode='constant', constant_values=0) recep_fields_h = [stride * i for i in range(out_height)] recep_fields_w = [stride * i for i in range(out_width)] input_conv = img2col(input_pad, recep_fields_h, recep_fields_w, kernel_h, kernel_w) input_conv = input_conv.reshape(batch, in_channel, -1, out_height, out_width) #out_grad, b_grad = self.add_bias.backward(out_grad, input_conv, bias) b_grad = np.sum(out_grad, axis=(0, 2, 3)) weights_conv = np.repeat(weights.reshape( 1, out_channel, in_channel * kernel_h * kernel_w), batch, axis=0) input_conv = input_conv.reshape(batch, in_channel * kernel_h * kernel_w, out_height * out_width) out_grad = out_grad.reshape(batch, out_channel, -1) w_grad = np.matmul(out_grad, input_conv.transpose(0, 2, 1)) w_grad = w_grad.reshape(batch, out_channel, in_channel, kernel_h, kernel_w) w_grad = np.sum(w_grad, axis=0) in_grad = np.matmul(weights_conv.transpose(0, 2, 1), out_grad) in_grad = in_grad.reshape(batch, in_channel, -1, out_height * out_width) input_pad_grad = np.zeros(input_pad.shape) idx = 0 for i in recep_fields_h: for j in recep_fields_w: input_pad_grad[:, :, i:i+kernel_h, j:j+kernel_w] += \ in_grad[:, :, :, idx].reshape( batch, in_channel, kernel_h, kernel_w) idx += 1 in_grad = input_pad_grad[:, :, pad:pad + in_height, pad:pad + in_width] ################################################################################# return in_grad, w_grad, b_grad