def backward(self, out_grad, input, weights, bias):
"""
# Arguments
out_grad: gradient to the forward output of conv layer, with shape (batch, out_channel, out_height, out_width)
input: numpy array with shape (batch, in_channel, in_height, in_width)
weights: numpy array with shape (out_channel, in_channel, kernel_h, kernel_w)
bias: numpy array with shape (out_channel)
# Returns
in_grad: gradient to the forward input of conv layer, with same shape as input
w_grad: gradient to weights, with same shape as weights
b_bias: gradient to bias, with same shape as bias
"""
kernel_h = self.conv_params['kernel_h'] # height of kernel
kernel_w = self.conv_params['kernel_w'] # width of kernel
pad = self.conv_params['pad']
stride = self.conv_params['stride']
in_channel = self.conv_params['in_channel']
out_channel = self.conv_params['out_channel']
batch, in_channel, in_height, in_width = input.shape
#################################################################################
# code here
opt_h = int(((in_height - kernel_h + pad) / stride) + 1)
opt_w = int(((in_width - kernel_w + pad) / stride) + 1)
in_grad = np.zeros((batch, in_channel, in_height + 2 * pad, in_width + 2 * pad))
# add padding to input
pad_scheme = (pad//2, pad - pad//2)
input_pad = np.pad(input, pad_width=((0,0), (0,0), pad_scheme, pad_scheme),
mode='constant', constant_values=0)
recep_fields_h = [stride*i for i in range(opt_h)]
recep_fields_w = [stride*i for i in range(opt_w)]
input_pool = img2col(input_pad, recep_fields_h,
recep_fields_w, kernel_h, kernel_w)
input_pool_grad = np.stack(map(lambda x: np.matmul(weights.reshape(out_channel, -1).T, x),
out_grad.reshape(batch, out_channel, -1)), axis=0)
flag = 0
for h in recep_fields_h:
for w in recep_fields_w:
grad = input_pool_grad[:,:,flag].reshape(batch, in_channel, kernel_h, kernel_w)
in_grad[:,:,h:h+kernel_h, w:w+kernel_w] += grad
flag += 1
in_grad = in_grad[:, :, pad:pad+in_height, pad:pad+in_width]
w_grad = sum(list(map(lambda x: np.matmul(x[0], x[1].T),
zip(out_grad.reshape(batch, out_channel, -1), input_pool)))) \
.reshape(weights.shape)
b_grad = out_grad.sum(axis=(0, 2, 3))
#################################################################################
return in_grad, w_grad, b_grad
def backward(self, out_grad, input):
"""
# Arguments
out_grad: gradient to the forward output of conv layer, with shape (batch, in_channel, out_height, out_width)
input: numpy array with shape (batch, in_channel, in_height, in_width)
# Returns
in_grad: gradient to the forward input of pool layer, with same shape as input
"""
pool_type = self.pool_params['pool_type']
pool_height = self.pool_params['pool_height']
pool_width = self.pool_params['pool_width']
stride = self.pool_params['stride']
pad = self.pool_params['pad']
batch, in_channel, in_height, in_width = input.shape
out_height = 1 + (in_height - pool_height + pad) // stride
out_width = 1 + (in_width - pool_width + pad) // stride
pad_scheme = (pad // 2, pad - pad // 2)
input_pad = np.pad(input,
pad_width=((0, 0), (0, 0), pad_scheme, pad_scheme),
mode='constant',
constant_values=0)
recep_fields_h = [stride * i for i in range(out_height)]
recep_fields_w = [stride * i for i in range(out_width)]
input_pool = img2col(input_pad, recep_fields_h, recep_fields_w,
pool_height, pool_width)
input_pool = input_pool.reshape(batch, in_channel, -1, out_height,
out_width)
if pool_type == 'max':
input_pool_grad = (input_pool == np.max(input_pool, axis=2, keepdims=True)) * \
out_grad[:, :, np.newaxis, :, :]
elif pool_type == 'avg':
scale = 1 / (pool_height * pool_width)
input_pool_grad = scale * \
np.repeat(out_grad[:, :, np.newaxis, :, :],
pool_height*pool_width, axis=2)
input_pool_grad = input_pool_grad.reshape(batch, in_channel, -1,
out_height * out_width)
input_pad_grad = np.zeros(input_pad.shape)
idx = 0
for i in recep_fields_h:
for j in recep_fields_w:
input_pad_grad[:, :, i:i+pool_height, j:j+pool_width] += \
input_pool_grad[:, :, :, idx].reshape(
batch, in_channel, pool_height, pool_width)
idx += 1
in_grad = input_pad_grad[:, :, pad:pad + in_height, pad:pad + in_width]
return in_grad
def forward(self, input, weights, bias):
"""
# Arguments
input: numpy array with shape (batch, in_channel, in_height, in_width)
weights: numpy array with shape (out_channel, in_channel, kernel_h, kernel_w)
bias: numpy array with shape (out_channel)
# Returns
output: numpy array with shape (batch, out_channel, out_height, out_width)
"""
kernel_h = self.conv_params['kernel_h'] # height of kernel
kernel_w = self.conv_params['kernel_w'] # width of kernel
pad = self.conv_params['pad']
stride = self.conv_params['stride']
in_channel = self.conv_params['in_channel']
out_channel = self.conv_params['out_channel']
batch, in_channel, in_height, in_width = input.shape
#####################################################################################
# code here
opt_h = int(((in_height - kernel_h + pad) / stride) + 1)
opt_w = int(((in_width - kernel_w + pad) / stride) + 1)
# add padding to input
pad_scheme = (pad//2, pad - pad//2)
input_pad = np.pad(input, pad_width=((0,0), (0,0), pad_scheme, pad_scheme),
mode='constant', constant_values=0)
recep_fields_h = [stride*i for i in range(opt_h)]
recep_fields_w = [stride*i for i in range(opt_w)]
input_pool = img2col(input_pad, recep_fields_h,
recep_fields_w, kernel_h, kernel_w)
weight_h = np.reshape(weights, (out_channel, in_channel * kernel_h * kernel_w))
# initialize outtput
output = np.zeros((batch, out_channel, opt_h * opt_w))
bias_h = np.zeros((out_channel, opt_h * opt_w))
for i in range(out_channel):
for j in range(opt_h * opt_w):
bias_h[i][j] = bias[i]
for i in range(batch):
output[i] = np.matmul(weight_h, input_pool[i]) + bias_h
# print(bias.shape)
# print(input_pool.shape)
# print(weight_h.shape)
output = output.reshape(batch, out_channel, opt_h, opt_w)
#####################################################################################
return output
def forward(self, input):
"""
# Arguments
input: numpy array with shape (batch, in_channel, in_height, in_width)
# Returns
output: numpy array with shape (batch, in_channel, out_height, out_width)
"""
pool_type = self.pool_params['pool_type']
pool_height = self.pool_params['pool_height']
pool_width = self.pool_params['pool_width']
stride = self.pool_params['stride']
pad = self.pool_params['pad']
batch, in_channel, in_height, in_width = input.shape
# get output shape
out_height = 1 + (in_height - pool_height + pad) // stride
out_width = 1 + (in_width - pool_width + pad) // stride
# do padding
pad_scheme = (pad // 2, pad - pad // 2)
input_pad = np.pad(input,
pad_width=((0, 0), (0, 0), pad_scheme, pad_scheme),
mode='constant',
constant_values=0)
# get initial nodes of receptive fields in height and width direction
recep_fields_h = [stride * i for i in range(out_height)]
recep_fields_w = [stride * i for i in range(out_width)]
# do img2col
input_pool = img2col(input_pad, recep_fields_h, recep_fields_w,
pool_height, pool_width)
#print(input_pool.shape) # (10, 48, 225)
# re-arrage to separate the channels, make every recep_field into a column
input_pool = input_pool.reshape(batch, in_channel, -1, out_height,
out_width)
#print(input_pool.shape) #(10, 3, 16, 15, 15)
if pool_type == 'max':
after_pool = np.max(input_pool, axis=2, keepdims=True)
elif pool_type == 'avg':
after_pool = np.mean(input_pool, axis=2, keepdims=True)
else:
raise ValueError("Only supports max or avg pooling.")
output = after_pool.reshape((batch, in_channel, out_height, out_width))
return output
def forward(self, input, weights, bias):
"""
# Arguments
input: numpy array with shape (batch, in_channel, in_height, in_width)
weights: numpy array with shape (out_channel, in_channel, kernel_h, kernel_w)
bias: numpy array with shape (out_channel)
# Returns
output: numpy array with shape (batch, out_channel, out_height, out_width)
"""
kernel_h = self.conv_params['kernel_h'] # height of kernel
kernel_w = self.conv_params['kernel_w'] # width of kernel
pad = self.conv_params['pad']
stride = self.conv_params['stride']
in_channel = self.conv_params['in_channel']
out_channel = self.conv_params['out_channel']
batch, in_channel, in_height, in_width = input.shape
#####################################################################################
# code here
out_height = 1 + (in_height - kernel_h + pad) // stride
out_width = 1 + (in_width - kernel_w + pad) // stride
pad_scheme = (pad // 2, pad - pad // 2)
input_pad = np.pad(input,
pad_width=((0, 0), (0, 0), pad_scheme, pad_scheme),
mode='constant',
constant_values=0)
recep_fields_h = [stride * i for i in range(out_height)]
recep_fields_w = [stride * i for i in range(out_width)]
input_conv = img2col(input_pad, recep_fields_h, recep_fields_w,
kernel_h, kernel_w)
input_conv = input_conv.reshape(batch,
in_channel * kernel_h * kernel_w,
out_height * out_width)
weights_conv = np.repeat(weights.reshape(
1, out_channel, in_channel * kernel_h * kernel_w),
batch,
axis=0)
out_conv = np.matmul(weights_conv, input_conv)
out_conv += bias.reshape(out_channel, -1)
output = out_conv.reshape(batch, out_channel, out_height, out_width)
#####################################################################################
return output
def forward(self, input, weights, bias):
"""
# Arguments
input: numpy array with shape (batch, in_channel, in_height, in_width)
weights: numpy array with shape (out_channel, in_channel, kernel_h, kernel_w)
bias: numpy array with shape (out_channel)
# Returns
output: numpy array with shape (batch, out_channel, out_height, out_width)
"""
kernel_h = self.conv_params['kernel_h'] # height of kernel
kernel_w = self.conv_params['kernel_w'] # width of kernel
pad = self.conv_params['pad']
stride = self.conv_params['stride']
in_channel = self.conv_params['in_channel']
out_channel = self.conv_params['out_channel']
batch, in_channel, in_height, in_width = input.shape
#####################################################################################
out_height = int((in_height + pad - kernel_h) / stride) + 1
out_width = int((in_width + pad - kernel_w) / stride) + 1
pad_scheme = (pad // 2, pad - pad // 2)
input_pad = np.pad(input,
pad_width=((0, 0), (0, 0), pad_scheme, pad_scheme),
mode='constant',
constant_values=0)
output = np.zeros((batch, out_channel, out_height, out_width))
recep_fields_h = [stride * i for i in range(out_height)]
recep_fields_w = [stride * i for i in range(out_width)]
X_col = img2col(input_pad, recep_fields_h, recep_fields_w, kernel_h,
kernel_w)
weights_reshaped = weights.reshape(out_channel, -1)
X_col_conv = np.stack(map(
lambda x: np.matmul(weights_reshaped, x) + bias.reshape(-1, 1),
X_col),
axis=0)
output = X_col_conv.reshape(batch, out_channel, out_height, out_width)
#####################################################################################
return output
def forward(self, input, weights, bias):
"""
# Arguments
input: numpy array with shape (batch, in_channel, in_height, in_width)
weights: numpy array with shape (out_channel, in_channel, kernel_h, kernel_w)
bias: numpy array with shape (out_channel)
# Returns
output: numpy array with shape (batch, out_channel, out_height, out_width)
"""
kernel_h = self.conv_params['kernel_h'] # height of kernel
kernel_w = self.conv_params['kernel_w'] # width of kernel
pad = self.conv_params['pad']
stride = self.conv_params['stride']
in_channel = self.conv_params['in_channel']
out_channel = self.conv_params['out_channel']
batch, in_channel, in_height, in_width = input.shape
# reshape kernels into row format
kernel_rows = weights.reshape(
(out_channel, in_channel * kernel_h * kernel_w))
# get output shape
out_height = 1 + (in_height - kernel_h + pad) // stride
out_width = 1 + (in_width - kernel_w + pad) // stride
# do padding
pad_scheme = (pad // 2, pad - pad // 2)
input_pad = np.pad(input,
pad_width=((0, 0), (0, 0), pad_scheme, pad_scheme),
mode='constant',
constant_values=0)
# get initial nodes of receptive fields in height and width direction
recep_fields_h = [stride * i for i in range(out_height)]
recep_fields_w = [stride * i for i in range(out_width)]
# do img2col
input_conv = img2col(input_pad, recep_fields_h, recep_fields_w,
kernel_h, kernel_w)
# matrix multiplication and add bias for every tensor in the batch
output = np.array([(np.dot(kernel_rows, i).T + bias).T.reshape(
(out_channel, out_height, out_width)) for i in input_conv])
return output
def forward(self, input):
"""
# Arguments
input: numpy array with shape (batch, in_channel, in_height, in_width)
# Returns
output: numpy array with shape (batch, in_channel, out_height, out_width)
"""
pool_type = self.pool_params['pool_type']
pool_height = self.pool_params['pool_height']
pool_width = self.pool_params['pool_width']
stride = self.pool_params['stride']
pad = self.pool_params['pad']
batch, in_channel, in_height, in_width = input.shape
#####################################################################################
# code here
out_height = 1 + (in_height - pool_height + pad) // stride
out_width = 1 + (in_width - pool_width + pad) // stride
pad_scheme = (pad // 2, pad - pad // 2)
input_pad = np.pad(input,
pad_width=((0, 0), (0, 0), pad_scheme, pad_scheme),
mode='constant',
constant_values=0)
recep_fields_h = [stride * i for i in range(out_height)]
recep_fields_w = [stride * i for i in range(out_width)]
input_pool = img2col(input_pad, recep_fields_h, recep_fields_w,
pool_height, pool_width)
input_pool = input_pool.reshape(batch, in_channel, -1, out_height,
out_width)
if pool_type == 'max':
output = np.max(input_pool, axis=2, keepdims=True)
elif pool_type == 'avg':
scale = 1 / (pool_height * pool_width)
output = scale * \
np.sum(input_pool, axis=2)
output = output.reshape(batch, in_channel, out_height, out_width)
# output = None
#####################################################################################
return output
def forward(self, input, weights, bias):
"""
# Arguments
input: numpy array with shape (batch, in_channel, in_height, in_width)
weights: numpy array with shape (out_channel, in_channel, kernel_h, kernel_w)
bias: numpy array with shape (out_channel)
# Returns
output: numpy array with shape (batch, out_channel, out_height, out_width)
"""
kernel_h = self.conv_params['kernel_h'] # height of kernel
kernel_w = self.conv_params['kernel_w'] # width of kernel
pad = self.conv_params['pad']
stride = self.conv_params['stride']
in_channel = self.conv_params['in_channel']
out_channel = self.conv_params['out_channel']
batch, in_channel, in_height, in_width = input.shape
out_height = 1 + (in_height - kernel_h + pad) // stride
out_width = 1 + (in_width - kernel_w + pad) // stride
pad_scheme = (pad // 2, pad - pad // 2)
input_pad = np.pad(input,
pad_width=((0, 0), (0, 0), pad_scheme, pad_scheme),
mode='constant',
constant_values=0)
# get initial nodes of receptive fields in height and width direction
recep_fields_h = [stride * i for i in range(out_height)]
recep_fields_w = [stride * i for i in range(out_width)]
x = img2col(input_pad, recep_fields_h, recep_fields_w, kernel_h,
kernel_w)
output = np.matmul(weights.reshape(out_channel, -1), x).reshape(
batch, out_channel, out_height,
out_width) + bias[np.newaxis, :, np.newaxis, np.newaxis]
return output
def backward(self, out_grad, input, weights, bias):
"""
# Arguments
out_grad: gradient to the forward output of conv layer, with shape (batch, out_channel, out_height, out_width)
input: numpy array with shape (batch, in_channel, in_height, in_width)
weights: numpy array with shape (out_channel, in_channel, kernel_h, kernel_w)
bias: numpy array with shape (out_channel)
# Returns
in_grad: gradient to the forward input of conv layer, with same shape as input
w_grad: gradient to weights, with same shape as weights
b_grad: gradient to bias, with same shape as bias
"""
kernel_h = self.conv_params['kernel_h'] # height of kernel
kernel_w = self.conv_params['kernel_w'] # width of kernel
pad = self.conv_params['pad']
stride = self.conv_params['stride']
in_channel = self.conv_params['in_channel']
out_channel = self.conv_params['out_channel']
batch, in_channel, in_height, in_width = input.shape
#################################################################################
in_grad = np.zeros_like(input)
w_grad = np.zeros_like(weights)
b_grad = np.zeros_like(bias)
out_height = int((in_height + pad - kernel_h) / stride) + 1
out_width = int((in_width + pad - kernel_w) / stride) + 1
b_grad = np.sum(out_grad, axis=(0, 2, 3))
pad_scheme = (pad // 2, pad - pad // 2)
input_pad = np.pad(input,
pad_width=((0, 0), (0, 0), pad_scheme, pad_scheme),
mode='constant',
constant_values=0)
recep_fields_h = [stride * i for i in range(out_height)]
recep_fields_w = [stride * i for i in range(out_width)]
X_col = img2col(input_pad, recep_fields_h, recep_fields_w, kernel_h,
kernel_w)
weights_reshaped = weights.reshape(out_channel, -1)
out_grad_reshaped = out_grad.reshape(batch, out_channel, -1)
#check out map lambda x
#https://book.pythontips.com/en/latest/map_filter.html
in_grad_col = np.stack(map(lambda x: np.matmul(weights_reshaped.T, x),
out_grad_reshaped),
axis=0)
#w_grad = out_grad_reshaped.dot(X_col.T).reshape(weights.shape)
input_pad_grad = np.zeros(
(batch, in_channel, in_height + 2 * pad, in_width + 2 * pad))
comb = zip(out_grad_reshaped, X_col)
grad_list = list(map(lambda x: np.matmul(x[0], x[1].T), comb))
w_grad = sum(grad_list)
#similar to backward pool
idx = 0
for i in recep_fields_h:
for j in recep_fields_w:
input_pad_grad[:, :, i:i + kernel_h, j:j + kernel_w] += \
in_grad_col[:, :, idx].reshape(batch, in_channel, kernel_h, kernel_w)
idx += 1
in_grad = input_pad_grad[:, :, pad:pad + in_height, pad:pad + in_width]
w_grad = w_grad.reshape(weights.shape)
#################################################################################
return in_grad, w_grad, b_grad
def backward(self, out_grad, input, weights, bias):
"""
# Arguments
out_grad: gradient to the forward output of conv layer, with shape (batch, out_channel, out_height, out_width)
input: numpy array with shape (batch, in_channel, in_height, in_width)
weights: numpy array with shape (out_channel, in_channel, kernel_h, kernel_w)
bias: numpy array with shape (out_channel)
# Returns
in_grad: gradient to the forward input of conv layer, with same shape as input
w_grad: gradient to weights, with same shape as weights
b_bias: gradient to bias, with same shape as bias
"""
kernel_h = self.conv_params['kernel_h'] # height of kernel
kernel_w = self.conv_params['kernel_w'] # width of kernel
pad = self.conv_params['pad']
stride = self.conv_params['stride']
in_channel = self.conv_params['in_channel']
out_channel = self.conv_params['out_channel']
batch, in_channel, in_height, in_width = input.shape
out_height = 1 + (in_height - kernel_h + pad) // stride
out_width = 1 + (in_width - kernel_w + pad) // stride
pad_scheme = (pad // 2, pad - pad // 2)
input_pad = np.pad(input,
pad_width=((0, 0), (0, 0), pad_scheme, pad_scheme),
mode='constant',
constant_values=0)
# get initial nodes of receptive fields in height and width direction
recep_fields_h = [stride * i for i in range(out_height)]
recep_fields_w = [stride * i for i in range(out_width)]
input_conv = img2col(input_pad, recep_fields_h, recep_fields_w,
kernel_h, kernel_w)
input_conv_grad = np.stack(map(
lambda x: np.matmul(weights.reshape(out_channel, -1).T, x),
out_grad.reshape(batch, out_channel, -1)),
axis=0)
input_pad_grad = np.zeros(
(batch, in_channel, in_height + 2 * pad, in_width + 2 * pad))
idx = 0
for i in recep_fields_h:
for j in recep_fields_w:
input_pad_grad[:, :, i:i+kernel_h, j:j+kernel_w] += \
input_conv_grad[:, :, idx].reshape(
batch, in_channel, kernel_h, kernel_w)
idx += 1
in_grad = input_pad_grad[:, :, pad:pad + in_height, pad:pad + in_width]
w_grad = sum(
list(
map(lambda x: np.matmul(x[0], x[1].T),
zip(out_grad.reshape(batch, out_channel, -1),
input_conv))))
w_grad = w_grad.reshape(weights.shape)
b_grad = out_grad.sum(axis=(0, 2, 3))
return in_grad, w_grad, b_grad
def backward(self, out_grad, input, weights, bias):
"""
# Arguments
out_grad: gradient to the forward output of conv layer, with shape (batch, out_channel, out_height, out_width)
input: numpy array with shape (batch, in_channel, in_height, in_width)
weights: numpy array with shape (out_channel, in_channel, kernel_h, kernel_w)
bias: numpy array with shape (out_channel)
# Returns
in_grad: gradient to the forward input of conv layer, with same shape as input
w_grad: gradient to weights, with same shape as weights
b_bias: gradient to bias, with same shape as bias
"""
kernel_h = self.conv_params['kernel_h'] # height of kernel
kernel_w = self.conv_params['kernel_w'] # width of kernel
pad = self.conv_params['pad']
stride = self.conv_params['stride']
in_channel = self.conv_params['in_channel']
out_channel = self.conv_params['out_channel']
batch, in_channel, in_height, in_width = input.shape
#################################################################################
# code here
# in_grad = none
# w_grad = none
# b_grad = none
out_height = 1 + (in_height - kernel_h + pad) // stride
out_width = 1 + (in_width - kernel_w + pad) // stride
pad_scheme = (pad // 2, pad - pad // 2)
input_pad = np.pad(input,
pad_width=((0, 0), (0, 0), pad_scheme, pad_scheme),
mode='constant',
constant_values=0)
recep_fields_h = [stride * i for i in range(out_height)]
recep_fields_w = [stride * i for i in range(out_width)]
input_conv = img2col(input_pad, recep_fields_h, recep_fields_w,
kernel_h, kernel_w)
input_conv = input_conv.reshape(batch, in_channel, -1, out_height,
out_width)
#out_grad, b_grad = self.add_bias.backward(out_grad, input_conv, bias)
b_grad = np.sum(out_grad, axis=(0, 2, 3))
weights_conv = np.repeat(weights.reshape(
1, out_channel, in_channel * kernel_h * kernel_w),
batch,
axis=0)
input_conv = input_conv.reshape(batch,
in_channel * kernel_h * kernel_w,
out_height * out_width)
out_grad = out_grad.reshape(batch, out_channel, -1)
w_grad = np.matmul(out_grad, input_conv.transpose(0, 2, 1))
w_grad = w_grad.reshape(batch, out_channel, in_channel, kernel_h,
kernel_w)
w_grad = np.sum(w_grad, axis=0)
in_grad = np.matmul(weights_conv.transpose(0, 2, 1), out_grad)
in_grad = in_grad.reshape(batch, in_channel, -1,
out_height * out_width)
input_pad_grad = np.zeros(input_pad.shape)
idx = 0
for i in recep_fields_h:
for j in recep_fields_w:
input_pad_grad[:, :, i:i+kernel_h, j:j+kernel_w] += \
in_grad[:, :, :, idx].reshape(
batch, in_channel, kernel_h, kernel_w)
idx += 1
in_grad = input_pad_grad[:, :, pad:pad + in_height, pad:pad + in_width]
#################################################################################
return in_grad, w_grad, b_grad
