class AffineReLUDropout(object):
def __init__(self, dropout_param=None):
"""
Args:
dropout_param: A dictionary with the following key(s):
- prob: Probability for each neuron to drop out, required
- seed: Seeding integer for random generator, optional
"""
self.affine_layer = Affine()
self.relu_layer = ReLU()
if dropout_param is not None:
self.dropout_layer = Dropout(prob=dropout_param['prob'],
seed=dropout_param.get('seed', None))
else:
self.dropout_layer = Dropout()
def forward_pass(self, x, w, b, mode='train'):
""" Performs forward propagation through affine, rectinfied linear unit, and dropout layers
Args:
x: Input
w: Weights
b: Bias
mode: 'train' or 'test'
Returns:
dropout_out: Output from Dropout layer
"""
affine_out = self.affine_layer.forward_pass(x, w, b)
relu_out = self.relu_layer.forward_pass(affine_out)
dropout_out = self.dropout_layer.forward_pass(relu_out, mode)
return dropout_out
def backward_pass(self, grad_out):
"""Performs back propagation through affine, rectinfied linear unit, and dropout layers
Args:
grad_out: Upstream gradient
Returns:
grad_x: Gradient w.r.t. input
grad_w: Gradient w.r.t. weight
grad_b: Gradient w.r.t. bias
"""
grad_dropout = self.dropout_layer.backward_pass(grad_out)
grad_relu = self.relu_layer.backward_pass(grad_dropout)
grad_x, grad_w, grad_b = self.affine_layer.backward_pass(grad_relu)
return grad_x, grad_w, grad_b
class AffineTest(unittest.TestCase):
def setUp(self):
self.num_inputs = 2
self.input_shape = (4, 5, 6)
self.output_dim = 3
self.input_size = self.num_inputs * np.prod(self.input_shape)
self.weight_size = self.output_dim * np.prod(self.input_shape)
self.layer = Affine()
def test_forward_pass(self):
x = np.linspace(-0.1, 0.5, num=self.input_size).reshape(self.num_inputs, *self.input_shape)
w = np.linspace(-0.2, 0.3, num=self.weight_size).reshape(np.prod(self.input_shape), self.output_dim)
b = np.linspace(-0.3, 0.1, num=self.output_dim)
output = self.layer.forward_pass(x, w, b)
correct_output = np.array([[ 1.49834967, 1.70660132, 1.91485297], [ 3.25553199, 3.5141327, 3.77273342]])
err = rel_error(output, correct_output)
# np.testing.asssert_almost_equal(output, correct_output, decimal=7)
# Error should be very close to 1e-9, or almost zero
self.assertAlmostEqual(err, 1e-9, places=2)
def test_bacward_pass(self):
np.random.seed(231)
x = np.random.randn(10, 2, 3)
w = np.random.randn(6, 5)
b = np.random.randn(5)
dout = np.random.randn(10, 5)
num_dx = eval_numerical_gradient_array(lambda x: self.layer.forward_pass(x, w, b), x, dout)
num_dw = eval_numerical_gradient_array(lambda w: self.layer.forward_pass(x, w, b), w, dout)
num_db = eval_numerical_gradient_array(lambda b: self.layer.forward_pass(x, w, b), b, dout)
dx, dw, db = self.layer.backward_pass(dout)
self.assertAlmostEqual(rel_error(num_dx, dx), 1e-9, places=2)
self.assertAlmostEqual(rel_error(num_dw, dw), 1e-9, places=2)
self.assertAlmostEqual(rel_error(num_db, db), 1e-9, places=2)
class AffineReLU(object):
def __init__(self):
self.affine_layer = Affine()
self.relu_layer = ReLU()
def forward_pass(self, x, w, b):
"""Performs forward propagation through affine and ReLU layers
Args:
x: Input
w: Weights
b: Bias
Returns:
relu_out: Output from ReLU layer
"""
affine_out = self.affine_layer.forward_pass(x, w, b)
relu_out = self.relu_layer.forward_pass(affine_out)
return relu_out
def backward_pass(self, grad_out):
"""Performs back propagation through affine and ReLU layers
Args:
grad_out: Upstream gradient
Returns:
grad_x: Gradient w.r.t. input
grad_w: Gradient w.r.t. weight
grad_b: Gradient w.r.t. bias
"""
grad_relu = self.relu_layer.backward_pass(grad_out)
grad_x, grad_w, grad_b = self.affine_layer.backward_pass(grad_relu)
return grad_x, grad_w, grad_b
class AffineBatchNormReLU(object):
def __init__(self, batch_norm_param=None):
"""
Optional argument:
batch_norm_param: A dictionary containing the following keys
- eps: constant for numeric stability, required
- momentum: constant for running mean/variance calculation, required
- running_mean: if input has shape (N, D), then this is array of shape (D,)
- running_var: if input has shape (N, D), then this is array of shape (D,)
"""
self.affine_layer = Affine()
self.relu_layer = ReLU()
if batch_norm_param is not None:
self.batch_norm_layer = BatchNorm(
eps=batch_norm_param['eps'],
momentum=batch_norm_param['momentum'],
running_mean=batch_norm_param.get('running_mean', None),
running_var=batch_norm_param.get('running_var', None))
else:
self.batch_norm_layer = BatchNorm()
def forward_pass(self, x, w, b, gamma, beta, mode='train'):
""" Performs forward propagation through affine, batch normalization, and rectinfied linear unit layers
Args:
x: Input
w: Weights
b: Bias
gamma: Scale factor
beta: Shifting factor
mode: 'train' or 'test'
Returns:
relu_out: Output from ReLU layer
"""
affine_out = self.affine_layer.forward_pass(x, w, b)
batch_norm_out = self.batch_norm_layer.forward_pass(
affine_out, gamma, beta, mode)
relu_out = self.relu_layer.forward_pass(batch_norm_out)
return relu_out
def backward_pass(self, grad_out):
"""Performs back propagation through affine, batch normalization, and rectinfied linear unit layers
Args:
grad_out: Upstream gradient
Returns:
grad_x: Gradient w.r.t. input
grad_w: Gradient w.r.t. weight
grad_b: Gradient w.r.t. bias
grad_gamma: Gradient w.r.t. gamma constant
grad_beta: Gradient w.r.t. beta constant
"""
grad_relu = self.relu_layer.backward_pass(grad_out)
grad_batch_norm, grad_gamma, grad_beta = self.batch_norm_layer.backward_pass(
grad_relu)
grad_x, grad_w, grad_b = self.affine_layer.backward_pass(
grad_batch_norm)
return grad_x, grad_w, grad_b, np.sum(grad_gamma), np.sum(grad_beta)