def test_model(num_feat, num_classes):
classifier = ClassifierModel()
classifier.add_layer('FC-1', FCLayer(num_feat, 2))
classifier.add_layer('Sigmoid', Sigmoid())
classifier.add_layer('FC-2', FCLayer(2, 5))
classifier.add_layer('ReLU', ReLU())
classifier.add_layer('FC-3', FCLayer(5, 3))
classifier.add_layer('tanh', Tanh())
classifier.add_layer('FC-4', FCLayer(3, num_classes))
classifier.add_layer('Softmax', SoftmaxLayer())
return classifier
def test_model(num_feat, num_classes):
classifier = CNN_Classifier()
classifier.add_layer(
'Conv-1', ConvolutionLayer(num_feat, 2, kernel_size=3, stride=1,
pad=1))
classifier.add_layer('ReLU', ReLU())
classifier.add_layer(
'Conv-2', ConvolutionLayer(2, 3, kernel_size=3, stride=1, pad=1))
classifier.add_layer('tanh', Tanh())
classifier.add_layer(
'Conv-3', ConvolutionLayer(3, 3, kernel_size=3, stride=1, pad=0))
classifier.add_layer('Sigmoid', Sigmoid())
classifier.add_layer('Max-pool - 1',
MaxPoolingLayer(kernel_size=2, stride=1))
classifier.add_layer('FC-4', FCLayer(12, num_classes))
classifier.add_layer('Softmax', SoftmaxLayer())
return classifier
# ReLU Layer
CNN.add_layer('ReLU Layer - 2', ReLU())
# Max-pool Layer
CNN.add_layer('Max-Pool Layer', MaxPoolingLayer(kernel_size=2, stride=2))
# FC Layer
CNN.add_layer('FC Example Layer - 1', FCLayer(input_dim=1568, output_dim=500))
# FC Layer
CNN.add_layer('FC Example Layer - 2', FCLayer(input_dim=500, output_dim=5))
# Softmax Layer
# 이 layer는 항상 마지막에 추가
CNN.add_layer('Softmax Layer', SoftmaxLayer())
# Model Architecture 출력
CNN.summary()
# Hyper-parameters
num_epochs = 10
learning_rate = 0.01
print_every = 1
# =========================================================================
batch_size = 128
train_accuracy = []
test_accuracy = []
# Difference should be
# => in the order of 1e-15 for forward
# => 0.0 for Backward, dW, db
print('Forward difference: ', (correct_fc_forward - fc_forward).sum())
print('Backward difference: ', (correct_fc_backward - fc_backward).sum())
print('dW difference: ', (correct_fc_dW - fc_dW).sum())
print('db difference: ', (correct_fc_db - fc_db).sum())
print()
# ===========================================================================
print(
'============================= 6. Softmax Layer ============================'
)
softmax_layer = SoftmaxLayer()
# Forward
"""
- Correct Softmax Forward -
[[2.48168930e-02 6.72851458e-01 3.02331649e-01]
[9.99831219e-01 1.23388975e-04 4.53922671e-05]
[1.38389653e-87 3.72007598e-44 1.00000000e+00]
[3.33333333e-01 3.33333333e-01 3.33333333e-01]]
"""
x = np.array([[-2., 1.3, 0.5], [-1., -10., -11.], [-100, 0, 100], [0, 0, 0]])
sm = softmax_layer.forward(x)
print(' Softmax ')
print('[Answer]')
print(correct_sm)
# Hyper-parameters
num_epochs = 100
learning_rate = 0.001
reg_lambda = 1e-8
print_every = 10
batch_size = 128
# Add layers
model.add_layer('FC-1', FCLayer(784, 500))
model.add_layer('sigmoid-1', Tanh())
model.add_layer('FC-2', FCLayer(500, 500))
model.add_layer('sigmoid-2', Tanh())
model.add_layer('FC-3',FCLayer(500,10))
model.add_layer('Softmax Layer', SoftmaxLayer())
# =========================================================================
assert dataset in ['mnist', 'fashion_mnist']
# Dataset
if dataset == 'mnist':
x_train, y_train, x_test, y_test = load_mnist('./data')
else:
x_train, y_train, x_test, y_test = load_fashion_mnist('./data')
x_train, x_test = np.squeeze(x_train), np.squeeze(x_test)
# Random 10% of train data as valid data
num_train = len(x_train)
perm = np.random.permutation(num_train)
def setUp(self):
self.softmax_layer = SoftmaxLayer()
class TestSoftmaxLayer(unittest.TestCase):
def setUp(self):
self.softmax_layer = SoftmaxLayer()
def test_softmax_layer_1_forward(self):
print('\n==================================')
print(' Test softmax layer forward ')
print('==================================')
np.random.seed(123)
x = np.random.randn(5, 5)
softmax_out = self.softmax_layer.forward(x)
correct_out = [
[0.06546572, 0.52558012, 0.25727246, 0.04298548, 0.10869621],
[0.52561239, 0.00890353, 0.06564194, 0.3574744, 0.04236773],
[0.07215113, 0.12940411, 0.63209441, 0.07509449, 0.09125586],
[0.02836542, 0.39760227, 0.39006297, 0.11953103, 0.06443832],
[0.20005518, 0.42494369, 0.03753939, 0.31014926, 0.02731249]
]
e = rel_error(correct_out, softmax_out)
print('Relative difference:', e)
self.assertTrue(e <= 1e-6)
out_sum = np.sum(softmax_out)
sum_e = out_sum - len(x)
print('Softmax sum error:', sum_e)
self.assertTrue(sum_e == 0)
def test_softmax_layer_2_ce_loss(self):
print('\n==================================')
print(' Test softmax layer ce loss ')
print('==================================')
np.random.seed(123)
x = np.random.randn(5, 5)
num_data, num_classes = x.shape
y_hat = self.softmax_layer.forward(x)
y = np.zeros_like(y_hat)
y_labels = np.random.permutation(num_data)
y[list(range(len(y_labels))), y_labels] = 1
loss = self.softmax_layer.ce_loss(y_hat, y)
correct_loss = 2.3052757961131616
e = rel_error(correct_loss, loss)
print('Relative difference:', e)
self.assertTrue(e <= 1e-11)
def test_softmax_layer_3_backward(self):
print('\n==================================')
print(' Test softmax layer backward ')
print('==================================')
np.random.seed(123)
x = np.random.randn(5, 5)
num_data, num_classes = x.shape
y_hat = self.softmax_layer.forward(x)
y = np.zeros_like(y_hat)
y_labels = np.random.permutation(num_data)
y[list(range(len(y_labels))), y_labels] = 1
loss = self.softmax_layer.ce_loss(y_hat, y)
dx = self.softmax_layer.backward(d_prev=1)
correct_dx = [
[0.01309314, 0.10511602, -0.14854551, 0.0085971, 0.02173924],
[0.10512248, 0.00178071, 0.01312839, 0.07149488, -0.19152645],
[0.01443023, 0.02588082, 0.12641888, -0.1849811, 0.01825117],
[-0.19432692, 0.07952045, 0.07801259, 0.02390621, 0.01288766],
[0.04001104, -0.11501126, 0.00750788, 0.06202985, 0.0054625]
]
e = rel_error(correct_dx, dx)
print('Relative difference:', e)
self.assertTrue(e <= 1e-6)
def runTest(self):
self.test_softmax_layer_1_forward()
self.test_softmax_layer_2_ce_loss()
self.test_softmax_layer_3_backward()