def test_chain_rule_factors(self):
image_data = [0.1, 0.3, 0.7]
image_layer = Image(image_data=image_data, shape=(3, 1), maximum=1)
softmax = SoftMax([1, 2])
model = Model(layers=[image_layer, softmax])
model.compile(build=True)
# bump and grind
p0 = model.value()
epsilon = 1e-6
crf_bumped = []
for i in range(len(image_data)):
model.update_data_layer([((d + epsilon) if i == j else d)
for j, d in enumerate(image_data)],
maximum=1)
p1 = model.value()
crf_bumped.append([(pp1 - pp0) / epsilon
for pp0, pp1 in zip(p0, p1)])
crf_bumped = np.array(crf_bumped).transpose()
crf_layer = softmax.chain_rule_factors(cache={})
crf_model = model.chain_rule_factors()
for g, j, m in zip(flatten(crf_bumped), flatten(crf_layer),
flatten(crf_model)):
print(f'{g:8.6f} {j:8.6f} {m:8.6f}')
for g, j, m in zip(flatten(crf_bumped), flatten(crf_layer),
flatten(crf_model)):
self.assertAlmostEqual(g, j, delta=0.001)
self.assertAlmostEqual(j, m, delta=0.000001)
def test_jacobian_category_3(self):
image_data = [0.2, 0.5, 0.3]
categories = [1, 2, 4]
model = Model(layers=[
Image(image_data=image_data, shape=(3, 1), maximum=1),
Category(categories),
])
model.compile(build=True)
v = model.value()
self.assertEqual(v, [max(image_data)])
p = model.predict()
self.assertEqual(p, [categories[int(np.argmax(np.array(image_data)))]])
pass
def test_softmax_tiny(self):
image_data = [0.2, 0.1]
categories = [1, 2, 3]
model = Model(layers=[
Image(image_data=image_data, shape=(2, 1), maximum=1),
SoftMax(categories),
])
model.compile(build=True)
v = model.value()
softmax_value = model._layers[1]._nodes[0].value({})
self.assertEqual(len(categories), len(softmax_value))
softmax_derivative = model._layers[1]._nodes[0].derivative({})
bumped_derivatives = finite_differences(model, False)
for i in range(6):
for j in range(3):
self.assertAlmostEqual(softmax_derivative[i][j],
list(bumped_derivatives.values())[i][j])
pass
def test_jacobian_dense_31_category_3(self):
image_data = [0.2, 0.5, 0.3]
categories = [1, 2, 4]
model = Model(layers=[
Image(image_data=image_data, shape=(3, 1), maximum=1),
Dense(reLU(), (3, 1)),
Category(categories),
])
self._run_derivatives_test(model)
v = model.value()
# compute value manually for this image_data:
manual_dense_layer = []
for i in range(3):
weights = model._layers[1]._nodes[i]._weights
manual_dense_layer.append(
sum([w.weight() * d for w, d in zip(weights, image_data)]))
max_value = max(manual_dense_layer)
imax_value = int(np.argmax(np.array(manual_dense_layer)))
self.assertEqual(v, [max_value])
p = model.predict()
self.assertEqual(p, [categories[imax_value]])
pass
pixels = 81
image = Image(image_data=[random.randint(59, 59) for _ in range(pixels)])
cnn_a = CNN(reLU(), (3, 3), name='a')
cnn_b = CNN(reLU(), (3, 3), name='b')
cnn_c = CNN(reLU(), (3, 3), name='c')
dense = Dense(reLU(), (10, 1), name='d')
category = Category([i for i in range(10)])
model = Model(layers=[
image,
cnn_a,
cnn_b,
cnn_c,
dense,
category,
])
model.compile(build=True)
v = model.value()
# v = model.probability()
analytic_jacobian = model.jacobian()
# finite_differences(model, False)
print(v)
model.update_data_layer([random.randint(199, 199) for _ in range(pixels)])
v = model.value()
# analytic_jacobian = model.jacobian()
finite_differences(model, False, nmax=500)
print(v)
pass
model.compile(build=True)
# ## Prepare images for training
mndata = MNISTData(path='../test/mnist')
images, labels = mndata.training()
train_images = []
train_labels = []
i = 0
train_n = len(images)
while i < len(images) and len(train_images) < train_n:
train_images.append(reduce_image(images[i]) if reduce else images[i])
train_labels.append(labels[i])
i += 1
# ## Stochastic Gradient Descent
gd = GradientDescent(model=model, learning_rate=0.1, labels=predict_labels)
gd.sgd(train_images, train_labels, iterations=20000)
# ## Test the model ###
test_images, test_labels = mndata.testing()
predictions = []
for i, (ti, tl) in enumerate(zip(test_images, test_labels)):
model.update_data_layer(reduce_image(ti) if reduce else ti)
p = model.predict()
if i % 100 == 0:
print(f'{p}, {tl}, {max(model.value()):6.4f}')
predictions.append(dict(label=tl, predicted_label=p, probability=max(model.value())))
predictions = pd.DataFrame(predictions)
predictions.to_csv(f'mnist_{datetime.datetime.now():%Y%m%dT%H%M}.csv')