def test_BatchGradientOptimizer_most_simple_case_big_batch():
node = SingleDirectionGradsStub(shape=[3], grad_value=2, forward_value=2)
loss = ConstantLoss(grads_value=1, loss_value=1)
optimizer = BatchGradientOptimizer(0.01, 100)
data_loader = GeneratorBasedLoader(generator_func=lambda: (1, 1))
parameters = node.get_learnable_parameters()
for _ in range(100):
optimizer.update_node_parameters(node, loss, data_loader)
np.testing.assert_array_almost_equal(parameters, [-1.0, -1.0, -1.0], 0.0001)
def test_BatchGradientOptimizer_random_data_shapes_batch_size_10(self):
for _ in range(50):
shape = []
for _ in range(np.random.randint(low=1, high=8)):
shape.append(np.random.randint(low=2, high=10))
node = SingleDirectionGradsStub(shape=shape, grad_value=2, forward_value=2)
loss = ConstantLoss(grads_value=1, loss_value=1)
optimizer = BatchGradientOptimizer(0.01, 10)
data_loader = GeneratorBasedLoader(generator_func=lambda: ([1], [1]))
parameters = node.get_learnable_parameters()
for _ in range(100):
optimizer.update_node_parameters(node, loss, data_loader)
delta = np.sum(np.abs(parameters - np.ones(shape=shape) * (-1)).ravel())
self.assertAlmostEqual(delta, 0, delta=1e-6)
def test_BatchGradientOptimizer_sphere_optimization(self):
for _ in range(10):
start_point = np.random.uniform(low=1.0, high=1.0, size=10)
end_point = np.zeros_like(start_point)
sphere_func = SphereFunc(start_point)
optimizer = BatchGradientOptimizer(0.25, 1)
loss = IdenticalLoss()
data_loader = StubDataLoader()
point = sphere_func.get_learnable_parameters()
iter_num = 0
while np.sum(np.abs(point - end_point)) > 1e-2:
iter_num += 1
if iter_num == 20000:
self.assertTrue(
False,
"Sphere minimum search with SGD takes too long time to converge",
)
optimizer.update_node_parameters(sphere_func, loss, data_loader)
sphere_func.point = point
self.assertTrue(True)
def test_BatchGradientOptimizer_Beale_func_optimization(self):
for _ in range(10):
noise = np.random.uniform(low=-0.75, high=0.75, size=2)
start_point = np.array([3.0, 0.5], dtype=np.float128) + noise
end_point = np.array([3.0, 0.5], dtype=np.float128)
func = BealeFunc(start_point)
optimizer = BatchGradientOptimizer(0.01, 1)
loss = IdenticalLoss()
data_loader = StubDataLoader()
point = func.get_learnable_parameters()
iter_num = 0
while np.sum(np.abs(point - end_point)) > 1e-2:
iter_num += 1
if iter_num == 50000:
self.assertTrue(
False,
"Beale function optimization with SGD takes too long time to converge",
)
optimizer.update_node_parameters(func, loss, data_loader)
func.point = point
self.assertTrue(True)
def test_linear_regression_BatchGradientOptimizer_with(self):
for _ in range(10):
dim = np.random.randint(low=1, high=25)
weights = np.random.normal(size=dim)
def get_data():
point = np.random.normal(size=dim - 1)
return point, np.dot(point, weights[:-1]) + weights[-1]
reg = LinearRegression(([dim - 1]))
loader = GeneratorBasedLoader(get_data)
loss = AbsoluteError()
opt = BatchGradientOptimizer(0.001, 25)
params = reg.get_learnable_parameters()
iter_num = 0
while np.sum(np.abs(weights - params)) > 0.01:
iter_num += 1
if iter_num == 20000:
self.assertTrue(
False,
"Linear regression with SGD takes too long time to converge",
)
opt.update_node_parameters(reg, loss, loader)
self.assertTrue(True)