def test_calculate_cost(self):
sys.stdout.write('FNN_trainer -> Performing calculate_cost test ... ')
sys.stdout.flush()
numx.random.seed(42)
l1 = FullConnLayer(2 * 2, 3 * 3)
l2 = FullConnLayer(3 * 3, 2 * 2)
l3 = FullConnLayer(2 * 2, 1 * 1)
model = MODEL.Model([l1, l2, l3])
trainer = TRAINER.GDTrainer(model)
model.forward_propagate(numx.arange(4).reshape(1, 4))
cost = model.calculate_cost(
[
numx.arange(9).reshape(1, 9),
numx.arange(4).reshape(1, 4),
numx.arange(1).reshape(1, 1)
], [CFct.SquaredError, CFct.SquaredError, CFct.SquaredError],
[1.0, 1.0, 1.0], [1.0, 1.0, 1.0],
[CFct.SquaredError, CFct.SquaredError, CFct.SquaredError],
[1.0, 1.0, 1.0])
assert numx.all(numx.abs(cost - 117.72346036) < self.epsilon)
cost = model.calculate_cost([
numx.arange(9).reshape(1, 9),
numx.arange(4).reshape(1, 4),
numx.arange(1).reshape(1, 1)
], [CFct.SquaredError, CFct.SquaredError, CFct.SquaredError],
[1.0, 1.0, 1.0], [1.0, 1.0, 1.0],
[None, None, None], [0.0, 0.0, 0.0])
assert numx.all(numx.abs(cost - 108.42118343) < self.epsilon)
cost = model.calculate_cost(
[None, None, numx.arange(1).reshape(1, 1)],
[None, None, CFct.SquaredError], [0.0, 0.0, 1.0], [0.0, 0.0, 1.0],
[None, None, None], [0.0, 0.0, 0.0])
assert numx.all(numx.abs(cost - 0.10778406) < self.epsilon)
print('successfully passed!')
sys.stdout.flush()
def test_check_setup(self):
sys.stdout.write('FNN_trainer -> Performing check_setup test ... ')
sys.stdout.flush()
numx.random.seed(42)
l1 = FullConnLayer(2 * 2, 3 * 3)
l2 = FullConnLayer(3 * 3, 2 * 2)
l3 = FullConnLayer(2 * 2, 1 * 1)
model = MODEL.Model([l1, l2, l3])
trainer = TRAINER.GDTrainer(model)
res = trainer.check_setup(
data=numx.arange(4).reshape(1, 4),
labels=[
numx.arange(9).reshape(1, 9),
numx.arange(4).reshape(1, 4),
numx.arange(1).reshape(1, 1)
],
epsilon=[0.01, 0.01, 0.01],
momentum=[0.09, 0.09, 0.09],
reg_L1Norm=[0.0002, 0.0002, 0.0002],
reg_L2Norm=[0.0002, 0.0002, 0.0002],
corruptor=None,
reg_costs=[1.0, 1.0, 1.0],
costs=[CFct.SquaredError, CFct.SquaredError, CFct.SquaredError],
reg_sparseness=[0.1, 0.1, 0.1],
desired_sparseness=[0.01, 0.01, 0.01],
costs_sparseness=[
CFct.SquaredError, CFct.SquaredError, CFct.SquaredError
],
update_offsets=[0.01, 0.01, 0.01],
restrict_gradient=[0.01, 0.01, 0.01],
restriction_norm='Mat')
assert numx.all(res)
print('successfully passed!')
sys.stdout.flush()
def test___init__(self):
sys.stdout.write('FNN_trainer -> Performing init test ... ')
sys.stdout.flush()
l1 = FullConnLayer(2 * 2, 3 * 3)
l2 = FullConnLayer(3 * 3, 2 * 2)
l3 = FullConnLayer(2 * 2, 1 * 1)
model = MODEL.Model([l1, l2, l3])
trainer = TRAINER.GDTrainer(model)
assert numx.all(trainer._old_grad[0][0].shape == (4, 9))
assert numx.all(trainer._old_grad[0][1].shape == (1, 9))
assert numx.all(trainer._old_grad[1][0].shape == (9, 4))
assert numx.all(trainer._old_grad[1][1].shape == (1, 4))
assert numx.all(trainer._old_grad[2][0].shape == (4, 1))
assert numx.all(trainer._old_grad[2][1].shape == (1, 1))
print('successfully passed!')
sys.stdout.flush()
def check(self, data, delta, act1, act2, act3, reg_sparseness,
desired_sparseness, cost_sparseness, reg_targets,
desired_targets, cost_targets, full):
connections = None
if full is False:
connections = generate_2d_connection_matrix(
6, 6, 3, 3, 2, 2, False)
model1 = FullConnLayer(6 * 6,
4 * 4,
activation_function=act1,
initial_weights='AUTO',
initial_bias=0.0,
initial_offset=0.0,
connections=connections,
dtype=numx.float64)
model2 = FullConnLayer(4 * 4,
5 * 5,
activation_function=act2,
initial_weights='AUTO',
initial_bias=0.0,
initial_offset=0.5,
dtype=numx.float64)
model3 = FullConnLayer(5 * 5,
6 * 6,
activation_function=act3,
initial_weights='AUTO',
initial_bias=0.0,
initial_offset=0.5,
dtype=numx.float64)
model = MODEL.Model([model1, model2, model3])
trainer = TRAINER.GDTrainer(model)
_, _, maxw, maxb = model.finit_differences(
delta, data, desired_targets, cost_targets, reg_targets,
desired_sparseness, cost_sparseness, reg_sparseness)
return numx.max([maxw, maxb])
def test_FNN_convergence(self):
sys.stdout.write(
'FNN_trainer -> Performing several convergences tests ... ')
sys.stdout.flush()
numx.random.seed(42)
x = numx.array([[0.0, 0.0], [0.0, 1.0], [1.0, 0.0], [1.0, 1.0]])
l = numx.array([[1, 0], [0, 1], [0, 1], [1, 0]
]) # 1,0 = 0, 0,1 = 1 otherwise Softmax would not work
for act_out in [AFct.SoftMax]:
for act_in in [
AFct.SoftSign, AFct.SoftPlus, AFct.Sigmoid,
AFct.HyperbolicTangent
]:
for cost in [
CFct.CrossEntropyError, CFct.SquaredError,
CFct.NegLogLikelihood
]:
l1 = FullConnLayer(input_dim=2,
output_dim=5,
activation_function=act_in,
initial_weights='AUTO',
initial_bias=0.0,
initial_offset=0.5,
connections=None)
l2 = FullConnLayer(input_dim=5,
output_dim=2,
activation_function=act_out,
initial_weights='AUTO',
initial_bias=0.0,
initial_offset=0.5,
connections=None)
model = MODEL.Model([l1, l2])
trainer = TRAINER.ADAGDTrainer(model)
for _ in range(1000):
trainer.train(data=x,
labels=[None, l],
epsilon=[0.3, 0.3],
reg_L1Norm=[0.000, 0.000],
reg_L2Norm=[0.000, 0.000],
corruptor=None,
reg_costs=[0.0, 1.0],
costs=[None, cost],
reg_sparseness=[0.0, 0.0],
desired_sparseness=[0.0, 0.0],
costs_sparseness=[None, None],
update_offsets=[0.1, 0.1],
restrict_gradient=0.0,
restriction_norm='Mat')
model.forward_propagate(x)
assert numx.all(trainer.calculate_errors(l) == 0)
l1 = FullConnLayer(input_dim=2,
output_dim=5,
activation_function=act_in,
initial_weights='AUTO',
initial_bias=0.0,
initial_offset=0.5,
connections=None)
l2 = FullConnLayer(input_dim=5,
output_dim=2,
activation_function=act_out,
initial_weights='AUTO',
initial_bias=0.0,
initial_offset=0.5,
connections=None)
model = MODEL.Model([l1, l2])
trainer = TRAINER.ADAGDTrainer(model)
for _ in range(1000):
trainer.train(
data=x,
labels=[None, l],
epsilon=[0.3, 0.3],
reg_L1Norm=[0.000, 0.000],
reg_L2Norm=[0.000, 0.000],
corruptor=None,
reg_costs=[0.0, 1.0],
costs=[None, cost],
reg_sparseness=[0.1, 0.0],
desired_sparseness=[0.1, 0.0],
costs_sparseness=[CFct.SquaredError, None],
update_offsets=[0.1, 0.1],
restrict_gradient=0.0,
restriction_norm='Mat')
model.forward_propagate(x)
assert numx.all(trainer.calculate_errors(l) == 0)
l1 = FullConnLayer(input_dim=2,
output_dim=5,
activation_function=act_in,
initial_weights='AUTO',
initial_bias=0.0,
initial_offset=0.5,
connections=None)
l2 = FullConnLayer(input_dim=5,
output_dim=2,
activation_function=act_out,
initial_weights='AUTO',
initial_bias=0.0,
initial_offset=0.5,
connections=None)
model = MODEL.Model([l1, l2])
trainer = TRAINER.GDTrainer(model)
for _ in range(2000):
trainer.train(
data=x,
labels=[None, l],
epsilon=[0.3, 0.3],
momentum=[0.9, 0.9],
reg_L1Norm=[0.000, 0.000],
reg_L2Norm=[0.000, 0.000],
corruptor=None,
reg_costs=[0.0, 1.0],
costs=[None, cost],
reg_sparseness=[0.1, 0.0],
desired_sparseness=[0.1, 0.0],
costs_sparseness=[CFct.SquaredError, None],
update_offsets=[0.1, 0.1],
restrict_gradient=[0.0, 0.0],
restriction_norm='Mat')
model.forward_propagate(x)
assert numx.all(trainer.calculate_errors(l) == 0)
print('successfully passed!')