def run_test(RNNCls, step, num_states, **cls_args):
layer_test(RNNCls(output_dim, **cls_args), [origin],
rnn([origin], step, output_dim, num_states))
# test dropout, just test if it can pass...
layer_test(RNNCls(output_dim, dropout_W=0.2, dropout_U=0.2, **cls_args),
[origin],
test_serialization=False)
# test return_sequences, go_backwards, unroll
layer_test(
RNNCls(output_dim,
return_sequences=True,
go_backwards=True,
unroll=True,
**cls_args), [origin],
rnn([origin],
step,
output_dim,
num_states,
return_sequences=True,
go_backwards=True))
# test stateful
rnn_layer = RNNCls(output_dim, stateful=True, **cls_args)
exp_output = rnn([origin], step, output_dim, num_states)
layer_test(rnn_layer, [origin], exp_output, input_args=dict(batch_size=1))
assert_allclose(B.eval(rnn_layer.states[0]), exp_output)
rnn_layer.reset_states()
assert not np.any(B.eval(rnn_layer.states[0]))
def test_sgd():
''' math:
Let W = [A, B], b = [C, D], y = [E, F]
MSE = 1/2*[(A+C-E)^2 + (B+D-F)^2]
dA, dB, dC, dD = (A+C-E), (B+D-F), (A+C-E), (B+D-F)
Assume E = 2*(A+C), F = 2*(B+D)
dA, dB, dC, dD = -(A+C), -(B+D), -(A+C), -(B+D)
A-=lr*dA, B-=lr*dB, C-=lr*dC, D-=lr*dD
'''
lr = 0.01
W = np.array([[1, 2]])
b = np.array([3, 4])
wpb = W+b
model = Sequential([Input(1), Dense(2, initial_weights=[W, b])])
optimizer = SGD(lr=lr)
model.compile(optimizer, 'mse')
model.fit([1], 2*wpb, nb_epoch=1)
expectedW = W+lr*wpb
expectedb = (b+lr*wpb).reshape((2,))
assert_allclose(B.eval(model.layers[1].W), expectedW)
assert_allclose(B.eval(model.layers[1].b), expectedb)
def feed_test(inp_layers,
oup_layers,
expected_output,
num_params,
multi_output=False):
inp_layers = to_list(inp_layers)
oup_layers = to_list(oup_layers)
model = Model(inp_layers, oup_layers)
model.compile('sgd', ['mse'] * len(oup_layers))
pred = model.predict([np.array([[1]])] * len(inp_layers))
if not multi_output:
expected_output = [expected_output]
for p, e in zip(pred, expected_output):
assert_allclose(p, e)
# use caller_name to avoid race condition when conducting parallel testing
caller_name = inspect.stack()[1][3]
arch_fname = '/tmp/arch_{}.json'.format(caller_name)
weight_fname = '/tmp/weight_{}.hkl'.format(caller_name)
model.compile('sgd', ['mse'] * len(oup_layers))
if len(model.trainable_params) == 0:
weight_fname = None
model.save_to_file(arch_fname, weight_fname, overwrite=True, indent=2)
model2 = Model.load_from_file(arch_fname, weight_fname)
model2.compile('sgd', ['mse'] * len(oup_layers))
assert len(model.trainable_params) == len(
model2.trainable_params) == num_params
for p1, p2 in zip(model.trainable_params, model2.trainable_params):
assert_allclose(B.eval(p1), B.eval(p1))
for r1, r2 in zip(model.regularizers.values(),
model2.regularizers.values()):
assert str(serialize(r1)) == str(serialize(r2))
for c1, c2 in zip(model.constraints.values(), model2.constraints.values()):
assert str(serialize(c1)) == str(serialize(c2))
def objectives_test(obj_fn,
np_fn,
mode=None,
np_pred=None,
np_true=None,
debug=False):
if mode == 'reg':
np_pred = np.array([[1, 0, 3], [4, 8, 6]])
np_true = np.array([[1, 2, 4], [7, 5, 9]])
elif mode == 'cls':
np_pred = np.array([[0.8, 0.9, 0.2], [0.7, 1e-9, 0.6]])
np_true = np.array([[1, 1, 1], [0, 0, 0]])
y_pred = B.variable(np_pred)
y_true = B.variable(np_true)
exp_output = np_fn(np_pred, np_true)
if debug:
print(obj_fn.__name__)
print("Expected: \n{}".format(exp_output))
print("Output: \n{}".format(B.eval(obj_fn(y_pred, y_true))))
assert_allclose(B.eval(B.mean(obj_fn(y_pred, y_true))),
np.mean(exp_output))
def init_test(init_fn,
shapes=[FC_SHAPE, CONV_SHAPE],
target_mean=None,
target_std=None,
target_max=None,
target_min=None):
for shape in shapes:
output = B.eval(init_fn(shape))
lim = 1e-2
if target_std is not None:
assert abs(output.std() - target_std) < lim
if target_mean is not None:
assert abs(output.mean() - target_mean) < lim
if target_max is not None:
assert abs(output.max() - target_max) < lim
if target_min is not None:
assert abs(output.min() - target_min) < lim
def test_random():
B.seed(1234)
print(B.eval(B.random_uniform((1, 2))))
def constraint_test(k_fn, np_fn, debug=False):
assert_allclose(B.eval(k_fn(B.variable(np_input))),
np_fn(np_input),
rtol=1e-05)
def test_constraints():
maxnorm = 2
m1 = create_model(dense(initial_weights=[W, b]))
m2 = create_model(
dense(initial_weights=[W, b],
constraints={
'W': MaxNorm(m=maxnorm, axis=1),
'b': MaxNorm(m=maxnorm, axis=0)
}))
m3 = create_model(
dense(initial_weights=[W, b],
constraints={'W': MaxNorm(m=maxnorm, axis=1)}))
m4 = create_model(
dense(initial_weights=[W, b],
constraints=[
MaxNorm(m=maxnorm, axis=1),
MaxNorm(m=maxnorm, axis=0)
]))
m5 = create_model(
dense(initial_weights=[W, b], constraints=[MaxNorm(m=maxnorm,
axis=1)]))
m6 = create_model(
Sequential([
dense(initial_weights=[W, b],
constraints=[
MaxNorm(m=maxnorm, axis=1),
MaxNorm(m=maxnorm, axis=0)
])
]))
m7 = create_model(
Sequential([
Sequential([
dense(initial_weights=[W, b],
constraints=[
MaxNorm(m=maxnorm, axis=1),
MaxNorm(m=maxnorm, axis=0)
])
])
]))
m1.fit([[1]], 5 * wpb, nb_epoch=1)
m2.fit([[1]], 5 * wpb, nb_epoch=1)
m3.fit([[1]], 5 * wpb, nb_epoch=1)
m4.fit([[1]], 5 * wpb, nb_epoch=1)
m5.fit([[1]], 5 * wpb, nb_epoch=1)
m6.fit([[1]], 5 * wpb, nb_epoch=1)
m7.fit([[1]], 5 * wpb, nb_epoch=1)
m1w = B.eval(m1.layers[1].W)
m1b = B.eval(m1.layers[1].b)
m1W_norm = np.sqrt(np.sum(np.square(m1w), axis=1))
m1b_norm = np.sqrt(np.sum(np.square(m1b), axis=0))
constraint_w = m1w * maxnorm / m1W_norm
constraint_b = m1b * maxnorm / m1b_norm
assert_allclose(B.eval(m2.layers[1].W), constraint_w)
assert_allclose(B.eval(m2.layers[1].b), constraint_b)
assert_allclose(B.eval(m3.layers[1].W), constraint_w)
assert_allclose(B.eval(m3.layers[1].b), m1b)
assert_allclose(B.eval(m4.layers[1].W), constraint_w)
assert_allclose(B.eval(m4.layers[1].b), constraint_b)
assert_allclose(B.eval(m5.layers[1].W), constraint_w)
assert_allclose(B.eval(m5.layers[1].b), m1b)
assert_allclose(B.eval(m6.layers[1].embedded_layers[0].W), constraint_w)
assert_allclose(B.eval(m6.layers[1].embedded_layers[0].b), constraint_b)
assert_allclose(
B.eval(m7.layers[1].embedded_layers[0].embedded_layers[0].W),
constraint_w)
assert_allclose(
B.eval(m7.layers[1].embedded_layers[0].embedded_layers[0].b),
constraint_b)