def test_gather(self):
shape = (10, 2, 3)
ref = np.arange(np.prod(shape)).reshape(shape)
ref_th = KTH.variable(ref)
ref_tf = KTF.variable(ref)
inds = [1, 3, 7, 9]
inds_th = KTH.variable(inds, dtype='int32')
inds_tf = KTF.variable(inds, dtype='int32')
th_z = KTH.gather(ref_th, inds_th)
th_result = KTH.eval(th_z)
tf_result = KTF.eval(KTF.gather(ref_tf, inds_tf))
assert_allclose(tf_result, th_result, atol=1e-05)
if hasattr(th_z, '_keras_shape'):
assert th_z._keras_shape == th_result.shape
# test theano shape inference when
# input shape has None entries
if K.backend() == 'theano':
x = K.placeholder(shape=(None, 3, 4))
indices = K.placeholder(shape=(5, 6), dtype='int32')
y = K.gather(x, indices)
assert y._keras_shape == (5, 6, 3, 4)
def test_value_manipulation(self):
val = np.random.random((4, 2))
xth = KTH.variable(val)
xtf = KTF.variable(val)
# get_value
valth = KTH.get_value(xth)
valtf = KTF.get_value(xtf)
assert valtf.shape == valth.shape
assert_allclose(valth, valtf, atol=1e-05)
# set_value
val = np.random.random((4, 2))
KTH.set_value(xth, val)
KTF.set_value(xtf, val)
valth = KTH.get_value(xth)
valtf = KTF.get_value(xtf)
assert valtf.shape == valth.shape
assert_allclose(valth, valtf, atol=1e-05)
# count_params
assert KTH.count_params(xth) == KTF.count_params(xtf)
# print_tensor
check_single_tensor_operation('print_tensor', ())
check_single_tensor_operation('print_tensor', (2,))
check_single_tensor_operation('print_tensor', (4, 3))
check_single_tensor_operation('print_tensor', (1, 2, 3))
val = np.random.random((3, 2))
xth = KTH.variable(val)
xtf = KTF.variable(val)
assert KTH.get_variable_shape(xth) == KTF.get_variable_shape(xtf)
def test_value_manipulation(self):
val = np.random.random((4, 2))
xth = KTH.variable(val)
xtf = KTF.variable(val)
# get_value
valth = KTH.get_value(xth)
valtf = KTF.get_value(xtf)
assert valtf.shape == valth.shape
assert_allclose(valth, valtf, atol=1e-05)
# set_value
val = np.random.random((4, 2))
KTH.set_value(xth, val)
KTF.set_value(xtf, val)
valth = KTH.get_value(xth)
valtf = KTF.get_value(xtf)
assert valtf.shape == valth.shape
assert_allclose(valth, valtf, atol=1e-05)
# count_params
assert KTH.count_params(xth) == KTF.count_params(xtf)
# print_tensor
check_single_tensor_operation('print_tensor', ())
check_single_tensor_operation('print_tensor', (2, ))
check_single_tensor_operation('print_tensor', (4, 3))
check_single_tensor_operation('print_tensor', (1, 2, 3))
val = np.random.random((3, 2))
xth = KTH.variable(val)
xtf = KTF.variable(val)
assert KTH.get_variable_shape(xth) == KTF.get_variable_shape(xtf)
def test_shape_operations(self):
# concatenate
xval = np.random.random((4, 3))
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
yval = np.random.random((4, 2))
yth = KTH.variable(yval)
ytf = KTF.variable(yval)
zth = KTH.eval(KTH.concatenate([xth, yth], axis=-1))
ztf = KTF.eval(KTF.concatenate([xtf, ytf], axis=-1))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
check_single_tensor_operation('reshape', (4, 2), shape=(8, 1))
check_single_tensor_operation('permute_dimensions', (4, 2, 3),
pattern=(2, 0, 1))
check_single_tensor_operation('repeat', (4, 1), n=3)
check_single_tensor_operation('flatten', (4, 1))
check_single_tensor_operation('expand_dims', (4, 3), dim=-1)
check_single_tensor_operation('expand_dims', (4, 3, 2), dim=1)
check_single_tensor_operation('squeeze', (4, 3, 1), axis=2)
check_single_tensor_operation('squeeze', (4, 1, 1), axis=1)
check_composed_tensor_operations('reshape', {'shape': (4, 3, 1, 1)},
'squeeze', {'axis': 2},
(4, 3, 1, 1))
def test_gather(self):
shape = (10, 2, 3)
ref = np.arange(np.prod(shape)).reshape(shape)
ref_th = KTH.variable(ref)
ref_tf = KTF.variable(ref)
inds = [1, 3, 7, 9]
inds_th = KTH.variable(inds, dtype='int32')
inds_tf = KTF.variable(inds, dtype='int32')
th_z = KTH.gather(ref_th, inds_th)
th_result = KTH.eval(th_z)
tf_result = KTF.eval(KTF.gather(ref_tf, inds_tf))
assert_allclose(tf_result, th_result, atol=1e-05)
if hasattr(th_z, '_keras_shape'):
assert th_z._keras_shape == th_result.shape
# test theano shape inference when
# input shape has None entries
if K.backend() == 'theano':
x = K.placeholder(shape=(None, 3, 4))
indices = K.placeholder(shape=(5, 6), dtype='int32')
y = K.gather(x, indices)
assert y._keras_shape == (5, 6, 3, 4)
def test_shape_operations(self):
# concatenate
xval = np.random.random((4, 3))
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
yval = np.random.random((4, 2))
yth = KTH.variable(yval)
ytf = KTF.variable(yval)
zth = KTH.eval(KTH.concatenate([xth, yth], axis=-1))
ztf = KTF.eval(KTF.concatenate([xtf, ytf], axis=-1))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
check_single_tensor_operation('reshape', (4, 2), shape=(8, 1))
check_single_tensor_operation('permute_dimensions', (4, 2, 3),
pattern=(2, 0, 1))
check_single_tensor_operation('repeat', (4, 1), n=3)
check_single_tensor_operation('flatten', (4, 1))
check_single_tensor_operation('batch_flatten', (20, 2, 5))
check_single_tensor_operation('expand_dims', (4, 3), axis=-1)
check_single_tensor_operation('expand_dims', (4, 3, 2), axis=1)
check_single_tensor_operation('squeeze', (4, 3, 1), axis=2)
check_single_tensor_operation('squeeze', (4, 1, 1), axis=1)
check_composed_tensor_operations('reshape', {'shape': (4, 3, 1, 1)},
'squeeze', {'axis': 2}, (4, 3, 1, 1))
def test_ctc_decode_greedy(self):
# Test adapted from tensorflow
"""Test two batch entries - best path decoder."""
max_time_steps = 6
seq_len_0 = 4
input_prob_matrix_0 = np.asarray(
[
[1.0, 0.0, 0.0, 0.0], # t=0
[0.0, 0.0, 0.4, 0.6], # t=1
[0.0, 0.0, 0.4, 0.6], # t=2
[0.0, 0.9, 0.1, 0.0], # t=3
[0.0, 0.0, 0.0, 0.0], # t=4 (ignored)
[0.0, 0.0, 0.0, 0.0],
], # t=5 (ignored)
dtype=np.float32,
)
input_log_prob_matrix_0 = np.log(input_prob_matrix_0)
seq_len_1 = 5
# dimensions are time x depth
input_prob_matrix_1 = np.asarray(
[
[0.1, 0.9, 0.0, 0.0], # t=0
[0.0, 0.9, 0.1, 0.0], # t=1
[0.0, 0.0, 0.1, 0.9], # t=2
[0.0, 0.9, 0.1, 0.1], # t=3
[0.9, 0.1, 0.0, 0.0], # t=4
[0.0, 0.0, 0.0, 0.0],
], # t=5 (ignored)
dtype=np.float32,
)
# len max_time_steps array of batch_size x depth matrices
inputs = [np.vstack([input_prob_matrix_0[t, :], input_prob_matrix_1[t, :]]) for t in range(max_time_steps)]
# change tensorflow order to keras backend order
inputs = KTF.variable(np.asarray(inputs).transpose((1, 0, 2)))
# batch_size length vector of sequence_lengths
input_length = KTF.variable(np.array([seq_len_0, seq_len_1], dtype=np.int32))
# batch_size length vector of negative log probabilities
log_prob_truth = np.array(
[np.sum(-np.log([1.0, 0.6, 0.6, 0.9])), np.sum(-np.log([0.9, 0.9, 0.9, 0.9, 0.9]))], np.float32
)[:, np.newaxis]
# keras output, unlike tensorflow, is a dense (not sparse) tensor
decode_truth = np.array([[0, 1, -1], [1, 1, 0]])
decode_pred_tf, log_prob_pred_tf = KTF.ctc_decode(inputs, input_length, greedy=True)
assert len(decode_pred_tf) == 1
decode_pred = KTF.eval(decode_pred_tf[0])
log_prob_pred = KTF.eval(log_prob_pred_tf)
assert np.alltrue(decode_truth == decode_pred)
assert np.allclose(log_prob_truth, log_prob_pred)
def test_ctc_decode_greedy(self):
# Test adapted from tensorflow
"""Test two batch entries - best path decoder."""
max_time_steps = 6
seq_len_0 = 4
input_prob_matrix_0 = np.asarray(
[[1.0, 0.0, 0.0, 0.0], # t=0
[0.0, 0.0, 0.4, 0.6], # t=1
[0.0, 0.0, 0.4, 0.6], # t=2
[0.0, 0.9, 0.1, 0.0], # t=3
[0.0, 0.0, 0.0, 0.0], # t=4 (ignored)
[0.0, 0.0, 0.0, 0.0]], # t=5 (ignored)
dtype=np.float32)
input_log_prob_matrix_0 = np.log(input_prob_matrix_0)
seq_len_1 = 5
# dimensions are time x depth
input_prob_matrix_1 = np.asarray(
[[0.1, 0.9, 0.0, 0.0], # t=0
[0.0, 0.9, 0.1, 0.0], # t=1
[0.0, 0.0, 0.1, 0.9], # t=2
[0.0, 0.9, 0.1, 0.1], # t=3
[0.9, 0.1, 0.0, 0.0], # t=4
[0.0, 0.0, 0.0, 0.0]], # t=5 (ignored)
dtype=np.float32)
# len max_time_steps array of batch_size x depth matrices
inputs = [np.vstack([input_prob_matrix_0[t, :],
input_prob_matrix_1[t, :]])
for t in range(max_time_steps)]
# change tensorflow order to keras backend order
inputs = KTF.variable(np.asarray(inputs).transpose((1, 0, 2)))
# batch_size length vector of sequence_lengths
input_length = KTF.variable(np.array([seq_len_0, seq_len_1], dtype=np.int32))
# batch_size length vector of negative log probabilities
log_prob_truth = np.array([
np.sum(-np.log([1.0, 0.6, 0.6, 0.9])),
np.sum(-np.log([0.9, 0.9, 0.9, 0.9, 0.9]))
], np.float32)[:, np.newaxis]
# keras output, unlike tensorflow, is a dense (not sparse) tensor
decode_truth = np.array([[0, 1, -1], [1, 1, 0]])
decode_pred_tf, log_prob_pred_tf = KTF.ctc_decode(inputs,
input_length,
greedy=True)
assert len(decode_pred_tf) == 1
decode_pred = KTF.eval(decode_pred_tf[0])
log_prob_pred = KTF.eval(log_prob_pred_tf)
assert np.alltrue(decode_truth == decode_pred)
assert np.allclose(log_prob_truth, log_prob_pred)
def test_ctc(self):
# simplified version of TensorFlow's test
label_lens = np.expand_dims(np.asarray([5, 4]), 1)
input_lens = np.expand_dims(np.asarray([5, 5]),
1) # number of timesteps
# the Theano and Tensorflow CTC code use different methods to ensure
# numerical stability. The Theano code subtracts out the max
# before the final log, so the results are different but scale
# identically and still train properly
loss_log_probs_tf = [3.34211, 5.42262]
loss_log_probs_th = [1.73308, 3.81351]
# dimensions are batch x time x categories
labels = np.asarray([[0, 1, 2, 1, 0], [0, 1, 1, 0, -1]])
inputs = np.asarray(
[[[0.633766, 0.221185, 0.0917319, 0.0129757, 0.0142857, 0.0260553],
[0.111121, 0.588392, 0.278779, 0.0055756, 0.00569609, 0.010436],
[
0.0357786, 0.633813, 0.321418, 0.00249248, 0.00272882,
0.0037688
],
[
0.0663296, 0.643849, 0.280111, 0.00283995, 0.0035545,
0.00331533
],
[
0.458235, 0.396634, 0.123377, 0.00648837, 0.00903441,
0.00623107
]],
[[0.30176, 0.28562, 0.0831517, 0.0862751, 0.0816851, 0.161508],
[0.24082, 0.397533, 0.0557226, 0.0546814, 0.0557528, 0.19549],
[0.230246, 0.450868, 0.0389607, 0.038309, 0.0391602, 0.202456],
[0.280884, 0.429522, 0.0326593, 0.0339046, 0.0326856, 0.190345],
[0.423286, 0.315517, 0.0338439, 0.0393744, 0.0339315, 0.154046]]
],
dtype=np.float32)
labels_tf = KTF.variable(labels, dtype="int32")
inputs_tf = KTF.variable(inputs, dtype="float32")
input_lens_tf = KTF.variable(input_lens, dtype="int32")
label_lens_tf = KTF.variable(label_lens, dtype="int32")
res = KTF.eval(
KTF.ctc_batch_cost(labels_tf, inputs_tf, input_lens_tf,
label_lens_tf))
assert_allclose(res[:, 0], loss_log_probs_tf, atol=1e-05)
labels_th = KTH.variable(labels, dtype="int32")
inputs_th = KTH.variable(inputs, dtype="float32")
input_lens_th = KTH.variable(input_lens, dtype="int32")
label_lens_th = KTH.variable(label_lens, dtype="int32")
res = KTH.eval(
KTH.ctc_batch_cost(labels_th, inputs_th, input_lens_th,
label_lens_th))
assert_allclose(res[0, :], loss_log_probs_th, atol=1e-05)
def test_rnn(self):
# implement a simple RNN
input_dim = 8
output_dim = 4
timesteps = 5
input_val = np.random.random((32, timesteps, input_dim))
init_state_val = np.random.random((32, output_dim))
W_i_val = np.random.random((input_dim, output_dim))
W_o_val = np.random.random((output_dim, output_dim))
def rnn_step_fn(input_dim, output_dim, K):
W_i = K.variable(W_i_val)
W_o = K.variable(W_o_val)
def step_function(x, states):
assert len(states) == 1
prev_output = states[0]
output = K.dot(x, W_i) + K.dot(prev_output, W_o)
return output, [output]
return step_function
th_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTH)
inputs = KTH.variable(input_val)
initial_states = [KTH.variable(init_state_val)]
last_output, outputs, new_states = KTH.rnn(th_rnn_step_fn,
inputs,
initial_states,
go_backwards=False,
mask=None)
th_last_output = KTH.eval(last_output)
th_outputs = KTH.eval(outputs)
assert len(new_states) == 1
th_state = KTH.eval(new_states[0])
tf_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTF)
inputs = KTF.variable(input_val)
initial_states = [KTF.variable(init_state_val)]
last_output, outputs, new_states = KTF.rnn(tf_rnn_step_fn,
inputs,
initial_states,
go_backwards=False,
mask=None)
tf_last_output = KTF.eval(last_output)
tf_outputs = KTF.eval(outputs)
assert len(new_states) == 1
tf_state = KTF.eval(new_states[0])
assert_allclose(tf_last_output, th_last_output, atol=1e-04)
assert_allclose(tf_outputs, th_outputs, atol=1e-04)
assert_allclose(tf_state, th_state, atol=1e-04)
def test_ctc_decode_beam_search(self):
"""Test one batch, two beams - hibernating beam search."""
depth = 6
seq_len_0 = 5
input_prob_matrix_0 = np.asarray(
[[0.30999, 0.309938, 0.0679938, 0.0673362, 0.0708352, 0.173908],
[0.215136, 0.439699, 0.0370931, 0.0393967, 0.0381581, 0.230517],
[0.199959, 0.489485, 0.0233221, 0.0251417, 0.0233289, 0.238763],
[0.279611, 0.452966, 0.0204795, 0.0209126, 0.0194803, 0.20655],
[0.51286, 0.288951, 0.0243026, 0.0220788, 0.0219297, 0.129878],
# Random entry added in at time=5
[0.155251, 0.164444, 0.173517, 0.176138, 0.169979, 0.160671]],
dtype=np.float32)
# len max_time_steps array of batch_size x depth matrices
inputs = ([input_prob_matrix_0[t, :][np.newaxis, :]
for t in range(seq_len_0)] + # Pad to max_time_steps = 8
2 * [np.zeros((1, depth), dtype=np.float32)])
inputs = KTF.variable(np.asarray(inputs).transpose((1, 0, 2)))
# batch_size length vector of sequence_lengths
input_length = KTF.variable(np.array([seq_len_0], dtype=np.int32))
# batch_size length vector of negative log probabilities
log_prob_truth = np.array([
0.584855, # output beam 0
0.389139 # output beam 1
], np.float32)[np.newaxis, :]
decode_truth = [np.array([1, 0]), np.array([0, 1, 0])]
beam_width = 2
top_paths = 2
decode_pred_tf, log_prob_pred_tf = KTF.ctc_decode(inputs,
input_length,
greedy=False,
beam_width=beam_width,
top_paths=top_paths)
assert len(decode_pred_tf) == top_paths
log_prob_pred = KTF.eval(log_prob_pred_tf)
for i in range(top_paths):
assert np.alltrue(decode_truth[i] == KTF.eval(decode_pred_tf[i]))
assert np.allclose(log_prob_truth, log_prob_pred)
def test_ctc_decode_beam_search(self):
"""Test one batch, two beams - hibernating beam search."""
depth = 6
seq_len_0 = 5
input_prob_matrix_0 = np.asarray(
[[0.30999, 0.309938, 0.0679938, 0.0673362, 0.0708352, 0.173908],
[0.215136, 0.439699, 0.0370931, 0.0393967, 0.0381581, 0.230517],
[0.199959, 0.489485, 0.0233221, 0.0251417, 0.0233289, 0.238763],
[0.279611, 0.452966, 0.0204795, 0.0209126, 0.0194803, 0.20655],
[0.51286, 0.288951, 0.0243026, 0.0220788, 0.0219297, 0.129878],
# Random entry added in at time=5
[0.155251, 0.164444, 0.173517, 0.176138, 0.169979, 0.160671]],
dtype=np.float32)
# len max_time_steps array of batch_size x depth matrices
inputs = ([input_prob_matrix_0[t, :][np.newaxis, :]
for t in range(seq_len_0)] + # Pad to max_time_steps = 8
2 * [np.zeros((1, depth), dtype=np.float32)])
inputs = KTF.variable(np.asarray(inputs).transpose((1, 0, 2)))
# batch_size length vector of sequence_lengths
input_length = KTF.variable(np.array([seq_len_0], dtype=np.int32))
# batch_size length vector of negative log probabilities
log_prob_truth = np.array([
0.584855, # output beam 0
0.389139 # output beam 1
], np.float32)[np.newaxis, :]
decode_truth = [np.array([1, 0]), np.array([0, 1, 0])]
beam_width = 2
top_paths = 2
decode_pred_tf, log_prob_pred_tf = KTF.ctc_decode(inputs,
input_length,
greedy=False,
beam_width=beam_width,
top_paths=top_paths)
assert len(decode_pred_tf) == top_paths
log_prob_pred = KTF.eval(log_prob_pred_tf)
for i in range(top_paths):
assert np.alltrue(decode_truth[i] == KTF.eval(decode_pred_tf[i]))
assert np.allclose(log_prob_truth, log_prob_pred)
def check_two_tensor_operation(function_name, x_input_shape, y_input_shape, **kwargs):
xval = np.random.random(x_input_shape) - 0.5
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
yval = np.random.random(y_input_shape) - 0.5
yth = KTH.variable(yval)
ytf = KTF.variable(yval)
zth = KTH.eval(getattr(KTH, function_name)(xth, yth, **kwargs))
ztf = KTF.eval(getattr(KTF, function_name)(xtf, ytf, **kwargs))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
def test_rnn(self):
# implement a simple RNN
input_dim = 8
output_dim = 4
timesteps = 5
input_val = np.random.random((32, timesteps, input_dim))
init_state_val = np.random.random((32, output_dim))
W_i_val = np.random.random((input_dim, output_dim))
W_o_val = np.random.random((output_dim, output_dim))
def rnn_step_fn(input_dim, output_dim, K):
W_i = K.variable(W_i_val)
W_o = K.variable(W_o_val)
def step_function(x, states):
assert len(states) == 1
prev_output = states[0]
output = K.dot(x, W_i) + K.dot(prev_output, W_o)
return output, [output]
return step_function
th_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTH)
inputs = KTH.variable(input_val)
initial_states = [KTH.variable(init_state_val)]
last_output, outputs, new_states = KTH.rnn(th_rnn_step_fn, inputs,
initial_states,
go_backwards=False,
masking=False)
th_last_output = KTH.eval(last_output)
th_outputs = KTH.eval(outputs)
assert len(new_states) == 1
th_state = KTH.eval(new_states[0])
tf_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTF)
inputs = KTF.variable(input_val)
initial_states = [KTF.variable(init_state_val)]
last_output, outputs, new_states = KTF.rnn(tf_rnn_step_fn, inputs,
initial_states,
go_backwards=False,
masking=False)
tf_last_output = KTF.eval(last_output)
tf_outputs = KTF.eval(outputs)
assert len(new_states) == 1
tf_state = KTF.eval(new_states[0])
assert_allclose(tf_last_output, th_last_output, atol=1e-04)
assert_allclose(tf_outputs, th_outputs, atol=1e-04)
assert_allclose(tf_state, th_state, atol=1e-04)
def check_two_tensor_operation(function_name, x_input_shape, y_input_shape,
**kwargs):
xval = np.random.random(x_input_shape) - 0.5
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
yval = np.random.random(y_input_shape) - 0.5
yth = KTH.variable(yval)
ytf = KTF.variable(yval)
zth = KTH.eval(getattr(KTH, function_name)(xth, yth, **kwargs))
ztf = KTF.eval(getattr(KTF, function_name)(xtf, ytf, **kwargs))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
def test_ctc(self):
# simplified version of TensorFlow's test
label_lens = np.expand_dims(np.asarray([5, 4]), 1)
input_lens = np.expand_dims(np.asarray([5, 5]), 1) # number of timesteps
# the Theano and Tensorflow CTC code use different methods to ensure
# numerical stability. The Theano code subtracts out the max
# before the final log, so the results are different but scale
# identically and still train properly
loss_log_probs_tf = [3.34211, 5.42262]
loss_log_probs_th = [1.73308, 3.81351]
# dimensions are batch x time x categories
labels = np.asarray([[0, 1, 2, 1, 0], [0, 1, 1, 0, -1]])
inputs = np.asarray(
[
[
[0.633766, 0.221185, 0.0917319, 0.0129757, 0.0142857, 0.0260553],
[0.111121, 0.588392, 0.278779, 0.0055756, 0.00569609, 0.010436],
[0.0357786, 0.633813, 0.321418, 0.00249248, 0.00272882, 0.0037688],
[0.0663296, 0.643849, 0.280111, 0.00283995, 0.0035545, 0.00331533],
[0.458235, 0.396634, 0.123377, 0.00648837, 0.00903441, 0.00623107],
],
[
[0.30176, 0.28562, 0.0831517, 0.0862751, 0.0816851, 0.161508],
[0.24082, 0.397533, 0.0557226, 0.0546814, 0.0557528, 0.19549],
[0.230246, 0.450868, 0.0389607, 0.038309, 0.0391602, 0.202456],
[0.280884, 0.429522, 0.0326593, 0.0339046, 0.0326856, 0.190345],
[0.423286, 0.315517, 0.0338439, 0.0393744, 0.0339315, 0.154046],
],
],
dtype=np.float32,
)
labels_tf = KTF.variable(labels, dtype="int32")
inputs_tf = KTF.variable(inputs, dtype="float32")
input_lens_tf = KTF.variable(input_lens, dtype="int32")
label_lens_tf = KTF.variable(label_lens, dtype="int32")
res = KTF.eval(KTF.ctc_batch_cost(labels_tf, inputs_tf, input_lens_tf, label_lens_tf))
assert_allclose(res[:, 0], loss_log_probs_tf, atol=1e-05)
labels_th = KTH.variable(labels, dtype="int32")
inputs_th = KTH.variable(inputs, dtype="float32")
input_lens_th = KTH.variable(input_lens, dtype="int32")
label_lens_th = KTH.variable(label_lens, dtype="int32")
res = KTH.eval(KTH.ctc_batch_cost(labels_th, inputs_th, input_lens_th, label_lens_th))
assert_allclose(res[0, :], loss_log_probs_th, atol=1e-05)
def weighted_categorical_crossentropy(weights):
"""
A weighted version of keras.objectives.categorical_crossentropy
Variables:
weights: numpy array of shape (C,) where C is the number of classes
Usage:
weights = np.array([0.5,2,10]) # Class one at 0.5, class 2 twice the normal weights, class 3 10x.
loss = weighted_categorical_crossentropy(weights)
model.compile(loss=loss,optimizer='adam')
"""
weights = K.variable(weights)
def loss(y_true, y_pred):
# scale predictions so that the class probas of each sample sum to 1
y_pred /= K.sum(y_pred, axis=-1, keepdims=True)
# clip to prevent NaN's and Inf's
y_pred = K.clip(y_pred, K.epsilon(), 1 - K.epsilon())
# calc
loss = y_true * K.log(y_pred) * weights
loss = -K.sum(loss, -1)
return loss
return loss
def test_moments(self):
input_shape = (10, 10, 10, 10)
x_0 = np.zeros(input_shape)
x_1 = np.ones(input_shape)
x_random = np.random.random(input_shape)
th_axes = [0, 2, 3]
tf_axes = [0, 1, 2]
for ip in [x_0, x_1, x_random]:
for axes in [th_axes, tf_axes]:
for keep_dims in [True, False]:
ip_th = KTH.variable(ip)
th_mean, th_var = KCTH.moments(ip_th,
axes,
keep_dims=keep_dims)
ip_tf = KTF.variable(ip)
tf_mean, tf_var = KCTF.moments(ip_tf,
axes,
keep_dims=keep_dims)
th_mean_val = KTH.eval(th_mean)
tf_mean_val = KTF.eval(tf_mean)
th_var_val = KTH.eval(th_var)
tf_var_val = KTF.eval(tf_var)
assert_allclose(th_mean_val, tf_mean_val, rtol=1e-4)
assert_allclose(th_var_val, tf_var_val, rtol=1e-4)
def check_composed_tensor_operations(
first_function_name,
first_function_args,
second_function_name,
second_function_args,
input_shape,
):
""" Creates a random tensor t0 with shape input_shape and compute
t1 = first_function_name(t0, **first_function_args)
t2 = second_function_name(t1, **second_function_args)
with both Theano and TensorFlow backends and ensures the answers match.
"""
val = np.random.random(input_shape) - 0.5
xth = KTH.variable(val)
xtf = KTF.variable(val)
yth = getattr(KCTH, first_function_name)(xth, **first_function_args)
ytf = getattr(KCTF, first_function_name)(xtf, **first_function_args)
zth = KTH.eval(
getattr(KCTH, second_function_name)(yth, **second_function_args))
ztf = KTF.eval(
getattr(KCTF, second_function_name)(ytf, **second_function_args))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
def test_depth_to_space(self):
for batch_size in [1, 2, 3]:
for scale in [2, 3]:
for channels in [1, 2, 3]:
for rows in [1, 2, 3]:
for cols in [1, 2, 3]:
if K.image_data_format() == 'channels_first':
arr = np.arange(batch_size * channels * scale * scale * rows * cols)\
.reshape((batch_size, channels * scale * scale, rows, cols))
elif K.image_data_format() == 'channels_last':
arr = np.arange(batch_size * rows * cols * scale * scale * channels) \
.reshape((batch_size, rows, cols, channels * scale * scale))
arr_tf = KTF.variable(arr)
arr_th = KTH.variable(arr)
if K.image_data_format() == 'channels_first':
expected = arr.reshape((batch_size, scale, scale, channels, rows, cols))\
.transpose((0, 3, 4, 1, 5, 2))\
.reshape((batch_size, channels, rows * scale, cols * scale))
elif K.image_data_format() == 'channels_last':
expected = arr.reshape((batch_size, rows, cols, scale, scale, channels)) \
.transpose((0, 1, 3, 2, 4, 5))\
.reshape((batch_size, rows * scale, cols * scale, channels))
tf_ans = KTF.eval(
KCTF.depth_to_space(arr_tf, scale))
th_ans = KTH.eval(
KCTH.depth_to_space(arr_th, scale))
assert tf_ans.shape == expected.shape
assert th_ans.shape == expected.shape
assert_allclose(expected, tf_ans, atol=1e-05)
assert_allclose(expected, th_ans, atol=1e-05)
def test_moments(self):
input_shape = (10, 10, 10, 10)
x_0 = np.zeros(input_shape)
x_1 = np.ones(input_shape)
x_random = np.random.random(input_shape)
th_axes = [0, 2, 3]
tf_axes = [0, 1, 2]
for ip in [x_0, x_1, x_random]:
for axes in [th_axes, tf_axes]:
for keep_dims in [True, False]:
ip_th = KTH.variable(ip)
th_mean, th_var = KCTH.moments(ip_th, axes, keep_dims=keep_dims)
ip_tf = KTF.variable(ip)
tf_mean, tf_var = KCTF.moments(ip_tf, axes, keep_dims=keep_dims)
ip_cntk = KCTK.variable(ip)
cntk_mean, cntk_var = KCNTK.moments(ip_cntk, axes, keep_dims=keep_dims)
th_mean_val = KTH.eval(th_mean)
tf_mean_val = KTF.eval(tf_mean)
cntk_mean_val = KCTK.eval(cntk_mean)
th_var_val = KTH.eval(th_var)
tf_var_val = KTF.eval(tf_var)
cntk_var_val = KCTK.eval(cntk_var)
# absolute tolerance needed when working with zeros
assert_allclose(th_mean_val, tf_mean_val, rtol=1e-4, atol=1e-10)
assert_allclose(th_var_val, tf_var_val, rtol=1e-4, atol=1e-10)
assert_allclose(th_mean_val, cntk_mean_val, rtol=1e-4, atol=1e-10)
assert_allclose(th_var_val, cntk_var_val, rtol=1e-4, atol=1e-10)
def binary_PTA(y_true, y_pred, threshold=K.variable(value=0.5)):
y_pred = K.cast(y_pred >= threshold, 'float32')
# P = total number of positive labels
P = K.sum(y_true)
# TP = total number of correct alerts, alerts from the positive class labels
TP = K.sum(y_pred * y_true)
return TP / P
def binary_PFA(y_true, y_pred, threshold=K.variable(value=0.5)):
y_pred = K.cast(y_pred >= threshold, 'float32')
# N = total number of negative labels
N = K.sum(1 - y_true)
# FP = total number of false alerts, alerts from the negative class labels
TN = K.sum((1 - y_pred) * (1 - y_true))
return TN / N
def test_gradient(self):
val = np.random.random((4, 2))
xth = KTH.variable(val)
xtf = KTF.variable(val)
expth = xth * KTH.exp(xth)
exptf = xtf * KTF.exp(xtf)
lossth = KTH.sum(expth)
losstf = KTF.sum(exptf)
zero_lossth = KTH.stop_gradient(lossth)
zero_losstf = KTF.stop_gradient(losstf)
gradth = KTH.gradients(lossth, [expth])
gradtf = KTF.gradients(losstf, [exptf])
zero_gradth = KTH.gradients(lossth + zero_lossth, [expth])
zero_gradtf = KTF.gradients(losstf + zero_losstf, [exptf])
zth = KTH.eval(gradth[0])
ztf = KTF.eval(gradtf[0])
zero_zth = KTH.eval(zero_gradth[0])
zero_ztf = KTF.eval(zero_gradtf[0])
assert zth.shape == ztf.shape
assert zero_zth.shape == zero_ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
assert_allclose(zero_zth, zero_ztf, atol=1e-05)
assert_allclose(zero_zth, zth, atol=1e-05)
assert_allclose(zero_ztf, ztf, atol=1e-05)
def get_initial_state(self, x):
input_shape = self.input_spec[0].shape
init_nb_row = input_shape[self.row_axis]
init_nb_col = input_shape[self.column_axis]
base_initial_state = K.zeros_like(
x) # (samples, timesteps) + image_shape
non_channel_axis = -1 if self.data_format == 'channels_first' else -2
for _ in range(2):
base_initial_state = K.sum(base_initial_state,
axis=non_channel_axis)
base_initial_state = K.sum(base_initial_state,
axis=1) # (samples, nb_channels)
initial_states = []
states_to_pass = ['r', 'c', 'e']
nlayers_to_pass = {u: self.nb_layers for u in states_to_pass}
if self.extrap_start_time is not None:
states_to_pass.append(
'ahat'
) # pass prediction in states so can use as actual for t+1 when extrapolating
nlayers_to_pass['ahat'] = 1
for u in states_to_pass:
for l in range(nlayers_to_pass[u]):
ds_factor = 2**l
nb_row = init_nb_row // ds_factor
nb_col = init_nb_col // ds_factor
if u in ['r', 'c']:
stack_size = self.R_stack_sizes[l]
elif u == 'e':
stack_size = 2 * self.stack_sizes[l]
elif u == 'ahat':
stack_size = self.stack_sizes[l]
output_size = stack_size * nb_row * nb_col # flattened size
reducer = K.zeros((input_shape[self.channel_axis],
output_size)) # (nb_channels, output_size)
initial_state = K.dot(base_initial_state,
reducer) # (samples, output_size)
if self.data_format == 'channels_first':
output_shp = (-1, stack_size, nb_row, nb_col)
else:
output_shp = (-1, nb_row, nb_col, stack_size)
initial_state = K.reshape(initial_state, output_shp)
initial_states += [initial_state]
if K._BACKEND == 'theano':
from theano import tensor as T
# There is a known issue in the Theano scan op when dealing with inputs whose shape is 1 along a dimension.
# In our case, this is a problem when training on grayscale images, and the below line fixes it.
initial_states = [
T.unbroadcast(init_state, 0, 1)
for init_state in initial_states
]
if self.extrap_start_time is not None:
initial_states += [
K.variable(0, int if K.backend() != 'tensorflow' else 'int32')
] # the last state will correspond to the current timestep
return initial_states
def yolo_head(graph, feats, anchors, num_classes):
with graph.as_default():
num_anchors = len(anchors)
anchors_tensor = K.reshape(K.variable(anchors),
[1, 1, 1, num_anchors, 2])
conv_dims = K.shape(feats)[1:3]
conv_height_index = K.arange(0, stop=conv_dims[0])
conv_width_index = K.arange(0, stop=conv_dims[1])
conv_height_index = K.tile(conv_height_index, [conv_dims[1]])
conv_width_index = K.tile(K.expand_dims(conv_width_index, 0),
[conv_dims[0], 1])
conv_width_index = K.flatten(K.transpose(conv_width_index))
conv_index = K.transpose(K.stack([conv_height_index,
conv_width_index]))
conv_index = K.reshape(conv_index,
[1, conv_dims[0], conv_dims[1], 1, 2])
conv_index = K.cast(conv_index, K.dtype(feats))
feats = K.reshape(
feats,
[-1, conv_dims[0], conv_dims[1], num_anchors, num_classes + 5])
conv_dims = K.cast(K.reshape(conv_dims, [1, 1, 1, 1, 2]),
K.dtype(feats))
box_xy = K.sigmoid(feats[..., :2])
box_wh = K.exp(feats[..., 2:4])
box_confidence = K.sigmoid(feats[..., 4:5])
box_class_probs = K.softmax(feats[..., 5:])
box_xy = (box_xy + conv_index) / conv_dims
box_wh = box_wh * anchors_tensor / conv_dims
return box_xy, box_wh, box_confidence, box_class_probs
def yolo_eval(graph,
yolo_outputs,
image_shape,
max_boxes=10,
score_threshold=.6,
iou_threshold=.5):
with graph.as_default():
box_xy, box_wh, box_confidence, box_class_probs = yolo_outputs
boxes = yolo_boxes_to_corners(graph, box_xy, box_wh)
boxes, scores, classes = yolo_filter_boxes(graph,
boxes,
box_confidence,
box_class_probs,
threshold=score_threshold)
# Scale boxes back to original image shape.
height = image_shape[0]
width = image_shape[1]
image_dims = K.stack([height, width, height, width])
image_dims = K.reshape(image_dims, [1, 4])
boxes = boxes * image_dims
# TODO: Something must be done about this ugly hack!
max_boxes_tensor = K.variable(max_boxes, dtype='int32')
K.get_session().run(tf.variables_initializer([max_boxes_tensor]))
nms_index = tf.image.non_max_suppression(boxes,
scores,
max_boxes_tensor,
iou_threshold=iou_threshold)
boxes = K.gather(boxes, nms_index)
scores = K.gather(scores, nms_index)
classes = K.gather(classes, nms_index)
return boxes, scores, classes
def test_function(self):
val = np.random.random((4, 2))
input_val = np.random.random((4, 2))
xth = KTH.variable(val)
xtf = KTF.variable(val)
yth = KTH.placeholder(ndim=2)
ytf = KTF.placeholder(ndim=2)
exp_th = KTH.square(xth) + yth
exp_tf = KTF.square(xtf) + ytf
update_th = xth * 2
update_tf = xtf * 2
fth = KTH.function([yth], [exp_th], updates=[(xth, update_th)])
ftf = KTF.function([ytf], [exp_tf], updates=[(xtf, update_tf)])
function_outputs_th = fth([input_val])[0]
function_outputs_tf = ftf([input_val])[0]
assert function_outputs_th.shape == function_outputs_tf.shape
assert_allclose(function_outputs_th, function_outputs_tf, atol=1e-05)
new_val_th = KTH.get_value(xth)
new_val_tf = KTF.get_value(xtf)
assert new_val_th.shape == new_val_tf.shape
assert_allclose(new_val_th, new_val_tf, atol=1e-05)
def test_gradient(self):
val = np.random.random((4, 2))
xth = KTH.variable(val)
xtf = KTF.variable(val)
expth = xth * KTH.exp(xth)
exptf = xtf * KTF.exp(xtf)
lossth = KTH.sum(expth)
losstf = KTF.sum(exptf)
zero_lossth = KTH.stop_gradient(lossth)
zero_losstf = KTF.stop_gradient(losstf)
gradth = KTH.gradients(lossth, [expth])
gradtf = KTF.gradients(losstf, [exptf])
zero_gradth = KTH.gradients(lossth + zero_lossth, [expth])
zero_gradtf = KTF.gradients(losstf + zero_losstf, [exptf])
zth = KTH.eval(gradth[0])
ztf = KTF.eval(gradtf[0])
zero_zth = KTH.eval(zero_gradth[0])
zero_ztf = KTF.eval(zero_gradtf[0])
assert zth.shape == ztf.shape
assert zero_zth.shape == zero_ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
assert_allclose(zero_zth, zero_ztf, atol=1e-05)
assert_allclose(zero_zth, zth, atol=1e-05)
assert_allclose(zero_ztf, ztf, atol=1e-05)
def test_nn_operations(self):
check_single_tensor_operation('relu', (4, 2), alpha=0.1, max_value=0.5)
check_single_tensor_operation('softmax', (4, 10))
check_single_tensor_operation('softplus', (4, 10))
check_single_tensor_operation('sigmoid', (4, 2))
check_single_tensor_operation('hard_sigmoid', (4, 2))
check_single_tensor_operation('tanh', (4, 2))
# dropout
val = np.random.random((100, 100))
xth = KTH.variable(val)
xtf = KTF.variable(val)
zth = KTH.eval(KTH.dropout(xth, level=0.2))
ztf = KTF.eval(KTF.dropout(xtf, level=0.2))
assert zth.shape == ztf.shape
# dropout patterns are different, only check mean
assert np.abs(zth.mean() - ztf.mean()) < 0.05
check_two_tensor_operation('binary_crossentropy', (4, 2), (4, 2), from_logits=True)
check_two_tensor_operation('categorical_crossentropy', (4, 2), (4, 2), from_logits=True)
check_two_tensor_operation('binary_crossentropy', (4, 2), (4, 2), from_logits=False)
check_two_tensor_operation('categorical_crossentropy', (4, 2), (4, 2), from_logits=False)
check_single_tensor_operation('l2_normalize', (4, 3), axis=-1)
check_single_tensor_operation('l2_normalize', (4, 3), axis=1)
def test_repeat_elements(self):
reps = 3
for ndims in [1, 2, 3]:
shape = np.arange(2, 2 + ndims)
arr = np.arange(np.prod(shape)).reshape(shape)
arr_th = KTH.variable(arr)
arr_tf = KTF.variable(arr)
for rep_axis in range(ndims):
np_rep = np.repeat(arr, reps, axis=rep_axis)
th_z = KTH.repeat_elements(arr_th, reps, axis=rep_axis)
th_rep = KTH.eval(th_z)
tf_rep = KTF.eval(
KTF.repeat_elements(arr_tf, reps, axis=rep_axis))
assert th_rep.shape == np_rep.shape
assert tf_rep.shape == np_rep.shape
assert_allclose(np_rep, th_rep, atol=1e-05)
assert_allclose(np_rep, tf_rep, atol=1e-05)
if hasattr(th_z, '_keras_shape'):
assert th_z._keras_shape == th_rep.shape
# test theano shape inference when
# input shape has None entries
if K.backend() == 'theano':
shape = list(shape)
shape[rep_axis] = None
x = K.placeholder(shape=shape)
y = K.repeat_elements(x, reps, axis=rep_axis)
assert y._keras_shape == tuple(shape)
def test_nn_operations(self):
check_single_tensor_operation('relu', (4, 2), alpha=0.1, max_value=0.5)
check_single_tensor_operation('softmax', (4, 10))
check_single_tensor_operation('softplus', (4, 10))
check_single_tensor_operation('elu', (4, 10), alpha=0.5)
check_single_tensor_operation('sigmoid', (4, 2))
check_single_tensor_operation('hard_sigmoid', (4, 2))
check_single_tensor_operation('tanh', (4, 2))
# dropout
val = np.random.random((100, 100))
xth = KTH.variable(val)
xtf = KTF.variable(val)
zth = KTH.eval(KTH.dropout(xth, level=0.2))
ztf = KTF.eval(KTF.dropout(xtf, level=0.2))
assert zth.shape == ztf.shape
# dropout patterns are different, only check mean
assert np.abs(zth.mean() - ztf.mean()) < 0.05
check_two_tensor_operation('binary_crossentropy', (4, 2), (4, 2),
from_logits=True)
check_two_tensor_operation('categorical_crossentropy', (4, 2), (4, 2),
from_logits=True)
check_two_tensor_operation('binary_crossentropy', (4, 2), (4, 2),
from_logits=False)
check_two_tensor_operation('categorical_crossentropy', (4, 2), (4, 2),
from_logits=False)
check_single_tensor_operation('l2_normalize', (4, 3), axis=-1)
check_single_tensor_operation('l2_normalize', (4, 3), axis=1)
def test_function(self):
val = np.random.random((4, 2))
input_val = np.random.random((4, 2))
xth = KTH.variable(val)
xtf = KTF.variable(val)
yth = KTH.placeholder(ndim=2)
ytf = KTF.placeholder(ndim=2)
exp_th = KTH.square(xth) + yth
exp_tf = KTF.square(xtf) + ytf
update_th = xth * 2
update_tf = xtf * 2
fth = KTH.function([yth], [exp_th], updates=[(xth, update_th)])
ftf = KTF.function([ytf], [exp_tf], updates=[(xtf, update_tf)])
function_outputs_th = fth([input_val])[0]
function_outputs_tf = ftf([input_val])[0]
assert function_outputs_th.shape == function_outputs_tf.shape
assert_allclose(function_outputs_th, function_outputs_tf, atol=1e-05)
new_val_th = KTH.get_value(xth)
new_val_tf = KTF.get_value(xtf)
assert new_val_th.shape == new_val_tf.shape
assert_allclose(new_val_th, new_val_tf, atol=1e-05)
def test_repeat_elements(self):
reps = 3
for ndims in [1, 2, 3]:
shape = np.arange(2, 2 + ndims)
arr = np.arange(np.prod(shape)).reshape(shape)
arr_th = KTH.variable(arr)
arr_tf = KTF.variable(arr)
for rep_axis in range(ndims):
np_rep = np.repeat(arr, reps, axis=rep_axis)
th_z = KTH.repeat_elements(arr_th, reps, axis=rep_axis)
th_rep = KTH.eval(th_z)
tf_rep = KTF.eval(
KTF.repeat_elements(arr_tf, reps, axis=rep_axis))
assert th_rep.shape == np_rep.shape
assert tf_rep.shape == np_rep.shape
assert_allclose(np_rep, th_rep, atol=1e-05)
assert_allclose(np_rep, tf_rep, atol=1e-05)
if hasattr(th_z, '_keras_shape'):
assert th_z._keras_shape == th_rep.shape
# test theano shape inference when
# input shape has None entries
if K.backend() == 'theano':
shape = list(shape)
shape[rep_axis] = None
x = K.placeholder(shape=shape)
y = K.repeat_elements(x, reps, axis=rep_axis)
assert y._keras_shape == tuple(shape)
def test_conv2d(self):
# TF kernel shape: (rows, cols, input_depth, depth)
# channels_first input shape: (n, input_depth, rows, cols)
for input_shape in [(2, 3, 4, 5), (2, 3, 5, 6)]:
for kernel_shape in [(2, 2, 3, 4), (4, 3, 3, 4)]:
for padding in ['valid', 'same']:
xval = np.random.random(input_shape)
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
kernel_val = np.random.random(kernel_shape) - 0.5
kernel_th = KTH.variable(convert_kernel(kernel_val))
kernel_tf = KTF.variable(kernel_val)
zth = KTH.eval(
KTH.conv2d(xth,
kernel_th,
data_format='channels_first'))
ztf = KTF.eval(
KTF.conv2d(xtf,
kernel_tf,
data_format='channels_first'))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
input_shape = (1, 6, 5, 3)
kernel_shape = (3, 3, 3, 2)
xval = np.random.random(input_shape)
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
kernel_val = np.random.random(kernel_shape) - 0.5
kernel_th = KTH.variable(convert_kernel(kernel_val))
kernel_tf = KTF.variable(kernel_val)
zth = KTH.eval(KTH.conv2d(xth, kernel_th, data_format='channels_last'))
ztf = KTF.eval(KTF.conv2d(xtf, kernel_tf, data_format='channels_last'))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
def test_gather(self):
shape = (10, 2, 3)
ref = np.arange(np.prod(shape)).reshape(shape)
ref_th = KTH.variable(ref)
ref_tf = KTF.variable(ref)
inds = [1, 3, 7, 9]
inds_th = KTH.variable(inds, dtype='int32')
inds_tf = KTF.variable(inds, dtype='int32')
th_z = KTH.gather(ref_th, inds_th)
th_result = KTH.eval(th_z)
tf_result = KTF.eval(KTF.gather(ref_tf, inds_tf))
assert_allclose(tf_result, th_result, atol=1e-05)
if hasattr(th_z, '_keras_shape'):
assert th_z._keras_shape == th_result.shape
def test_conv3d(self):
# TH input shape: (samples, input_depth, conv_dim1, conv_dim2, conv_dim3)
# TF input shape: (samples, conv_dim1, conv_dim2, conv_dim3, input_depth)
# TH kernel shape: (depth, input_depth, x, y, z)
# TF kernel shape: (x, y, z, input_depth, depth)
# test in data_format = channels_first
for input_shape in [(2, 3, 4, 5, 4), (2, 3, 5, 4, 6)]:
for kernel_shape in [(2, 2, 2, 3, 4), (3, 2, 4, 3, 4)]:
xval = np.random.random(input_shape)
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
kernel_val = np.random.random(kernel_shape) - 0.5
kernel_th = KTH.variable(convert_kernel(kernel_val))
kernel_tf = KTF.variable(kernel_val)
zth = KTH.eval(
KTH.conv3d(xth, kernel_th, data_format='channels_first'))
ztf = KTF.eval(
KTF.conv3d(xtf, kernel_tf, data_format='channels_first'))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
# test in data_format = channels_last
input_shape = (1, 2, 2, 2, 1)
kernel_shape = (2, 2, 2, 1, 1)
xval = np.random.random(input_shape)
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
kernel_val = np.random.random(kernel_shape) - 0.5
kernel_th = KTH.variable(convert_kernel(kernel_val))
kernel_tf = KTF.variable(kernel_val)
zth = KTH.eval(KTH.conv3d(xth, kernel_th, data_format='channels_last'))
ztf = KTF.eval(KTF.conv3d(xtf, kernel_tf, data_format='channels_last'))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
def test_extract(self):
for input_shape in [(1, 3, 40, 40), (1, 3, 10, 10)]:
for kernel_shape in [2, 5]:
xval = np.random.random(input_shape)
kernel = [kernel_shape, kernel_shape]
strides = [kernel_shape, kernel_shape]
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
ztf = KTF.eval(
KCTF.extract_image_patches(xtf,
kernel,
strides,
dim_ordering='th',
border_mode="valid"))
zth = KTH.eval(
KCTH.extract_image_patches(xth,
kernel,
strides,
dim_ordering='th',
border_mode="valid"))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-02)
for input_shape in [(1, 40, 40, 3), (1, 10, 10, 3)]:
for kernel_shape in [2, 5]:
xval = np.random.random(input_shape)
kernel = [kernel_shape, kernel_shape]
strides = [kernel_shape, kernel_shape]
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
ztf = KTF.eval(
KCTF.extract_image_patches(xtf,
kernel,
strides,
dim_ordering='tf',
border_mode="same"))
zth = KTH.eval(
KCTH.extract_image_patches(xth,
kernel,
strides,
dim_ordering='tf',
border_mode="same"))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-02)
def test_extract(self):
for input_shape in [(1, 3, 40, 40), (1, 3, 10, 10)]:
for kernel_shape in [2, 5]:
xval = np.random.random(input_shape)
kernel = [kernel_shape, kernel_shape]
strides = [kernel_shape, kernel_shape]
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
ztf = KTF.eval(
KCTF.extract_image_patches(xtf,
kernel,
strides,
data_format='channels_first',
padding='valid'))
zth = KTH.eval(
KCTH.extract_image_patches(xth,
kernel,
strides,
data_format='channels_first',
padding='valid'))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-02)
for input_shape in [(1, 40, 40, 3), (1, 10, 10, 3)]:
for kernel_shape in [2, 5]:
xval = np.random.random(input_shape)
kernel = [kernel_shape, kernel_shape]
strides = [kernel_shape, kernel_shape]
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
ztf = KTF.eval(
KCTF.extract_image_patches(xtf,
kernel,
strides,
data_format='channels_last',
padding='same'))
zth = KTH.eval(
KCTH.extract_image_patches(xth,
kernel,
strides,
data_format='channels_last',
padding='same'))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-02)
def test_in_top_k(self):
batch_size = 20
num_classes = 10
# Random prediction test case
predictions = np.random.random(
(batch_size, num_classes)).astype('float32')
targets = np.random.randint(num_classes,
size=batch_size,
dtype='int32')
predictions_th = KTH.variable(predictions, dtype='float32')
targets_th = KTH.variable(targets, dtype='int32')
predictions_tf = KTF.variable(predictions, dtype='float32')
targets_tf = KTF.variable(targets, dtype='int32')
for k in range(1, num_classes + 1):
res_th = KTH.eval(KTH.in_top_k(predictions_th, targets_th, k))
res_tf = KTF.eval(KTF.in_top_k(predictions_tf, targets_tf, k))
assert res_th.shape == res_tf.shape
assert_allclose(res_th, res_tf, atol=1e-05)
# Identical prediction test case:
# randomly set half of the predictions to an identical value
num_identical = num_classes // 2
for i in range(batch_size):
idx_identical = np.random.choice(num_classes,
size=num_identical,
replace=False)
predictions[i, idx_identical] = predictions[i, 0]
targets = np.zeros(batch_size, dtype='int32')
predictions_th = KTH.variable(predictions, dtype='float32')
targets_th = KTH.variable(targets, dtype='int32')
predictions_tf = KTF.variable(predictions, dtype='float32')
targets_tf = KTF.variable(targets, dtype='int32')
for k in range(1, num_classes + 1):
res_th = KTH.eval(KTH.in_top_k(predictions_th, targets_th, k))
res_tf = KTF.eval(KTF.in_top_k(predictions_tf, targets_tf, k))
assert res_th.shape == res_tf.shape
assert_allclose(res_th, res_tf, atol=1e-05)
def test_conv3d(self):
# TH input shape: (samples, input_depth, conv_dim1, conv_dim2, conv_dim3)
# TF input shape: (samples, conv_dim1, conv_dim2, conv_dim3, input_depth)
# TH kernel shape: (depth, input_depth, x, y, z)
# TF kernel shape: (x, y, z, input_depth, depth)
# test in dim_ordering = th
for input_shape in [(2, 3, 4, 5, 4), (2, 3, 5, 4, 6)]:
for kernel_shape in [(4, 3, 2, 2, 2), (4, 3, 3, 2, 4)]:
xval = np.random.random(input_shape)
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
kernel_val = np.random.random(kernel_shape) - 0.5
kernel_th = KTH.variable(convert_kernel(kernel_val))
kernel_tf = KTF.variable(kernel_val)
zth = KTH.eval(KTH.conv3d(xth, kernel_th))
ztf = KTF.eval(KTF.conv3d(xtf, kernel_tf))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
# test in dim_ordering = tf
input_shape = (1, 2, 2, 2, 1)
kernel_shape = (2, 2, 2, 1, 1)
xval = np.random.random(input_shape)
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
kernel_val = np.random.random(kernel_shape) - 0.5
kernel_th = KTH.variable(convert_kernel(kernel_val, dim_ordering='tf'))
kernel_tf = KTF.variable(kernel_val)
zth = KTH.eval(KTH.conv3d(xth, kernel_th, dim_ordering='tf'))
ztf = KTF.eval(KTF.conv3d(xtf, kernel_tf, dim_ordering='tf'))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
def check_single_tensor_operation(function_name, input_shape, **kwargs):
val = np.random.random(input_shape) - 0.5
xth = KTH.variable(val)
xtf = KTF.variable(val)
zth = KTH.eval(getattr(KCTH, function_name)(xth, **kwargs))
ztf = KTF.eval(getattr(KCTF, function_name)(xtf, **kwargs))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
def test_tile(self):
shape = (3, 4)
arr = np.arange(np.prod(shape)).reshape(shape)
arr_th = KTH.variable(arr)
arr_tf = KTF.variable(arr)
n = (2, 1)
th_rep = KTH.eval(KTH.tile(arr_th, n))
tf_rep = KTF.eval(KTF.tile(arr_tf, n))
assert_allclose(tf_rep, th_rep, atol=1e-05)
def test_tile(self):
shape = (3, 4)
arr = np.arange(np.prod(shape)).reshape(shape)
arr_th = KTH.variable(arr)
arr_tf = KTF.variable(arr)
n = (2, 1)
th_rep = KTH.eval(KTH.tile(arr_th, n))
tf_rep = KTF.eval(KTF.tile(arr_tf, n))
assert_allclose(tf_rep, th_rep, atol=1e-05)
def check_single_tensor_operation(function_name, input_shape, **kwargs):
val = np.random.random(input_shape) - 0.5
xth = KTH.variable(val)
xtf = KTF.variable(val)
zth = KTH.eval(getattr(KTH, function_name)(xth, **kwargs))
ztf = KTF.eval(getattr(KTF, function_name)(xtf, **kwargs))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
def test_shape_operations(self):
# concatenate
xval = np.random.random((4, 3))
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
yval = np.random.random((4, 2))
yth = KTH.variable(yval)
ytf = KTF.variable(yval)
zth = KTH.eval(KTH.concatenate([xth, yth], axis=-1))
ztf = KTF.eval(KTF.concatenate([xtf, ytf], axis=-1))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
check_single_tensor_operation("reshape", (4, 2), shape=(8, 1))
check_single_tensor_operation("permute_dimensions", (4, 2, 3), pattern=(2, 0, 1))
check_single_tensor_operation("repeat", (4, 1), n=3)
check_single_tensor_operation("flatten", (4, 1))
check_single_tensor_operation("expand_dims", (4, 3), dim=-1)
check_single_tensor_operation("expand_dims", (4, 3, 2), dim=1)
check_single_tensor_operation("squeeze", (4, 3, 1), axis=2)
def test_conv2d(self):
# TF kernel shape: (rows, cols, input_depth, depth)
# channels_first input shape: (n, input_depth, rows, cols)
for input_shape in [(2, 3, 4, 5), (2, 3, 5, 6)]:
for kernel_shape in [(2, 2, 3, 4), (4, 3, 3, 4)]:
for padding in ['valid', 'same']:
xval = np.random.random(input_shape)
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
kernel_val = np.random.random(kernel_shape) - 0.5
kernel_th = KTH.variable(convert_kernel(kernel_val))
kernel_tf = KTF.variable(kernel_val)
zth = KTH.eval(KTH.conv2d(xth, kernel_th, data_format='channels_first'))
ztf = KTF.eval(KTF.conv2d(xtf, kernel_tf, data_format='channels_first'))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
input_shape = (1, 6, 5, 3)
kernel_shape = (3, 3, 3, 2)
xval = np.random.random(input_shape)
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
kernel_val = np.random.random(kernel_shape) - 0.5
kernel_th = KTH.variable(convert_kernel(kernel_val))
kernel_tf = KTF.variable(kernel_val)
zth = KTH.eval(KTH.conv2d(xth, kernel_th, data_format='channels_last'))
ztf = KTF.eval(KTF.conv2d(xtf, kernel_tf, data_format='channels_last'))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
def test_conv2d(self):
# TH kernel shape: (depth, input_depth, rows, cols)
# TF kernel shape: (rows, cols, input_depth, depth)
for input_shape in [(2, 3, 4, 5), (2, 3, 5, 6)]:
for kernel_shape in [(4, 3, 2, 2), (4, 3, 3, 4)]:
xval = np.random.random(input_shape)
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
kernel_val = np.random.random(kernel_shape) - 0.5
kernel_th = KTH.variable(convert_kernel(kernel_val))
kernel_tf = KTF.variable(kernel_val)
zth = KTH.eval(KTH.conv2d(xth, kernel_th))
ztf = KTF.eval(KTF.conv2d(xtf, kernel_tf))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
input_shape = (1, 6, 5, 3)
kernel_shape = (3, 3, 3, 2)
xval = np.random.random(input_shape)
xth = KTH.variable(xval)
xtf = KTF.variable(xval)
kernel_val = np.random.random(kernel_shape) - 0.5
kernel_th = KTH.variable(convert_kernel(kernel_val, dim_ordering='tf'))
kernel_tf = KTF.variable(kernel_val)
zth = KTH.eval(KTH.conv2d(xth, kernel_th, dim_ordering='tf'))
ztf = KTF.eval(KTF.conv2d(xtf, kernel_tf, dim_ordering='tf'))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
def test_switch(self):
val = np.random.random()
xth = KTH.variable(val)
xth = KTH.switch(xth >= 0.5, xth * 0.1, xth * 0.2)
xtf = KTF.variable(val)
xtf = KTF.switch(xtf >= 0.5, xtf * 0.1, xtf * 0.2)
zth = KTH.eval(xth)
ztf = KTF.eval(xtf)
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
def test_in_top_k(self):
batch_size = 20
num_classes = 10
# Random prediction test case
predictions = np.random.random((batch_size, num_classes)).astype('float32')
targets = np.random.randint(num_classes, size=batch_size, dtype='int32')
predictions_th = KTH.variable(predictions, dtype='float32')
targets_th = KTH.variable(targets, dtype='int32')
predictions_tf = KTF.variable(predictions, dtype='float32')
targets_tf = KTF.variable(targets, dtype='int32')
for k in range(1, num_classes + 1):
res_th = KTH.eval(KTH.in_top_k(predictions_th, targets_th, k))
res_tf = KTF.eval(KTF.in_top_k(predictions_tf, targets_tf, k))
assert res_th.shape == res_tf.shape
assert_allclose(res_th, res_tf, atol=1e-05)
# Identical prediction test case:
# randomly set half of the predictions to an identical value
num_identical = num_classes // 2
for i in range(batch_size):
idx_identical = np.random.choice(num_classes, size=num_identical, replace=False)
predictions[i, idx_identical] = predictions[i, 0]
targets = np.zeros(batch_size, dtype='int32')
predictions_th = KTH.variable(predictions, dtype='float32')
targets_th = KTH.variable(targets, dtype='int32')
predictions_tf = KTF.variable(predictions, dtype='float32')
targets_tf = KTF.variable(targets, dtype='int32')
for k in range(1, num_classes + 1):
res_th = KTH.eval(KTH.in_top_k(predictions_th, targets_th, k))
res_tf = KTF.eval(KTF.in_top_k(predictions_tf, targets_tf, k))
assert res_th.shape == res_tf.shape
assert_allclose(res_th, res_tf, atol=1e-05)
def test_tile(self):
shape = (3, 4)
arr = np.arange(np.prod(shape)).reshape(shape)
arr_th = KTH.variable(arr)
arr_tf = KTF.variable(arr)
n = (2, 1)
th_z = KTH.tile(arr_th, n)
th_rep = KTH.eval(th_z)
tf_rep = KTF.eval(KTF.tile(arr_tf, n))
assert_allclose(tf_rep, th_rep, atol=1e-05)
if hasattr(th_z, '_keras_shape'):
assert th_z._keras_shape == th_rep.shape
def test_repeat_elements(self):
reps = 3
for ndims in [1, 2, 3]:
shape = np.arange(2, 2 + ndims)
arr = np.arange(np.prod(shape)).reshape(shape)
arr_th = KTH.variable(arr)
arr_tf = KTF.variable(arr)
for rep_axis in range(ndims):
np_rep = np.repeat(arr, reps, axis=rep_axis)
th_rep = KTH.eval(KTH.repeat_elements(arr_th, reps, axis=rep_axis))
tf_rep = KTF.eval(KTF.repeat_elements(arr_tf, reps, axis=rep_axis))
assert th_rep.shape == np_rep.shape
assert tf_rep.shape == np_rep.shape
assert_allclose(np_rep, th_rep, atol=1e-05)
assert_allclose(np_rep, tf_rep, atol=1e-05)
def test_rnn_no_states(self):
# implement a simple RNN without states
input_dim = 8
output_dim = 4
timesteps = 5
input_val = np.random.random((32, timesteps, input_dim))
W_i_val = np.random.random((input_dim, output_dim))
def rnn_step_fn(input_dim, output_dim, K):
W_i = K.variable(W_i_val)
def step_function(x, states):
assert len(states) == 0
output = K.dot(x, W_i)
return output, []
return step_function
# test default setup
th_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTH)
th_inputs = KTH.variable(input_val)
th_initial_states = []
last_output, outputs, new_states = KTH.rnn(th_rnn_step_fn, th_inputs,
th_initial_states,
go_backwards=False,
mask=None)
th_last_output = KTH.eval(last_output)
th_outputs = KTH.eval(outputs)
assert len(new_states) == 0
tf_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTF)
tf_inputs = KTF.variable(input_val)
tf_initial_states = []
last_output, outputs, new_states = KTF.rnn(tf_rnn_step_fn, tf_inputs,
tf_initial_states,
go_backwards=False,
mask=None)
tf_last_output = KTF.eval(last_output)
tf_outputs = KTF.eval(outputs)
assert len(new_states) == 0
assert_allclose(tf_last_output, th_last_output, atol=1e-04)
assert_allclose(tf_outputs, th_outputs, atol=1e-04)
def check_composed_tensor_operations(first_function_name, first_function_args,
second_function_name, second_function_args,
input_shape):
''' Creates a random tensor t0 with shape input_shape and compute
t1 = first_function_name(t0, **first_function_args)
t2 = second_function_name(t1, **second_function_args)
with both Theano and TensorFlow backends and ensures the answers match.
'''
val = np.random.random(input_shape) - 0.5
xth = KTH.variable(val)
xtf = KTF.variable(val)
yth = getattr(KTH, first_function_name)(xth, **first_function_args)
ytf = getattr(KTF, first_function_name)(xtf, **first_function_args)
zth = KTH.eval(getattr(KTH, second_function_name)(yth, **second_function_args))
ztf = KTF.eval(getattr(KTF, second_function_name)(ytf, **second_function_args))
assert zth.shape == ztf.shape
assert_allclose(zth, ztf, atol=1e-05)
def test_value_manipulation(self):
val = np.random.random((4, 2))
xth = KTH.variable(val)
xtf = KTF.variable(val)
# get_value
valth = KTH.get_value(xth)
valtf = KTF.get_value(xtf)
assert valtf.shape == valth.shape
assert_allclose(valth, valtf, atol=1e-05)
# set_value
val = np.random.random((4, 2))
KTH.set_value(xth, val)
KTF.set_value(xtf, val)
valth = KTH.get_value(xth)
valtf = KTF.get_value(xtf)
assert valtf.shape == valth.shape
assert_allclose(valth, valtf, atol=1e-05)
# count_params
assert KTH.count_params(xth) == KTF.count_params(xtf)
def test_tile(self):
shape = (3, 4)
arr = np.arange(np.prod(shape)).reshape(shape)
arr_th = KTH.variable(arr)
arr_tf = KTF.variable(arr)
n = (2, 1)
th_z = KTH.tile(arr_th, n)
th_rep = KTH.eval(th_z)
tf_rep = KTF.eval(KTF.tile(arr_tf, n))
assert_allclose(tf_rep, th_rep, atol=1e-05)
if hasattr(th_z, '_keras_shape'):
assert th_z._keras_shape == th_rep.shape
# test theano shape inference when
# input shape has None entries
if K.backend() == 'theano':
x = K.placeholder(shape=(None, 4))
n = 2
y = KTH.tile(x, n)
assert y._keras_shape == (None, 8)
n = (4, 3)
y = K.tile(x, n)
assert y._keras_shape == (None, 12)
def test_rnn(self):
# implement a simple RNN
input_dim = 8
output_dim = 4
timesteps = 5
input_val = np.random.random((32, timesteps, input_dim))
init_state_val = np.random.random((32, output_dim))
W_i_val = np.random.random((input_dim, output_dim))
W_o_val = np.random.random((output_dim, output_dim))
def rnn_step_fn(input_dim, output_dim, K):
W_i = K.variable(W_i_val)
W_o = K.variable(W_o_val)
def step_function(x, states):
assert len(states) == 1
prev_output = states[0]
output = K.dot(x, W_i) + K.dot(prev_output, W_o)
return output, [output]
return step_function
th_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTH)
th_inputs = KTH.variable(input_val)
th_initial_states = [KTH.variable(init_state_val)]
last_output, outputs, new_states = KTH.rnn(th_rnn_step_fn, th_inputs,
th_initial_states,
go_backwards=False,
mask=None)
th_last_output = KTH.eval(last_output)
th_outputs = KTH.eval(outputs)
assert len(new_states) == 1
th_state = KTH.eval(new_states[0])
tf_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTF)
tf_inputs = KTF.variable(input_val)
tf_initial_states = [KTF.variable(init_state_val)]
last_output, outputs, new_states = KTF.rnn(tf_rnn_step_fn, tf_inputs,
tf_initial_states,
go_backwards=False,
mask=None)
tf_last_output = KTF.eval(last_output)
tf_outputs = KTF.eval(outputs)
assert len(new_states) == 1
tf_state = KTF.eval(new_states[0])
assert_allclose(tf_last_output, th_last_output, atol=1e-04)
assert_allclose(tf_outputs, th_outputs, atol=1e-04)
assert_allclose(tf_state, th_state, atol=1e-04)
# test unroll
unrolled_last_output, unrolled_outputs, unrolled_new_states = KTH.rnn(
th_rnn_step_fn, th_inputs,
th_initial_states,
go_backwards=False,
mask=None,
unroll=True,
input_length=timesteps)
unrolled_th_last_output = KTH.eval(unrolled_last_output)
unrolled_th_outputs = KTH.eval(unrolled_outputs)
assert len(unrolled_new_states) == 1
unrolled_th_state = KTH.eval(unrolled_new_states[0])
assert_allclose(th_last_output, unrolled_th_last_output, atol=1e-04)
assert_allclose(th_outputs, unrolled_th_outputs, atol=1e-04)
assert_allclose(th_state, unrolled_th_state, atol=1e-04)
# test unroll with backwards = True
bwd_last_output, bwd_outputs, bwd_new_states = KTH.rnn(
th_rnn_step_fn, th_inputs,
th_initial_states,
go_backwards=True,
mask=None)
bwd_th_last_output = KTH.eval(bwd_last_output)
bwd_th_outputs = KTH.eval(bwd_outputs)
assert len(bwd_new_states) == 1
bwd_th_state = KTH.eval(bwd_new_states[0])
bwd_unrolled_last_output, bwd_unrolled_outputs, bwd_unrolled_new_states = KTH.rnn(
th_rnn_step_fn, th_inputs,
th_initial_states,
go_backwards=True,
mask=None,
unroll=True,
input_length=timesteps)
bwd_unrolled_th_last_output = KTH.eval(bwd_unrolled_last_output)
bwd_unrolled_th_outputs = KTH.eval(bwd_unrolled_outputs)
assert len(bwd_unrolled_new_states) == 1
bwd_unrolled_th_state = KTH.eval(bwd_unrolled_new_states[0])
assert_allclose(bwd_th_last_output, bwd_unrolled_th_last_output, atol=1e-04)
assert_allclose(bwd_th_outputs, bwd_unrolled_th_outputs, atol=1e-04)
assert_allclose(bwd_th_state, bwd_unrolled_th_state, atol=1e-04)
# test unroll with masking
np_mask = np.random.randint(2, size=(32, timesteps))
th_mask = KTH.variable(np_mask)
masked_last_output, masked_outputs, masked_new_states = KTH.rnn(
th_rnn_step_fn, th_inputs,
th_initial_states,
go_backwards=False,
mask=th_mask)
masked_th_last_output = KTH.eval(masked_last_output)
masked_th_outputs = KTH.eval(masked_outputs)
assert len(masked_new_states) == 1
masked_th_state = KTH.eval(masked_new_states[0])
unrolled_masked_last_output, unrolled_masked_outputs, unrolled_masked_new_states = KTH.rnn(
th_rnn_step_fn, th_inputs,
th_initial_states,
go_backwards=False,
mask=th_mask,
unroll=True,
input_length=timesteps)
unrolled_masked_th_last_output = KTH.eval(unrolled_masked_last_output)
unrolled_masked_th_outputs = KTH.eval(unrolled_masked_outputs)
assert len(unrolled_masked_new_states) == 1
unrolled_masked_th_state = KTH.eval(unrolled_masked_new_states[0])
assert_allclose(unrolled_masked_th_last_output, masked_th_last_output, atol=1e-04)
assert_allclose(unrolled_masked_th_outputs, masked_th_outputs, atol=1e-04)
assert_allclose(unrolled_masked_th_state, masked_th_state, atol=1e-04)