def get_output(self, train=False):
# input shape: (nb_samples, time (padded with zeros), input_dim)
X = self.get_input(train)
self.h_t = X[1]
self.h_s = X[0]
if self.feed_state:
self.h_init = X[2]
self.P_j = K.dot(self.h_s, self.W_s)
if self.stateful and not train:
initial_states = self.states
else:
initial_states = self.get_initial_states(self.h_s)
if self.feed_state:
initial_states[0] = self.h_init
last_output, outputs, states = K.rnn(self.step, self.h_t, initial_states)
if self.stateful:
self.updates = []
for i in range(len(states)):
self.updates.append((self.states[i], states[i]))
if self.return_sequences:
return outputs
else:
return last_output
def test_rnn(self):
# implement a simple RNN
input_dim = 8
output_dim = 4
timesteps = 5
input_val = np.random.random((32, timesteps, input_dim))
init_state_val = np.random.random((32, output_dim))
W_i_val = np.random.random((input_dim, output_dim))
W_o_val = np.random.random((output_dim, output_dim))
def rnn_step_fn(input_dim, output_dim, K):
W_i = K.variable(W_i_val)
W_o = K.variable(W_o_val)
def step_function(x, states):
assert len(states) == 1
prev_output = states[0]
output = K.dot(x, W_i) + K.dot(prev_output, W_o)
return output, [output]
return step_function
th_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTH)
inputs = KTH.variable(input_val)
initial_states = [KTH.variable(init_state_val)]
last_output, outputs, new_states = KTH.rnn(th_rnn_step_fn,
inputs,
initial_states,
go_backwards=False,
mask=None)
th_last_output = KTH.eval(last_output)
th_outputs = KTH.eval(outputs)
assert len(new_states) == 1
th_state = KTH.eval(new_states[0])
tf_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTF)
inputs = KTF.variable(input_val)
initial_states = [KTF.variable(init_state_val)]
last_output, outputs, new_states = KTF.rnn(tf_rnn_step_fn,
inputs,
initial_states,
go_backwards=False,
mask=None)
tf_last_output = KTF.eval(last_output)
tf_outputs = KTF.eval(outputs)
assert len(new_states) == 1
tf_state = KTF.eval(new_states[0])
assert_allclose(tf_last_output, th_last_output, atol=1e-04)
assert_allclose(tf_outputs, th_outputs, atol=1e-04)
assert_allclose(tf_state, th_state, atol=1e-04)
def test_rnn(self):
# implement a simple RNN
input_dim = 8
output_dim = 4
timesteps = 5
input_val = np.random.random((32, timesteps, input_dim))
init_state_val = np.random.random((32, output_dim))
W_i_val = np.random.random((input_dim, output_dim))
W_o_val = np.random.random((output_dim, output_dim))
def rnn_step_fn(input_dim, output_dim, K):
W_i = K.variable(W_i_val)
W_o = K.variable(W_o_val)
def step_function(x, states):
assert len(states) == 1
prev_output = states[0]
output = K.dot(x, W_i) + K.dot(prev_output, W_o)
return output, [output]
return step_function
th_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTH)
inputs = KTH.variable(input_val)
initial_states = [KTH.variable(init_state_val)]
last_output, outputs, new_states = KTH.rnn(th_rnn_step_fn, inputs,
initial_states,
go_backwards=False,
masking=False)
th_last_output = KTH.eval(last_output)
th_outputs = KTH.eval(outputs)
assert len(new_states) == 1
th_state = KTH.eval(new_states[0])
tf_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTF)
inputs = KTF.variable(input_val)
initial_states = [KTF.variable(init_state_val)]
last_output, outputs, new_states = KTF.rnn(tf_rnn_step_fn, inputs,
initial_states,
go_backwards=False,
masking=False)
tf_last_output = KTF.eval(last_output)
tf_outputs = KTF.eval(outputs)
assert len(new_states) == 1
tf_state = KTF.eval(new_states[0])
assert_allclose(tf_last_output, th_last_output, atol=1e-04)
assert_allclose(tf_outputs, th_outputs, atol=1e-04)
assert_allclose(tf_state, th_state, atol=1e-04)
def test_rnn_no_states(self):
# implement a simple RNN without states
input_dim = 8
output_dim = 4
timesteps = 5
input_val = np.random.random((32, timesteps, input_dim))
W_i_val = np.random.random((input_dim, output_dim))
def rnn_step_fn(input_dim, output_dim, K):
W_i = K.variable(W_i_val)
def step_function(x, states):
assert len(states) == 0
output = K.dot(x, W_i)
return output, []
return step_function
# test default setup
th_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTH)
th_inputs = KTH.variable(input_val)
th_initial_states = []
last_output, outputs, new_states = KTH.rnn(th_rnn_step_fn,
th_inputs,
th_initial_states,
go_backwards=False,
mask=None)
th_last_output = KTH.eval(last_output)
th_outputs = KTH.eval(outputs)
assert len(new_states) == 0
tf_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTF)
tf_inputs = KTF.variable(input_val)
tf_initial_states = []
last_output, outputs, new_states = KTF.rnn(tf_rnn_step_fn,
tf_inputs,
tf_initial_states,
go_backwards=False,
mask=None)
tf_last_output = KTF.eval(last_output)
tf_outputs = KTF.eval(outputs)
assert len(new_states) == 0
assert_allclose(tf_last_output, th_last_output, atol=1e-04)
assert_allclose(tf_outputs, th_outputs, atol=1e-04)
def test_rnn_no_states(self):
# implement a simple RNN without states
input_dim = 8
output_dim = 4
timesteps = 5
input_val = np.random.random((32, timesteps, input_dim))
W_i_val = np.random.random((input_dim, output_dim))
def rnn_step_fn(input_dim, output_dim, K):
W_i = K.variable(W_i_val)
def step_function(x, states):
assert len(states) == 0
output = K.dot(x, W_i)
return output, []
return step_function
# test default setup
th_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTH)
th_inputs = KTH.variable(input_val)
th_initial_states = []
last_output, outputs, new_states = KTH.rnn(th_rnn_step_fn, th_inputs,
th_initial_states,
go_backwards=False,
mask=None)
th_last_output = KTH.eval(last_output)
th_outputs = KTH.eval(outputs)
assert len(new_states) == 0
tf_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTF)
tf_inputs = KTF.variable(input_val)
tf_initial_states = []
last_output, outputs, new_states = KTF.rnn(tf_rnn_step_fn, tf_inputs,
tf_initial_states,
go_backwards=False,
mask=None)
tf_last_output = KTF.eval(last_output)
tf_outputs = KTF.eval(outputs)
assert len(new_states) == 0
assert_allclose(tf_last_output, th_last_output, atol=1e-04)
assert_allclose(tf_outputs, th_outputs, atol=1e-04)
def test_rnn(self):
# implement a simple RNN
input_dim = 8
output_dim = 4
timesteps = 5
input_val = np.random.random((32, timesteps, input_dim))
init_state_val = np.random.random((32, output_dim))
W_i_val = np.random.random((input_dim, output_dim))
W_o_val = np.random.random((output_dim, output_dim))
def rnn_step_fn(input_dim, output_dim, K):
W_i = K.variable(W_i_val)
W_o = K.variable(W_o_val)
def step_function(x, states):
assert len(states) == 1
prev_output = states[0]
output = K.dot(x, W_i) + K.dot(prev_output, W_o)
return output, [output]
return step_function
th_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTH)
th_inputs = KTH.variable(input_val)
th_initial_states = [KTH.variable(init_state_val)]
last_output, outputs, new_states = KTH.rnn(th_rnn_step_fn, th_inputs,
th_initial_states,
go_backwards=False,
mask=None)
th_last_output = KTH.eval(last_output)
th_outputs = KTH.eval(outputs)
assert len(new_states) == 1
th_state = KTH.eval(new_states[0])
tf_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTF)
tf_inputs = KTF.variable(input_val)
tf_initial_states = [KTF.variable(init_state_val)]
last_output, outputs, new_states = KTF.rnn(tf_rnn_step_fn, tf_inputs,
tf_initial_states,
go_backwards=False,
mask=None)
tf_last_output = KTF.eval(last_output)
tf_outputs = KTF.eval(outputs)
assert len(new_states) == 1
tf_state = KTF.eval(new_states[0])
assert_allclose(tf_last_output, th_last_output, atol=1e-04)
assert_allclose(tf_outputs, th_outputs, atol=1e-04)
assert_allclose(tf_state, th_state, atol=1e-04)
# test unroll
unrolled_last_output, unrolled_outputs, unrolled_new_states = KTH.rnn(
th_rnn_step_fn, th_inputs,
th_initial_states,
go_backwards=False,
mask=None,
unroll=True,
input_length=timesteps)
unrolled_th_last_output = KTH.eval(unrolled_last_output)
unrolled_th_outputs = KTH.eval(unrolled_outputs)
assert len(unrolled_new_states) == 1
unrolled_th_state = KTH.eval(unrolled_new_states[0])
assert_allclose(th_last_output, unrolled_th_last_output, atol=1e-04)
assert_allclose(th_outputs, unrolled_th_outputs, atol=1e-04)
assert_allclose(th_state, unrolled_th_state, atol=1e-04)
# test unroll with backwards = True
bwd_last_output, bwd_outputs, bwd_new_states = KTH.rnn(
th_rnn_step_fn, th_inputs,
th_initial_states,
go_backwards=True,
mask=None)
bwd_th_last_output = KTH.eval(bwd_last_output)
bwd_th_outputs = KTH.eval(bwd_outputs)
assert len(bwd_new_states) == 1
bwd_th_state = KTH.eval(bwd_new_states[0])
bwd_unrolled_last_output, bwd_unrolled_outputs, bwd_unrolled_new_states = KTH.rnn(
th_rnn_step_fn, th_inputs,
th_initial_states,
go_backwards=True,
mask=None,
unroll=True,
input_length=timesteps)
bwd_unrolled_th_last_output = KTH.eval(bwd_unrolled_last_output)
bwd_unrolled_th_outputs = KTH.eval(bwd_unrolled_outputs)
assert len(bwd_unrolled_new_states) == 1
bwd_unrolled_th_state = KTH.eval(bwd_unrolled_new_states[0])
assert_allclose(bwd_th_last_output, bwd_unrolled_th_last_output, atol=1e-04)
assert_allclose(bwd_th_outputs, bwd_unrolled_th_outputs, atol=1e-04)
assert_allclose(bwd_th_state, bwd_unrolled_th_state, atol=1e-04)
# test unroll with masking
np_mask = np.random.randint(2, size=(32, timesteps))
th_mask = KTH.variable(np_mask)
masked_last_output, masked_outputs, masked_new_states = KTH.rnn(
th_rnn_step_fn, th_inputs,
th_initial_states,
go_backwards=False,
mask=th_mask)
masked_th_last_output = KTH.eval(masked_last_output)
masked_th_outputs = KTH.eval(masked_outputs)
assert len(masked_new_states) == 1
masked_th_state = KTH.eval(masked_new_states[0])
unrolled_masked_last_output, unrolled_masked_outputs, unrolled_masked_new_states = KTH.rnn(
th_rnn_step_fn, th_inputs,
th_initial_states,
go_backwards=False,
mask=th_mask,
unroll=True,
input_length=timesteps)
unrolled_masked_th_last_output = KTH.eval(unrolled_masked_last_output)
unrolled_masked_th_outputs = KTH.eval(unrolled_masked_outputs)
assert len(unrolled_masked_new_states) == 1
unrolled_masked_th_state = KTH.eval(unrolled_masked_new_states[0])
assert_allclose(unrolled_masked_th_last_output, masked_th_last_output, atol=1e-04)
assert_allclose(unrolled_masked_th_outputs, masked_th_outputs, atol=1e-04)
assert_allclose(unrolled_masked_th_state, masked_th_state, atol=1e-04)
def test_rnn(self):
# implement a simple RNN
input_dim = 8
output_dim = 4
timesteps = 5
input_val = np.random.random((32, timesteps, input_dim))
init_state_val = np.random.random((32, output_dim))
W_i_val = np.random.random((input_dim, output_dim))
W_o_val = np.random.random((output_dim, output_dim))
def rnn_step_fn(input_dim, output_dim, K):
W_i = K.variable(W_i_val)
W_o = K.variable(W_o_val)
def step_function(x, states):
assert len(states) == 1
prev_output = states[0]
output = K.dot(x, W_i) + K.dot(prev_output, W_o)
return output, [output]
return step_function
# test default setup
th_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTH)
th_inputs = KTH.variable(input_val)
th_initial_states = [KTH.variable(init_state_val)]
last_output, outputs, new_states = KTH.rnn(th_rnn_step_fn,
th_inputs,
th_initial_states,
go_backwards=False,
mask=None)
th_last_output = KTH.eval(last_output)
th_outputs = KTH.eval(outputs)
assert len(new_states) == 1
th_state = KTH.eval(new_states[0])
tf_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTF)
tf_inputs = KTF.variable(input_val)
tf_initial_states = [KTF.variable(init_state_val)]
last_output, outputs, new_states = KTF.rnn(tf_rnn_step_fn,
tf_inputs,
tf_initial_states,
go_backwards=False,
mask=None)
tf_last_output = KTF.eval(last_output)
tf_outputs = KTF.eval(outputs)
assert len(new_states) == 1
tf_state = KTF.eval(new_states[0])
assert_allclose(tf_last_output, th_last_output, atol=1e-04)
assert_allclose(tf_outputs, th_outputs, atol=1e-04)
assert_allclose(tf_state, th_state, atol=1e-04)
# test unroll
unrolled_last_output, unrolled_outputs, unrolled_new_states = KTH.rnn(
th_rnn_step_fn,
th_inputs,
th_initial_states,
go_backwards=False,
mask=None,
unroll=True,
input_length=timesteps)
unrolled_th_last_output = KTH.eval(unrolled_last_output)
unrolled_th_outputs = KTH.eval(unrolled_outputs)
assert len(unrolled_new_states) == 1
unrolled_th_state = KTH.eval(unrolled_new_states[0])
assert_allclose(th_last_output, unrolled_th_last_output, atol=1e-04)
assert_allclose(th_outputs, unrolled_th_outputs, atol=1e-04)
assert_allclose(th_state, unrolled_th_state, atol=1e-04)
# test go_backwards
th_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTH)
th_inputs = KTH.variable(input_val)
th_initial_states = [KTH.variable(init_state_val)]
last_output, outputs, new_states = KTH.rnn(th_rnn_step_fn,
th_inputs,
th_initial_states,
go_backwards=True,
mask=None)
th_last_output = KTH.eval(last_output)
th_outputs = KTH.eval(outputs)
assert len(new_states) == 1
th_state = KTH.eval(new_states[0])
tf_rnn_step_fn = rnn_step_fn(input_dim, output_dim, KTF)
tf_inputs = KTF.variable(input_val)
tf_initial_states = [KTF.variable(init_state_val)]
last_output, outputs, new_states = KTF.rnn(tf_rnn_step_fn,
tf_inputs,
tf_initial_states,
go_backwards=True,
mask=None)
tf_last_output = KTF.eval(last_output)
tf_outputs = KTF.eval(outputs)
assert len(new_states) == 1
tf_state = KTF.eval(new_states[0])
assert_allclose(tf_last_output, th_last_output, atol=1e-04)
assert_allclose(tf_outputs, th_outputs, atol=1e-04)
assert_allclose(tf_state, th_state, atol=1e-04)
# test unroll with backwards = True
bwd_last_output, bwd_outputs, bwd_new_states = KTH.rnn(
th_rnn_step_fn,
th_inputs,
th_initial_states,
go_backwards=True,
mask=None)
bwd_th_last_output = KTH.eval(bwd_last_output)
bwd_th_outputs = KTH.eval(bwd_outputs)
assert len(bwd_new_states) == 1
bwd_th_state = KTH.eval(bwd_new_states[0])
bwd_unrolled_last_output, bwd_unrolled_outputs, bwd_unrolled_new_states = KTH.rnn(
th_rnn_step_fn,
th_inputs,
th_initial_states,
go_backwards=True,
mask=None,
unroll=True,
input_length=timesteps)
bwd_unrolled_th_last_output = KTH.eval(bwd_unrolled_last_output)
bwd_unrolled_th_outputs = KTH.eval(bwd_unrolled_outputs)
assert len(bwd_unrolled_new_states) == 1
bwd_unrolled_th_state = KTH.eval(bwd_unrolled_new_states[0])
assert_allclose(bwd_th_last_output,
bwd_unrolled_th_last_output,
atol=1e-04)
assert_allclose(bwd_th_outputs, bwd_unrolled_th_outputs, atol=1e-04)
assert_allclose(bwd_th_state, bwd_unrolled_th_state, atol=1e-04)
# test unroll with masking
np_mask = np.random.randint(2, size=(32, timesteps))
th_mask = KTH.variable(np_mask)
masked_last_output, masked_outputs, masked_new_states = KTH.rnn(
th_rnn_step_fn,
th_inputs,
th_initial_states,
go_backwards=False,
mask=th_mask)
masked_th_last_output = KTH.eval(masked_last_output)
masked_th_outputs = KTH.eval(masked_outputs)
assert len(masked_new_states) == 1
masked_th_state = KTH.eval(masked_new_states[0])
unrolled_masked_last_output, unrolled_masked_outputs, unrolled_masked_new_states = KTH.rnn(
th_rnn_step_fn,
th_inputs,
th_initial_states,
go_backwards=False,
mask=th_mask,
unroll=True,
input_length=timesteps)
unrolled_masked_th_last_output = KTH.eval(unrolled_masked_last_output)
unrolled_masked_th_outputs = KTH.eval(unrolled_masked_outputs)
assert len(unrolled_masked_new_states) == 1
unrolled_masked_th_state = KTH.eval(unrolled_masked_new_states[0])
assert_allclose(unrolled_masked_th_last_output,
masked_th_last_output,
atol=1e-04)
assert_allclose(unrolled_masked_th_outputs,
masked_th_outputs,
atol=1e-04)
assert_allclose(unrolled_masked_th_state, masked_th_state, atol=1e-04)