def unroll_lstm(fn, sequences, outputs_info, non_sequences, n_steps, go_backwards=False):
if not isinstance(sequences, (list, tuple)):
sequences = [sequences]
# When backwards reverse the recursion direction
counter = range(n_steps)
if go_backwards:
counter = counter[::-1]
output = []
prev_vals = outputs_info
for i in counter:
step_input = [s[i] for s in sequences] + prev_vals + non_sequences
out_ = fn(*step_input)
# The returned values from step can be either a TensorVariable,
# a list, or a tuple. Below, we force it to always be a list.
if type(out_) is not list:
out_ = [out_]
output.append(out_)
prev_vals = output[-1]
# iterate over each scan output and convert it to same format as scan:
# [[output11, output12,...output1n],
# [output21, output22,...output2n],...]
output_scan = []
for i in range(len(output[0])):
l = map(lambda x: x[i], output)
output_scan.append(cgt.stack(l))
return output_scan
def make_prediction(self, max_label_length, ground_labels_basis_btc):
context_i_bf = parameter(init_array(IIDGaussian(0.1), (self.batch_size, self.feature_size)), name=None)
state_i_bf = parameter(init_array(IIDGaussian(0.1), (self.batch_size, self.decoder_size)), name=None)
char_list = []
for iter_step in range(0, max_label_length): #Is this right?
prev_out_bc = ground_labels_basis_btc[:, iter_step, :]
state_i_bf = self.get_decoder_state(context_i_bf, prev_out_bc, state_i_bf)
context_i_bf = self.get_context(state_i_bf)
this_character_dist = self.get_character_distribution(state_i_bf, context_i_bf)
char_list.append(cgt.argmax(this_character_dist, axis=1))
final = cgt.dimshuffle(cgt.stack(char_list), [1, 0])
return final
def pyramidLayer(nn_input, temporal_resolution_decrease=2):
"""
Batch by time by features. Decreases temporal resolution and increases feature dimension by a resolution decrease factor.
"""
t_steps = cgt.infer_shape(nn_input)[1]
if t_steps % temporal_resolution_decrease != 0:
raise ValueError('number of timesteps is not divisable by resolution decrease!')
out_list = []
for iter_step in range(0, t_steps, temporal_resolution_decrease):
concentrate_list = []
for sub_iter_step in range(0, temporal_resolution_decrease):
concentrate_list.append(nn_input[:, iter_step + sub_iter_step, :])
out_list.append(cgt.concatenate(concentrate_list, axis=1))
return cgt.dimshuffle(cgt.stack(out_list), [1, 0, 2])
def temporalDenseLayer(nn_input, num_units, activation=rectify, w_init=XavierNormal(), bias_init=Constant(0)):
"""
Batch by time by features.
"""
if len(nn_input.shape) > 3:
nn_input = nn_input.reshape([nn_input.shape[0], nn_input.shape[1], nn_input.shape[2:]])
dims = cgt.infer_shape(nn_input)
temporal_dims = dims[1]
feature_dims = dims[2]
affine_underbelly = Affine(feature_dims, num_units, weight_init=w_init, bias_init=bias_init)
out_list = []
for iter_step in range(0, temporal_dims):
input_slice = nn_input[:, iter_step, :]
out_list.append(activation(affine_underbelly(input_slice)))
return cgt.dimshuffle(cgt.stack(out_list), [1, 0, 2])
def unroll_recurrence(step_function, input_to_unroll_tbf, hid_init, n_steps, go_backwards=False):
# When backwards reverse the recursion direction
step_counter = range(n_steps)
if go_backwards:
step_counter = step_counter[::-1]
output_t_bh = []
prev_out_bh = hid_init
for step in step_counter:
step_input_bh = [input_to_unroll_tbf[step]] + prev_out_bh
step_out_bh = step_function(*step_input_bh)
step_out_bh = [step_out_bh]
output_t_bh.append(step_out_bh)
prev_out_bh = step_out_bh
l_t_bh = map(lambda x: x[0], output_t_bh)
return cgt.stack(l_t_bh)
def get_context_backup(self, prev_state_bf):
state_step_bf = cgt.sigmoid(self.states_mlp_bf(prev_state_bf))
product_list = []
for time_step in range(0, 3):
inner_product = cgt.sum(state_step_bf*self.features_post_mlp_btf[:, time_step, :], axis=1)
product_list.append(inner_product)
st = cgt.stack(product_list)
st = cgt.dimshuffle(st, [1, 0])
softmax_weights = softmax(st)
sum = None
for time_step in range(0, 3):
softmax_t_step = cgt.dimshuffle(softmax_weights[:, time_step], [0, 'x'])
if sum is None:
sum = cgt.broadcast('*', softmax_t_step, self.features_post_mlp_btf[:, time_step, :], 'x1,xx')
else:
sum += cgt.broadcast('*', softmax_t_step, self.features_post_mlp_btf[:, time_step, :], 'x1,xx')
return sum
def test_stack():
x = cgt.scalar()
y = cgt.scalar()
z = cgt.scalar()
s0 = cgt.stack([x, y, z], axis=0)
assert cgt.numeric_eval(s0, {x: 1, y: 2, z: 3}).shape == (3, )
x = cgt.vector()
y = cgt.vector()
z = cgt.vector()
v0 = cgt.stack([x, y, z], axis=0)
assert cgt.numeric_eval(v0, {
x: np.zeros(2),
y: np.zeros(2),
z: np.zeros(2)
}).shape == (3, 2)
v1 = cgt.stack([x, y, z], axis=1)
assert cgt.numeric_eval(v1, {
x: np.zeros(2),
y: np.ones(2),
z: np.zeros(2)
}).shape == (2, 3)
x = cgt.matrix()
y = cgt.matrix()
z = cgt.matrix()
m0 = cgt.stack([x, y, z], axis=0)
assert cgt.numeric_eval(m0, {
x: np.zeros((2, 4)),
y: np.zeros((2, 4)),
z: np.zeros((2, 4))
}).shape == (3, 2, 4)
m1 = cgt.stack([x, y, z], axis=1)
assert cgt.numeric_eval(m1, {
x: np.zeros((2, 4)),
y: np.zeros((2, 4)),
z: np.zeros((2, 4))
}).shape == (2, 3, 4)
m2 = cgt.stack([x, y, z], axis=2)
assert cgt.numeric_eval(m2, {
x: np.zeros((2, 4)),
y: np.zeros((2, 4)),
z: np.zeros((2, 4))
}).shape == (2, 4, 3)
def test_stack():
x = cgt.scalar()
y = cgt.scalar()
z = cgt.scalar()
s0 = cgt.stack([x, y, z], axis=0)
assert cgt.numeric_eval(s0, {x: 1, y: 2, z: 3}).shape == (3,)
x = cgt.vector()
y = cgt.vector()
z = cgt.vector()
v0 = cgt.stack([x, y, z], axis=0)
assert cgt.numeric_eval(v0, {x: np.zeros(2), y: np.zeros(2), z: np.zeros(2)}).shape == (3, 2)
v1 = cgt.stack([x, y, z], axis=1)
assert cgt.numeric_eval(v1, {x: np.zeros(2), y: np.ones(2), z: np.zeros(2)}).shape == (2, 3)
x = cgt.matrix()
y = cgt.matrix()
z = cgt.matrix()
m0 = cgt.stack([x, y, z], axis=0)
assert cgt.numeric_eval(m0, {x: np.zeros((2, 4)), y: np.zeros((2, 4)), z: np.zeros((2, 4))}).shape == (3, 2, 4)
m1 = cgt.stack([x, y, z], axis=1)
assert cgt.numeric_eval(m1, {x: np.zeros((2, 4)), y: np.zeros((2, 4)), z: np.zeros((2, 4))}).shape == (2, 3, 4)
m2 = cgt.stack([x, y, z], axis=2)
assert cgt.numeric_eval(m2, {x: np.zeros((2, 4)), y: np.zeros((2, 4)), z: np.zeros((2, 4))}).shape == (2, 4, 3)
