def forward(self, X, dropout_ratio, train=True, on_gpu=False,
wPauseMask=False, pause_indices=None):
"""
Given input batch X this function propagates forward through the network,
forward and backward through-time and returns a vector representation of the sequences
In:
X: 3D array of type int32
(first dimension must be time, second dimension batch_samples,
and third index is the size of the input space)
Out:
vector representations of all sequences in the batch
"""
if wPauseMask:
assert pause_indices is not None
if on_gpu:
X = cuda.to_gpu(X.astype(np.int32))
if pause_indices is not None:
pause_indices = cuda.to_gpu(pause_indices.astype(np.int32))
T, batchsize, D = X.shape
assert D == 1
states_tm1 = self.make_initial_states(batchsize=batchsize,
on_gpu=on_gpu,
train=train)
states_tp1 = self.make_initial_states(batchsize=batchsize,
on_gpu=on_gpu,
train=train)
h0_list = []
h2_forward_list = []
#forward through time
indices_list = []
for t in range(T):
x_data = X[t].reshape(batchsize,D)
if wPauseMask:
indices = Variable(x_data[:,0], volatile=not train)
indices_list.append(indices)
if t > 0:
h2_forward_list.append(nF.pauseMask(states_tm1['h2'], indices,
pause_indices))
x = Variable(x_data, volatile=not train)
h0 = self.id_to_h0(x)
h0_list.append(h0)
states_tm1 = self.get_forward_states(h0, states_tm1,
dropout_ratio=dropout_ratio,
train=train)
h2_forward_list.append(states_tm1['h2'])
#backward through time
h2_backward_deque = deque([])
for t in reversed(range(T)):
indices = indices_list[t]
if t<(T-1):
h2_backward_deque.appendleft(nF.pauseMask(states_tp1['h2'],
indices, pause_indices))
states_tp1 = self.get_backward_states(h0_list[t], states_tp1,
dropout_ratio=dropout_ratio,
train=train)
h2_backward_deque.appendleft(states_tp1['h2'])
h3_list = []
for h2_f, h2_b in zip(h2_forward_list, h2_backward_deque):
h3 = self.h2_to_h3(h2_f, h2_b)
h3_list.append(h3)
states_tm1_2 = self.make_initial_states2(batchsize=batchsize,
on_gpu=on_gpu,
train=train)
states_tp1_2 = self.make_initial_states2(batchsize=batchsize,
on_gpu=on_gpu,
train=train)
#forward through time
for h3 in h3_list:
states_tm1_2 = self.get_forward_states2(h3, states_tm1_2)
#backward through time
for h3 in reversed(h3_list):
states_tp1_2 = self.get_backward_states2(h3, states_tp1_2)
#get final outputs
h5 = self.h4_to_h5(states_tm1_2['h4'], states_tp1_2['h4'])
y = self.h5_to_y(h5)
return y
def forward(self, X, whitespace_indices, train, on_gpu, ID_col, **kwargs):
"""
Given input batch X this function propagates forward through the network,
forward and backward through-time and returns a vector representation of the sequences
In:
X: 3D array of type int32
(first dimension must be time, second dimension batch_samples,
and third index is the size of the input space)
Out:
vector representations of all sequences in the batch
"""
if on_gpu:
X = cuda.to_gpu(X.astype(np.int32))
whitespace_indices = cuda.to_gpu(whitespace_indices.astype(np.int32))
T, batchsize, D = X.shape
assert D == 1
states_tm1 = self.make_initial_states(batchsize=batchsize,
on_gpu=on_gpu,
train=train)
states_tp1 = self.make_initial_states(batchsize=batchsize,
on_gpu=on_gpu,
train=train)
h0_list = []
h2_forward_list = []
#forward through time
indices_list = []
inputs_list = []
for t in range(T):
x_data = X[t].reshape(batchsize,D)
x = Variable(x_data, volatile=not train)
indices = Variable(x_data[:,0], volatile=not train)
indices_list.append(indices)
inputs_list.append(x)
h0 = self.id_to_h0(x)
states_tm1 = self.get_forward_states(h0, states_tm1,
train=train)
h0_list.append(h0)
if t < (T-1):
h2_forward_list.append(nF.pauseMask(states_tm1['h2'], indices,
whitespace_indices))
else:
h2_forward_list.append(states_tm1['h2'])
if len(h2_forward_list) == 0:
h2_forward_list.append(states_tm1['h2'])
#backward through time
h2_backward_deque = deque([])
for t in reversed(range(T)):
x = inputs_list[t]
indices = indices_list[t]
states_tp1 = self.get_backward_states(h0_list[t], states_tp1,
train=train)
if t > 0:
h2_backward_deque.appendleft(nF.pauseMask(states_tp1['h2'], indices,
whitespace_indices))
else:
h2_backward_deque.appendleft(states_tp1['h2'])
if len(h2_backward_deque) == 0:
h2_backward_deque.appendleft(states_tp1['h2'])
#attention function / context
c = self.get_context(h2_forward_list, h2_backward_deque,
train=train)
#get final outputs
h4 = self.c_to_h4(c)
output_func = getattr(self, 'h4_to_'+ID_col)
y = output_func(h4)
return y
def forward(self, X, dropout_ratio, train=True, on_gpu=False, wPauseMask=False, pause_indices=None):
"""
Given input batch X this function propagates forward through the network,
forward and backward through-time and returns a vector representation of the sequences
In:
X: 3D array of type int32
(first dimension must be time, second dimension batch_samples,
and third index is the size of the input space)
Out:
vector representations of all sequences in the batch
"""
if wPauseMask:
assert pause_indices is not None
if on_gpu:
X = cuda.to_gpu(X.astype(np.int32))
if pause_indices is not None:
pause_indices = cuda.to_gpu(pause_indices.astype(np.int32))
T, batchsize, D = X.shape
assert D == 1
states_tm1 = self.make_initial_states(batchsize=batchsize, on_gpu=on_gpu, train=train)
states_tp1 = self.make_initial_states(batchsize=batchsize, on_gpu=on_gpu, train=train)
h0_list = []
h2_forward_list = []
# forward through time
indices_list = []
for t in range(T):
x_data = X[t].reshape(batchsize, D)
if wPauseMask:
indices = Variable(x_data[:, 0], volatile=not train)
indices_list.append(indices)
if t > 0:
h2_forward_list.append(nF.pauseMask(states_tm1["h2"], indices, pause_indices))
x = Variable(x_data, volatile=not train)
h0 = self.id_to_h0(x)
h0_list.append(h0)
states_tm1 = self.get_forward_states(h0, states_tm1, dropout_ratio=dropout_ratio, train=train)
h2_forward_list.append(states_tm1["h2"])
# backward through time
h2_backward_deque = deque([])
for t in reversed(range(T)):
indices = indices_list[t]
if t < (T - 1):
h2_backward_deque.appendleft(nF.pauseMask(states_tp1["h2"], indices, pause_indices))
states_tp1 = self.get_backward_states(h0_list[t], states_tp1, dropout_ratio=dropout_ratio, train=train)
h2_backward_deque.appendleft(states_tp1["h2"])
h3_list = []
for h2_f, h2_b in zip(h2_forward_list, h2_backward_deque):
h3 = self.h2_to_h3(h2_f, h2_b)
h3_list.append(h3)
states_tm1_2 = self.make_initial_states2(batchsize=batchsize, on_gpu=on_gpu, train=train)
states_tp1_2 = self.make_initial_states2(batchsize=batchsize, on_gpu=on_gpu, train=train)
# forward through time
for h3 in h3_list:
states_tm1_2 = self.get_forward_states2(h3, states_tm1_2)
# backward through time
for h3 in reversed(h3_list):
states_tp1_2 = self.get_backward_states2(h3, states_tp1_2)
# get final outputs
h5 = self.h4_to_h5(states_tm1_2["h4"], states_tp1_2["h4"])
y = self.h5_to_y(h5)
return y
