def forward(self, X, dropout_ratio, train=True, on_gpu=False, wPauseMask=False, pause_indices=None): """ Given input batch X this function propagates forward through the network, forward and backward through-time and returns a vector representation of the sequences In: X: 3D array of type int32 (first dimension must be time, second dimension batch_samples, and third index is the size of the input space) Out: vector representations of all sequences in the batch """ if wPauseMask: assert pause_indices is not None if on_gpu: X = cuda.to_gpu(X.astype(np.int32)) if pause_indices is not None: pause_indices = cuda.to_gpu(pause_indices.astype(np.int32)) T, batchsize, D = X.shape assert D == 1 states_tm1 = self.make_initial_states(batchsize=batchsize, on_gpu=on_gpu, train=train) states_tp1 = self.make_initial_states(batchsize=batchsize, on_gpu=on_gpu, train=train) h0_list = [] h2_forward_list = [] #forward through time indices_list = [] for t in range(T): x_data = X[t].reshape(batchsize,D) if wPauseMask: indices = Variable(x_data[:,0], volatile=not train) indices_list.append(indices) if t > 0: h2_forward_list.append(nF.pauseMask(states_tm1['h2'], indices, pause_indices)) x = Variable(x_data, volatile=not train) h0 = self.id_to_h0(x) h0_list.append(h0) states_tm1 = self.get_forward_states(h0, states_tm1, dropout_ratio=dropout_ratio, train=train) h2_forward_list.append(states_tm1['h2']) #backward through time h2_backward_deque = deque([]) for t in reversed(range(T)): indices = indices_list[t] if t<(T-1): h2_backward_deque.appendleft(nF.pauseMask(states_tp1['h2'], indices, pause_indices)) states_tp1 = self.get_backward_states(h0_list[t], states_tp1, dropout_ratio=dropout_ratio, train=train) h2_backward_deque.appendleft(states_tp1['h2']) h3_list = [] for h2_f, h2_b in zip(h2_forward_list, h2_backward_deque): h3 = self.h2_to_h3(h2_f, h2_b) h3_list.append(h3) states_tm1_2 = self.make_initial_states2(batchsize=batchsize, on_gpu=on_gpu, train=train) states_tp1_2 = self.make_initial_states2(batchsize=batchsize, on_gpu=on_gpu, train=train) #forward through time for h3 in h3_list: states_tm1_2 = self.get_forward_states2(h3, states_tm1_2) #backward through time for h3 in reversed(h3_list): states_tp1_2 = self.get_backward_states2(h3, states_tp1_2) #get final outputs h5 = self.h4_to_h5(states_tm1_2['h4'], states_tp1_2['h4']) y = self.h5_to_y(h5) return y
def forward(self, X, whitespace_indices, train, on_gpu, ID_col, **kwargs): """ Given input batch X this function propagates forward through the network, forward and backward through-time and returns a vector representation of the sequences In: X: 3D array of type int32 (first dimension must be time, second dimension batch_samples, and third index is the size of the input space) Out: vector representations of all sequences in the batch """ if on_gpu: X = cuda.to_gpu(X.astype(np.int32)) whitespace_indices = cuda.to_gpu(whitespace_indices.astype(np.int32)) T, batchsize, D = X.shape assert D == 1 states_tm1 = self.make_initial_states(batchsize=batchsize, on_gpu=on_gpu, train=train) states_tp1 = self.make_initial_states(batchsize=batchsize, on_gpu=on_gpu, train=train) h0_list = [] h2_forward_list = [] #forward through time indices_list = [] inputs_list = [] for t in range(T): x_data = X[t].reshape(batchsize,D) x = Variable(x_data, volatile=not train) indices = Variable(x_data[:,0], volatile=not train) indices_list.append(indices) inputs_list.append(x) h0 = self.id_to_h0(x) states_tm1 = self.get_forward_states(h0, states_tm1, train=train) h0_list.append(h0) if t < (T-1): h2_forward_list.append(nF.pauseMask(states_tm1['h2'], indices, whitespace_indices)) else: h2_forward_list.append(states_tm1['h2']) if len(h2_forward_list) == 0: h2_forward_list.append(states_tm1['h2']) #backward through time h2_backward_deque = deque([]) for t in reversed(range(T)): x = inputs_list[t] indices = indices_list[t] states_tp1 = self.get_backward_states(h0_list[t], states_tp1, train=train) if t > 0: h2_backward_deque.appendleft(nF.pauseMask(states_tp1['h2'], indices, whitespace_indices)) else: h2_backward_deque.appendleft(states_tp1['h2']) if len(h2_backward_deque) == 0: h2_backward_deque.appendleft(states_tp1['h2']) #attention function / context c = self.get_context(h2_forward_list, h2_backward_deque, train=train) #get final outputs h4 = self.c_to_h4(c) output_func = getattr(self, 'h4_to_'+ID_col) y = output_func(h4) return y
def forward(self, X, dropout_ratio, train=True, on_gpu=False, wPauseMask=False, pause_indices=None): """ Given input batch X this function propagates forward through the network, forward and backward through-time and returns a vector representation of the sequences In: X: 3D array of type int32 (first dimension must be time, second dimension batch_samples, and third index is the size of the input space) Out: vector representations of all sequences in the batch """ if wPauseMask: assert pause_indices is not None if on_gpu: X = cuda.to_gpu(X.astype(np.int32)) if pause_indices is not None: pause_indices = cuda.to_gpu(pause_indices.astype(np.int32)) T, batchsize, D = X.shape assert D == 1 states_tm1 = self.make_initial_states(batchsize=batchsize, on_gpu=on_gpu, train=train) states_tp1 = self.make_initial_states(batchsize=batchsize, on_gpu=on_gpu, train=train) h0_list = [] h2_forward_list = [] # forward through time indices_list = [] for t in range(T): x_data = X[t].reshape(batchsize, D) if wPauseMask: indices = Variable(x_data[:, 0], volatile=not train) indices_list.append(indices) if t > 0: h2_forward_list.append(nF.pauseMask(states_tm1["h2"], indices, pause_indices)) x = Variable(x_data, volatile=not train) h0 = self.id_to_h0(x) h0_list.append(h0) states_tm1 = self.get_forward_states(h0, states_tm1, dropout_ratio=dropout_ratio, train=train) h2_forward_list.append(states_tm1["h2"]) # backward through time h2_backward_deque = deque([]) for t in reversed(range(T)): indices = indices_list[t] if t < (T - 1): h2_backward_deque.appendleft(nF.pauseMask(states_tp1["h2"], indices, pause_indices)) states_tp1 = self.get_backward_states(h0_list[t], states_tp1, dropout_ratio=dropout_ratio, train=train) h2_backward_deque.appendleft(states_tp1["h2"]) h3_list = [] for h2_f, h2_b in zip(h2_forward_list, h2_backward_deque): h3 = self.h2_to_h3(h2_f, h2_b) h3_list.append(h3) states_tm1_2 = self.make_initial_states2(batchsize=batchsize, on_gpu=on_gpu, train=train) states_tp1_2 = self.make_initial_states2(batchsize=batchsize, on_gpu=on_gpu, train=train) # forward through time for h3 in h3_list: states_tm1_2 = self.get_forward_states2(h3, states_tm1_2) # backward through time for h3 in reversed(h3_list): states_tp1_2 = self.get_backward_states2(h3, states_tp1_2) # get final outputs h5 = self.h4_to_h5(states_tm1_2["h4"], states_tp1_2["h4"]) y = self.h5_to_y(h5) return y