def transduce( self, es: 'expression_seqs.ExpressionSequence' ) -> 'expression_seqs.ExpressionSequence': mask = es.mask # first layer forward_es = self.forward_layers[0].transduce(es) rev_backward_es = self.backward_layers[0].transduce( expression_seqs.ReversedExpressionSequence(es)) for layer_i in range(1, len(self.forward_layers)): new_forward_es = self.forward_layers[layer_i].transduce([ forward_es, expression_seqs.ReversedExpressionSequence(rev_backward_es) ]) rev_backward_es = expression_seqs.ExpressionSequence( self.backward_layers[layer_i].transduce([ expression_seqs.ReversedExpressionSequence(forward_es), rev_backward_es ]).as_list(), mask=mask) forward_es = new_forward_es self._final_states = [ transducers.FinalTransducerState(dy.concatenate([self.forward_layers[layer_i].get_final_states()[0].main_expr(), self.backward_layers[layer_i].get_final_states()[ 0].main_expr()]), dy.concatenate([self.forward_layers[layer_i].get_final_states()[0].cell_expr(), self.backward_layers[layer_i].get_final_states()[ 0].cell_expr()])) \ for layer_i in range(len(self.forward_layers))] return expression_seqs.ExpressionSequence(expr_list=[ dy.concatenate([forward_es[i], rev_backward_es[-i - 1]]) for i in range(len(forward_es)) ], mask=mask)
def transduce(self, es: expression_seqs.ExpressionSequence) -> expression_seqs.ExpressionSequence: """ returns the list of output Expressions obtained by adding the given inputs to the current state, one by one, to both the forward and backward RNNs, and concatenating. Args: es: an ExpressionSequence """ es_list = [es] for layer_i, (fb, bb) in enumerate(self.builder_layers): reduce_factor = self._reduce_factor_for_layer(layer_i) if es_list[0].mask is None: mask_out = None else: mask_out = es_list[0].mask.lin_subsampled(reduce_factor) if self.downsampling_method=="concat" and es_list[0].sent_len() % reduce_factor != 0: raise ValueError(f"For 'concat' subsampling, sequence lengths must be multiples of the total reduce factor, " f"but got sequence length={es_list[0].sent_len()} for reduce_factor={reduce_factor}. " f"Set Batcher's pad_src_to_multiple argument accordingly.") fs = fb.transduce(es_list) bs = bb.transduce([expression_seqs.ReversedExpressionSequence(es_item) for es_item in es_list]) if layer_i < len(self.builder_layers) - 1: if self.downsampling_method=="skip": es_list = [expression_seqs.ExpressionSequence(expr_list=fs[::reduce_factor], mask=mask_out), expression_seqs.ExpressionSequence(expr_list=bs[::reduce_factor][::-1], mask=mask_out)] elif self.downsampling_method=="concat": es_len = es_list[0].sent_len() es_list_fwd = [] es_list_bwd = [] for i in range(0, es_len, reduce_factor): for j in range(reduce_factor): if i==0: es_list_fwd.append([]) es_list_bwd.append([]) es_list_fwd[j].append(fs[i+j]) es_list_bwd[j].append(bs[es_list[0].sent_len()-reduce_factor+j-i]) es_list = [expression_seqs.ExpressionSequence(expr_list=es_list_fwd[j], mask=mask_out) for j in range(reduce_factor)] + \ [expression_seqs.ExpressionSequence(expr_list=es_list_bwd[j], mask=mask_out) for j in range(reduce_factor)] else: raise RuntimeError(f"unknown downsampling_method {self.downsampling_method}") else: # concat final outputs ret_es = expression_seqs.ExpressionSequence( expr_list=[tt.concatenate([f, b]) for f, b in zip(fs, expression_seqs.ReversedExpressionSequence(bs))], mask=mask_out) self._final_states = [transducers.FinalTransducerState(tt.concatenate([fb.get_final_states()[0].main_expr(), bb.get_final_states()[0].main_expr()]), tt.concatenate([fb.get_final_states()[0].cell_expr(), bb.get_final_states()[0].cell_expr()])) \ for (fb, bb) in self.builder_layers] return ret_es
def transduce(self, es): mask = es.mask # first layer forward_es = self.forward_layers[0].transduce(es) rev_backward_es = self.backward_layers[0].transduce( expression_seqs.ReversedExpressionSequence(es)) # TODO: concat input of each layer to its output; or, maybe just add standard residual connections for layer_i in range(1, len(self.forward_layers)): new_forward_es = self.forward_layers[layer_i].transduce([ forward_es, expression_seqs.ReversedExpressionSequence(rev_backward_es) ]) mask_out = mask if mask_out is not None and new_forward_es.mask.np_arr.shape != mask_out.np_arr.shape: mask_out = mask_out.lin_subsampled(trg_len=len(new_forward_es)) rev_backward_es = expression_seqs.ExpressionSequence( self.backward_layers[layer_i].transduce([ expression_seqs.ReversedExpressionSequence(forward_es), rev_backward_es ]).as_list(), mask=mask_out) forward_es = new_forward_es self._final_states = [ transducers.FinalTransducerState(dy.concatenate([self.forward_layers[layer_i].get_final_states()[0].main_expr(), self.backward_layers[layer_i].get_final_states()[ 0].main_expr()]), dy.concatenate([self.forward_layers[layer_i].get_final_states()[0].cell_expr(), self.backward_layers[layer_i].get_final_states()[ 0].cell_expr()])) \ for layer_i in range(len(self.forward_layers))] mask_out = mask if mask_out is not None and forward_es.mask.np_arr.shape != mask_out.np_arr.shape: mask_out = mask_out.lin_subsampled(trg_len=len(forward_es)) return expression_seqs.ExpressionSequence(expr_list=[ dy.concatenate([forward_es[i], rev_backward_es[-i - 1]]) for i in range(len(forward_es)) ], mask=mask_out)
def transduce(self, es: expression_seqs.ExpressionSequence) -> expression_seqs.ExpressionSequence: for layer_i, (fb, bb) in enumerate(self.lstm_layers): fs = fb.transduce(es) bs = bb.transduce(expression_seqs.ReversedExpressionSequence(es)) interleaved = [] if es.mask is None: mask = None else: mask = es.mask.lin_subsampled(0.5) # upsample the mask to encompass interleaved fwd / bwd expressions for pos in range(len(fs)): interleaved.append(fs[pos]) interleaved.append(bs[-pos-1]) projected = expression_seqs.ExpressionSequence(expr_list=interleaved, mask=mask) projected = self.nin_layers[layer_i].transduce(projected) assert math.ceil(len(es) / float(self.stride))==len(projected), \ f"mismatched len(es)=={len(es)}, stride=={self.stride}, len(projected)=={len(projected)}" es = projected self._final_states = [transducers.FinalTransducerState(projected[-1])] return projected
def transduce( self, expr_sequence: expression_seqs.ExpressionSequence ) -> expression_seqs.ExpressionSequence: # first layer forward_es = self.forward_layers[0].transduce(expr_sequence) rev_backward_es = self.backward_layers[0].transduce( expression_seqs.ReversedExpressionSequence(expr_sequence)) for layer_i in range(1, len(self.forward_layers)): concat_fwd = expression_seqs.ExpressionSequence(expr_list=[ dy.concatenate([fwd_expr, bwd_expr]) for fwd_expr, bwd_expr in zip( forward_es.as_list(), reversed(rev_backward_es.as_list())) ]) concat_bwd = expression_seqs.ExpressionSequence(expr_list=[ dy.concatenate([fwd_expr, bwd_expr]) for fwd_expr, bwd_expr in zip(reversed(forward_es.as_list()), rev_backward_es.as_list()) ]) new_forward_es = self.forward_layers[layer_i].transduce(concat_fwd) rev_backward_es = self.backward_layers[layer_i].transduce( concat_bwd) forward_es = new_forward_es self._final_states = [ transducers.FinalTransducerState(dy.concatenate([self.forward_layers[layer_i].get_final_states()[0].main_expr(), self.backward_layers[layer_i].get_final_states()[ 0].main_expr()]), dy.concatenate([self.forward_layers[layer_i].get_final_states()[0].cell_expr(), self.backward_layers[layer_i].get_final_states()[ 0].cell_expr()])) \ for layer_i in range(len(self.forward_layers))] return expression_seqs.ExpressionSequence(expr_list=[ dy.concatenate([forward_es[i], rev_backward_es[-i - 1]]) for i in range(len(forward_es)) ])