def transduce(self, sent: ExpressionSequence) -> ExpressionSequence:
if self.pos_encoding_type == "trigonometric":
if self.position_encoding_block is None or self.position_encoding_block.shape[
2] < sent.sent_len():
self.initialize_position_encoding(
int(sent.sent_len() * 1.2),
self.input_dim if self.pos_encoding_combine == "add" else
self.pos_encoding_size)
encoding = dy.inputTensor(
self.position_encoding_block[0, :, :sent.sent_len()])
elif self.pos_encoding_type == "embedding":
encoding = self.positional_embedder.embed_sent(
sent.sent_len()).as_tensor()
if self.pos_encoding_type:
if self.pos_encoding_combine == "add":
sent = ExpressionSequence(expr_tensor=sent.as_tensor() +
encoding,
mask=sent.mask)
else: # concat
sent = ExpressionSequence(expr_tensor=dy.concatenate(
[sent.as_tensor(), encoding]),
mask=sent.mask)
elif self.pos_encoding_type:
raise ValueError(f"unknown encoding type {self.pos_encoding_type}")
for module in self.modules:
enc_sent = module.transduce(sent)
sent = enc_sent
self._final_states = [transducers.FinalTransducerState(sent[-1])]
return sent
def __call__(self, x: ExpressionSequence) -> tt.Tensor:
"""
Move the time-dimension of an input expression into the batch dimension via a reshape.
Args:
x: expression of dimensions ((hidden, timesteps), batch_size)
Returns:
expression of dimensions ((hidden,), timesteps*batch_size)
"""
batch_size = x[0].dim()[1]
model_dim = x[0].dim()[0][0]
seq_len = x.sent_len()
total_words = seq_len * batch_size
input_tensor = x.as_tensor()
return dy.reshape(input_tensor, (model_dim, ), batch_size=total_words)
def transduce(
self, src: expression_seqs.ExpressionSequence
) -> expression_seqs.ExpressionSequence:
sent_len = src.sent_len()
embeddings = dy.strided_select(dy.parameter(self.embedder), [1, 1],
[0, 0], [self.input_dim, sent_len])
if self.op == 'sum':
output = embeddings + src.as_tensor()
elif self.op == 'concat':
output = dy.concatenate([embeddings, src.as_tensor()])
else:
raise ValueError(
f'Illegal op {op} in PositionalTransducer (options are "sum"/"concat")'
)
if self.train and self.dropout > 0.0:
output = dy.dropout(output, self.dropout)
output_seq = expression_seqs.ExpressionSequence(expr_tensor=output,
mask=src.mask)
self._final_states = [transducers.FinalTransducerState(output_seq[-1])]
return output_seq
def transduce(self, x: ExpressionSequence) -> ExpressionSequence:
seq_len = x.sent_len()
batch_size = x[0].dim()[1]
att_mask = None
if self.diagonal_mask_width is not None:
if self.diagonal_mask_width is None:
att_mask = np.zeros((seq_len, seq_len))
else:
att_mask = np.ones((seq_len, seq_len))
for i in range(seq_len):
from_i = max(0, i - self.diagonal_mask_width // 2)
to_i = min(seq_len, i + self.diagonal_mask_width // 2 + 1)
att_mask[from_i:to_i, from_i:to_i] = 0.0
mid = self.self_attn(x=x,
att_mask=att_mask,
batch_mask=x.mask.np_arr if x.mask else None,
p=self.dropout)
if self.downsample_factor > 1:
seq_len = int(math.ceil(seq_len / float(self.downsample_factor)))
hidden_dim = mid.dim()[0][0]
out_mask = x.mask
if self.downsample_factor > 1 and out_mask is not None:
out_mask = out_mask.lin_subsampled(
reduce_factor=self.downsample_factor)
if self.ff_lstm:
mid_re = dy.reshape(mid, (hidden_dim, seq_len),
batch_size=batch_size)
out = self.feed_forward.transduce(
ExpressionSequence(expr_tensor=mid_re, mask=out_mask))
out = dy.reshape(out.as_tensor(), (hidden_dim, ),
batch_size=seq_len * batch_size)
else:
out = self.feed_forward.transduce(mid, p=self.dropout)
self._recent_output = out
return ExpressionSequence(expr_tensor=dy.reshape(
out, (out.dim()[0][0], seq_len), batch_size=batch_size),
mask=out_mask)