def transduce(self, sent: ExpressionSequence) -> ExpressionSequence:
if self.pos_encoding_type == "trigonometric":
if self.position_encoding_block is None or self.position_encoding_block.shape[
2] < len(sent):
self.initialize_position_encoding(
int(len(sent) * 1.2),
self.input_dim if self.pos_encoding_combine == "add" else
self.pos_encoding_size)
encoding = dy.inputTensor(
self.position_encoding_block[0, :, :len(sent)])
elif self.pos_encoding_type == "embedding":
encoding = self.positional_embedder.embed_sent(
len(sent)).as_tensor()
if self.pos_encoding_type:
if self.pos_encoding_combine == "add":
sent = ExpressionSequence(expr_tensor=sent.as_tensor() +
encoding,
mask=sent.mask)
else: # concat
sent = ExpressionSequence(expr_tensor=dy.concatenate(
[sent.as_tensor(), encoding]),
mask=sent.mask)
elif self.pos_encoding_type:
raise ValueError(f"unknown encoding type {self.pos_encoding_type}")
for module in self.modules:
enc_sent = module.transduce(sent)
sent = enc_sent
self._final_states = [transducer.FinalTransducerState(sent[-1])]
return sent
def transduce(self, src: ExpressionSequence) -> ExpressionSequence:
sent_len = len(src)
embeddings = dy.strided_select(dy.parameter(self.embedder), [1,1], [0,0], [self.input_dim, sent_len])
if self.op == 'sum':
output = embeddings + src.as_tensor()
elif self.op == 'concat':
output = dy.concatenate([embeddings, src.as_tensor()])
else:
raise ValueError(f'Illegal op {op} in PositionalTransducer (options are "sum"/"concat")')
output_seq = ExpressionSequence(expr_tensor=output, mask=src.mask)
self._final_states = [FinalTransducerState(output_seq[-1])]
return output_seq
def transduce(self, src: ExpressionSequence) -> ExpressionSequence:
src = src.as_tensor()
src_height = src.dim()[0][0]
src_width = src.dim()[0][1]
# src_channels = 1
batch_size = src.dim()[1]
# convolution and pooling layers
# src dim is ((40, 1000), 128)
src = padding(src, self.filter_width[0]+3)
l1 = dy.rectify(dy.conv2d(src, dy.parameter(self.filters1), stride = [self.stride[0], self.stride[0]], is_valid = True)) # ((1, 1000, 64), 128)
pool1 = dy.maxpooling2d(l1, (1, 4), (1,2), is_valid = True) #((1, 499, 64), 128)
pool1 = padding(pool1, self.filter_width[1]+3)
l2 = dy.rectify(dy.conv2d(pool1, dy.parameter(self.filters2), stride = [self.stride[1], self.stride[1]], is_valid = True))# ((1, 499, 512), 128)
pool2 = dy.maxpooling2d(l2, (1, 4), (1,2), is_valid = True)#((1, 248, 512), 128)
pool2 = padding(pool2, self.filter_width[2])
l3 = dy.rectify(dy.conv2d(pool2, dy.parameter(self.filters3), stride = [self.stride[2], self.stride[2]], is_valid = True))# ((1, 248, 1024), 128)
pool3 = dy.max_dim(l3, d = 1)
my_norm = dy.l2_norm(pool3) + 1e-6
output = dy.cdiv(pool3,my_norm)
output = dy.reshape(output, (self.num_filters[2],), batch_size = batch_size)
return ExpressionSequence(expr_tensor=output)
def transduce(self, embed_sent: ExpressionSequence) -> ExpressionSequence:
src = embed_sent.as_tensor()
sent_len = src.dim()[0][1]
batch_size = src.dim()[1]
pad_size = (self.window_receptor -
1) / 2 #TODO adapt it also for even window size
src = dy.concatenate([
dy.zeroes((self.input_dim, pad_size), batch_size=batch_size), src,
dy.zeroes((self.input_dim, pad_size), batch_size=batch_size)
],
d=1)
padded_sent_len = sent_len + 2 * pad_size
conv1 = dy.parameter(self.pConv1)
bias1 = dy.parameter(self.pBias1)
src_chn = dy.reshape(src, (self.input_dim, padded_sent_len, 1),
batch_size=batch_size)
cnn_layer1 = dy.conv2d_bias(src_chn, conv1, bias1, stride=[1, 1])
hidden_layer = dy.reshape(cnn_layer1, (self.internal_dim, sent_len, 1),
batch_size=batch_size)
if self.non_linearity is 'linear':
hidden_layer = hidden_layer
elif self.non_linearity is 'tanh':
hidden_layer = dy.tanh(hidden_layer)
elif self.non_linearity is 'relu':
hidden_layer = dy.rectify(hidden_layer)
elif self.non_linearity is 'sigmoid':
hidden_layer = dy.logistic(hidden_layer)
for conv_hid, bias_hid in self.builder_layers:
hidden_layer = dy.conv2d_bias(hidden_layer,
dy.parameter(conv_hid),
dy.parameter(bias_hid),
stride=[1, 1])
hidden_layer = dy.reshape(hidden_layer,
(self.internal_dim, sent_len, 1),
batch_size=batch_size)
if self.non_linearity is 'linear':
hidden_layer = hidden_layer
elif self.non_linearity is 'tanh':
hidden_layer = dy.tanh(hidden_layer)
elif self.non_linearity is 'relu':
hidden_layer = dy.rectify(hidden_layer)
elif self.non_linearity is 'sigmoid':
hidden_layer = dy.logistic(hidden_layer)
last_conv = dy.parameter(self.last_conv)
last_bias = dy.parameter(self.last_bias)
output = dy.conv2d_bias(hidden_layer,
last_conv,
last_bias,
stride=[1, 1])
output = dy.reshape(output, (sent_len, self.output_dim),
batch_size=batch_size)
output_seq = ExpressionSequence(expr_tensor=output)
self._final_states = [FinalTransducerState(output_seq[-1])]
return output_seq
def transduce(self, seq: ExpressionSequence) -> ExpressionSequence:
seq_tensor = self.child.transduce(seq).as_tensor() + seq.as_tensor()
if self.layer_norm:
d = seq_tensor.dim()
seq_tensor = dy.reshape(seq_tensor, (d[0][0],), batch_size=d[0][1]*d[1])
seq_tensor = dy.layer_norm(seq_tensor, self.ln_g, self.ln_b)
seq_tensor = dy.reshape(seq_tensor, d[0], batch_size=d[1])
return ExpressionSequence(expr_tensor=seq_tensor)
def exprseq_pooling(self, exprseq):
# Reduce to vector
exprseq = ExpressionSequence(expr_tensor=exprseq.mask.add_to_tensor_expr(exprseq.as_tensor(),-1e10), mask=exprseq.mask)
if exprseq.expr_tensor != None:
if len(exprseq.expr_tensor.dim()[0]) > 1:
return dy.max_dim(exprseq.expr_tensor, d=1)
else:
return exprseq.expr_tensor
else:
return dy.emax(exprseq.expr_list)
def transduce(self, x: ExpressionSequence) -> ExpressionSequence:
x_T = x.as_transposed_tensor()
scores = x_T * dy.parameter(self.W)
if x.mask is not None:
scores = x.mask.add_to_tensor_expr(scores, multiplicator=-100.0, time_first=True)
if self.pos_enc_max:
seq_len = x_T.dim()[0][0]
pos_enc = self.pos_enc[:seq_len,:]
scores = dy.cmult(scores, dy.inputTensor(pos_enc))
attention = dy.softmax(scores)
output_expr = x.as_tensor() * attention
return expression_sequence.ExpressionSequence(expr_tensor=output_expr, mask=None)
def transduce(self, src: ExpressionSequence) -> ExpressionSequence:
src = src.as_tensor()
src_height = src.dim()[0][0]
src_width = 1
batch_size = src.dim()[1]
W = dy.parameter(self.pW)
b = dy.parameter(self.pb)
src = dy.reshape(src, (src_height, src_width), batch_size=batch_size) # ((276, 80, 3), 1)
# convolution and pooling layers
l1 = (W*src)+b
output = dy.cdiv(l1,dy.sqrt(dy.squared_norm(l1)))
return ExpressionSequence(expr_tensor=output)
def transduce(self, src: ExpressionSequence) -> ExpressionSequence:
src = src.as_tensor()
# convolutional layer
src = padding(src, src.dim()[0][0], src.dim()[0][1], self.filter_width, self.stride, src.dim()[1])
l1 = dy.rectify(dy.conv2d(src, dy.parameter(self.filter_conv), stride = [self.stride, self.stride], is_valid = True))
timestep = l1.dim()[0][1]
features = l1.dim()[0][2]
batch_size = l1.dim()[1]
# transpose l1 to be (timesetp, dim), but keep the batch_size.
rhn_in = dy.reshape(l1, (timestep, features), batch_size = batch_size)
rhn_in = [dy.pick(rhn_in, i) for i in range(timestep)]
for l in range(self.rhn_num_hidden_layers):
rhn_out = []
# initialize a random vector for the first state vector, keep the same batch size.
prev_state = dy.parameter(self.init[l])
# begin recurrent high way network
for t in range(timestep):
for m in range(0, self.rhn_microsteps):
H = dy.affine_transform([dy.parameter(self.recur[l][m][1]), dy.parameter(self.recur[l][m][0]), prev_state])
T = dy.affine_transform([dy.parameter(self.recur[l][m][3]), dy.parameter(self.recur[l][m][2]), prev_state])
if m == 0:
H += dy.parameter(self.linear[l][0]) * rhn_in[t]
T += dy.parameter(self.linear[l][1]) * rhn_in[t]
H = dy.tanh(H)
T = dy.logistic(T)
prev_state = dy.cmult(1 - T, prev_state) + dy.cmult(T, H) # ((1024, ), batch_size)
rhn_out.append(prev_state)
if self.residual and l>0:
rhn_out = [sum(x) for x in zip(rhn_out, rhn_in)]
rhn_in = rhn_out
# Compute the attention-weighted average of the activations
rhn_in = dy.concatenate_cols(rhn_in)
scores = dy.transpose(dy.parameter(self.attention[0][1]))*dy.tanh(dy.parameter(self.attention[0][0])*rhn_in) # ((1,510), batch_size)
scores = dy.reshape(scores, (scores.dim()[0][1],), batch_size = scores.dim()[1])
attn_out = rhn_in*dy.softmax(scores) # # rhn_in.as_tensor() is ((1024,510), batch_size) softmax is ((510,), batch_size)
return ExpressionSequence(expr_tensor = attn_out)
def transduce(self, src: ExpressionSequence) -> ExpressionSequence:
output = self.transform(src.as_tensor())
output_seq = ExpressionSequence(expr_tensor=output)
self._final_states = [FinalTransducerState(output_seq[-1])]
return output_seq