def transduce(
self, expr_seq: expression_seqs.ExpressionSequence
) -> expression_seqs.ExpressionSequence:
if expr_seq.dim()[1] > 1:
raise ValueError(
f"LatticeLSTMTransducer requires batch size 1, got {expr_seq.dim()[1]}"
)
lattice = self.cur_src[0]
Wx_iog = dy.parameter(self.p_Wx_iog)
Wh_iog = dy.parameter(self.p_Wh_iog)
b_iog = dy.parameter(self.p_b_iog)
Wx_f = dy.parameter(self.p_Wx_f)
Wh_f = dy.parameter(self.p_Wh_f)
b_f = dy.parameter(self.p_b_f)
h = []
c = []
batch_size = expr_seq.dim()[1]
if self.dropout_rate > 0.0 and self.train:
self.set_dropout_masks(batch_size=batch_size)
for node_i in range(lattice.sent_len()):
cur_node = lattice.nodes[node_i]
val = expr_seq[node_i]
if self.dropout_rate > 0.0 and self.train:
val = dy.cmult(val, self.dropout_mask_x)
i_ft_list = []
if len(cur_node.nodes_prev) == 0:
tmp_iog = dy.affine_transform([b_iog, Wx_iog, val])
else:
h_tilde = sum(h[pred] for pred in cur_node.nodes_prev)
tmp_iog = dy.affine_transform(
[b_iog, Wx_iog, val, Wh_iog, h_tilde])
for pred in cur_node.nodes_prev:
i_ft_list.append(
dy.logistic(
dy.affine_transform(
[b_f, Wx_f, val, Wh_f, h[pred]])))
i_ait = dy.pick_range(tmp_iog, 0, self.hidden_dim)
i_aot = dy.pick_range(tmp_iog, self.hidden_dim,
self.hidden_dim * 2)
i_agt = dy.pick_range(tmp_iog, self.hidden_dim * 2,
self.hidden_dim * 3)
i_it = dy.logistic(i_ait)
i_ot = dy.logistic(i_aot)
i_gt = dy.tanh(i_agt)
if len(cur_node.nodes_prev) == 0:
c.append(dy.cmult(i_it, i_gt))
else:
fc = dy.cmult(i_ft_list[0], c[cur_node.nodes_prev[0]])
for i in range(1, len(cur_node.nodes_prev)):
fc += dy.cmult(i_ft_list[i], c[cur_node.nodes_prev[i]])
c.append(fc + dy.cmult(i_it, i_gt))
h_t = dy.cmult(i_ot, dy.tanh(c[-1]))
if self.dropout_rate > 0.0 and self.train:
h_t = dy.cmult(h_t, self.dropout_mask_h)
h.append(h_t)
self._final_states = [transducers.FinalTransducerState(h[-1], c[-1])]
return expression_seqs.ExpressionSequence(expr_list=h)
def transduce(self, src: ExpressionSequence) -> ExpressionSequence:
src = src.as_tensor()
src_height = src.dim()[0][0]
src_width = src.dim()[0][1]
# src_channels = 1
batch_size = src.dim()[1]
# convolution and pooling layers
# src dim is ((40, 1000), 128)
src = padding(src, self.filter_width[0]+3)
l1 = dy.rectify(dy.conv2d(src, dy.parameter(self.filters1), stride = [self.stride[0], self.stride[0]], is_valid = True)) # ((1, 1000, 64), 128)
pool1 = dy.maxpooling2d(l1, (1, 4), (1,2), is_valid = True) #((1, 499, 64), 128)
pool1 = padding(pool1, self.filter_width[1]+3)
l2 = dy.rectify(dy.conv2d(pool1, dy.parameter(self.filters2), stride = [self.stride[1], self.stride[1]], is_valid = True))# ((1, 499, 512), 128)
pool2 = dy.maxpooling2d(l2, (1, 4), (1,2), is_valid = True)#((1, 248, 512), 128)
pool2 = padding(pool2, self.filter_width[2])
l3 = dy.rectify(dy.conv2d(pool2, dy.parameter(self.filters3), stride = [self.stride[2], self.stride[2]], is_valid = True))# ((1, 248, 1024), 128)
pool3 = dy.max_dim(l3, d = 1)
my_norm = dy.l2_norm(pool3) + 1e-6
output = dy.cdiv(pool3,my_norm)
output = dy.reshape(output, (self.num_filters[2],), batch_size = batch_size)
return ExpressionSequence(expr_tensor=output)
def transduce(self, src: ExpressionSequence) -> ExpressionSequence:
src = src.as_tensor()
# convolutional layer
src = padding(src,
src.dim()[0][0],
src.dim()[0][1], self.filter_width, self.stride,
src.dim()[1])
l1 = dy.rectify(
dy.conv2d(src,
dy.parameter(self.filter_conv),
stride=[self.stride, self.stride],
is_valid=True))
timestep = l1.dim()[0][1]
features = l1.dim()[0][2]
batch_size = l1.dim()[1]
# transpose l1 to be (timesetp, dim), but keep the batch_size.
rhn_in = dy.reshape(l1, (timestep, features), batch_size=batch_size)
rhn_in = [dy.pick(rhn_in, i) for i in range(timestep)]
for l in range(self.rhn_num_hidden_layers):
rhn_out = []
# initialize a random vector for the first state vector, keep the same batch size.
prev_state = dy.parameter(self.init[l])
# begin recurrent high way network
for t in range(timestep):
for m in range(0, self.rhn_microsteps):
H = dy.affine_transform([
dy.parameter(self.recur[l][m][1]),
dy.parameter(self.recur[l][m][0]), prev_state
])
T = dy.affine_transform([
dy.parameter(self.recur[l][m][3]),
dy.parameter(self.recur[l][m][2]), prev_state
])
if m == 0:
H += dy.parameter(self.linear[l][0]) * rhn_in[t]
T += dy.parameter(self.linear[l][1]) * rhn_in[t]
H = dy.tanh(H)
T = dy.logistic(T)
prev_state = dy.cmult(1 - T, prev_state) + dy.cmult(
T, H) # ((1024, ), batch_size)
rhn_out.append(prev_state)
if self.residual and l > 0:
rhn_out = [sum(x) for x in zip(rhn_out, rhn_in)]
rhn_in = rhn_out
# Compute the attention-weighted average of the activations
rhn_in = dy.concatenate_cols(rhn_in)
scores = dy.transpose(dy.parameter(self.attention[0][1])) * dy.tanh(
dy.parameter(self.attention[0][0]) *
rhn_in) # ((1,510), batch_size)
scores = dy.reshape(scores, (scores.dim()[0][1], ),
batch_size=scores.dim()[1])
attn_out = rhn_in * dy.softmax(
scores
) # # rhn_in.as_tensor() is ((1024,510), batch_size) softmax is ((510,), batch_size)
return ExpressionSequence(expr_tensor=attn_out)
def transduce(self, src: ExpressionSequence) -> ExpressionSequence:
src = src.as_tensor()
src_height = src.dim()[0][0]
src_width = 1
batch_size = src.dim()[1]
W = dy.parameter(self.pW)
b = dy.parameter(self.pb)
src = dy.reshape(src, (src_height, src_width), batch_size=batch_size) # ((276, 80, 3), 1)
# convolution and pooling layers
l1 = (W*src)+b
output = dy.cdiv(l1,dy.sqrt(dy.squared_norm(l1)))
return ExpressionSequence(expr_tensor=output)
def transduce(
self, expr_seq: expression_seqs.ExpressionSequence
) -> expression_seqs.ExpressionSequence:
"""
transduce the sequence, applying masks if given (masked timesteps simply copy previous h / c)
Args:
expr_seq: expression sequence (will be accessed via tensor_expr)
Return:
expression sequence
"""
if isinstance(expr_seq, list):
mask_out = expr_seq[0].mask
seq_len = len(expr_seq[0])
batch_size = expr_seq[0].dim()[1]
tensors = [e.as_tensor() for e in expr_seq]
input_tensor = dy.reshape(dy.concatenate(tensors),
(seq_len, 1, self.input_dim),
batch_size=batch_size)
else:
mask_out = expr_seq.mask
seq_len = len(expr_seq)
batch_size = expr_seq.dim()[1]
input_tensor = dy.reshape(dy.transpose(expr_seq.as_tensor()),
(seq_len, 1, self.input_dim),
batch_size=batch_size)
if self.dropout > 0.0 and self.train:
input_tensor = dy.dropout(input_tensor, self.dropout)
proj_inp = dy.conv2d_bias(input_tensor,
dy.parameter(self.p_f),
dy.parameter(self.p_b),
stride=(self.stride, 1),
is_valid=False)
reduced_seq_len = proj_inp.dim()[0][0]
proj_inp = dy.transpose(
dy.reshape(proj_inp, (reduced_seq_len, self.hidden_dim * 3),
batch_size=batch_size))
# proj_inp dims: (hidden, 1, seq_len), batch_size
if self.stride > 1 and mask_out is not None:
mask_out = mask_out.lin_subsampled(trg_len=reduced_seq_len)
h = [dy.zeroes(dim=(self.hidden_dim, 1), batch_size=batch_size)]
c = [dy.zeroes(dim=(self.hidden_dim, 1), batch_size=batch_size)]
for t in range(reduced_seq_len):
f_t = dy.logistic(
dy.strided_select(proj_inp, [], [0, t],
[self.hidden_dim, t + 1]))
o_t = dy.logistic(
dy.strided_select(proj_inp, [], [self.hidden_dim, t],
[self.hidden_dim * 2, t + 1]))
z_t = dy.tanh(
dy.strided_select(proj_inp, [], [self.hidden_dim * 2, t],
[self.hidden_dim * 3, t + 1]))
if self.dropout > 0.0 and self.train:
retention_rate = 1.0 - self.dropout
dropout_mask = dy.random_bernoulli((self.hidden_dim, 1),
retention_rate,
batch_size=batch_size)
f_t = 1.0 - dy.cmult(
dropout_mask, 1.0 - f_t
) # TODO: would be easy to make a zoneout dynet operation to save memory
i_t = 1.0 - f_t
if t == 0:
c_t = dy.cmult(i_t, z_t)
else:
c_t = dy.cmult(f_t, c[-1]) + dy.cmult(i_t, z_t)
h_t = dy.cmult(
o_t, c_t) # note: LSTM would use dy.tanh(c_t) instead of c_t
if mask_out is None or np.isclose(
np.sum(mask_out.np_arr[:, t:t + 1]), 0.0):
c.append(c_t)
h.append(h_t)
else:
c.append(
mask_out.cmult_by_timestep_expr(c_t, t, True) +
mask_out.cmult_by_timestep_expr(c[-1], t, False))
h.append(
mask_out.cmult_by_timestep_expr(h_t, t, True) +
mask_out.cmult_by_timestep_expr(h[-1], t, False))
self._final_states = [transducers.FinalTransducerState(dy.reshape(h[-1], (self.hidden_dim,), batch_size=batch_size), \
dy.reshape(c[-1], (self.hidden_dim,),
batch_size=batch_size))]
return expression_seqs.ExpressionSequence(expr_list=h[1:],
mask=mask_out)