def forward(self, src, src_length, trg, trg_length):
# Encoder
_, enc_final_state = self.encoder(src, src_length)
# Build distribution
z_mean, z_log_var = self.build_distribution(enc_final_state)
# Decoder
latent_z = self.sampling(z_mean, z_log_var)
dec_first_hidden_cell = self.fc(latent_z)
dec_first_hidden, dec_first_cell = paddle.split(dec_first_hidden_cell,
2,
axis=-1)
if self.num_layers > 1:
dec_first_hidden = paddle.split(dec_first_hidden, self.num_layers)
dec_first_cell = paddle.split(dec_first_cell, self.num_layers)
else:
dec_first_hidden = [dec_first_hidden]
dec_first_cell = [dec_first_cell]
dec_initial_states = [[h, c]
for h, c in zip(dec_first_hidden, dec_first_cell)
]
dec_output = self.decoder(trg, dec_initial_states, latent_z)
kl_loss = self.calc_kl_dvg(z_mean, z_log_var)
trg_mask = (self.PAD_ID != trg).astype(paddle.get_default_dtype())
return kl_loss, dec_output, trg_mask
def forward(self, x):
x = self.conv1(x)
if self.radix > 1:
splited = paddle.split(x, num_or_sections=self.radix, axis=1)
gap = paddle.add_n(splited)
else:
gap = x
gap = self.avg_pool2d(gap)
gap = self.conv2(gap)
atten = self.conv3(gap)
atten = self.rsoftmax(atten)
if self.radix > 1:
attens = paddle.split(atten, num_or_sections=self.radix, axis=1)
y = paddle.add_n([
paddle.multiply(split, att)
for (att, split) in zip(attens, splited)
])
else:
y = paddle.multiply(x, atten)
return y
def forward(self, combined_features):
"""
combined_features: (batch_size, (sparse_field_num + 1), embedding_size)
W_Query: (embedding_size, factor_dim * att_head_num)
(b, f, e) * (e, d*h) -> (b, f, d*h)
"""
# (b, f, d*h)
querys = paddle.matmul(combined_features, self.W_Query)
keys = paddle.matmul(combined_features, self.W_Key)
values = paddle.matmul(combined_features, self.W_Value)
b, f, d_h = querys.shape
# (h, b, f, d) <- (b, f, d)
querys = paddle.stack(paddle.split(querys, self.att_head_num, axis=2))
keys = paddle.stack(paddle.split(keys, self.att_head_num, axis=2))
values = paddle.stack(paddle.split(values, self.att_head_num, axis=2))
# (h, b, f, f)
inner_product = paddle.matmul(querys, keys, transpose_y=True)
inner_product /= self.att_factor_dim**0.5
normalized_att_scores = fun.softmax(inner_product)
# (h, b, f, d)
result = paddle.matmul(normalized_att_scores, values)
result = paddle.concat(paddle.split(result, self.att_head_num, axis=0),
axis=-1)
# (b, f, h * d)
result = paddle.squeeze(result, axis=0)
result += paddle.matmul(combined_features, self.W_Res)
# (b, f * h * d)
result = paddle.reshape(result, shape=(b, f * d_h))
m_vec = self.dnn_layer(result)
return m_vec
def test_out1(self):
with fluid.dygraph.guard():
input_1 = np.random.random([4, 6, 6]).astype("int32")
# input is a variable which shape is [4, 6, 6]
input = paddle.to_tensor(input_1)
x0, x1, x2 = paddle.split(input, num_or_sections=3, axis=1)
x0_out = x0.numpy()
x1_out = x1.numpy()
x2_out = x2.numpy()
ex_x0, ex_x1, ex_x2 = np.split(input_1, 3, axis=1)
with _test_eager_guard():
# input is a variable which shape is [4, 6, 6]
input = paddle.to_tensor(input_1)
input.stop_gradient = False
x0, x1, x2 = paddle.split(input, num_or_sections=3, axis=1)
eager_x0_out = x0.numpy()
eager_x1_out = x1.numpy()
eager_x2_out = x2.numpy()
loss = x0.sum()
loss.backward()
manul_grad = np.zeros_like(input_1)
manul_grad[:, :2, :] = 1
self.assertTrue(np.allclose(input.gradient(), manul_grad))
self.assertTrue(np.allclose(ex_x0, eager_x0_out))
self.assertTrue(np.allclose(ex_x1, eager_x1_out))
self.assertTrue(np.allclose(ex_x2, eager_x2_out))
self.assertTrue(np.allclose(ex_x0, x0_out))
self.assertTrue(np.allclose(ex_x1, x1_out))
self.assertTrue(np.allclose(ex_x2, x2_out))
def forward(self, input):
# PROJ
if isinstance(input, list):
data_in = paddle.concat([input[0], input[1]], axis=1)
else:
data_in = input
if self.has_proj:
c1x1_w = self.c1x1_w_func(data_in)
data_o1, data_o2 = paddle.split(
c1x1_w, num_or_sections=[self.num_1x1_c, 2 * self.inc], axis=1)
else:
data_o1 = input[0]
data_o2 = input[1]
c1x1_a = self.c1x1_a_func(data_in)
c3x3_b = self.c3x3_b_func(c1x1_a)
c1x1_c = self.c1x1_c_func(c3x3_b)
c1x1_c1, c1x1_c2 = paddle.split(
c1x1_c, num_or_sections=[self.num_1x1_c, self.inc], axis=1)
# OUTPUTS
summ = paddle.add(x=data_o1, y=c1x1_c1)
dense = paddle.concat([data_o2, c1x1_c2], axis=1)
# tensor, channels
return [summ, dense]
def bev_box_encode(boxes, anchors, encode_angle_to_vector=False, smooth_dim=False):
"""box encode for VoxelNet
Args:
boxes ([N, 7] Tensor): normal boxes: x, y, z, l, w, h, r
anchors ([N, 7] Tensor): anchors
"""
xa, ya, wa, la, ra = paddle.split(anchors, 5, axis=-1)
xg, yg, wg, lg, rg = paddle.split(boxes, 5, axis=-1)
diagonal = paddle.sqrt(la**2 + wa**2)
xt = (xg - xa) / diagonal
yt = (yg - ya) / diagonal
if smooth_dim:
lt = lg / la - 1
wt = wg / wa - 1
else:
lt = paddle.log(lg / la)
wt = paddle.log(wg / wa)
if encode_angle_to_vector:
rgx = paddle.cos(rg)
rgy = paddle.sin(rg)
rax = paddle.cos(ra)
ray = paddle.sin(ra)
rtx = rgx - rax
rty = rgy - ray
return paddle.concat([xt, yt, wt, lt, rtx, rty], axis=-1)
else:
rt = rg - ra
return paddle.concat([xt, yt, wt, lt, rt], axis=-1)
def batch_predict(
model,
data_loader,
rel_vocab,
word_pad_index,
word_bos_index,
word_eos_index,
):
model.eval()
arcs, rels = [], []
for inputs in data_loader():
if args.encoding_model.startswith("ernie") or args.encoding_model == "lstm-pe":
words = inputs[0]
words, feats = flat_words(words)
s_arc, s_rel, words = model(words, feats)
else:
words, feats = inputs
s_arc, s_rel, words = model(words, feats)
mask = paddle.logical_and(
paddle.logical_and(words != word_pad_index, words != word_bos_index),
words != word_eos_index,
)
lens = paddle.sum(paddle.cast(mask, "int32"), axis=-1)
arc_preds, rel_preds = decode(s_arc, s_rel, mask)
arcs.extend(paddle.split(paddle.masked_select(arc_preds, mask), lens.numpy().tolist()))
rels.extend(paddle.split(paddle.masked_select(rel_preds, mask), lens.numpy().tolist()))
arcs = [[str(s) for s in seq.numpy().tolist()] for seq in arcs]
rels = [rel_vocab.to_tokens(seq.numpy().tolist()) for seq in rels]
return arcs, rels
def forward(self, node_feat, edge_feat):
# get size
num_tasks = node_feat.shape[0]
num_data = node_feat.shape[1]
# get eye matrix (batch_size x 2 x node_size x node_size)
diag_mask = 1.0 - paddle.expand(
paddle.eye(num_data),
[num_tasks, self.edge_dim, num_data, num_data])
# set diagonal as zero and normalize
edge_feat = F.normalize(edge_feat * diag_mask, p=1, axis=-1)
# compute attention and aggregate
aggr_feat = paddle.bmm(
paddle.concat(paddle.split(edge_feat, 2, 1),
self.edge_dim).squeeze(1), node_feat)
node_feat = paddle.transpose(
paddle.concat(
[node_feat,
paddle.concat(paddle.split(aggr_feat, 2, 1), -1)], -1),
(0, 2, 1))
# non-linear transform
node_feat = paddle.transpose(self.network(node_feat.unsqueeze(-1)),
(0, 2, 1, 3)).squeeze(-1)
return node_feat
def forward(self, x):
x0 = self.linear0(x[0])
x1 = self.linear1(x[1])
bs = x1.shape[0]
if self.dropout_input > 0:
x0 = F.dropout(x0, p=self.dropout_input, training=self.training)
x1 = F.dropout(x1, p=self.dropout_input, training=self.training)
x0_chunks = paddle.split(x0, self.chunks, -1)
x1_chunks = paddle.split(x1, self.chunks, -1)
zs = []
for x0_c, x1_c, m0, m1 in zip(x0_chunks, x1_chunks, self.merge_linears0,
self.merge_linears1):
m = m0(x0_c) * m1(x1_c) # bs x split_size*rank
m = m.reshape([bs, self.rank, -1])
z = paddle.sum(m, 1)
if self.pos_norm == 'before_cat':
z = paddle.sqrt(F.relu(z)) - paddle.sqrt(F.relu(-z))
z = F.normalize(z)
zs.append(z)
z = paddle.concat(zs, 1)
if self.pos_norm == 'after_cat':
z = paddle.sqrt(F.relu(z)) - paddle.sqrt(F.relu(-z))
z = F.normalize(z)
if self.dropout_pre_lin > 0:
z = F.dropout(z, p=self.dropout_pre_lin, training=self.training)
z = self.linear_out(z)
if self.dropout_output > 0:
z = F.dropout(z, p=self.dropout_output, training=self.training)
return z
def forward(self, x, adj_matrix, previous_x=None):
# x: [len_tokens, d]
# adj_matrix: [len_tokens, len_tokens]
len_tokens = x.shape[0]
if previous_x is not None:
x_new = self.leakyReLU(
self.fc(paddle.concat([previous_x, x], axis=0))).unsqueeze(0)
else:
x_new = self.leakyReLU(
self.fc(x).unsqueeze(0)) # [1, len_tokens, d]
q = self.leakyReLU(self.qfc(x_new))
x_ = paddle.concat(paddle.split(x_new, 3, axis=2), axis=0)
q_ = paddle.concat(paddle.split(q, 3, axis=2), axis=0)
outputs = paddle.matmul(q_, x_.transpose(
[0, 2, 1])) / 10.0 # [3, len_tokens, len_tokens]
tmp_adj_matrix = (adj_matrix == 0).expand(
shape=[3, adj_matrix.shape[0], adj_matrix.shape[1]])
outputs = model_utils.mask_fill(input=outputs,
mask=tmp_adj_matrix,
value=-1e9)
outputs = self.dropout(F.softmax(outputs, axis=-1))
outputs = paddle.matmul(outputs, x_)
outputs = paddle.concat(paddle.split(outputs, 3, axis=0), axis=2)
if previous_x is not None:
outputs = paddle.split(outputs, 2, axis=1)[1]
outputs = x.unsqueeze(0) + outputs
ret = x + self.dropout(
self.leakyReLU(self.final_fc(outputs).squeeze(0)))
return ret
def __call__(self, pbox, gbox, iou_weight=1., loc_reweight=None):
x1, y1, x2, y2 = paddle.split(pbox, num_or_sections=4, axis=-1)
x1g, y1g, x2g, y2g = paddle.split(gbox, num_or_sections=4, axis=-1)
box1 = [x1, y1, x2, y2]
box2 = [x1g, y1g, x2g, y2g]
iou, overlap, union = self.bbox_overlap(box1, box2, self.eps)
xc1 = paddle.minimum(x1, x1g)
yc1 = paddle.minimum(y1, y1g)
xc2 = paddle.maximum(x2, x2g)
yc2 = paddle.maximum(y2, y2g)
area_c = (xc2 - xc1) * (yc2 - yc1) + self.eps
miou = iou - ((area_c - union) / area_c)
if loc_reweight is not None:
loc_reweight = paddle.reshape(loc_reweight, shape=(-1, 1))
loc_thresh = 0.9
giou = 1 - (1 - loc_thresh
) * miou - loc_thresh * miou * loc_reweight
else:
giou = 1 - miou
if self.reduction == 'none':
loss = giou
elif self.reduction == 'sum':
loss = paddle.sum(giou * iou_weight)
else:
loss = paddle.mean(giou * iou_weight)
return loss * self.loss_weight
def forward(self, predicts, labels):
l_score, l_geo, l_mask = labels[1:]
f_score = predicts['f_score']
f_geo = predicts['f_geo']
dice_loss = self.dice_loss(f_score, l_score, l_mask)
#smoooth_l1_loss
channels = 8
l_geo_split = paddle.split(
l_geo, num_or_sections=channels + 1, axis=1)
f_geo_split = paddle.split(f_geo, num_or_sections=channels, axis=1)
smooth_l1 = 0
for i in range(0, channels):
geo_diff = l_geo_split[i] - f_geo_split[i]
abs_geo_diff = paddle.abs(geo_diff)
smooth_l1_sign = paddle.less_than(abs_geo_diff, l_score)
smooth_l1_sign = paddle.cast(smooth_l1_sign, dtype='float32')
in_loss = abs_geo_diff * abs_geo_diff * smooth_l1_sign + \
(abs_geo_diff - 0.5) * (1.0 - smooth_l1_sign)
out_loss = l_geo_split[-1] / channels * in_loss * l_score
smooth_l1 += out_loss
smooth_l1_loss = paddle.mean(smooth_l1 * l_score)
dice_loss = dice_loss * 0.01
total_loss = dice_loss + smooth_l1_loss
losses = {"loss":total_loss, \
"dice_loss":dice_loss,\
"smooth_l1_loss":smooth_l1_loss}
return losses
def bev_box_decode(box_encodings, anchors, encode_angle_to_vector=False, smooth_dim=False):
"""box decode for VoxelNet in lidar
Args:
boxes ([N, 7] Tensor): normal boxes: x, y, z, w, l, h, r
anchors ([N, 7] Tensor): anchors
"""
xa, ya, wa, la, ra = paddle.split(anchors, 5, axis=-1)
if encode_angle_to_vector:
xt, yt, wt, lt, rtx, rty = paddle.split(
box_encodings, 1, axis=-1)
else:
xt, yt, wt, lt, rt = paddle.split(box_encodings, 5, axis=-1)
# xt, yt, zt, wt, lt, ht, rt = paddle.split(box_encodings, 1, axis=-1)
diagonal = paddle.sqrt(la**2 + wa**2)
xg = xt * diagonal + xa
yg = yt * diagonal + ya
if smooth_dim:
lg = (lt + 1) * la
wg = (wt + 1) * wa
else:
lg = paddle.exp(lt) * la
wg = paddle.exp(wt) * wa
if encode_angle_to_vector:
rax = paddle.cos(ra)
ray = paddle.sin(ra)
rgx = rtx + rax
rgy = rty + ray
rg = atan2(rgy, rgx)
else:
rg = rt + ra
return paddle.concat([xg, yg, wg, lg, rg], axis=-1)
def forward(self, x):
xx = paddle.split(x, self.splitted_in_channels, axis=self.axis)
xx = paddle.split(x, self.splitted_in_channels, axis=self.axis)
out = [
conv_i(x_i) for x_i, conv_i in zip(xx, self._sub_layers.values())
]
x = paddle.concat(tuple(out), axis=self.axis)
return x
def split(x, batch_size, dim=0):
if isinstance(batch_size, int):
if batch_size > x.shape[dim]:
return [x] #do nothing
return [
convertTensor(y)
for y in paddle.split(x, x.shape[dim] // batch_size, dim)
]
else:
return [convertTensor(y) for y in paddle.split(x, batch_size, dim)]
def __call__(self, pbox, gbox, iou_weight=1.):
x1, y1, x2, y2 = paddle.split(pbox, num_or_sections=4, axis=-1)
x1g, y1g, x2g, y2g = paddle.split(gbox, num_or_sections=4, axis=-1)
cx = (x1 + x2) / 2
cy = (y1 + y2) / 2
w = x2 - x1
h = y2 - y1
cxg = (x1g + x2g) / 2
cyg = (y1g + y2g) / 2
wg = x2g - x1g
hg = y2g - y1g
x2 = paddle.maximum(x1, x2)
y2 = paddle.maximum(y1, y2)
# A and B
xkis1 = paddle.maximum(x1, x1g)
ykis1 = paddle.maximum(y1, y1g)
xkis2 = paddle.minimum(x2, x2g)
ykis2 = paddle.minimum(y2, y2g)
# A or B
xc1 = paddle.minimum(x1, x1g)
yc1 = paddle.minimum(y1, y1g)
xc2 = paddle.maximum(x2, x2g)
yc2 = paddle.maximum(y2, y2g)
intsctk = (xkis2 - xkis1) * (ykis2 - ykis1)
intsctk = intsctk * paddle.greater_than(
xkis2, xkis1) * paddle.greater_than(ykis2, ykis1)
unionk = (x2 - x1) * (y2 - y1) + (x2g - x1g) * (
y2g - y1g) - intsctk + self.eps
iouk = intsctk / unionk
# DIOU term
dist_intersection = (cx - cxg) * (cx - cxg) + (cy - cyg) * (cy - cyg)
dist_union = (xc2 - xc1) * (xc2 - xc1) + (yc2 - yc1) * (yc2 - yc1)
diou_term = (dist_intersection + self.eps) / (dist_union + self.eps)
# CIOU term
ciou_term = 0
if self.use_complete_iou_loss:
ar_gt = wg / hg
ar_pred = w / h
arctan = paddle.atan(ar_gt) - paddle.atan(ar_pred)
ar_loss = 4. / np.pi / np.pi * arctan * arctan
alpha = ar_loss / (1 - iouk + ar_loss + self.eps)
alpha.stop_gradient = True
ciou_term = alpha * ar_loss
diou = paddle.mean((1 - iouk + ciou_term + diou_term) * iou_weight)
return diou * self.loss_weight
def resize_mat(self, x, t):
n, c, s, s1 = x.shape
assert s == s1
if t <= 1:
return x
x = paddle.reshape(x, (n * c, -1, 1, 1))
x = x * paddle.eye(t, t, dtype=x.dtype)
x = paddle.reshape(x, (n * c, s, s, t, t))
x = paddle.concat(paddle.split(x, 1, axis=1), axis=3)
x = paddle.concat(paddle.split(x, 1, axis=2), axis=4)
x = paddle.reshape(x, (n, c, s * t, s * t))
return x
def forward(self, raw_features, coarse_volumes):
n_views_rendering = coarse_volumes.shape[1]
raw_features = paddle.split(raw_features,
num_or_sections=raw_features.shape[1],
axis=1)
volume_weights = []
for i in range(n_views_rendering):
raw_feature = paddle.squeeze(raw_features[i], axis=1)
# print("torch.Size([batch_size, 9, 32, 32, 32]) ---",raw_feature.shape)
volume_weight = self.layer1(raw_feature)
# print("torch.Size([batch_size, 16, 32, 32, 32]) ---",volume_weight.shape) #
volume_weight = self.layer2(volume_weight)
# print("torch.Size([batch_size, 8, 32, 32, 32]) ---",volume_weight.shape) #
volume_weight = self.layer3(volume_weight)
# print("torch.Size([batch_size, 4, 32, 32, 32]) ---",volume_weight.shape) #
volume_weight = self.layer4(volume_weight)
# print("torch.Size([batch_size, 2, 32, 32, 32]) ---",volume_weight.shape) #
volume_weight = self.layer5(volume_weight)
# print("torch.Size([batch_size, 1, 32, 32, 32]) ---",volume_weight.shape) #
volume_weight = paddle.squeeze(volume_weight, axis=1)
# print("torch.Size([batch_size, 32, 32, 32]) ---",volume_weight.shape) #
volume_weights.append(volume_weight)
volume_weights = paddle.transpose(paddle.stack(volume_weights),
perm=[1, 0, 2, 3, 4])
volume_weights = paddle.nn.functional.softmax(volume_weights, axis=1)
# print("torch.Size([batch_size, n_views, 32, 32, 32]) ---",volume_weights.shape) #
# print("torch.Size([batch_size, n_views, 32, 32, 32]) ---",coarse_volumes.shape) #
coarse_volumes = coarse_volumes * volume_weights
coarse_volumes = paddle.sum(coarse_volumes, axis=1)
return paddle.nn.clip(coarse_volumes, min=0, max=1)
def forward(self, q, v, size):
B, h, N, Ch = q.shape
H, W = size
assert N == 1 + H * W
# Convolutional relative position encoding.
# Shape: [B, h, H*W, Ch].
q_img = q[:, :, 1:, :]
# Shape: [B, h, H*W, Ch].
v_img = v[:, :, 1:, :]
# Shape: [B, h, H*W, Ch] -> [B, h*Ch, H, W].
v_img = v_img.reshape((B, h, H, W, Ch))
v_img = v_img.transpose((0, 1, 4, 2, 3))
v_img = v_img.flatten(1, 2)
# v_img = rearrange(v_img, 'B h (H W) Ch -> B (h Ch) H W', H=H, W=W)
# Split according to channels.
v_img_list = paddle.split(v_img, self.channel_splits, axis=1)
conv_v_img_list = [
conv(x) for conv, x in zip(self.conv_list, v_img_list)
]
conv_v_img = paddle.concat(conv_v_img_list, axis=1)
# Shape: [B, h*Ch, H, W] -> [B, h, H*W, Ch].
conv_v_img = conv_v_img.reshape((B, h, Ch, H, W))
conv_v_img = conv_v_img.transpose((0, 1, 3, 4, 2))
conv_v_img = conv_v_img.flatten(2, 3)
# conv_v_img = rearrange(conv_v_img, 'B (h Ch) H W -> B h (H W) Ch', h=h)
EV_hat_img = q_img * conv_v_img
zero = paddle.zeros((B, h, 1, Ch), dtype=q.dtype)
# Shape: [B, h, N, Ch].
EV_hat = paddle.concat((zero, EV_hat_img), axis=2)
return EV_hat
def forward(self, x):
self.training = True
B, N, C = x.shape
kv = self.kv(x)
kv = paddle.reshape(kv, [B, N, self.num_heads, -1])
k, v = paddle.split(kv, [self.key_dim, self.d], axis=3)
k = paddle.transpose(k, perm=[0, 2, 1, 3]) # BHNC
v = paddle.transpose(v, perm=[0, 2, 1, 3])
q = paddle.reshape(
self.q(x), [B, self.resolution_2, self.num_heads, self.key_dim])
q = paddle.transpose(q, perm=[0, 2, 1, 3])
if self.training:
attention_biases = cal_attention_biases(self.attention_biases,
self.attention_bias_idxs)
else:
attention_biases = self.ab
attn = (paddle.matmul(q, paddle.transpose(
k, perm=[0, 1, 3, 2]))) * self.scale + attention_biases
attn = F.softmax(attn)
x = paddle.reshape(
paddle.transpose(paddle.matmul(attn, v), perm=[0, 2, 1, 3]),
[B, -1, self.dh])
x = self.proj(x)
return x
def forward(self, rendering_images):
# print("rendering_images.shape", rendering_images.shape) # torch.Size([batch_size, n_views, img_c, img_h, img_w])
rendering_images = paddle.transpose(
rendering_images, perm=[
1, 0, 2, 3, 4
]) # pytorch:rendering_images.permute(1, 0, 2, 3, 4).contiguous()
# print("after transpose shape", rendering_images.shape) # [2, 4, 3, 224, 224]
rendering_images = paddle.split(
rendering_images,
num_or_sections=rendering_images.shape[0],
axis=0) # return list @@ len() = num_or_sections 跟pytorch区别大
# print("after split len", len(rendering_images))
image_features = []
for img in rendering_images:
features = self.vgg(paddle.squeeze(img, axis=0))
# print(features.shape) # torch.Size([batch_size, 512, 28, 28])
features = self.layer1(features)
# print(features.shape) # torch.Size([batch_size, 512, 28, 28])
features = self.layer2(features)
# print(features.shape) # torch.Size([batch_size, 256, 6, 6])
features = self.layer3(features)
# print(features.shape) # torch.Size([batch_size, 128, 4, 4])
image_features.append(features)
image_features = paddle.stack(image_features)
# print(image_features.shape)
image_features = paddle.transpose(image_features, perm=[1, 0, 2, 3, 4])
# print(image_features.shape) # torch.Size([batch_size, n_views, 128, 4, 4])
return image_features
def forward(self, image_features):
image_features = paddle.transpose(image_features, perm=[1, 0, 2, 3, 4])
image_features = paddle.split(image_features,
num_or_sections=image_features.shape[0],
axis=0)
gen_voxels = []
raw_features = []
for features in image_features:
gen_voxel = paddle.reshape(features, [-1, 256, 2, 2, 2])
# print("torch.Size([batch_size, 256, 2, 2, 2]) ---", gen_voxel.shape)
gen_voxel = self.layer1(gen_voxel)
# print("torch.Size([batch_size, 128, 4, 4, 4]) ---", gen_voxel.shape)
gen_voxel = self.layer2(gen_voxel)
# print("torch.Size([batch_size, 64, 8, 8, 8]) ---", gen_voxel.shape)
gen_voxel = self.layer3(gen_voxel)
# print("torch.Size([batch_size, 32, 16, 16, 16]) ---", gen_voxel.shape)
gen_voxel = self.layer4(gen_voxel)
# print("torch.Size([batch_size, 8, 32, 32, 32]) ---", gen_voxel.shape)
raw_feature = gen_voxel
gen_voxel = self.layer5(gen_voxel)
# print("torch.Size([batch_size, 1, 32, 32, 32]) ---", gen_voxel.shape)
raw_feature = paddle.concat(x=[raw_feature, gen_voxel], axis=1)
# print("torch.Size([batch_size, 9, 32, 32, 32]) ---",raw_feature.shape)
gen_voxels.append(paddle.squeeze(gen_voxel, axis=1))
raw_features.append(raw_feature)
gen_voxels = paddle.transpose(paddle.stack(gen_voxels),
perm=[1, 0, 2, 3, 4])
raw_features = paddle.transpose(paddle.stack(raw_features),
perm=[1, 0, 2, 3, 4, 5])
# print("torch.Size([batch_size, n_views, 32, 32, 32]) ---", gen_voxels.shape)
# print("torch.Size([batch_size, n_views, 9, 32, 32, 32]) ---",raw_features.shape)
return raw_features, gen_voxels
def forward(
self,
input_ids=None,
token_type_ids=None,
position_ids=None,
inputs_embeds=None,
start_positions=None,
end_positions=None,
output_hidden_states=None,
return_dict=None,
):
outputs = self.fnet(
input_ids,
token_type_ids=token_type_ids,
position_ids=position_ids,
inputs_embeds=inputs_embeds,
output_hidden_states=output_hidden_states,
return_dict=return_dict,
)
sequence_output = outputs[0] if not return_dict else outputs[
"last_hidden_state"]
logits = self.qa_outputs(sequence_output)
start_logits, end_logits = paddle.split(logits,
num_or_sections=1,
axis=-1)
start_logits = start_logits.squeeze(axis=-1)
end_logits = start_logits.squeeze(axis=-1)
if return_dict:
return {
"start_logits": start_logits,
"end_logits": end_logits,
"hidden_states": outputs["all_hidden_states"],
}
return start_logits, end_logits
def forward(self, feats):
assert len(feats) == len(self.fpn_strides), \
"The size of feats is not equal to size of fpn_strides"
anchors, anchor_points, num_anchors_list, stride_tensor =\
generate_anchors_for_grid_cell(
feats, self.fpn_strides, self.grid_cell_scale,
self.grid_cell_offset)
anchor_centers_split = paddle.split(anchor_points / stride_tensor,
num_anchors_list)
cls_score_list, bbox_pred_list = [], []
for feat, scale_reg, anchor_centers, stride in zip(
feats, self.scales_regs, anchor_centers_split,
self.fpn_strides):
b, _, h, w = get_static_shape(feat)
inter_feats = []
for inter_conv in self.inter_convs:
feat = F.relu(inter_conv(feat))
inter_feats.append(feat)
feat = paddle.concat(inter_feats, axis=1)
# task decomposition
avg_feat = F.adaptive_avg_pool2d(feat, (1, 1))
cls_feat = self.cls_decomp(feat, avg_feat)
reg_feat = self.reg_decomp(feat, avg_feat)
# cls prediction and alignment
cls_logits = self.tood_cls(cls_feat)
if self.use_align_head:
cls_prob = F.relu(self.cls_prob_conv1(feat))
cls_prob = F.sigmoid(self.cls_prob_conv2(cls_prob))
cls_score = (F.sigmoid(cls_logits) * cls_prob).sqrt()
else:
cls_score = F.sigmoid(cls_logits)
cls_score_list.append(cls_score.flatten(2).transpose([0, 2, 1]))
# reg prediction and alignment
reg_dist = scale_reg(self.tood_reg(reg_feat).exp())
reg_dist = reg_dist.flatten(2).transpose([0, 2, 1])
reg_bbox = batch_distance2bbox(anchor_centers.unsqueeze(0),
reg_dist)
if self.use_align_head:
reg_offset = F.relu(self.reg_offset_conv1(feat))
reg_offset = self.reg_offset_conv2(reg_offset)
reg_bbox = reg_bbox.transpose([0, 2, 1]).reshape([b, 4, h, w])
anchor_centers = anchor_centers.reshape([1, h, w, 2])
bbox_pred = self._reg_grid_sample(reg_bbox, reg_offset,
anchor_centers)
bbox_pred = bbox_pred.flatten(2).transpose([0, 2, 1])
else:
bbox_pred = reg_bbox
if not self.training:
bbox_pred *= stride
bbox_pred_list.append(bbox_pred)
cls_score_list = paddle.concat(cls_score_list, axis=1)
bbox_pred_list = paddle.concat(bbox_pred_list, axis=1)
return cls_score_list, bbox_pred_list, anchors, num_anchors_list, stride_tensor
def ctcloss(self, f_char, tcl_pos, tcl_mask, tcl_label, label_t):
f_char = paddle.transpose(f_char, [0, 2, 3, 1])
tcl_pos = paddle.reshape(tcl_pos, [-1, 3])
tcl_pos = paddle.cast(tcl_pos, dtype=int)
f_tcl_char = paddle.gather_nd(f_char, tcl_pos)
f_tcl_char = paddle.reshape(f_tcl_char,
[-1, 64, 37]) # len(Lexicon_Table)+1
f_tcl_char_fg, f_tcl_char_bg = paddle.split(f_tcl_char, [36, 1],
axis=2)
f_tcl_char_bg = f_tcl_char_bg * tcl_mask + (1.0 - tcl_mask) * 20.0
b, c, l = tcl_mask.shape
tcl_mask_fg = paddle.expand(x=tcl_mask, shape=[b, c, 36 * l])
tcl_mask_fg.stop_gradient = True
f_tcl_char_fg = f_tcl_char_fg * tcl_mask_fg + (1.0 -
tcl_mask_fg) * (-20.0)
f_tcl_char_mask = paddle.concat([f_tcl_char_fg, f_tcl_char_bg], axis=2)
f_tcl_char_ld = paddle.transpose(f_tcl_char_mask, (1, 0, 2))
N, B, _ = f_tcl_char_ld.shape
input_lengths = paddle.to_tensor([N] * B, dtype='int64')
cost = paddle.nn.functional.ctc_loss(log_probs=f_tcl_char_ld,
labels=tcl_label,
input_lengths=input_lengths,
label_lengths=label_t,
blank=self.pad_num,
reduction='none')
cost = cost.mean()
return cost
def _gather_topk_pyramid(self, gt2anchor_distances, num_anchors_list,
pad_gt_mask):
pad_gt_mask = pad_gt_mask.tile([1, 1, self.topk]).astype(paddle.bool)
gt2anchor_distances_list = paddle.split(gt2anchor_distances,
num_anchors_list,
axis=-1)
num_anchors_index = np.cumsum(num_anchors_list).tolist()
num_anchors_index = [
0,
] + num_anchors_index[:-1]
is_in_topk_list = []
topk_idxs_list = []
for distances, anchors_index in zip(gt2anchor_distances_list,
num_anchors_index):
num_anchors = distances.shape[-1]
topk_metrics, topk_idxs = paddle.topk(distances,
self.topk,
axis=-1,
largest=False)
topk_idxs_list.append(topk_idxs + anchors_index)
topk_idxs = paddle.where(pad_gt_mask, topk_idxs,
paddle.zeros_like(topk_idxs))
is_in_topk = F.one_hot(topk_idxs, num_anchors).sum(axis=-2)
is_in_topk = paddle.where(is_in_topk > 1,
paddle.zeros_like(is_in_topk),
is_in_topk)
is_in_topk_list.append(is_in_topk.astype(
gt2anchor_distances.dtype))
is_in_topk_list = paddle.concat(is_in_topk_list, axis=-1)
topk_idxs_list = paddle.concat(topk_idxs_list, axis=-1)
return is_in_topk_list, topk_idxs_list
def get_model(self, main_prog, startup_program, rank, indata=None):
with fluid.program_guard(main_prog, startup_program):
seed = os.getpid()
np.random.seed(seed)
in_feat = 2
n_expert = 2
world_size = 2
tot_expert = n_expert * world_size
local_expert_count = np.random.randint(
1, 4, size=tot_expert).astype("int")
fwd_expert_count = sum(local_expert_count)
local_input_buf = np.random.rand(fwd_expert_count,
in_feat).astype("float32")
local_expert_count = paddle.to_tensor(local_expert_count)
local_input_buf = paddle.to_tensor(local_input_buf)
global_expert_count = []
paddle.distributed.alltoall(
paddle.split(
local_expert_count, 2, axis=0),
global_expert_count)
global_expert_count = paddle.concat(global_expert_count, axis=0)
local_input_buf.stop_gradient = False
output = paddle.distributed.utils.global_scatter(
local_input_buf, local_expert_count, global_expert_count)
output.stop_gradient = False
c = output * output
c.backward()
return [output.numpy(), local_input_buf.grad.numpy()]
def forward(self, inputs):
out = self.branch2a(inputs)
feature_split = paddle.split(out, self.scales, 1)
out_split = []
for i in range(self.scales - 1):
if i == 0 or self.stride == 2:
out_split.append(self.branch2b[i](feature_split[i]))
else:
out_split.append(self.branch2b[i](paddle.add(
feature_split[i], out_split[-1])))
if self.stride == 1:
out_split.append(feature_split[-1])
else:
out_split.append(F.avg_pool2d(feature_split[-1], 3, self.stride,
1))
out = self.branch2c(paddle.concat(out_split, 1))
if self.shortcut:
short = inputs
else:
short = self.branch1(inputs)
out = paddle.add(out, short)
out = F.relu(out)
return out
def _fuse_prepare_qkv(self, query):
mix_layer = self.qkv_proj(query)
mix_layer = paddle.reshape_(mix_layer,
[0, 0, self.num_heads, 3 * self.head_dim])
mix_layer = paddle.transpose(mix_layer, [0, 2, 1, 3])
q, k, v = paddle.split(mix_layer, num_or_sections=3, axis=-1)
return q, k, v
def get_model(self, main_prog, startup_program, rank):
with fluid.program_guard(main_prog, startup_program):
fleet.init(is_collective=True)
np.random.seed(2020)
np_array = np.random.rand(1000, 16)
data = paddle.static.data(name='tindata',
shape=[10, 1000],
dtype="float32")
paddle.distributed.broadcast(data, src=0)
data = paddle.split(data, 2, axis=1)[rank]
if rank == 0:
param_attr = paddle.fluid.ParamAttr(
initializer=paddle.fluid.initializer.NumpyArrayInitializer(
np_array[0:500, :]), )
else:
param_attr = paddle.fluid.ParamAttr(
initializer=paddle.fluid.initializer.NumpyArrayInitializer(
np_array[500:1000, :]), )
linear_out = paddle.distributed.split(
data,
size=(1000, 16),
operation='linear',
axis=0,
num_partitions=2,
weight_attr=param_attr,
bias_attr=True,
)
return [linear_out]
