def get_feature_by_coordinate(self, x, coord, offset_h, offset_w,
padded_x_w):
x = paddle.reshape(x, [0, 0, -1])
index = paddle.cast(
coord[:, :, :, :self.N] * padded_x_w,
dtype='int64') + coord[:, :, :, self.N:] # offset_x*w + offset_y
index = paddle.unsqueeze(index, 1)
index = paddle.tile(index, [1, self.in_channel, 1, 1, 1])
index = paddle.reshape(index, (0, 0, -1))
x_range = list(range(3))
dim = 2
x_range[0] = dim
x_range[dim] = 0
x_swaped = paddle.transpose(x, perm=x_range)
index_range = list(range(3))
index_range[0] = dim
index_range[dim] = 0
index_swaped = paddle.transpose(index, perm=index_range)
x_shape = layers.shape(x_swaped)
index_shape = layers.shape(index_swaped)
prod = paddle.prod(x_shape[1:], keepdim=True)
x_swaped_flattend = paddle.reshape(x_swaped, [-1])
index_swaped_flattend = paddle.reshape(index_swaped, [-1])
index_swaped_flattend *= prod
bias = paddle.arange(start=0, end=prod, step=1, dtype='float32')
bias = paddle.tile(bias, index_shape[0])
index_swaped_flattend += bias
gathered = paddle.gather(x_swaped_flattend, index_swaped_flattend)
gathered = paddle.reshape(gathered, layers.shape(index_swaped))
x_offset = paddle.transpose(gathered, perm=x_range)
x_offset = paddle.reshape(
x_offset, (-1, self.in_channel, offset_h, offset_w, self.N))
return x_offset
def calc_square_dist(point_feat_a, point_feat_b, norm=True):
"""Calculating square distance between a and b.
Args:
point_feat_a (Tensor): (B, N, C) Feature vector of each point.
point_feat_b (Tensor): (B, M, C) Feature vector of each point.
norm (Bool): Whether to normalize the distance.
Default: True.
Returns:
Tensor: (B, N, M) Distance between each pair points.
"""
length_a = point_feat_a.shape[1]
length_b = point_feat_b.shape[1]
num_channel = point_feat_a.shape[-1]
# [bs, n, 1]
a_square = paddle.sum(point_feat_a.unsqueeze(2).pow(2), axis=-1)
# [bs, 1, m]
b_square = paddle.sum(point_feat_b.unsqueeze(1).pow(2), axis=-1)
a_square = paddle.tile(a_square, (1, 1, length_b)) # [bs, n, m]
b_square = paddle.tile(b_square, (1, length_a, 1)) # [bs, n, m]
coor = paddle.matmul(point_feat_a, point_feat_b.transpose((1, 2)))
dist = a_square + b_square - 2 * coor
if norm:
dist = paddle.sqrt(dist) / num_channel
return dist
def forward(self, inputs, encoder_word_pos, gsrm_word_pos):
b, c, h, w = inputs.shape
conv_features = paddle.reshape(inputs, shape=[-1, c, h * w])
conv_features = paddle.transpose(conv_features, perm=[0, 2, 1])
# transformer encoder
b, t, c = conv_features.shape
enc_inputs = [conv_features, encoder_word_pos, None]
word_features = self.wrap_encoder_for_feature(enc_inputs)
# pvam
b, t, c = word_features.shape
word_features = self.fc0(word_features)
word_features_ = paddle.reshape(word_features, [-1, 1, t, c])
word_features_ = paddle.tile(word_features_,
[1, self.max_length, 1, 1])
word_pos_feature = self.emb(gsrm_word_pos)
word_pos_feature_ = paddle.reshape(word_pos_feature,
[-1, self.max_length, 1, c])
word_pos_feature_ = paddle.tile(word_pos_feature_, [1, 1, t, 1])
y = word_pos_feature_ + word_features_
y = F.tanh(y)
attention_weight = self.fc1(y)
attention_weight = paddle.reshape(attention_weight,
shape=[-1, self.max_length, t])
attention_weight = F.softmax(attention_weight, axis=-1)
pvam_features = paddle.matmul(attention_weight,
word_features) #[b, max_length, c]
return pvam_features
def forward(self):
input_feature = self.input('Input', 0)
input_shape = paddle.shape(input_feature)
n, c, h, w = paddle.tensor.split(input_shape, num_or_sections=4)
x_ctr = paddle.arange(start=0, end=w, step=1, dtype=input_feature.dtype)
y_ctr = paddle.arange(start=0, end=h, step=1, dtype=input_feature.dtype)
x_ctr = x_ctr * self.strides[0] + self.offset * (self.strides[0] - 1)
y_ctr = y_ctr * self.strides[1] + self.offset * (self.strides[1] - 1)
tensor_one = paddle.ones(shape=[1], dtype='int64')
tensor_len_shape = paddle.full(
shape=[1], fill_value=len(self.shapes), dtype='int64')
x_ctr = paddle.reshape(x_ctr, shape=(1, -1))
y_ctr = paddle.reshape(y_ctr, shape=(1, -1))
x_ctr = paddle.tile(x_ctr, repeat_times=(h, tensor_one))
y_ctr = paddle.tile(y_ctr, repeat_times=(w, tensor_one))
y_ctr = paddle.transpose(y_ctr, perm=[1, 0])
centers = paddle.stack([x_ctr, y_ctr], axis=-1)
centers = paddle.tensor.unsqueeze(centers, axis=[2])
centers = paddle.tile(centers, repeat_times=(1, 1, len(self.shapes), 2))
shape_tensor = paddle.assign(np.array(self.shapes).astype('float32'))
anchors = centers + shape_tensor
variance_tensor = paddle.assign(
np.asarray(self.variances).astype('float32'))
vars = paddle.reshape(variance_tensor, shape=[1, 1, 1, -1])
vars = paddle.tile(
vars, repeat_times=(h, w, tensor_len_shape, tensor_one))
return {'Anchors': [anchors], 'Variances': [vars]}
def _sample(self, n_samples=1, **kwargs):
if n_samples > 1:
_shape = fluid.layers.shape(self._mean)
_shape = fluid.layers.concat([paddle.to_tensor([n_samples], dtype="int32"), _shape])
_len = len(self._std.shape)
_std = paddle.tile(self._std, repeat_times=[n_samples, *_len*[1]])
_mean = paddle.tile(self._mean, repeat_times=[n_samples, *_len*[1]])
else:
_shape = fluid.layers.shape(self._mean)
_std = self._std + 0.
_mean = self._mean + 0.
if self.is_reparameterized:
epsilon = paddle.normal(name='sample',
shape=_shape,
mean=0.0,
std=1.0)
sample_ = _mean + _std * epsilon
else:
_mean.stop_gradient = True
_std.stop_gradient = True
epsilon = paddle.normal(name='sample',
shape=_shape,
mean=0.0,
std=1.0)
sample_ = _mean + _std * epsilon
sample_.stop_gradient = False
self.sample_cache = sample_
if n_samples > 1:
assert(sample_.shape[0] == n_samples)
return sample_
def create_sparse_motions(self, source_image, kp_driving, kp_source):
"""
Eq 4. in the paper T_{s<-d}(z)
"""
bs, _, h, w = source_image.shape
identity_grid = make_coordinate_grid((h, w),
type=kp_source['value'].dtype)
identity_grid = identity_grid.reshape([1, 1, h, w, 2])
coordinate_grid = identity_grid - kp_driving['value'].reshape(
[bs, self.num_kp, 1, 1, 2])
if 'jacobian' in kp_driving:
jacobian = paddle.matmul(kp_source['jacobian'],
paddle.inverse(kp_driving['jacobian']))
jacobian = jacobian.unsqueeze(-3).unsqueeze(-3)
jacobian = paddle.tile(jacobian, [1, 1, h, w, 1, 1])
coordinate_grid = paddle.matmul(jacobian,
coordinate_grid.unsqueeze(-1))
coordinate_grid = coordinate_grid.squeeze(-1)
driving_to_source = coordinate_grid + kp_source['value'].reshape(
[bs, self.num_kp, 1, 1, 2])
#adding background feature
identity_grid = paddle.tile(identity_grid, (bs, 1, 1, 1, 1))
sparse_motions = paddle.concat([identity_grid, driving_to_source],
axis=1)
return sparse_motions
def query_ball_point(radius, nsample, xyz, new_xyz):
"""
Input:
radius: local region radius
nsample: max sample number in local region
xyz: all points, [B, N, 3]
new_xyz: query points, [B, S, 3]
Return:
group_idx: grouped points index, [B, S, nsample]
"""
B, N, C = xyz.shape
_, S, _ = new_xyz.shape
group_idx = paddle.tile(paddle.arange(N).reshape([1, 1, N]), [B, S, 1])
sqrdists = square_distance(new_xyz, xyz)
mask = sqrdists > radius ** 2
group_idx_np = group_idx.numpy()
mask_np = mask.numpy()
group_idx_np[mask_np] = N
group_idx = paddle.to_tensor(group_idx_np)
group_idx = group_idx.sort(axis=-1)[:, :, :nsample]
group_first = paddle.tile(group_idx[:, :, 0].reshape([B, S, 1]), [1, 1, nsample])
mask = group_idx == N
group_idx_np = group_idx.numpy()
group_first_np = group_first.numpy()
mask_np = mask.numpy()
group_idx_np[mask_np] = group_first_np[mask_np]
group_idx = paddle.to_tensor(group_idx_np)
return group_idx
def test_api(self):
with program_guard(Program(), Program()):
repeat_times = [2, 2]
x1 = fluid.layers.data(name='x1', shape=[4], dtype="int32")
out = paddle.tile(x1, repeat_times)
positive_2 = fluid.layers.fill_constant([1],
dtype="int32",
value=2)
out2 = paddle.tile(x1, repeat_times=[positive_2, 2])
def make_animation(self,
source_image,
driving_video,
generator,
kp_detector,
relative=True,
adapt_movement_scale=True):
with paddle.no_grad():
predictions = []
source = paddle.to_tensor(source_image[np.newaxis].astype(
np.float32)).transpose([0, 3, 1, 2])
driving = paddle.to_tensor(
np.array(driving_video).astype(np.float32)).transpose(
[0, 3, 1, 2])
kp_source = kp_detector(source)
kp_driving_initial = kp_detector(driving[0:1])
kp_source_batch = {}
kp_source_batch["value"] = paddle.tile(
kp_source["value"], repeat_times=[self.batch_size, 1, 1])
kp_source_batch["jacobian"] = paddle.tile(
kp_source["jacobian"], repeat_times=[self.batch_size, 1, 1, 1])
source = paddle.tile(source,
repeat_times=[self.batch_size, 1, 1, 1])
begin_idx = 0
for frame_idx in tqdm(
range(
int(np.ceil(float(driving.shape[0]) /
self.batch_size)))):
frame_num = min(self.batch_size, driving.shape[0] - begin_idx)
driving_frame = driving[begin_idx:begin_idx + frame_num]
kp_driving = kp_detector(driving_frame)
kp_source_img = {}
kp_source_img["value"] = kp_source_batch["value"][0:frame_num]
kp_source_img["jacobian"] = kp_source_batch["jacobian"][
0:frame_num]
kp_norm = normalize_kp(
kp_source=kp_source,
kp_driving=kp_driving,
kp_driving_initial=kp_driving_initial,
use_relative_movement=relative,
use_relative_jacobian=relative,
adapt_movement_scale=adapt_movement_scale)
out = generator(source[0:frame_num],
kp_source=kp_source_img,
kp_driving=kp_norm)
img = np.transpose(out['prediction'].numpy(),
[0, 2, 3, 1]) * 255.0
if self.face_enhancement:
img = self.faceenhancer.enhance_from_batch(img)
predictions.append(img)
begin_idx += frame_num
return np.concatenate(predictions)
def _meshgrid(self, x, y, w, h, row_major=True):
xx = paddle.tile(paddle.reshape(x, (1, -1)), [h, 1])
yy = paddle.tile(paddle.reshape(y, (-1, 1)), [1, w])
xx = paddle.reshape(xx, (-1, ))
yy = paddle.reshape(yy, (-1, ))
if row_major:
return xx, yy
else:
return yy, xx
def concatenation(self, theta_x, phi_x):
h = theta_x.shape[2]
w = phi_x.shape[3]
theta_x = paddle.tile(theta_x, [1, 1, 1, w])
phi_x = paddle.tile(phi_x, [1, 1, h, 1])
concat_feature = paddle.concat([theta_x, phi_x], axis=1)
pairwise_weight = self.concat_project(concat_feature)
n, _, h, w = pairwise_weight.shape
pairwise_weight = paddle.reshape(pairwise_weight, [n, h, w])
pairwise_weight /= pairwise_weight.shape[-1]
return pairwise_weight
def forward(self, observed):
self.observe(observed)
x = self.observed['x']
h = paddle.tile(x, [self.n_particles, *len(x.shape) * [1]])
batch_size = x.shape[0]
for i, (n_in, n_out) in enumerate(
zip(self.layer_sizes[:-1], self.layer_sizes[1:])):
w = self.sn('Normal',
name="w" + str(i),
mean=fluid.layers.zeros(shape=[n_out, n_in + 1],
dtype='float32'),
std=fluid.layers.ones(shape=[n_out, n_in + 1],
dtype='float32'),
group_ndims=2,
n_samples=self.n_particles,
reduce_mean_dims=[0])
w = fluid.layers.unsqueeze(w, axes=[1])
w = paddle.tile(w, [1, batch_size, 1, 1])
h = paddle.concat([
h,
fluid.layers.ones(shape=[*h.shape[:-1], 1], dtype='float32')
], -1)
h = paddle.reshape(h, h.shape + [1])
p = fluid.layers.sqrt(paddle.to_tensor(h.shape[2],
dtype='float32'))
h = paddle.matmul(w, h) / p
h = paddle.squeeze(h, [-1])
if i < len(self.layer_sizes) - 2:
h = paddle.nn.ReLU()(h)
y_mean = fluid.layers.squeeze(h, [2])
y = self.observed['y']
y_pred = fluid.layers.reduce_mean(y_mean, [0])
self.cache['rmse'] = fluid.layers.sqrt(
fluid.layers.reduce_mean((y - y_pred)**2))
self.sn(
'Normal',
name='y',
mean=y_mean,
logstd=self.y_logstd,
reparameterize=True,
reduce_mean_dims=[0, 1],
multiplier=456,
) ## training data size
return self
def __init__(self, channels, scale):
super(AntiAliasInterpolation2d, self).__init__()
sigma = (1 / scale - 1) / 2
kernel_size = 2 * round(sigma * 4) + 1
self.ka = kernel_size // 2
self.kb = self.ka - 1 if kernel_size % 2 == 0 else self.ka
kernel_size = [kernel_size, kernel_size]
sigma = [sigma, sigma]
# The gaussian kernel is the product of the
# gaussian function of each dimension.
kernel = 1
meshgrids = paddle.meshgrid(
[paddle.arange(size, dtype='float32') for size in kernel_size])
for size, std, mgrid in zip(kernel_size, sigma, meshgrids):
mean = (size - 1) / 2
kernel *= paddle.exp(-(mgrid - mean)**2 / (2 * std**2 + 1e-9))
# Make sure sum of values in gaussian kernel equals 1.
kernel = kernel / paddle.sum(kernel)
# Reshape to depthwise convolutional weight
kernel = kernel.reshape([1, 1, *kernel.shape])
kernel = paddle.tile(kernel, [channels, *[1] * (kernel.dim() - 1)])
self.register_buffer('weight', kernel)
self.groups = channels
self.scale = scale
def prepare_inputs_for_generation(self,
decoder_input_ids,
attention_mask=None,
encoder_output=None,
use_cache=True,
cache=None,
**kwargs):
if encoder_output is not None:
expand_size = int(decoder_input_ids.shape[0] /
encoder_output.shape[0])
if expand_size > 1:
index = paddle.tile(
paddle.arange(encoder_output.shape[0]).unsqueeze(-1),
[1, expand_size]).reshape([-1])
encoder_output = paddle.index_select(encoder_output, index)
if use_cache and cache is None:
if encoder_output is None:
raise ValueError(
"Encoder output can not be none if `use_cache` is True")
cache = self.decoder.decoder.gen_cache(memory=encoder_output)
if cache is not None:
decoder_input_ids = decoder_input_ids[:, -1:]
return {
"input_ids":
None, # during prediction, Encoder_output is provided, do not need input_ids.
"decoder_input_ids": decoder_input_ids,
"encoder_output": encoder_output,
"attention_mask": attention_mask,
"use_cache": use_cache,
"cache": cache
}
def prepare_inputs_for_generation(self,
decoder_input_ids,
attention_mask=None,
encoder_output=None,
use_cache=True,
cache=None,
**kwargs):
"""
Prepare inputs for decoder to generate sentences.
Return:
dict: A dictionary containing necessary inputs for generating next token.
"""
if encoder_output is not None:
expand_size = int(decoder_input_ids.shape[0] /
encoder_output.shape[0])
if expand_size > 1:
index = paddle.tile(
paddle.arange(encoder_output.shape[0]).unsqueeze(-1),
[1, expand_size]).reshape([-1])
encoder_output = paddle.index_select(encoder_output, index)
if cache is not None:
decoder_input_ids = decoder_input_ids[:, -1:]
return {
"input_ids":
None, # during prediction, Encoder_output is provided, do not need input_ids.
"decoder_input_ids": decoder_input_ids,
"encoder_output": encoder_output,
"attention_mask": attention_mask,
"use_cache": use_cache,
"cache": cache
}
def forward(self, x):
b, c, h, w = x.shape
x = paddle.reshape(x, [b, c, h * w])
mu = paddle.tile(self.mu, [b, 1, 1])
with paddle.no_grad():
for i in range(self.stage_num):
x_t = paddle.transpose(x, [0, 2, 1])
z = paddle.bmm(x_t, mu)
z = F.softmax(z, axis=2)
z_ = F.normalize(z, axis=1, p=1)
mu = paddle.bmm(x, z_)
mu = F.normalize(mu, axis=1, p=2)
z_t = paddle.transpose(z, [0, 2, 1])
x = paddle.matmul(mu, z_t)
x = paddle.reshape(x, [b, c, h, w])
if self.training:
mu = paddle.mean(mu, 0, keepdim=True)
if paddle.distributed.get_world_size() > 1:
paddle.distributed.reduce(
mu / paddle.distributed.get_world_size(), 0)
mu = F.normalize(mu, axis=1, p=2)
self.mu = self.mu * (1 - self.momentum) + mu * self.momentum
return x
def forward(self, inputs):
xyz = paddle.to_tensor(inputs[0])
cls_label = Categorical(inputs[1], self.num_classes)
# Set Abstraction layers
B, C, N = xyz.shape
if self.normal_channel:
l0_points = xyz
l0_xyz = xyz[:, :3, :]
else:
l0_points = xyz
l0_xyz = xyz
l1_xyz, l1_points = self.sa1(l0_xyz, l0_points)
l2_xyz, l2_points = self.sa2(l1_xyz, l1_points)
l3_xyz, l3_points = self.sa3(l2_xyz, l2_points)
# Feature Propagation layers
l2_points = self.fp3(l2_xyz, l3_xyz, l2_points, l3_points)
l1_points = self.fp2(l1_xyz, l2_xyz, l1_points, l2_points)
cls_label_one_hot = paddle.tile(
cls_label.reshape([B, self.num_classes, 1]), [1, 1, N])
l0_points = self.fp1(
l0_xyz, l1_xyz,
paddle.concat([cls_label_one_hot, l0_xyz, l0_points], 1),
l1_points)
# FC layers
feat = F.relu(self.bn1(self.conv1(l0_points)))
x = self.drop1(feat)
x = self.conv2(x)
x = x.transpose([0, 2, 1])
return x
def forward(self, inputs):
inputs = paddle.to_tensor(inputs[0])
batchsize = inputs.shape[0]
t_net = self.input_transform_net(inputs)
t_net = paddle.squeeze(t_net, axis=-1)
t_net = self.input_fc(t_net)
t_net = paddle.reshape(t_net, [batchsize, 3, 3])
x = paddle.transpose(inputs, (0, 2, 1))
x = paddle.matmul(x, t_net)
x = paddle.transpose(x, (0, 2, 1))
x = self.mlp_1(x)
t_net = self.feature_transform_net(x)
t_net = paddle.squeeze(t_net, axis=-1)
t_net = self.feature_fc(t_net)
t_net = paddle.reshape(t_net, [batchsize, 64, 64])
x = paddle.squeeze(x, axis=-1)
x = paddle.transpose(x, (0, 2, 1))
x = paddle.matmul(x, t_net)
x = paddle.transpose(x, (0, 2, 1))
point_feat = x
x = self.mlp_2(x)
x = paddle.max(x, axis=-1, keepdim=True)
global_feat_expand = paddle.tile(x, [1, 1, self.max_point])
x = paddle.concat([point_feat, global_feat_expand], axis=1)
x = self.seg_net(x)
x = paddle.squeeze(x, axis=-1)
x = paddle.transpose(x, (0, 2, 1))
return x
def forward(self, x):
x_shape = paddle.shape(x)
x = x.flatten(2)
mu = paddle.tile(self.mu, [x_shape[0], 1, 1])
with paddle.no_grad():
for i in range(self.stage_num):
x_t = paddle.transpose(x, [0, 2, 1])
z = paddle.bmm(x_t, mu)
z = F.softmax(z, axis=2)
z_ = F.normalize(z, axis=1, p=1)
mu = paddle.bmm(x, z_)
mu = F.normalize(mu, axis=1, p=2)
z_t = paddle.transpose(z, [0, 2, 1])
x = paddle.matmul(mu, z_t)
x = paddle.reshape(x, [0, self.c, x_shape[2], x_shape[3]])
if self.training:
mu = paddle.mean(mu, 0, keepdim=True)
mu = F.normalize(mu, axis=1, p=2)
mu = self.mu * (1 - self.momentum) + mu * self.momentum
if paddle.distributed.get_world_size() > 1:
mu = paddle.distributed.all_reduce(mu)
mu /= paddle.distributed.get_world_size()
self.mu = mu
return x
def __call__(self, x, index):
if self.dim < 0:
self.dim += len(x.shape)
x_range = list(range(len(x.shape)))
x_range[0] = self.dim
x_range[self.dim] = 0
x_swaped = paddle.transpose(x, perm=x_range)
index_range = list(range(len(index.shape)))
index_range[0] = self.dim
index_range[self.dim] = 0
index_swaped = paddle.transpose(index, perm=index_range)
dtype = index.dtype
x_shape = paddle.shape(x_swaped)
index_shape = paddle.shape(index_swaped)
prod = paddle.cast(paddle.prod(x_shape), dtype=dtype) / x_shape[0]
x_swaped_flattend = paddle.flatten(x_swaped)
index_swaped_flattend = paddle.flatten(index_swaped)
index_swaped_flattend *= prod
bias = paddle.arange(start=0, end=prod, dtype=dtype)
bias = paddle.reshape(bias, x_shape[1:])
bias = paddle.crop(bias, index_shape[1:])
bias = paddle.flatten(bias)
bias = paddle.tile(bias, [index_shape[0]])
index_swaped_flattend += bias
gathered = paddle.index_select(x_swaped_flattend, index_swaped_flattend)
gathered = paddle.reshape(gathered, index_swaped.shape)
out = paddle.transpose(gathered, perm=x_range)
return out
def expand_inputs_for_generation(input_ids,
expand_size,
attention_mask=None,
**model_kwargs):
index = paddle.tile(
paddle.arange(input_ids.shape[0]).unsqueeze(-1),
[1, expand_size]).reshape([-1])
input_ids = paddle.index_select(input_ids, index)
if attention_mask is not None:
model_kwargs["attention_mask"] = paddle.index_select(
attention_mask, index)
if "token_type_ids" in model_kwargs:
token_type_ids = model_kwargs["token_type_ids"]
model_kwargs["token_type_ids"] = paddle.index_select(
token_type_ids, index)
if "position_ids" in model_kwargs:
position_ids = model_kwargs["position_ids"]
model_kwargs["position_ids"] = paddle.index_select(
position_ids, index)
return input_ids, model_kwargs
def generate_relative_positions_embeddings(self,
length,
depth,
max_relative_position=127):
vocab_size = max_relative_position * 2 + 1
range_vec = paddle.arange(length)
range_mat = paddle.tile(range_vec, repeat_times=[length]).reshape(
(length, length))
distance_mat = range_mat - paddle.t(range_mat)
distance_mat_clipped = paddle.clip(distance_mat.astype('float32'),
-max_relative_position,
max_relative_position)
final_mat = distance_mat_clipped + max_relative_position
embeddings_table = np.zeros([vocab_size, depth])
for pos in range(vocab_size):
for i in range(depth // 2):
embeddings_table[pos, 2 * i] = np.sin(
pos / np.power(10000, 2 * i / depth))
embeddings_table[pos, 2 * i + 1] = np.cos(
pos / np.power(10000, 2 * i / depth))
embeddings_table_tensor = paddle.to_tensor(embeddings_table,
dtype='float32')
flat_relative_positions_matrix = final_mat.reshape((-1, ))
one_hot_relative_positions_matrix = paddle.nn.functional.one_hot(
flat_relative_positions_matrix.astype('int64'),
num_classes=vocab_size)
embeddings = paddle.matmul(one_hot_relative_positions_matrix,
embeddings_table_tensor)
my_shape = final_mat.shape
my_shape.append(depth)
embeddings = embeddings.reshape(my_shape)
return embeddings
def forward(self, item_his_emb, seq_len):
"""forward
Args:
item_his_emb : [B, seqlen, dim]
seq_len : [B, 1]
"""
batch_size = item_his_emb.shape[0]
seq_len_tile = paddle.tile(seq_len, [1, self.k_max])
mask = self.sequence_mask(seq_len_tile, self.maxlen)
pad = paddle.ones_like(mask, dtype="float32") * (-2**32 + 1)
# S*e
low_capsule_new = paddle.matmul(item_his_emb,
self.bilinear_mapping_matrix)
low_capsule_new_nograd = paddle.assign(low_capsule_new)
low_capsule_new_nograd.stop_gradient = True
B = paddle.tile(self.routing_logits,
[paddle.shape(item_his_emb)[0], 1, 1])
for i in range(self.iters - 1):
B_mask = paddle.where(mask, B, pad)
# print(B_mask)
W = F.softmax(B_mask, axis=1)
high_capsule_tmp = paddle.matmul(W, low_capsule_new_nograd)
high_capsule = self.squash(high_capsule_tmp)
B_delta = paddle.matmul(high_capsule,
low_capsule_new_nograd,
transpose_y=True)
B += B_delta / paddle.maximum(
paddle.norm(B_delta, p=2, axis=-1, keepdim=True),
paddle.ones_like(B_delta))
B_mask = paddle.where(mask, B, pad)
W = F.softmax(B_mask, axis=1)
# paddle.static.Print(W)
high_capsule_tmp = paddle.matmul(W, low_capsule_new)
# high_capsule_tmp.stop_gradient = False
high_capsule = self.squash(high_capsule_tmp)
# high_capsule.stop_gradient = False
return high_capsule, W, seq_len
def get_offset_coordinate(self, offset, dtype, offset_shape):
kernel_grid_origin_x = paddle.arange(
0,
self.kernel_size + (self.kernel_size - 1) * (self.dilation - 1),
step=self.dilation,
dtype=dtype)
kernel_grid_origin_x = kernel_grid_origin_x.unsqueeze(1)
kernel_grid_origin_x = paddle.tile(kernel_grid_origin_x,
[1, self.kernel_size])
kernel_grid_origin_y = paddle.arange(
0,
self.kernel_size + (self.kernel_size - 1) * (self.dilation - 1),
step=self.dilation,
dtype=dtype)
kernel_grid_origin_y = kernel_grid_origin_y.unsqueeze(0)
kernel_grid_origin_y = paddle.tile(kernel_grid_origin_y,
[self.kernel_size, 1])
kernel_grid_origin_x = paddle.reshape(kernel_grid_origin_x, [-1])
kernel_grid_origin_y = paddle.reshape(kernel_grid_origin_y, [-1])
kernel_grid_origin = paddle.concat(
[kernel_grid_origin_x, kernel_grid_origin_y], -1)
kernel_grid_origin = paddle.reshape(kernel_grid_origin,
(1, 2 * self.N, 1, 1))
kernel_offset_x = paddle.arange(0,
offset_shape[2] * self.stride,
step=self.stride,
dtype=dtype)
kernel_offset_x = kernel_offset_x.unsqueeze(1)
kernel_offset_x = paddle.expand(kernel_offset_x, offset_shape[2:])
kernel_offset_y = paddle.arange(0,
offset_shape[3] * self.stride,
step=self.stride,
dtype=dtype)
kernel_offset_y = kernel_offset_y.unsqueeze(0)
kernel_offset_y = paddle.expand(kernel_offset_y, offset_shape[2:])
kernel_offset_x = kernel_offset_x.unsqueeze([0, 1])
kernel_offset_x = paddle.tile(kernel_offset_x, (1, self.N, 1, 1))
kernel_offset_y = kernel_offset_y.unsqueeze([0, 1])
kernel_offset_y = paddle.tile(kernel_offset_y, (1, self.N, 1, 1))
kernel_offset = paddle.concat([kernel_offset_x, kernel_offset_y], 1)
offset = offset + paddle.cast(kernel_offset, 'float32') + paddle.cast(
kernel_grid_origin, 'float32')
return offset
def __call__(self, input):
if not self.coord_conv:
return input
b = input.shape[0]
h = input.shape[2]
w = input.shape[3]
eps = 1e-3
x_range = paddle.arange(0, w - eps, 1., dtype='float32') / (w - 1) * 2.0 - 1
y_range = paddle.arange(0, h - eps, 1., dtype='float32') / (h - 1) * 2.0 - 1
# x_range = paddle.to_tensor(x_range, place=input.place)
# y_range = paddle.to_tensor(y_range, place=input.place)
x_range = paddle.reshape(x_range, (1, 1, 1, -1)) # [1, 1, 1, w]
y_range = paddle.reshape(y_range, (1, 1, -1, 1)) # [1, 1, h, 1]
x_range = paddle.tile(x_range, [b, 1, h, 1]) # [b, 1, h, w]
y_range = paddle.tile(y_range, [b, 1, 1, w]) # [b, 1, h, w]
offset = paddle.concat([input, x_range, y_range], axis=1)
return offset
def make_coordinate_grid(spatial_size, type='float32'):
"""
Create a meshgrid [-1,1] x [-1,1] of given spatial_size.
"""
h, w = spatial_size
x = paddle.arange(w, dtype=type) #.type(type)
y = paddle.arange(h, dtype=type) #.type(type)
x = (2 * (x / (w - 1)) - 1)
y = (2 * (y / (h - 1)) - 1)
yy = paddle.tile(y.reshape([-1, 1]), [1, w])
xx = paddle.tile(x.reshape([1, -1]), [h, 1])
meshed = paddle.concat([xx.unsqueeze(2), yy.unsqueeze(2)], 2)
return meshed
def force_decoding(self, beam_search_output, beam_search_state, trg_word,
trg_length, time):
batch_size = paddle.shape(beam_search_output.predicted_ids)[0]
beam_size = paddle.shape(beam_search_output.predicted_ids)[1]
ids_dtype = beam_search_output.predicted_ids.dtype
scores_dtype = beam_search_output.scores.dtype
parent_ids = paddle.zeros(shape=[batch_size, 1], dtype=ids_dtype)
scores = paddle.ones(shape=[batch_size, beam_size],
dtype=scores_dtype) * -10e9
scores = paddle.scatter(
scores.flatten(),
paddle.arange(0,
batch_size * beam_size,
step=beam_size,
dtype=scores_dtype),
paddle.zeros([batch_size])).reshape([batch_size, beam_size])
force_position = paddle.unsqueeze(trg_length > time, [1])
# NOTE: When the date type of the input of paddle.tile is bool
# and enable static mode, its stop_gradient must be True .
force_position.stop_gradient = True
force_position = paddle.tile(force_position, [1, beam_size])
crt_trg_word = paddle.slice(trg_word,
axes=[1],
starts=[time],
ends=[time + 1])
crt_trg_word = paddle.tile(crt_trg_word, [1, beam_size])
predicted_ids = paddle.where(force_position, crt_trg_word,
beam_search_output.predicted_ids)
scores = paddle.where(force_position, scores,
beam_search_output.scores)
parent_ids = paddle.where(force_position, parent_ids,
beam_search_output.parent_ids)
cell_states = beam_search_state.cell_states
log_probs = paddle.where(force_position, scores,
beam_search_state.log_probs)
finished = beam_search_state.finished
lengths = beam_search_state.lengths
return self.OutputWrapper(scores, predicted_ids,
parent_ids), self.StateWrapper(
cell_states, log_probs, finished,
lengths)
def get_loss(self, scores, deltas, targets, rois, bbox_weight):
"""
scores (Tensor): scores from bbox head outputs
deltas (Tensor): deltas from bbox head outputs
targets (list[List[Tensor]]): bbox targets containing tgt_labels, tgt_bboxes and tgt_gt_inds
rois (List[Tensor]): RoIs generated in each batch
"""
# TODO: better pass args
tgt_labels, tgt_bboxes, tgt_gt_inds = targets
tgt_labels = paddle.concat(
tgt_labels) if len(tgt_labels) > 1 else tgt_labels[0]
tgt_labels = tgt_labels.cast('int64')
tgt_labels.stop_gradient = True
loss_bbox_cls = F.cross_entropy(input=scores,
label=tgt_labels,
reduction='mean')
# bbox reg
cls_agnostic_bbox_reg = deltas.shape[1] == 4
fg_inds = paddle.nonzero(
paddle.logical_and(tgt_labels >= 0,
tgt_labels < self.num_classes)).flatten()
cls_name = 'loss_bbox_cls'
reg_name = 'loss_bbox_reg'
loss_bbox = {}
if cls_agnostic_bbox_reg:
reg_delta = paddle.gather(deltas, fg_inds)
else:
fg_gt_classes = paddle.gather(tgt_labels, fg_inds)
reg_row_inds = paddle.arange(fg_gt_classes.shape[0]).unsqueeze(1)
reg_row_inds = paddle.tile(reg_row_inds, [1, 4]).reshape([-1, 1])
reg_col_inds = 4 * fg_gt_classes.unsqueeze(1) + paddle.arange(4)
reg_col_inds = reg_col_inds.reshape([-1, 1])
reg_inds = paddle.concat([reg_row_inds, reg_col_inds], axis=1)
reg_delta = paddle.gather(deltas, fg_inds)
reg_delta = paddle.gather_nd(reg_delta, reg_inds).reshape([-1, 4])
rois = paddle.concat(rois) if len(rois) > 1 else rois[0]
tgt_bboxes = paddle.concat(
tgt_bboxes) if len(tgt_bboxes) > 1 else tgt_bboxes[0]
reg_target = bbox2delta(rois, tgt_bboxes, bbox_weight)
reg_target = paddle.gather(reg_target, fg_inds)
reg_target.stop_gradient = True
loss_bbox_reg = paddle.abs(reg_delta -
reg_target).sum() / tgt_labels.shape[0]
loss_bbox[cls_name] = loss_bbox_cls
loss_bbox[reg_name] = loss_bbox_reg
return loss_bbox
def forward(self, src_word, trg_word):
src_max_len = paddle.shape(src_word)[-1]
trg_max_len = paddle.shape(trg_word)[-1]
base_attn_bias = paddle.cast(
src_word == self.bos_id,
dtype=paddle.get_default_dtype()).unsqueeze([1, 2]) * -1e9
src_slf_attn_bias = base_attn_bias
src_slf_attn_bias.stop_gradient = True
trg_slf_attn_bias = paddle.tensor.triu(
(paddle.ones(
(trg_max_len, trg_max_len),
dtype=paddle.get_default_dtype()) * -np.inf),
1)
trg_slf_attn_bias.stop_gradient = True
trg_src_attn_bias = paddle.tile(base_attn_bias, [1, 1, trg_max_len, 1])
src_pos = paddle.cast(
src_word != self.bos_id, dtype="int64") * paddle.arange(
start=0, end=src_max_len)
trg_pos = paddle.cast(
trg_word != self.bos_id, dtype="int64") * paddle.arange(
start=0, end=trg_max_len)
src_emb = self.src_word_embedding(src_word)
src_pos_emb = self.src_pos_embedding(src_pos)
src_emb = src_emb + src_pos_emb
enc_input = F.dropout(
src_emb, p=self.dropout,
training=self.training) if self.dropout else src_emb
with paddle.static.amp.fp16_guard():
if self.waitk >= src_max_len or self.waitk == -1:
# Full sentence
enc_outputs = [
self.encoder(
enc_input, src_mask=src_slf_attn_bias)
]
else:
# Wait-k policy
enc_outputs = []
for i in range(self.waitk, src_max_len + 1):
enc_output = self.encoder(
enc_input[:, :i, :],
src_mask=src_slf_attn_bias[:, :, :, :i])
enc_outputs.append(enc_output)
trg_emb = self.trg_word_embedding(trg_word)
trg_pos_emb = self.trg_pos_embedding(trg_pos)
trg_emb = trg_emb + trg_pos_emb
dec_input = F.dropout(
trg_emb, p=self.dropout,
training=self.training) if self.dropout else trg_emb
dec_output = self.decoder(
dec_input,
enc_outputs,
tgt_mask=trg_slf_attn_bias,
memory_mask=trg_src_attn_bias)
predict = self.linear(dec_output)
return predict
def test_api(self):
with fluid.dygraph.guard(paddle.XPUPlace(0)):
np_x = np.random.random([12, 14]).astype("float32")
x = paddle.to_tensor(np_x)
positive_2 = np.array([2]).astype("int32")
positive_2 = paddle.to_tensor(positive_2)
repeat_times = np.array([2, 3]).astype("int32")
repeat_times = paddle.to_tensor(repeat_times)
out_1 = paddle.tile(x, repeat_times=[2, 3])
out_2 = paddle.tile(x, repeat_times=[positive_2, 3])
out_3 = paddle.tile(x, repeat_times=repeat_times)
assert np.array_equal(out_1.numpy(), np.tile(np_x, (2, 3)))
assert np.array_equal(out_2.numpy(), np.tile(np_x, (2, 3)))
assert np.array_equal(out_3.numpy(), np.tile(np_x, (2, 3)))
