def __init__(self,
dim,
window_size,
num_heads,
qkv_bias=True,
qk_scale=None,
attn_drop=0.,
proj_drop=0.):
super().__init__()
self.dim = dim
self.window_size = window_size # Wh, Ww
self.num_heads = num_heads
head_dim = dim // num_heads
self.scale = qk_scale or head_dim**-0.5
# define a parameter table of relative position bias
self.relative_position_bias_table = self.create_parameter(
shape=((2 * window_size[0] - 1) * (2 * window_size[1] - 1),
num_heads),
default_initializer=zeros_)
self.add_parameter("relative_position_bias_table",
self.relative_position_bias_table)
# get pair-wise relative position index for each token inside the window
coords_h = paddle.arange(self.window_size[0])
coords_w = paddle.arange(self.window_size[1])
coords = paddle.stack(paddle.meshgrid([coords_h,
coords_w])) # 2, Wh, Ww
coords_flatten = paddle.flatten(coords, 1) # 2, Wh*Ww
coords_flatten_1 = coords_flatten.unsqueeze(axis=2)
coords_flatten_2 = coords_flatten.unsqueeze(axis=1)
relative_coords = coords_flatten_1 - coords_flatten_2
relative_coords = relative_coords.transpose([1, 2, 0])
relative_coords[:, :,
0] += self.window_size[0] - 1 # shift to start from 0
relative_coords[:, :, 1] += self.window_size[1] - 1
relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1
relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww
self.register_buffer("relative_position_index",
relative_position_index)
self.qkv = nn.Linear(dim, dim * 3, bias_attr=qkv_bias)
self.attn_drop = nn.Dropout(attn_drop)
self.proj = nn.Linear(dim, dim)
self.proj_drop = nn.Dropout(proj_drop)
trunc_normal_(self.relative_position_bias_table)
self.softmax = nn.Softmax(axis=-1)
def build_P_paddle(self, I_r_size):
I_r_height, I_r_width = I_r_size
I_r_grid_x = (paddle.arange(-I_r_width, I_r_width, 2, dtype='float64')
+ 1.0) / paddle.to_tensor(np.array([I_r_width]))
I_r_grid_y = (
paddle.arange(-I_r_height, I_r_height, 2, dtype='float64') +
1.0) / paddle.to_tensor(np.array([I_r_height]))
# P: self.I_r_width x self.I_r_height x 2
P = paddle.stack(paddle.meshgrid(I_r_grid_x, I_r_grid_y), axis=2)
P = paddle.transpose(P, perm=[1, 0, 2])
# n (= self.I_r_width x self.I_r_height) x 2
return P.reshape([-1, 2])
def compute_loss(gold_seq,
scores,
index_to_token_maps,
gold_tok_to_id,
noise=0.00000001):
""" Computes the loss of a gold sequence given scores.
Args:
gold_seq (`list`): A sequence of gold tokens.
scores (`list`): Expressions representing the scores of
potential output tokens for each token in gold_seq.
index_to_token_maps (`list`): Maps from index in the
sequence to a dictionary mapping from a string to a set of integers.
gold_tok_to_id (`func`): Maps from the gold token
and some lookup function to the indices in the probability distribution
where the gold token occurs.
noise (`float`, optional): The amount of noise to add to the loss.
Returns:
`Tensor`: representing the sum of losses over the sequence.
"""
assert len(gold_seq) == len(scores) == len(index_to_token_maps)
losses = []
predicted_sql = []
for i, gold_tok in enumerate(gold_seq):
score = scores[i]
token_map = index_to_token_maps[i]
gold_indices = gold_tok_to_id(gold_tok, token_map)
assert len(gold_indices) > 0
noise_i = noise
'''
if len(gold_indices) == 1:
noise_i = 0
'''
probdist = score
prob_of_tok = paddle.sum(
paddle.index_select(probdist, paddle.to_tensor(gold_indices)))
if prob_of_tok < noise_i:
prob_of_tok = prob_of_tok + noise_i
elif prob_of_tok > 1 - noise_i:
prob_of_tok = prob_of_tok - noise_i
losses.append(-paddle.log(prob_of_tok))
return paddle.sum(paddle.stack(losses))
def forward(self, featmap_size, stride=16):
# featmap_size*stride project it to original area
feat_h = featmap_size[0]
feat_w = featmap_size[1]
shift_x = paddle.arange(0, feat_w, 1, 'int32') * stride
shift_y = paddle.arange(0, feat_h, 1, 'int32') * stride
shift_xx, shift_yy = self._meshgrid(shift_x, shift_y)
shifts = paddle.stack([shift_xx, shift_yy, shift_xx, shift_yy],
axis=-1)
all_anchors = self.base_anchors[:, :] + shifts[:, :]
all_anchors = all_anchors.reshape([feat_h * feat_w, 4])
return all_anchors
def get_reference_points(spatial_shapes, valid_ratios):
valid_ratios = valid_ratios.unsqueeze(1)
reference_points = []
for i, (H, W) in enumerate(spatial_shapes.tolist()):
ref_y, ref_x = paddle.meshgrid(paddle.linspace(0.5, H - 0.5, H),
paddle.linspace(0.5, W - 0.5, W))
ref_y = ref_y.flatten().unsqueeze(0) / (valid_ratios[:, :, i, 1] *
H)
ref_x = ref_x.flatten().unsqueeze(0) / (valid_ratios[:, :, i, 0] *
W)
reference_points.append(paddle.stack((ref_x, ref_y), axis=-1))
reference_points = paddle.concat(reference_points, 1).unsqueeze(2)
reference_points = reference_points * valid_ratios
return reference_points
def __call__(self, im_crops):
paddle.disable_static()
try:
im_batch = self._preprocess(im_crops)
# im_batch = im_batch.to(self.device)
features = [] # self.net(im_batch)
for f in im_batch:
features.append(
torch.squeeze(
self.net(torch.unsqueeze(np.moveaxis(f, -1, 0),
axis=0))))
return torch.stack(features, axis=0).numpy()
finally:
paddle.enable_static()
def encode_text(self, text):
x = self.token_embedding(text)
x = x + self.positional_embedding
x = self.transformer(x)
x = self.ln_final(x)
select = []
index = zip(paddle.arange(x.shape[0]).numpy(), text.argmax(axis=-1).numpy())
for i, j in index:
select.append(x[int(i), int(j)])
x = paddle.stack(select) @ self.text_projection
return x
def rbox2poly(rrects):
"""
rrect:[x_ctr,y_ctr,w,h,angle]
to
poly:[x0,y0,x1,y1,x2,y2,x3,y3]
"""
N = paddle.shape(rrects)[0]
x_ctr = rrects[:, 0]
y_ctr = rrects[:, 1]
width = rrects[:, 2]
height = rrects[:, 3]
angle = rrects[:, 4]
tl_x, tl_y, br_x, br_y = -width * 0.5, -height * 0.5, width * 0.5, height * 0.5
normal_rects = paddle.stack(
[tl_x, br_x, br_x, tl_x, tl_y, tl_y, br_y, br_y], axis=0)
normal_rects = paddle.reshape(normal_rects, [2, 4, N])
normal_rects = paddle.transpose(normal_rects, [2, 0, 1])
sin, cos = paddle.sin(angle), paddle.cos(angle)
# M.shape=[N,2,2]
M = paddle.stack([cos, -sin, sin, cos], axis=0)
M = paddle.reshape(M, [2, 2, N])
M = paddle.transpose(M, [2, 0, 1])
# polys:[N,8]
polys = paddle.matmul(M, normal_rects)
polys = paddle.transpose(polys, [2, 1, 0])
polys = paddle.reshape(polys, [-1, N])
polys = paddle.transpose(polys, [1, 0])
tmp = paddle.stack(
[x_ctr, y_ctr, x_ctr, y_ctr, x_ctr, y_ctr, x_ctr, y_ctr], axis=1)
polys = polys + tmp
return polys
def paddle_default_data_collator(
features: List[InputDataClass]) -> Dict[str, Any]:
if not isinstance(features[0], (dict, BatchEncoding)):
features = [vars(f) for f in features]
first = features[0]
batch = {}
# Special handling for labels.
# Ensure that tensor is created with the correct type
# (it should be automatically the case, but let's make sure of it.)
if "label" in first and first["label"] is not None:
label = first["label"].item() if isinstance(
first["label"], paddle.Tensor) else first["label"]
dtype = 'int64' if isinstance(label, int) else 'float32'
batch["labels"] = paddle.to_tensor([f["label"] for f in features],
dtype=dtype)
elif "label_ids" in first and first["label_ids"] is not None:
if isinstance(first["label_ids"], paddle.Tensor):
batch["labels"] = paddle.stack([f["label_ids"] for f in features])
else:
dtype = 'int64' if type(
first["label_ids"][0]) is int else 'float32'
batch["labels"] = paddle.to_tensor(
[f["label_ids"] for f in features], dtype=dtype)
# Handling of all other possible keys.
# Again, we will use the first element to figure out which key/values are not None for this model.
for k, v in first.items():
if k not in ("label",
"label_ids") and v is not None and not isinstance(v, str):
if isinstance(v, paddle.Tensor):
batch[k] = paddle.stack([f[k] for f in features])
else:
batch[k] = paddle.to_tensor([f[k] for f in features])
return batch
def get_score(self, head, rel, tail):
re_head, im_head = paddle.chunk(head, chunks=2, axis=-1)
re_tail, im_tail = paddle.chunk(tail, chunks=2, axis=-1)
phase_rel = rel / (self.emb_init / np.pi)
re_rel, im_rel = paddle.cos(phase_rel), paddle.sin(phase_rel)
re_score = re_rel * re_tail + im_rel * im_tail
im_score = re_rel * im_tail - im_rel * re_tail
re_score = re_score - re_head
im_score = im_score - im_head
score = paddle.stack([re_score, im_score], axis=0)
score = self.gamma - paddle.sum(paddle.norm(score, p=2, axis=0),
axis=-1)
return score
def forward(self, g_list, bond_feat):
h_list = []
for k in range(self.num_angle):
h = self.conv_layer[k](g_list[k], bond_feat)
h_list.append(h)
if self.merge == 'cat':
feat_h = paddle.concat(h_list, axis=-1)
if self.merge == 'mean':
feat_h = paddle.mean(paddle.stack(h_list, axis=-1), axis=1)
if self.merge == 'sum':
feat_h = paddle.sum(paddle.stack(h_list, axis=-1), axis=1)
if self.merge == 'max':
feat_h = paddle.max(paddle.stack(h_list, axis=-1), axis=1)
if self.merge == 'cat_max':
feat_h = paddle.stack(h_list, axis=-1)
feat_max = paddle.max(feat_h, dim=1)[0]
feat_max = paddle.reshape(feat_max, [-1, 1, self.hidden_dim])
feat_h = paddle.reshape(feat_h * feat_max,
[-1, self.num_angle * self.hidden_dim])
if self.activation:
feat_h = self.activation(feat_h)
return feat_h
def _train(self, inputs, labels):
"""
Args:
inputs (TYPE): NULL
labels (TYPE)
"""
enc_results = self.encoder(inputs)
lst_loss = []
for orig_inputs, label_info, enc_result in zip(inputs['orig_inputs'],
labels, enc_results):
loss = self.decoder.compute_loss(orig_inputs, label_info,
enc_result)
lst_loss.append(loss)
return paddle.mean(paddle.stack(lst_loss, axis=0), axis=0)
def pointer_choice(self, node_type, logits, attention_logits):
"""pointer_choice"""
# Group them based on pointer map
pointer_logprobs = self.model.pointer_infer(node_type, logits)
pointer_map = self.desc_enc.pointer_maps.get(node_type)
if not pointer_map:
return pointer_logprobs
pointer_logprobs = dict(pointer_logprobs)
return [
(orig_index, paddle.logsumexp(
paddle.stack(tuple(pointer_logprobs[i] for i in mapped_indices), axis=0),
axis=0))
for orig_index, mapped_indices in pointer_map.items()
]
def delta2rbox(rrois,
deltas,
means=[0, 0, 0, 0, 0],
stds=[1, 1, 1, 1, 1],
wh_ratio_clip=1e-6):
"""
:param rrois: (cx, cy, w, h, theta)
:param deltas: (dx, dy, dw, dh, dtheta)
:param means:
:param stds:
:param wh_ratio_clip:
:return:
"""
means = paddle.to_tensor(means)
stds = paddle.to_tensor(stds)
deltas = paddle.reshape(deltas, [-1, deltas.shape[-1]])
denorm_deltas = deltas * stds + means
dx = denorm_deltas[:, 0]
dy = denorm_deltas[:, 1]
dw = denorm_deltas[:, 2]
dh = denorm_deltas[:, 3]
dangle = denorm_deltas[:, 4]
max_ratio = np.abs(np.log(wh_ratio_clip))
dw = paddle.clip(dw, min=-max_ratio, max=max_ratio)
dh = paddle.clip(dh, min=-max_ratio, max=max_ratio)
rroi_x = rrois[:, 0]
rroi_y = rrois[:, 1]
rroi_w = rrois[:, 2]
rroi_h = rrois[:, 3]
rroi_angle = rrois[:, 4]
gx = dx * rroi_w * paddle.cos(rroi_angle) - dy * rroi_h * paddle.sin(
rroi_angle) + rroi_x
gy = dx * rroi_w * paddle.sin(rroi_angle) + dy * rroi_h * paddle.cos(
rroi_angle) + rroi_y
gw = rroi_w * dw.exp()
gh = rroi_h * dh.exp()
ga = np.pi * dangle + rroi_angle
ga = (ga + np.pi / 4) % np.pi - np.pi / 4
ga = paddle.to_tensor(ga)
gw = paddle.to_tensor(gw, dtype='float32')
gh = paddle.to_tensor(gh, dtype='float32')
bboxes = paddle.stack([gx, gy, gw, gh, ga], axis=-1)
return bboxes
def forward(self, input_ids, position_ids=None, attention_mask=None):
r"""
The GPTForSequenceClassification forward method, overrides the __call__() special method.
Args:
input_ids (Tensor):
See :class:`GPTModel`.
position_ids(Tensor, optional):
See :class:`GPTModel`.
attention_mask (list, optional):
See :class:`GPTModel`.
Returns:
Tensor: Returns tensor `logits`, a tensor of the input text classification logits.
Shape as `[batch_size, num_classes]` and dtype as float32.
Example:
.. code-block::
import paddle
from paddlenlp.transformers import GPTForSequenceClassification, GPTTokenizer
tokenizer = GPTTokenizer.from_pretrained('gpt2-medium-en')
model = GPTForSequenceClassification.from_pretrained('gpt2-medium-en')
inputs = tokenizer("Welcome to use PaddlePaddle and PaddleNLP!", return_token_type_ids=False)
inputs = {k:paddle.to_tensor([v]) for (k, v) in inputs.items()}
logits = model(**inputs)
"""
# sequence_output shape [bs, seq_len, hidden_size]
sequence_output = self.gpt(input_ids,
position_ids=position_ids,
attention_mask=attention_mask)
# logits shape [bs, seq_len, num_class]
logits = self.score(sequence_output)
# padding index maybe 0
eos_token_id = self.gpt.config.get("eos_token_id", 0)
# sequence_lengths shape [bs,]
sequence_lengths = (input_ids != eos_token_id).astype("int64").sum(
axis=-1) - 1
pooled_logits = logits.gather_nd(
paddle.stack(
[paddle.arange(sequence_output.shape[0]), sequence_lengths],
axis=-1))
return pooled_logits
def forward(self, graph, feat):
"""Forward
Args:
graph: hetergeneous graph built by pgl.HeterGraph.
inputs: node features/representation from graph/previous layer.
"""
if self.num_bases < self.num_rels:
weight = paddle.transpose(self.weight, perm=[1, 0, 2])
weight = paddle.matmul(self.w_comp, weight)
weight = paddle.transpose(weight, perm=[1, 0, 2])
else:
weight = self.weight
def send_func(src_feat, dst_feat, edge_feat):
"""
send function
"""
return src_feat
def recv_func(msg):
"""
receive function
"""
return msg.reduce_mean(msg['h'])
feat_list = []
for idx, etype in enumerate(self.etypes):
sub_g = graph[graph.edge_types[idx]]
sub_g.tensor()
if self.norm:
norm = GF.degree_norm(sub_g)
feat = feat * norm
w = weight[idx, :, :].squeeze()
h = paddle.matmul(feat, w)
msg = sub_g.send(send_func, src_feat={'h':h})
h = sub_g.recv(recv_func, msg)
feat_list.append(h)
h = paddle.stack(feat_list, axis=0)
h = paddle.sum(h, axis=0)
if self.act == 'relu':
Act = paddle.nn.ReLU()
h = Act(h)
else:
Act = paddle.nn.Sigmoid()
h = Act(h)
return h
def forward(self, inputs):
x = self.bn1(inputs)
x = paddle.reshape(x, [1, 3 * 16 * 16])
x = self.ln1(x)
x = self.fc1(x)
x = paddle.fluid.layers.unsqueeze(input=x, axes=[2])
x = self.relu1(x)
y = paddle.fluid.layers.fill_constant(x.shape,
dtype=paddle.float32,
value=1)
x = paddle.stack([x, y], axis=3)
x = paddle.slice(x, axes=[0], starts=[0], ends=[1])
x = paddle.exp(x)
y += paddle.fluid.layers.uniform_random(y.shape)
y = paddle.fluid.layers.reduce_mean(y, dim=1, keep_dim=True)
return x + y
def get_gt_mask_from_polygons(gt_poly, pad_mask):
out_gt_mask = []
for polygons, padding in zip(gt_poly, pad_mask):
height, width = int(padding[:, 0].sum()), int(padding[0, :].sum())
masks = []
for obj_poly in polygons:
rles = mask_util.frPyObjects(obj_poly, height, width)
rle = mask_util.merge(rles)
masks.append(
paddle.to_tensor(mask_util.decode(rle)).astype('float32'))
masks = paddle.stack(masks)
masks_pad = paddle.zeros(
[masks.shape[0], pad_mask.shape[1], pad_mask.shape[2]])
masks_pad[:, :height, :width] = masks
out_gt_mask.append(masks_pad)
return out_gt_mask
def _create_n_head_attn_mask(self, attn_mask, batch_size):
# attn_mask shape: [B, T, 1]
# concat an data_mask, shape: [B, M + T, 1]
data_mask = paddle.concat([
paddle.ones(shape=[batch_size, self.memory_len, 1],
dtype=attn_mask.dtype), attn_mask
],
axis=1)
data_mask.stop_gradient = True
# create a self_attn_mask, shape: [B, T, M + T]
self_attn_mask = paddle.matmul(attn_mask, data_mask, transpose_y=True)
self_attn_mask = (self_attn_mask - 1) * 1e8
n_head_self_attn_mask = paddle.stack([self_attn_mask] * self.n_head,
axis=1)
n_head_self_attn_mask.stop_gradient = True
return n_head_self_attn_mask
def _compute_locatioins_by_level(self, fpn_stride, feature):
shape_fm = feature.shape
h, w = shape_fm[2], shape_fm[3]
shift_x = paddle.arange(0, w * fpn_stride, fpn_stride)
shift_y = paddle.arange(0, h * fpn_stride, fpn_stride)
shift_x = paddle.unsqueeze(shift_x, axis=0)
shift_y = paddle.unsqueeze(shift_y, axis=1)
shift_x = paddle.expand_as(shift_x, feature[0, 0, :, :])
shift_y = paddle.expand_as(shift_y, feature[0, 0, :, :])
shift_x.stop_gradient = True
shift_y.stop_gradient = True
shift_x = paddle.reshape(shift_x, shape=[-1])
shift_y = paddle.reshape(shift_y, shape=[-1])
location = paddle.stack([shift_x, shift_y], axis=-1) + fpn_stride / 2
location.stop_gradient = True
return location
def __setitem__(self, key, value):
if isinstance(key,tuple):
if len(key)==2:
return super(Tensor,self).__setitem__(key,value)
if isinstance(key,int):
return super(Tensor, self).__setitem__(key, value)
def convert_key_to_inttensor(key):
if isinstance(key, np.ndarray):
# print(max(args),min(args),self.shape,len(args),len(set(args)) )
key=paddorch.from_numpy(key).long()
# print("converted numpy", type(args))
if isinstance(key,paddle.Tensor):
if key.dtype==paddle.fluid.core.VarDesc.VarType.BOOL:
key = paddle.masked_select(paddle.arange(len(key)), key)
elif key.dtype==paddle.fluid.core.VarDesc.VarType.INT32 or key.dtype==paddle.fluid.core.VarDesc.VarType.INT64:
return key
else:
return key.astype("int32")
if isinstance(key,int):
return paddorch.LongTensor(np.array([key]))
if isinstance(key,list):
key = paddorch.from_numpy(key).long()
return key
if isinstance(key, np.ndarray) or isinstance(key,paddle.Tensor):
key = convert_key_to_inttensor(key)
elif isinstance(key,Iterable) :
if isinstance(key[0],slice):
return super(Tensor, self).__setitem__(key, value)
key2=[]
for i in range(len(key)):
key2.append(convert_key_to_inttensor(key[i]))
key=paddle.stack(key2,axis=1)
if len(key2)==1:
key= key.reshape([-1])
else:
key=convert_key_to_inttensor(key)
if key.shape[0]==0: ##empty selection, do nothing
return self
if not isinstance(value,paddle.Tensor):
value=paddle.ones_like(key)*float(value)
return paddle.scatter_(self,key,value)
def forward(self, sparse_inputs, dense_inputs):
# Embedding
sparse_inputs_concat = paddle.concat(
sparse_inputs, axis=1) # [batch_size, sparse_feature_number]
sparse_embeddings = self.embedding(
sparse_inputs_concat
) # [batch_size, sparse_feature_number, sparse_feature_dim]
dense_inputs_re = paddle.unsqueeze(dense_inputs, axis=2)
dense_embeddings = paddle.multiply(dense_inputs_re, self.dense_w)
feat_embeddings = paddle.concat(
[sparse_embeddings, dense_embeddings], 1
) # [batch_size, dense_feature_number + feature_number, dense_feature_dim]
feat_embeddings = paddle.reshape(
feat_embeddings,
[-1, self.num_fields * self.num_fields, self.feature_dim])
# inputs: emb_inputs, shape = (B, N^2, E) if squared else (B, N, E)
# output: pooled_inputs, shape = (B, N^2, 1)
pooled_inputs = self.pooling(feat_embeddings)
# Flatten pooled_inputs
# inputs: pooled_inputs, shape = (B, N^2, 1)
# output: pooled_inputs, shape = (B, N^2)
pooled_inputs = paddle.flatten(
pooled_inputs, start_axis=1, stop_axis=-1, name=None)
# Calculate attention weight with dense layer forwardly
# inputs: pooled_inputs, shape = (B, N^2)
# output: attn_w, shape = (B, N^2)
attn_w = self.fc(pooled_inputs)
# Unflatten attention weights and apply it to emb_inputs
# inputs: attn_w, shape = (B, N^2)
# inputs: emb_inputs, shape = (B, N^2, E)
# output: outputs, shape = (B, N^2, E)
attn_w = paddle.tile(attn_w, repeat_times=[self.feature_dim])
attn_w = paddle.split(attn_w, num_or_sections=self.feature_dim, axis=1)
attn_w = paddle.stack(attn_w, axis=2)
# Multiply attentional weights on field embedding tensors
outputs = paddle.multiply(feat_embeddings, attn_w) # (B, N^2, E)
return outputs
def forward(self, x, condition):
"""Probability density estimation of random variable x given the
condition.
Parameters
-----------
x : Tensor [shape=(batch_size, time_steps)]
The audio.
condition : Tensor [shape=(batch_size, condition channel, time_steps)]
The local condition (mel spectrogram here).
Returns
--------
z : Tensor [shape=(batch_size, time_steps)]
The transformed random variable.
log_det_jacobian: Tensor [shape=(1,)]
The log determinant of the jacobian of the transformation from x
to z.
"""
# x: (B, T)
# condition: (B, C, T) upsampled condition
x, condition = self._trim(x, condition)
# to (B, C, h, T//h) layout
x = paddle.unsqueeze(
paddle.transpose(fold(x, self.n_group), [0, 2, 1]), 1)
condition = paddle.transpose(
fold(condition, self.n_group), [0, 1, 3, 2])
# flows
logs_list = []
for i, layer in enumerate(self):
x, (logs, b) = layer(x, condition)
logs_list.append(logs)
# permute paddle has no shuffle dim
x = geo.shuffle_dim(x, 2, perm=self.perms[i])
condition = geo.shuffle_dim(condition, 2, perm=self.perms[i])
z = paddle.squeeze(x, 1) # (B, H, W)
batch_size = z.shape[0]
z = paddle.reshape(paddle.transpose(z, [0, 2, 1]), [batch_size, -1])
log_det_jacobian = paddle.sum(paddle.stack(logs_list))
return z, log_det_jacobian
def lovasz_softmax_flat(probas, labels, classes='present'):
"""
Multi-class Lovasz-Softmax loss.
Args:
probas (Tensor): Shape is [P, C], class probabilities at each prediction (between 0 and 1).
labels (Tensor): Shape is [P], ground truth labels (between 0 and C - 1).
classes (str|list): 'all' for all, 'present' for classes present in labels, or a list of classes to average.
"""
if probas.numel() == 0:
# only void pixels, the gradients should be 0
return probas * 0.
C = probas.shape[1]
losses = []
classes_to_sum = list(range(C)) if classes in ['all', 'present'
] else classes
for c in classes_to_sum:
fg = paddle.cast(labels == c, probas.dtype) # foreground for class c
if classes == 'present' and fg.sum() == 0:
continue
fg.stop_gradient = True
if C == 1:
if len(classes_to_sum) > 1:
raise ValueError('Sigmoid output possible only with 1 class')
class_pred = probas[:, 0]
else:
class_pred = probas[:, c]
errors = paddle.abs(fg - class_pred)
errors_sorted, perm = paddle.fluid.core.ops.argsort(
errors, 'axis', 0, 'descending', True)
errors_sorted.stop_gradient = False
fg_sorted = paddle.gather(fg, perm)
fg_sorted.stop_gradient = True
grad = lovasz_grad(fg_sorted)
grad.stop_gradient = True
loss = paddle.sum(errors_sorted * grad)
losses.append(loss)
if len(classes_to_sum) == 1:
return losses[0]
losses_tensor = paddle.stack(losses)
mean_loss = paddle.mean(losses_tensor)
return mean_loss
def paste_mask(self, masks, boxes, im_h, im_w):
# paste each mask on image
x0, y0, x1, y1 = paddle.split(boxes, 4, axis=1)
masks = paddle.unsqueeze(masks, [0, 1])
img_y = paddle.arange(0, im_h, dtype='float32') + 0.5
img_x = paddle.arange(0, im_w, dtype='float32') + 0.5
img_y = (img_y - y0) / (y1 - y0) * 2 - 1
img_x = (img_x - x0) / (x1 - x0) * 2 - 1
img_x = paddle.unsqueeze(img_x, [1])
img_y = paddle.unsqueeze(img_y, [2])
N = boxes.shape[0]
gx = paddle.expand(img_x, [N, img_y.shape[1], img_x.shape[2]])
gy = paddle.expand(img_y, [N, img_y.shape[1], img_x.shape[2]])
grid = paddle.stack([gx, gy], axis=3)
img_masks = F.grid_sample(masks, grid, align_corners=False)
return img_masks[:, 0]
def generate_anchor(self, nGh, nGw, anchor_wh):
nA = len(anchor_wh)
yv, xv = paddle.meshgrid([paddle.arange(nGh), paddle.arange(nGw)])
mesh = paddle.stack((xv, yv),
axis=0).cast(dtype='float32') # 2 x nGh x nGw
meshs = paddle.tile(mesh, [nA, 1, 1, 1])
anchor_offset_mesh = anchor_wh[:, :, None][:, :, :, None].repeat(
int(nGh), axis=-2).repeat(int(nGw), axis=-1)
anchor_offset_mesh = paddle.to_tensor(
anchor_offset_mesh.astype(np.float32))
# nA x 2 x nGh x nGw
anchor_mesh = paddle.concat([meshs, anchor_offset_mesh], axis=1)
anchor_mesh = paddle.transpose(anchor_mesh,
[0, 2, 3, 1]) # (nA x nGh x nGw) x 4
return anchor_mesh
def segment_padding(data, segment_ids):
"""
Segment padding operator.
This operator padding the input elements which with the same index in 'segment_ids' to a common length ,
and reshape its into [uniq_segment_id, max_padding, dim].
Args:
data (tensor): a tensor, available data type float32, float64.
segment_ids (tensor): a 1-d tensor, which have the same size
with the first dimension of input data.
available data type is int32, int64.
Returns:
output (Tensor): the padding result with shape [uniq_segment_id, max_padding, dim].
seq_len (Tensor): the numbers of elements grouped same segment_ids
index: The index of elements for gather_nd or scatter_nd operation
Examples:
.. code-block:: python
import paddle
import pgl
data = paddle.to_tensor([[1, 2, 3], [3, 2, 1], [4, 5, 6]], dtype='float32')
segment_ids = paddle.to_tensor([0, 0, 1], dtype='int64')
output, seq_len, index = pgl.math.segment_padding(data, segment_ids)
"""
idx_a = segment_ids
idx_b = paddle.arange(paddle.shape(segment_ids)[0])
temp_idx = paddle.ones_like(segment_ids, dtype='float32')
segment_len = segment_sum(temp_idx, segment_ids).astype('int32')
max_padding = paddle.max(segment_len)
segment_shift = get_index_from_counts(segment_len)[:-1]
segment_shift = paddle.gather(segment_shift, segment_ids)
idx_b = idx_b - segment_shift
index = paddle.stack([idx_a, idx_b], axis=1)
shape = [paddle.shape(segment_len)[0], max_padding, data.shape[-1]]
output = paddle.scatter_nd(index, data, shape)
return output, segment_len, index
def collate_fn(batch):
a2a_gs, b2a_gs, b2b_gs_l, feats, types, counts, labels = map(
list, zip(*batch))
a2a_g = pgl.Graph.batch(a2a_gs).tensor()
b2a_g = pgl.BiGraph.batch(b2a_gs).tensor()
b2b_gl = [
pgl.Graph.batch([g[i] for g in b2b_gs_l]).tensor()
for i in range(len(b2b_gs_l[0]))
]
feats = paddle.concat(
[paddle.to_tensor(f, dtype='float32') for f in feats])
types = paddle.concat([paddle.to_tensor(t) for t in types])
counts = paddle.stack([paddle.to_tensor(c) for c in counts], axis=1)
labels = paddle.to_tensor(np.array(labels), dtype='float32')
return a2a_g, b2a_g, b2b_gl, feats, types, counts, labels
def forward(self, outputs, labels):
predicts = outputs['maps']
predicts = F.interpolate(predicts, scale_factor=4)
texts = predicts[:, 0, :, :]
kernels = predicts[:, 1:, :, :]
gt_texts, gt_kernels, training_masks = labels[1:]
# text loss
selected_masks = self.ohem_batch(texts, gt_texts, training_masks)
loss_text = self.dice_loss(texts, gt_texts, selected_masks)
iou_text = iou((texts > 0).astype('int64'),
gt_texts,
training_masks,
reduce=False)
losses = dict(loss_text=loss_text, iou_text=iou_text)
# kernel loss
loss_kernels = []
if self.kernel_sample_mask == 'gt':
selected_masks = gt_texts * training_masks
elif self.kernel_sample_mask == 'pred':
selected_masks = (
F.sigmoid(texts) > 0.5).astype('float32') * training_masks
for i in range(kernels.shape[1]):
kernel_i = kernels[:, i, :, :]
gt_kernel_i = gt_kernels[:, i, :, :]
loss_kernel_i = self.dice_loss(kernel_i, gt_kernel_i,
selected_masks)
loss_kernels.append(loss_kernel_i)
loss_kernels = paddle.mean(paddle.stack(loss_kernels, axis=1), axis=1)
iou_kernel = iou((kernels[:, -1, :, :] > 0).astype('int64'),
gt_kernels[:, -1, :, :],
training_masks * gt_texts,
reduce=False)
losses.update(dict(loss_kernels=loss_kernels, iou_kernel=iou_kernel))
loss = self.alpha * loss_text + (1 - self.alpha) * loss_kernels
losses['loss'] = loss
if self.reduction == 'sum':
losses = {x: paddle.sum(v) for x, v in losses.items()}
elif self.reduction == 'mean':
losses = {x: paddle.mean(v) for x, v in losses.items()}
return losses
def forward(self, bond_types_batch, type_count_batch, bond_feat):
"""
Input example:
bond_types_batch: [0,0,2,0,1,2] + [0,0,2,0,1,2] + [2]
type_count_batch: [[3, 3, 0], [1, 1, 0], [2, 2, 1]] # [num_type, batch_size]
"""
bond_feat = self.fc_1(
paddle.reshape(bond_feat, [-1, self.num_angle * self.bond_dim]))
inter_mat_list = []
for type_i in range(self.num_type):
type_i_index = paddle.masked_select(paddle.arange(len(bond_feat)),
bond_types_batch == type_i)
if paddle.sum(type_count_batch[type_i]) == 0:
inter_mat_list.append(
paddle.to_tensor(np.array([0.] *
len(type_count_batch[type_i])),
dtype='float32'))
continue
bond_feat_type_i = paddle.gather(bond_feat, type_i_index)
graph_bond_index = op.get_index_from_counts(
type_count_batch[type_i])
# graph_bond_id = generate_segment_id_from_index(graph_bond_index)
graph_bond_id = generate_segment_id(graph_bond_index)
graph_feat_type_i = math.segment_pool(bond_feat_type_i,
graph_bond_id,
pool_type='sum')
mat_flat_type_i = self.fc_2(graph_feat_type_i).squeeze(1)
# print(graph_bond_id)
# print(graph_bond_id.shape, graph_feat_type_i.shape, mat_flat_type_i.shape)
my_pad = nn.Pad1D(padding=[
0, len(type_count_batch[type_i]) - len(mat_flat_type_i)
],
value=-1e9)
mat_flat_type_i = my_pad(mat_flat_type_i)
inter_mat_list.append(mat_flat_type_i)
inter_mat_batch = paddle.stack(inter_mat_list,
axis=1) # [batch_size, num_type]
inter_mat_mask = paddle.ones_like(inter_mat_batch) * -1e9
inter_mat_batch = paddle.where(
type_count_batch.transpose([1, 0]) > 0, inter_mat_batch,
inter_mat_mask)
inter_mat_batch = self.softmax(inter_mat_batch)
return inter_mat_batch
