def label_smooth_loss(self, scores, labels, **kwargs):
labels = F.one_hot(labels, self.num_classes)
labels = F.label_smooth(labels, epsilon=self.ls_eps)
labels = paddle.squeeze(labels, axis=1)
loss = self.loss_func(scores, labels, soft_label=True, **kwargs)
return loss
def TestInit(self):
with self.assertRaises(ValueError):
paddle.distribution.Dirichlet(
paddle.squeeze(self.concentration))
def __call__(self,
seg_preds,
seg_masks,
cate_labels,
cate_scores,
sum_masks=None):
# sort and keep top nms_pre
sort_inds = self._sort_score(cate_scores, self.pre_nms_top_n)
seg_masks = paddle.gather(seg_masks, index=sort_inds)
seg_preds = paddle.gather(seg_preds, index=sort_inds)
sum_masks = paddle.gather(sum_masks, index=sort_inds)
cate_scores = paddle.gather(cate_scores, index=sort_inds)
cate_labels = paddle.gather(cate_labels, index=sort_inds)
seg_masks = paddle.flatten(seg_masks, start_axis=1, stop_axis=-1)
# inter.
inter_matrix = paddle.mm(seg_masks, paddle.transpose(seg_masks, [1, 0]))
n_samples = paddle.shape(cate_labels)
# union.
sum_masks_x = paddle.expand(sum_masks, shape=[n_samples, n_samples])
# iou.
iou_matrix = (inter_matrix / (
sum_masks_x + paddle.transpose(sum_masks_x, [1, 0]) - inter_matrix))
iou_matrix = paddle.triu(iou_matrix, diagonal=1)
# label_specific matrix.
cate_labels_x = paddle.expand(cate_labels, shape=[n_samples, n_samples])
label_matrix = paddle.cast(
(cate_labels_x == paddle.transpose(cate_labels_x, [1, 0])),
'float32')
label_matrix = paddle.triu(label_matrix, diagonal=1)
# IoU compensation
compensate_iou = paddle.max((iou_matrix * label_matrix), axis=0)
compensate_iou = paddle.expand(
compensate_iou, shape=[n_samples, n_samples])
compensate_iou = paddle.transpose(compensate_iou, [1, 0])
# IoU decay
decay_iou = iou_matrix * label_matrix
# matrix nms
if self.kernel == 'gaussian':
decay_matrix = paddle.exp(-1 * self.sigma * (decay_iou**2))
compensate_matrix = paddle.exp(-1 * self.sigma *
(compensate_iou**2))
decay_coefficient = paddle.min(decay_matrix / compensate_matrix,
axis=0)
elif self.kernel == 'linear':
decay_matrix = (1 - decay_iou) / (1 - compensate_iou)
decay_coefficient = paddle.min(decay_matrix, axis=0)
else:
raise NotImplementedError
# update the score.
cate_scores = cate_scores * decay_coefficient
y = paddle.zeros(shape=paddle.shape(cate_scores), dtype='float32')
keep = paddle.where(cate_scores >= self.update_threshold, cate_scores,
y)
keep = paddle.nonzero(keep)
keep = paddle.squeeze(keep, axis=[1])
# Prevent empty and increase fake data
keep = paddle.concat(
[keep, paddle.cast(paddle.shape(cate_scores)[0] - 1, 'int64')])
seg_preds = paddle.gather(seg_preds, index=keep)
cate_scores = paddle.gather(cate_scores, index=keep)
cate_labels = paddle.gather(cate_labels, index=keep)
# sort and keep top_k
sort_inds = self._sort_score(cate_scores, self.post_nms_top_n)
seg_preds = paddle.gather(seg_preds, index=sort_inds)
cate_scores = paddle.gather(cate_scores, index=sort_inds)
cate_labels = paddle.gather(cate_labels, index=sort_inds)
return seg_preds, cate_scores, cate_labels
def predict(model,
model_path,
val_dataset,
image_list,
image_dir=None,
save_dir='output',
custom_color=None):
"""
predict and visualize the image_list.
Args:
model (nn.Layer): Used to predict for input image.
model_path (str): The path of pretrained model.
val_dataset (paddle.io.Dataset): Used to read validation information.
image_list (list): A list of image path to be predicted.
image_dir (str, optional): The root directory of the images predicted. Default: None.
save_dir (str, optional): The directory to save the visualized results. Default: 'output'.
custom_color (list, optional): Save images with a custom color map. Default: None, use paddleseg's default color map.
"""
utils.utils.load_entire_model(model, model_path)
model.eval()
nranks = paddle.distributed.get_world_size()
local_rank = paddle.distributed.get_rank()
if nranks > 1:
img_lists = partition_list(image_list, nranks)
else:
img_lists = [image_list]
added_saved_dir = os.path.join(save_dir, 'added_prediction')
pred_saved_dir = os.path.join(save_dir, 'pseudo_color_prediction')
transforms = val_dataset.transforms
cut_height = val_dataset.cut_height
postprocessor = tusimple_processor.TusimpleProcessor(
num_classes=val_dataset.num_classes,
cut_height=cut_height,
save_dir=save_dir)
logger.info("Start to predict...")
progbar_pred = progbar.Progbar(target=len(img_lists[0]), verbose=1)
color_map = visualize.get_color_map_list(256, custom_color=custom_color)
with paddle.no_grad():
for i, im_path in enumerate(img_lists[local_rank]):
im = cv2.imread(im_path)
ori_shape = im.shape[:2]
im, _ = transforms(im)
im = im[np.newaxis, ...]
im = paddle.to_tensor(im)
pred = infer.inference(model,
im,
ori_shape=ori_shape,
transforms=transforms.transforms)
# get lane points
postprocessor.predict(pred[1], im_path)
pred = paddle.squeeze(pred[0])
pred = pred.numpy().astype('uint8')
# get the saved name
if image_dir is not None:
im_file = im_path.replace(image_dir, '')
else:
im_file = os.path.basename(im_path)
if im_file[0] == '/' or im_file[0] == '\\':
im_file = im_file[1:]
# save added image
added_image = utils.visualize.visualize(im_path,
pred,
color_map,
weight=0.6)
added_image_path = os.path.join(added_saved_dir, im_file)
mkdir(added_image_path)
cv2.imwrite(added_image_path, added_image)
# save pseudo color prediction
pred_mask = utils.visualize.get_pseudo_color_map(pred, color_map)
pred_saved_path = os.path.join(
pred_saved_dir,
os.path.splitext(im_file)[0] + ".png")
mkdir(pred_saved_path)
pred_mask.save(pred_saved_path)
# pred_im = utils.visualize(im_path, pred, weight=0.0)
# pred_saved_path = os.path.join(pred_saved_dir, im_file)
# mkdir(pred_saved_path)
# cv2.imwrite(pred_saved_path, pred_im)
progbar_pred.update(i + 1)
def test_tensor_patch_method(self):
paddle.disable_static()
x_np = np.random.uniform(-1, 1, [2, 3]).astype(self.dtype)
y_np = np.random.uniform(-1, 1, [2, 3]).astype(self.dtype)
z_np = np.random.uniform(-1, 1, [6, 9]).astype(self.dtype)
x = paddle.to_tensor(x_np)
y = paddle.to_tensor(y_np)
z = paddle.to_tensor(z_np)
a = paddle.to_tensor([[1, 1], [2, 2], [3, 3]])
b = paddle.to_tensor([[1, 1], [2, 2], [3, 3]])
# 1. Unary operation for Tensor
self.assertEqual(x.dim(), 2)
self.assertEqual(x.ndimension(), 2)
self.assertEqual(x.ndim, 2)
self.assertEqual(x.size, 6)
self.assertEqual(x.numel(), 6)
self.assertTrue(np.array_equal(x.exp().numpy(), paddle.exp(x).numpy()))
self.assertTrue(
np.array_equal(x.tanh().numpy(),
paddle.tanh(x).numpy()))
self.assertTrue(
np.array_equal(x.atan().numpy(),
paddle.atan(x).numpy()))
self.assertTrue(np.array_equal(x.abs().numpy(), paddle.abs(x).numpy()))
m = x.abs()
self.assertTrue(
np.array_equal(m.sqrt().numpy(),
paddle.sqrt(m).numpy()))
self.assertTrue(
np.array_equal(m.rsqrt().numpy(),
paddle.rsqrt(m).numpy()))
self.assertTrue(
np.array_equal(x.ceil().numpy(),
paddle.ceil(x).numpy()))
self.assertTrue(
np.array_equal(x.floor().numpy(),
paddle.floor(x).numpy()))
self.assertTrue(np.array_equal(x.cos().numpy(), paddle.cos(x).numpy()))
self.assertTrue(
np.array_equal(x.acos().numpy(),
paddle.acos(x).numpy()))
self.assertTrue(
np.array_equal(x.asin().numpy(),
paddle.asin(x).numpy()))
self.assertTrue(np.array_equal(x.sin().numpy(), paddle.sin(x).numpy()))
self.assertTrue(
np.array_equal(x.sinh().numpy(),
paddle.sinh(x).numpy()))
self.assertTrue(
np.array_equal(x.cosh().numpy(),
paddle.cosh(x).numpy()))
self.assertTrue(
np.array_equal(x.round().numpy(),
paddle.round(x).numpy()))
self.assertTrue(
np.array_equal(x.reciprocal().numpy(),
paddle.reciprocal(x).numpy()))
self.assertTrue(
np.array_equal(x.square().numpy(),
paddle.square(x).numpy()))
self.assertTrue(
np.array_equal(x.rank().numpy(),
paddle.rank(x).numpy()))
self.assertTrue(
np.array_equal(x[0].t().numpy(),
paddle.t(x[0]).numpy()))
d = paddle.to_tensor([[1.2285208, 1.3491015, 1.4899898],
[1.30058, 1.0688717, 1.4928783],
[1.0958099, 1.3724753, 1.8926544]])
d = d.matmul(d.t())
self.assertTrue(
np.array_equal(d.cholesky().numpy(),
paddle.cholesky(d).numpy()))
self.assertTrue(
np.array_equal(x.is_empty().numpy(),
paddle.is_empty(x).numpy()))
self.assertTrue(
np.array_equal(x.isfinite().numpy(),
paddle.isfinite(x).numpy()))
self.assertTrue(
np.array_equal(
x.cast('int32').numpy(),
paddle.cast(x, 'int32').numpy()))
self.assertTrue(
np.array_equal(
x.expand([3, 2, 3]).numpy(),
paddle.expand(x, [3, 2, 3]).numpy()))
self.assertTrue(
np.array_equal(
x.tile([2, 2]).numpy(),
paddle.tile(x, [2, 2]).numpy()))
self.assertTrue(
np.array_equal(x.flatten().numpy(),
paddle.flatten(x).numpy()))
index = paddle.to_tensor([0, 1])
self.assertTrue(
np.array_equal(
x.gather(index).numpy(),
paddle.gather(x, index).numpy()))
index = paddle.to_tensor([[0, 1], [1, 2]])
self.assertTrue(
np.array_equal(
x.gather_nd(index).numpy(),
paddle.gather_nd(x, index).numpy()))
self.assertTrue(
np.array_equal(
x.reverse([0, 1]).numpy(),
paddle.reverse(x, [0, 1]).numpy()))
self.assertTrue(
np.array_equal(
a.reshape([3, 2]).numpy(),
paddle.reshape(a, [3, 2]).numpy()))
self.assertTrue(
np.array_equal(
x.slice([0, 1], [0, 0], [1, 2]).numpy(),
paddle.slice(x, [0, 1], [0, 0], [1, 2]).numpy()))
self.assertTrue(
np.array_equal(
x.split(2)[0].numpy(),
paddle.split(x, 2)[0].numpy()))
m = paddle.to_tensor(
np.random.uniform(-1, 1, [1, 6, 1, 1]).astype(self.dtype))
self.assertTrue(
np.array_equal(
m.squeeze([]).numpy(),
paddle.squeeze(m, []).numpy()))
self.assertTrue(
np.array_equal(
m.squeeze([1, 2]).numpy(),
paddle.squeeze(m, [1, 2]).numpy()))
m = paddle.to_tensor([2, 3, 3, 1, 5, 3], 'float32')
self.assertTrue(
np.array_equal(m.unique()[0].numpy(),
paddle.unique(m)[0].numpy()))
self.assertTrue(
np.array_equal(
m.unique(return_counts=True)[1],
paddle.unique(m, return_counts=True)[1]))
self.assertTrue(np.array_equal(x.flip([0]), paddle.flip(x, [0])))
self.assertTrue(np.array_equal(x.unbind(0), paddle.unbind(x, 0)))
self.assertTrue(np.array_equal(x.roll(1), paddle.roll(x, 1)))
self.assertTrue(np.array_equal(x.cumsum(1), paddle.cumsum(x, 1)))
m = paddle.to_tensor(1)
self.assertTrue(np.array_equal(m.increment(), paddle.increment(m)))
m = x.abs()
self.assertTrue(np.array_equal(m.log(), paddle.log(m)))
self.assertTrue(np.array_equal(x.pow(2), paddle.pow(x, 2)))
self.assertTrue(np.array_equal(x.reciprocal(), paddle.reciprocal(x)))
# 2. Binary operation
self.assertTrue(
np.array_equal(x.divide(y).numpy(),
paddle.divide(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.matmul(y, True, False).numpy(),
paddle.matmul(x, y, True, False).numpy()))
self.assertTrue(
np.array_equal(
x.norm(p='fro', axis=[0, 1]).numpy(),
paddle.norm(x, p='fro', axis=[0, 1]).numpy()))
self.assertTrue(
np.array_equal(x.dist(y).numpy(),
paddle.dist(x, y).numpy()))
self.assertTrue(
np.array_equal(x.cross(y).numpy(),
paddle.cross(x, y).numpy()))
m = x.expand([2, 2, 3])
n = y.expand([2, 2, 3]).transpose([0, 2, 1])
self.assertTrue(
np.array_equal(m.bmm(n).numpy(),
paddle.bmm(m, n).numpy()))
self.assertTrue(
np.array_equal(
x.histogram(5, -1, 1).numpy(),
paddle.histogram(x, 5, -1, 1).numpy()))
self.assertTrue(
np.array_equal(x.equal(y).numpy(),
paddle.equal(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.greater_equal(y).numpy(),
paddle.greater_equal(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.greater_than(y).numpy(),
paddle.greater_than(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.less_equal(y).numpy(),
paddle.less_equal(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.less_than(y).numpy(),
paddle.less_than(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.not_equal(y).numpy(),
paddle.not_equal(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.equal_all(y).numpy(),
paddle.equal_all(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.allclose(y).numpy(),
paddle.allclose(x, y).numpy()))
m = x.expand([2, 2, 3])
self.assertTrue(
np.array_equal(
x.expand_as(m).numpy(),
paddle.expand_as(x, m).numpy()))
index = paddle.to_tensor([2, 1, 0])
self.assertTrue(
np.array_equal(
a.scatter(index, b).numpy(),
paddle.scatter(a, index, b).numpy()))
# 3. Bool tensor operation
x = paddle.to_tensor([[True, False], [True, False]])
y = paddle.to_tensor([[False, False], [False, True]])
self.assertTrue(
np.array_equal(
x.logical_and(y).numpy(),
paddle.logical_and(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.logical_not(y).numpy(),
paddle.logical_not(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.logical_or(y).numpy(),
paddle.logical_or(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.logical_xor(y).numpy(),
paddle.logical_xor(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.logical_and(y).numpy(),
paddle.logical_and(x, y).numpy()))
a = paddle.to_tensor([[1, 2], [3, 4]])
b = paddle.to_tensor([[4, 3], [2, 1]])
self.assertTrue(
np.array_equal(
x.where(a, b).numpy(),
paddle.where(x, a, b).numpy()))
self.assertTrue(inspect.ismethod(a.dot))
self.assertTrue(inspect.ismethod(a.logsumexp))
self.assertTrue(inspect.ismethod(a.multiplex))
self.assertTrue(inspect.ismethod(a.prod))
self.assertTrue(inspect.ismethod(a.scale))
self.assertTrue(inspect.ismethod(a.stanh))
self.assertTrue(inspect.ismethod(a.add_n))
self.assertTrue(inspect.ismethod(a.max))
self.assertTrue(inspect.ismethod(a.maximum))
self.assertTrue(inspect.ismethod(a.min))
self.assertTrue(inspect.ismethod(a.minimum))
self.assertTrue(inspect.ismethod(a.floor_divide))
self.assertTrue(inspect.ismethod(a.remainder))
self.assertTrue(inspect.ismethod(a.floor_mod))
self.assertTrue(inspect.ismethod(a.multiply))
self.assertTrue(inspect.ismethod(a.logsumexp))
self.assertTrue(inspect.ismethod(a.inverse))
self.assertTrue(inspect.ismethod(a.log1p))
self.assertTrue(inspect.ismethod(a.erf))
self.assertTrue(inspect.ismethod(a.addmm))
self.assertTrue(inspect.ismethod(a.clip))
self.assertTrue(inspect.ismethod(a.trace))
self.assertTrue(inspect.ismethod(a.kron))
self.assertTrue(inspect.ismethod(a.isinf))
self.assertTrue(inspect.ismethod(a.isnan))
self.assertTrue(inspect.ismethod(a.concat))
self.assertTrue(inspect.ismethod(a.broadcast_to))
self.assertTrue(inspect.ismethod(a.scatter_nd_add))
self.assertTrue(inspect.ismethod(a.scatter_nd))
self.assertTrue(inspect.ismethod(a.shard_index))
self.assertTrue(inspect.ismethod(a.chunk))
self.assertTrue(inspect.ismethod(a.stack))
self.assertTrue(inspect.ismethod(a.strided_slice))
self.assertTrue(inspect.ismethod(a.unsqueeze))
self.assertTrue(inspect.ismethod(a.unstack))
self.assertTrue(inspect.ismethod(a.argmax))
self.assertTrue(inspect.ismethod(a.argmin))
self.assertTrue(inspect.ismethod(a.argsort))
self.assertTrue(inspect.ismethod(a.masked_select))
self.assertTrue(inspect.ismethod(a.topk))
self.assertTrue(inspect.ismethod(a.index_select))
self.assertTrue(inspect.ismethod(a.nonzero))
self.assertTrue(inspect.ismethod(a.sort))
self.assertTrue(inspect.ismethod(a.index_sample))
self.assertTrue(inspect.ismethod(a.mean))
self.assertTrue(inspect.ismethod(a.std))
self.assertTrue(inspect.ismethod(a.numel))
def get_seg_single(self, cate_preds, seg_preds, kernel_preds, featmap_size,
im_shape, scale_factor):
"""
The code of this function is based on:
https://github.com/WXinlong/SOLO/blob/master/mmdet/models/anchor_heads/solov2_head.py#L385
"""
h = paddle.cast(im_shape[0], 'int32')[0]
w = paddle.cast(im_shape[1], 'int32')[0]
upsampled_size_out = [featmap_size[0] * 4, featmap_size[1] * 4]
y = paddle.zeros(shape=paddle.shape(cate_preds), dtype='float32')
inds = paddle.where(cate_preds > self.score_threshold, cate_preds, y)
inds = paddle.nonzero(inds)
cate_preds = paddle.reshape(cate_preds, shape=[-1])
# Prevent empty and increase fake data
ind_a = paddle.cast(paddle.shape(kernel_preds)[0], 'int64')
ind_b = paddle.zeros(shape=[1], dtype='int64')
inds_end = paddle.unsqueeze(paddle.concat([ind_a, ind_b]), 0)
inds = paddle.concat([inds, inds_end])
kernel_preds_end = paddle.ones(
shape=[1, self.kernel_out_channels], dtype='float32')
kernel_preds = paddle.concat([kernel_preds, kernel_preds_end])
cate_preds = paddle.concat(
[cate_preds, paddle.zeros(
shape=[1], dtype='float32')])
# cate_labels & kernel_preds
cate_labels = inds[:, 1]
kernel_preds = paddle.gather(kernel_preds, index=inds[:, 0])
cate_score_idx = paddle.add(inds[:, 0] * self.cate_out_channels,
cate_labels)
cate_scores = paddle.gather(cate_preds, index=cate_score_idx)
size_trans = np.power(self.seg_num_grids, 2)
strides = []
for _ind in range(len(self.segm_strides)):
strides.append(
paddle.full(
shape=[int(size_trans[_ind])],
fill_value=self.segm_strides[_ind],
dtype="int32"))
strides = paddle.concat(strides)
strides = paddle.concat(
[strides, paddle.zeros(
shape=[1], dtype='int32')])
strides = paddle.gather(strides, index=inds[:, 0])
# mask encoding.
kernel_preds = paddle.unsqueeze(kernel_preds, [2, 3])
seg_preds = F.conv2d(seg_preds, kernel_preds)
seg_preds = F.sigmoid(paddle.squeeze(seg_preds, [0]))
seg_masks = seg_preds > self.mask_threshold
seg_masks = paddle.cast(seg_masks, 'float32')
sum_masks = paddle.sum(seg_masks, axis=[1, 2])
y = paddle.zeros(shape=paddle.shape(sum_masks), dtype='float32')
keep = paddle.where(sum_masks > strides, sum_masks, y)
keep = paddle.nonzero(keep)
keep = paddle.squeeze(keep, axis=[1])
# Prevent empty and increase fake data
keep_other = paddle.concat(
[keep, paddle.cast(paddle.shape(sum_masks)[0] - 1, 'int64')])
keep_scores = paddle.concat(
[keep, paddle.cast(paddle.shape(sum_masks)[0], 'int64')])
cate_scores_end = paddle.zeros(shape=[1], dtype='float32')
cate_scores = paddle.concat([cate_scores, cate_scores_end])
seg_masks = paddle.gather(seg_masks, index=keep_other)
seg_preds = paddle.gather(seg_preds, index=keep_other)
sum_masks = paddle.gather(sum_masks, index=keep_other)
cate_labels = paddle.gather(cate_labels, index=keep_other)
cate_scores = paddle.gather(cate_scores, index=keep_scores)
# mask scoring.
seg_mul = paddle.cast(seg_preds * seg_masks, 'float32')
seg_scores = paddle.sum(seg_mul, axis=[1, 2]) / sum_masks
cate_scores *= seg_scores
# Matrix NMS
seg_preds, cate_scores, cate_labels = self.mask_nms(
seg_preds, seg_masks, cate_labels, cate_scores, sum_masks=sum_masks)
ori_shape = im_shape[:2] / scale_factor + 0.5
ori_shape = paddle.cast(ori_shape, 'int32')
seg_preds = F.interpolate(
paddle.unsqueeze(seg_preds, 0),
size=upsampled_size_out,
mode='bilinear',
align_corners=False,
align_mode=0)
seg_preds = paddle.slice(
seg_preds, axes=[2, 3], starts=[0, 0], ends=[h, w])
seg_masks = paddle.squeeze(
F.interpolate(
seg_preds,
size=ori_shape[:2],
mode='bilinear',
align_corners=False,
align_mode=0),
axis=[0])
seg_masks = paddle.cast(seg_masks > self.mask_threshold, 'uint8')
return seg_masks, cate_labels, cate_scores
def view_api_processing(self, var):
return paddle.squeeze(var)
def train():
paddle.disable_static()
n_gpus = dist.get_world_size()
rank = dist.get_rank()
if n_gpus > 1:
dist.init_parallel_env()
args = parse_args()
vocab = load_vocab(args.vocab_file, args.max_characters_per_token)
elmo = ELMo(args.batch_size,
args.char_embed_dim,
args.projection_dim,
vocab.size,
dropout=args.dropout,
num_layers=args.num_layers,
num_highways=args.num_highways,
char_vocab_size=vocab.char_size)
if n_gpus > 1:
elmo = paddle.DataParallel(elmo)
elmo.train()
gloabl_norm_clip = nn.ClipGradByGlobalNorm(args.max_grad_norm)
optimizer = paddle.optimizer.Adagrad(learning_rate=args.lr,
parameters=elmo.parameters(),
initial_accumulator_value=1.0,
grad_clip=gloabl_norm_clip)
elmo_loss = ELMoLoss()
# Loads pre-trained parameters.
if args.init_from_ckpt:
weight_state_dict = paddle.load(args.init_from_ckpt + '.pdparams')
opt_state_dict = paddle.load(args.init_from_ckpt + '.pdopt')
elmo.set_state_dict(weight_state_dict)
optimizer.set_state_dict(opt_state_dict)
print("Loaded checkpoint from %s" % args.init_from_ckpt)
train_dataset = OneBillionWordDataset(args.train_data_path,
vocab,
args.batch_size,
args.unroll_steps,
n_gpus,
rank,
mode='train',
shuffle=True,
seed=args.random_seed)
# FIXME(xiemoyuan): When DataLoader support setting batch_size to None,
# setting batch_size to None.
train_dataloader = DataLoader(train_dataset,
return_list=True,
batch_size=1)
n_tokens_per_batch = args.batch_size * args.unroll_steps * n_gpus
n_steps_per_epoch = int(train_dataset.number_of_tokens /
n_tokens_per_batch)
n_steps_total = args.epochs * n_steps_per_epoch
if rank == 0:
print("Training for %s epochs and %s steps" %
(args.epochs, n_steps_total))
total_time = 0.0
batch_start_time = time.time()
for step, inputs in enumerate(train_dataloader, start=1):
# FIXME(xiemoyuan): When DataLoader support setting batch_size to None,
# deleting the operation of squeeze.
for j in range(len(inputs)):
inputs[j] = paddle.squeeze(inputs[j], axis=0)
ids, next_ids, ids_reverse, next_ids_reverse = inputs
outputs = elmo([ids, ids_reverse])
loss = elmo_loss(outputs, [next_ids, next_ids_reverse])
ppl = paddle.exp(loss)
loss *= args.unroll_steps
loss.backward()
optimizer.step()
optimizer.clear_grad()
total_time += (time.time() - batch_start_time)
if rank == 0:
if step % args.log_freq == 0:
print(
"step %d/%d - loss: %.4f - Perplexity: %.4f - %.3fs/step" %
(step, n_steps_total, loss.numpy()[0], ppl.numpy()[0],
total_time / args.log_freq))
total_time = 0.0
if step % args.save_freq == 0:
save_params(elmo, optimizer, args.save_dir, step)
if step == n_steps_total:
# training done
save_params(elmo, optimizer, args.save_dir, 'final')
break
batch_start_time = time.time()
def predict(model,
model_path,
transforms,
image_list,
image_dir=None,
save_dir='output',
aug_pred=False,
scales=1.0,
flip_horizontal=True,
flip_vertical=False,
is_slide=False,
stride=None,
crop_size=None,
custom_color=None):
"""
predict and visualize the image_list.
Args:
model (nn.Layer): Used to predict for input image.
model_path (str): The path of pretrained model.
transforms (transform.Compose): Preprocess for input image.
image_list (list): A list of image path to be predicted.
image_dir (str, optional): The root directory of the images predicted. Default: None.
save_dir (str, optional): The directory to save the visualized results. Default: 'output'.
aug_pred (bool, optional): Whether to use mulit-scales and flip augment for predition. Default: False.
scales (list|float, optional): Scales for augment. It is valid when `aug_pred` is True. Default: 1.0.
flip_horizontal (bool, optional): Whether to use flip horizontally augment. It is valid when `aug_pred` is True. Default: True.
flip_vertical (bool, optional): Whether to use flip vertically augment. It is valid when `aug_pred` is True. Default: False.
is_slide (bool, optional): Whether to predict by sliding window. Default: False.
stride (tuple|list, optional): The stride of sliding window, the first is width and the second is height.
It should be provided when `is_slide` is True.
crop_size (tuple|list, optional): The crop size of sliding window, the first is width and the second is height.
It should be provided when `is_slide` is True.
custom_color (list, optional): Save images with a custom color map. Default: None, use paddleseg's default color map.
"""
utils.utils.load_entire_model(model, model_path)
model.eval()
nranks = paddle.distributed.get_world_size()
local_rank = paddle.distributed.get_rank()
if nranks > 1:
img_lists = partition_list(image_list, nranks)
else:
img_lists = [image_list]
added_saved_dir = os.path.join(save_dir, 'added_prediction')
pred_saved_dir = os.path.join(save_dir, 'pseudo_color_prediction')
logger.info("Start to predict...")
progbar_pred = progbar.Progbar(target=len(img_lists[0]), verbose=1)
color_map = visualize.get_color_map_list(256, custom_color=custom_color)
with paddle.no_grad():
for i, im_path in enumerate(img_lists[local_rank]):
im = cv2.imread(im_path)
ori_shape = im.shape[:2]
im, _ = transforms(im)
im = im[np.newaxis, ...]
im = paddle.to_tensor(im)
if aug_pred:
pred, _ = infer.aug_inference(model,
im,
ori_shape=ori_shape,
transforms=transforms.transforms,
scales=scales,
flip_horizontal=flip_horizontal,
flip_vertical=flip_vertical,
is_slide=is_slide,
stride=stride,
crop_size=crop_size)
else:
pred, _ = infer.inference(model,
im,
ori_shape=ori_shape,
transforms=transforms.transforms,
is_slide=is_slide,
stride=stride,
crop_size=crop_size)
pred = paddle.squeeze(pred)
pred = pred.numpy().astype('uint8')
# get the saved name
if image_dir is not None:
im_file = im_path.replace(image_dir, '')
else:
im_file = os.path.basename(im_path)
if im_file[0] == '/' or im_file[0] == '\\':
im_file = im_file[1:]
# save added image
added_image = utils.visualize.visualize(im_path,
pred,
color_map,
weight=0.6)
added_image_path = os.path.join(added_saved_dir, im_file)
mkdir(added_image_path)
cv2.imwrite(added_image_path, added_image)
# save pseudo color prediction
pred_mask = utils.visualize.get_pseudo_color_map(pred, color_map)
pred_saved_path = os.path.join(
pred_saved_dir,
os.path.splitext(im_file)[0] + ".png")
mkdir(pred_saved_path)
pred_mask.save(pred_saved_path)
# pred_im = utils.visualize(im_path, pred, weight=0.0)
# pred_saved_path = os.path.join(pred_saved_dir, im_file)
# mkdir(pred_saved_path)
# cv2.imwrite(pred_saved_path, pred_im)
progbar_pred.update(i + 1)
def dnn_model_define(user_input,
unit_id_emb,
node_emb_size=24,
fea_groups="20,20,10,10,2,2,2,1,1,1",
active_op='prelu',
use_batch_norm=True,
with_att=False):
fea_groups = [int(s) for s in fea_groups.split(',')]
total_group_length = np.sum(np.array(fea_groups))
print "fea_groups", fea_groups, "total_group_length", total_group_length, "eb_dim", node_emb_size
layer_data = []
# start att
if with_att:
print("TDM Attention DNN")
att_user_input = paddle.concat(
user_input, axis=1) # [bs, total_group_length, emb_size]
att_node_input = fluid.layers.expand(
unit_id_emb, expand_times=[1, total_group_length, 1])
att_din = paddle.concat(
[att_user_input, att_user_input * att_node_input, att_node_input],
axis=2)
att_active_op = 'prelu'
att_layer_arr = []
att_layer1 = FullyConnected3D(3 * node_emb_size,
36,
active_op=att_active_op,
version=1)
att_layer_arr.append(att_layer1)
att_layer2 = FullyConnected3D(36,
1,
active_op=att_active_op,
version=2)
att_layer_arr.append(att_layer2)
layer_data.append(att_din)
for layer in att_layer_arr:
layer_data.append(layer.call(layer_data[-1]))
att_dout = layer_data[-1]
att_dout = fluid.layers.expand(att_dout,
expand_times=[1, 1, node_emb_size])
user_input = att_user_input * att_dout
else:
print("TDM DNN")
user_input = paddle.concat(
user_input, axis=1) # [bs, total_group_length, emb_size]
# end att
idx = 0
grouped_user_input = []
for group_length in fea_groups:
block_before_sum = paddle.slice(user_input,
axes=[1],
starts=[idx],
ends=[idx + group_length])
block = paddle.sum(block_before_sum, axis=1) / group_length
grouped_user_input.append(block)
idx += group_length
grouped_user_input = paddle.concat(grouped_user_input,
axis=1) # [bs, 10 * emb_size]
din = paddle.concat(
[grouped_user_input,
paddle.squeeze(unit_id_emb, axis=1)], axis=1)
net_version = "d"
layer_arr = []
layer1 = paddle_dnn_layer(11 * node_emb_size,
128,
active_op=active_op,
use_batch_norm=use_batch_norm,
version="%d_%s" % (1, net_version))
layer_arr.append(layer1)
layer2 = paddle_dnn_layer(128,
64,
active_op=active_op,
use_batch_norm=use_batch_norm,
version="%d_%s" % (2, net_version))
layer_arr.append(layer2)
layer3 = paddle_dnn_layer(64,
32,
active_op=active_op,
use_batch_norm=use_batch_norm,
version="%d_%s" % (3, net_version))
layer_arr.append(layer3)
layer4 = paddle_dnn_layer(32,
2,
active_op='',
use_batch_norm=False,
version="%d_%s" % (4, net_version))
layer_arr.append(layer4)
layer_data.append(din)
for layer in layer_arr:
layer_data.append(layer.call(layer_data[-1]))
dout = layer_data[-1]
return dout
def GetBaselineOut(self):
paddle.disable_static(place=paddle.CUDAPlace(0))
tensor_query = paddle.to_tensor(self.query, stop_gradient=False)
cache_kv = None
if self.has_cache_kv:
cache_kv = paddle.to_tensor(self.cache_kv, stop_gradient=False)
if self.has_attn_mask:
attn_mask = paddle.to_tensor(self.attn_mask, stop_gradient=False)
else:
attn_mask = None
residual = tensor_query
ln1_out = tensor_query
if self.pre_layer_norm:
ln1_out = self.norm1(tensor_query)
q = self.q_proj(ln1_out)
q = tensor.reshape(x=q, shape=[0, 0, self.num_heads, self.head_dim])
q_out = tensor.transpose(x=q, perm=[0, 2, 1, 3])
k = self.k_proj(ln1_out)
v = self.v_proj(ln1_out)
k = tensor.reshape(x=k, shape=[0, 0, self.num_heads, self.head_dim])
k_out = tensor.transpose(x=k, perm=[0, 2, 1, 3])
v = tensor.reshape(x=v, shape=[0, 0, self.num_heads, self.head_dim])
v_out = tensor.transpose(x=v, perm=[0, 2, 1, 3])
if self.has_cache_kv:
# [1, B, n_head, cache_seq_len, head_dim]
cache_k, cache_v = paddle.split(cache_kv, 2)
cache_k = paddle.squeeze(cache_k, axis=0)
cache_v = paddle.squeeze(cache_v, axis=0)
# [B, n_head, cache_seq_len + seq_len, head_dim]
# out_seq_len = cache_seq_len + seq_len
k_out = paddle.concat([cache_k, k_out], axis=-2)
v_out = paddle.concat([cache_v, v_out], axis=-2)
# [B, n_head, seq_len, head_dim] * [B, n_head, out_seq_len, head_dim]
# --> [B, n_head, seq_len, out_seq_len]
qk_out = layers.matmul(x=q_out,
y=k_out,
transpose_y=True,
alpha=self.head_dim**-0.5)
if attn_mask is not None:
attn_mask = _convert_attention_mask(attn_mask, qk_out.dtype)
attn_mask_out = qk_out + attn_mask
softmax_out = F.softmax(attn_mask_out)
else:
softmax_out = F.softmax(qk_out)
if self.dropout_prob:
dropout_out = F.dropout(softmax_out,
self.dropout_prob,
training=self.training,
mode="upscale_in_train")
# [B, n_head, seq_len, out_seq_len] * [B, n_head, out_seq_len, head_dim]
# --> [B, n_head, seq_len, head_dim]
qktv_out = tensor.matmul(dropout_out, v_out)
else:
qktv_out = tensor.matmul(softmax_out, v_out)
fmha_out = tensor.transpose(qktv_out, perm=[0, 2, 1, 3])
out_linear_in = tensor.reshape(
x=fmha_out, shape=[0, 0, fmha_out.shape[2] * fmha_out.shape[3]])
out = self.out_proj(out_linear_in)
residual_out = residual + self.dropout(out)
if not self.pre_layer_norm:
final_out = self.norm1(residual_out)
else:
final_out = residual_out
if self.has_cache_kv:
return final_out
paddle.autograd.backward([final_out], [paddle.to_tensor(self.dout)],
retain_graph=True)
return final_out, tensor_query.grad
def predict(self,
images: Union[str, np.ndarray],
batch_size: int = 1,
visualization: bool = True,
save_path: str = 'seg_result') -> List[np.ndarray]:
'''
Obtain segmentation results.
Args:
images(list[str|np.array]): Content image path or BGR image.
batch_size(int): Batch size for prediciton.
visualization(bool): Whether to save colorized images.
save_path(str) : Path to save colorized images.
Returns:
output(list[np.ndarray]) : The segmentation mask.
'''
self.eval()
result = []
total_num = len(images)
loop_num = int(np.ceil(total_num / batch_size))
for iter_id in range(loop_num):
batch_data = []
handle_id = iter_id * batch_size
for image_id in range(batch_size):
try:
image, _ = self.transform(images[handle_id + image_id])
batch_data.append(image)
except:
pass
batch_image = np.array(batch_data).astype('float32')
pred = self(paddle.to_tensor(batch_image))
pred = paddle.argmax(pred[0], axis=1, keepdim=True, dtype='int32')
for num in range(pred.shape[0]):
if isinstance(images[handle_id + num], str):
image = cv2.imread(images[handle_id + num])
else:
image = images[handle_id + num]
h, w, c = image.shape
pred_final = utils.reverse_transform(
pred[num:num + 1], (h, w), self.transforms.transforms)
pred_final = paddle.squeeze(pred_final)
pred_final = pred_final.numpy().astype('uint8')
if visualization:
added_image = utils.visualize(images[handle_id + num],
pred_final,
weight=0.6)
pred_mask = utils.get_pseudo_color_map(pred_final)
pred_image_path = os.path.join(save_path, 'image',
str(time.time()) + ".png")
pred_mask_path = os.path.join(save_path, 'mask',
str(time.time()) + ".png")
if not os.path.exists(os.path.dirname(pred_image_path)):
os.makedirs(os.path.dirname(pred_image_path))
if not os.path.exists(os.path.dirname(pred_mask_path)):
os.makedirs(os.path.dirname(pred_mask_path))
cv2.imwrite(pred_image_path, added_image)
pred_mask.save(pred_mask_path)
result.append(pred_final)
return result
def get_odm_loss(self, odm_target, s2anet_head_out):
(labels, label_weights, bbox_targets, bbox_weights, pos_inds,
neg_inds) = odm_target
fam_cls_branch_list, fam_reg_branch_list, odm_cls_branch_list, odm_reg_branch_list = s2anet_head_out
odm_cls_losses = []
odm_bbox_losses = []
st_idx = 0
featmap_sizes = [self.featmap_sizes[e] for e in self.featmap_sizes]
num_total_samples = len(pos_inds) + len(
neg_inds) if self.sampling else len(pos_inds)
num_total_samples = max(1, num_total_samples)
for idx, feat_size in enumerate(featmap_sizes):
feat_anchor_num = feat_size[0] * feat_size[1]
# step1: get data
feat_labels = labels[st_idx:st_idx + feat_anchor_num]
feat_label_weights = label_weights[st_idx:st_idx + feat_anchor_num]
feat_bbox_targets = bbox_targets[st_idx:st_idx +
feat_anchor_num, :]
feat_bbox_weights = bbox_weights[st_idx:st_idx +
feat_anchor_num, :]
st_idx += feat_anchor_num
# step2: calc cls loss
feat_labels = feat_labels.reshape(-1)
feat_label_weights = feat_label_weights.reshape(-1)
odm_cls_score = odm_cls_branch_list[idx]
odm_cls_score = paddle.squeeze(odm_cls_score, axis=0)
odm_cls_score1 = odm_cls_score
# gt_classes 0~14(data), feat_labels 0~14, sigmoid_focal_loss need class>=1
feat_labels = paddle.to_tensor(feat_labels)
feat_labels_one_hot = paddle.nn.functional.one_hot(
feat_labels, self.cls_out_channels + 1)
feat_labels_one_hot = feat_labels_one_hot[:, 1:]
feat_labels_one_hot.stop_gradient = True
num_total_samples = paddle.to_tensor(num_total_samples,
dtype='float32',
stop_gradient=True)
odm_cls = F.sigmoid_focal_loss(odm_cls_score1,
feat_labels_one_hot,
normalizer=num_total_samples,
reduction='none')
feat_label_weights = feat_label_weights.reshape(
feat_label_weights.shape[0], 1)
feat_label_weights = np.repeat(feat_label_weights,
self.cls_out_channels,
axis=1)
feat_label_weights = paddle.to_tensor(feat_label_weights)
feat_label_weights.stop_gradient = True
odm_cls = odm_cls * feat_label_weights
odm_cls_total = paddle.sum(odm_cls)
odm_cls_losses.append(odm_cls_total)
# # step3: regression loss
feat_bbox_targets = paddle.to_tensor(feat_bbox_targets,
dtype='float32')
feat_bbox_targets = paddle.reshape(feat_bbox_targets, [-1, 5])
feat_bbox_targets.stop_gradient = True
odm_bbox_pred = odm_reg_branch_list[idx]
odm_bbox_pred = paddle.squeeze(odm_bbox_pred, axis=0)
odm_bbox_pred = paddle.reshape(odm_bbox_pred, [-1, 5])
odm_bbox = self.smooth_l1_loss(odm_bbox_pred, feat_bbox_targets)
loss_weight = paddle.to_tensor(self.reg_loss_weight,
dtype='float32',
stop_gradient=True)
odm_bbox = paddle.multiply(odm_bbox, loss_weight)
feat_bbox_weights = paddle.to_tensor(feat_bbox_weights,
stop_gradient=True)
odm_bbox = odm_bbox * feat_bbox_weights
odm_bbox_total = paddle.sum(odm_bbox) / num_total_samples
odm_bbox_losses.append(odm_bbox_total)
odm_cls_loss = paddle.add_n(odm_cls_losses)
odm_cls_loss = odm_cls_loss * 2.0
odm_reg_loss = paddle.add_n(odm_bbox_losses)
return odm_cls_loss, odm_reg_loss
def update(self, batch_id, data, model):
"""update metrics during each iter
"""
self.video_num += 1
seq_dataset = data
seq_name = seq_dataset.seq_name
logger.info('Prcessing Seq {} [{}/{}]:'.format(seq_name,
self.video_num,
self.total_video_num))
seq_dataloader = DataLoader(seq_dataset,
return_list=True,
batch_size=1,
shuffle=False,
num_workers=0)
seq_total_time = 0
seq_total_frame = 0
ref_embeddings = []
ref_masks = []
prev_embedding = []
prev_mask = []
with paddle.no_grad():
for frame_idx, samples in enumerate(seq_dataloader):
time_start = time.time()
all_preds = []
join_label = None
for aug_idx in range(len(samples)):
if len(ref_embeddings) <= aug_idx:
ref_embeddings.append([])
ref_masks.append([])
prev_embedding.append(None)
prev_mask.append(None)
sample = samples[aug_idx]
ref_emb = ref_embeddings[aug_idx]
ref_m = ref_masks[aug_idx]
prev_emb = prev_embedding[aug_idx]
prev_m = prev_mask[aug_idx]
current_img = sample['current_img']
if 'current_label' in sample.keys():
current_label = sample['current_label']
current_label = paddle.to_tensor(current_label)
else:
current_label = None
obj_num = sample['meta']['obj_num']
imgname = sample['meta']['current_name']
ori_height = sample['meta']['height']
ori_width = sample['meta']['width']
current_img = current_img
obj_num = obj_num
bs, _, h, w = current_img.shape
data_batch = [
ref_emb, ref_m, prev_emb, prev_m, current_img,
[ori_height, ori_width], obj_num
]
all_pred, current_embedding = model(data_batch,
mode='test')
if frame_idx == 0:
if current_label is None:
logger.info(
"No first frame label in Seq {}.".format(
seq_name))
ref_embeddings[aug_idx].append(current_embedding)
ref_masks[aug_idx].append(current_label)
prev_embedding[aug_idx] = current_embedding
prev_mask[aug_idx] = current_label
else:
if sample['meta']['flip']: #False
all_pred = self.flip_tensor(all_pred, 3)
# In YouTube-VOS, not all the objects appear in the first frame for the first time. Thus, we
# have to introduce new labels for new objects, if necessary.
if not sample['meta']['flip'] and not (
current_label is None) and join_label is None:
join_label = paddle.cast(current_label,
dtype='int64')
all_preds.append(all_pred)
if current_label is not None:
ref_embeddings[aug_idx].append(current_embedding)
prev_embedding[aug_idx] = current_embedding
if frame_idx > 0:
all_preds = paddle.concat(all_preds, axis=0)
all_preds = paddle.mean(
all_preds, axis=0) #average results if augmentation
pred_label = paddle.argmax(all_preds, axis=0)
if join_label is not None:
join_label = paddle.squeeze(paddle.squeeze(join_label,
axis=0),
axis=0)
keep = paddle.cast((join_label == 0), dtype="int64")
pred_label = pred_label * keep + join_label * (1 -
keep)
pred_label = pred_label
current_label = paddle.reshape(
pred_label, shape=[1, 1, ori_height, ori_width])
flip_pred_label = self.flip_tensor(pred_label, 1)
flip_current_label = paddle.reshape(
flip_pred_label, shape=[1, 1, ori_height, ori_width])
for aug_idx in range(len(samples)):
if join_label is not None:
if samples[aug_idx]['meta']['flip']:
ref_masks[aug_idx].append(flip_current_label)
else:
ref_masks[aug_idx].append(current_label)
if samples[aug_idx]['meta']['flip']:
prev_mask[aug_idx] = flip_current_label
else:
prev_mask[
aug_idx] = current_label #update prev_mask
one_frametime = time.time() - time_start
seq_total_time += one_frametime
seq_total_frame += 1
obj_num = obj_num.numpy()[0].item()
logger.info('Frame: {}, Obj Num: {}, Time: {}'.format(
imgname[0], obj_num, one_frametime))
self.save_mask(
pred_label,
os.path.join(self.result_root, seq_name,
imgname[0].split('.')[0] + '.png'))
else:
one_frametime = time.time() - time_start
seq_total_time += one_frametime
logger.info('Ref Frame: {}, Time: {}'.format(
imgname[0], one_frametime))
del (ref_embeddings)
del (ref_masks)
del (prev_embedding)
del (prev_mask)
del (seq_dataset)
del (seq_dataloader)
seq_avg_time_per_frame = seq_total_time / seq_total_frame
self.total_time += seq_total_time
self.total_frame += seq_total_frame
total_avg_time_per_frame = self.total_time / self.total_frame
self.total_sfps += seq_avg_time_per_frame
avg_sfps = self.total_sfps / (batch_id + 1)
logger.info("Seq {} FPS: {}, Total FPS: {}, FPS per Seq: {}".format(
seq_name, 1. / seq_avg_time_per_frame,
1. / total_avg_time_per_frame, 1. / avg_sfps))
def local_response_norm(x,
size,
alpha=1e-4,
beta=0.75,
k=1.,
data_format="NCHW",
name=None):
r"""
Local Response Normalization performs a type of "lateral inhibition" by normalizing over local input regions.
For more information, please refer to `ImageNet Classification with Deep Convolutional Neural Networks <https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf>`_
The formula is as follows:
.. math::
Output(i, x, y) = Input(i, x, y) / \\left(k + \\alpha \\sum\\limits^{\\min(C-1, i + size/2)}_{j = \\max(0, i - size/2)}(Input(j, x, y))^2\\right)^{\\beta}
In the above equation:
- :math:`size` : The number of channels to sum over.
- :math:`k` : The offset (avoid being divided by 0).
- :math:`\\alpha` : The scaling parameter.
- :math:`\\beta` : The exponent parameter.
Args:
x (Tensor): The input 3-D/4-D/5-D tensor. The data type is float32.
size (int): The number of channels to sum over.
alpha (float, optional): The scaling parameter, positive. Default:1e-4
beta (float, optional): The exponent, positive. Default:0.75
k (float, optional): An offset, positive. Default: 1.0
data_format (str, optional): Specify the data format of the input, and the data format of the output
will be consistent with that of the input. An optional string from:
If x is 3-D Tensor, the string could be `"NCL"` or `"NLC"` . When it is `"NCL"`,
the data is stored in the order of: `[batch_size, input_channels, feature_length]`.
If x is 4-D Tensor, the string could be `"NCHW"`, `"NHWC"`. When it is `"NCHW"`,
the data is stored in the order of: `[batch_size, input_channels, input_height, input_width]`.
If x is 5-D Tensor, the string could be `"NCDHW"`, `"NDHWC"` . When it is `"NCDHW"`,
the data is stored in the order of: `[batch_size, input_channels, input_depth, input_height, input_width]`.
name (str, optional): Name for the operation (optional, default is None). For more information,
please refer to :ref:`api_guide_Name`.
Returns:
A tensor storing the transformation result with the same shape and data type as input.
Examples:
.. code-block:: python
import paddle
x = paddle.rand(shape=(3, 3, 112, 112), dtype="float32")
y = paddle.nn.functional.local_response_norm(x, size=5)
print(y.shape) # [3, 3, 112, 112]
"""
if not in_dygraph_mode():
check_variable_and_dtype(x, 'x', ['float32'], 'local_response_norm')
if data_format not in ['NCL', 'NLC', 'NCHW', 'NHWC', 'NCDHW', 'NDHWC']:
raise ValueError(
"data_format should be in one of [NCL, NCHW, NCDHW, NLC, NHWC, NDHWC], " \
"but got {}".format(data_format))
sizes = x.shape
dim = len(sizes)
if dim < 3:
raise ValueError(
'Expected 3D or higher dimensionality input, but got {} dimensions'
.format(dim))
channel_last = True if data_format[-1] == "C" else False
div = paddle.unsqueeze(paddle.multiply(x, x), axis=1)
if not channel_last:
pad4d_shape = [0, 0, size // 2, (size - 1) // 2]
pool2d_shape = (size, 1)
reshape_shape = [sizes[0], 1, sizes[1], sizes[2], -1]
pad5d_shape = [0, 0, 0, 0, size // 2, (size - 1) // 2]
pool3d_shape = (size, 1, 1)
else:
pad4d_shape = [size // 2, (size - 1) // 2, 0, 0]
pool2d_shape = (1, size)
reshape_shape = [sizes[0], 1, sizes[1], -1, sizes[-1]]
pad5d_shape = [size // 2, (size - 1) // 2, 0, 0, 0, 0]
pool3d_shape = (1, 1, size)
if dim == 3:
div = paddle.nn.functional.pad(div, pad=pad4d_shape)
div = paddle.nn.functional.avg_pool2d(div,
kernel_size=pool2d_shape,
stride=1)
div = paddle.squeeze(div, axis=1)
else:
div = paddle.reshape(div, shape=reshape_shape)
div = paddle.nn.functional.pad(div,
pad=pad5d_shape,
data_format='NCDHW')
div = paddle.nn.functional.avg_pool3d(div,
kernel_size=pool3d_shape,
stride=1)
div = paddle.reshape(paddle.squeeze(div, axis=1), sizes)
div = paddle.scale(div, scale=alpha, bias=k)
div = paddle.pow(div, beta)
res = paddle.divide(x, div, name=name)
return res
def forward(self, sparse_inputs, dense_inputs):
# -------------------- first order term --------------------
sparse_inputs_concat = paddle.concat(
sparse_inputs, axis=1) # [batch_size, sparse_feature_number]
sparse_emb_one = self.embedding_one(
sparse_inputs_concat) # [batch_size, sparse_feature_number, 1]
_dense_emb_one = paddle.multiply(
dense_inputs,
self.dense_w_one) # [batch_size, dense_feature_number]
dense_emb_one = paddle.unsqueeze(
_dense_emb_one, axis=2) # [batch_size, dense_feature_number, 1]
y_first_order = paddle.sum(sparse_emb_one, 1) + paddle.sum(
dense_emb_one, 1)
# -------------------- Field-embedded second order term --------------------
sparse_embeddings = self.embedding(
sparse_inputs_concat
) # [batch_size, sparse_feature_number, sparse_feature_dim]
dense_inputs_re = (dense_inputs * 1e5 + 1e6 + 2).astype(
'int64') # [batch_size, dense_feature_number]
dense_embeddings = self.embedding(
dense_inputs_re
) # [batch_size, dense_feature_number, dense_feature_dim]
feat_embeddings = paddle.concat(
[sparse_embeddings, dense_embeddings], 1
) # [batch_size, dense_feature_number + sparse_feature_number, dense_feature_dim]
pairwise_inner_prods = []
for fi, fj in itertools.combinations(
range(self.num_fields), 2
): # self.num_fields = 39, dense_feature_number + sparse_num_field
field_pair_id = str(fi) + "-" + str(fj)
feat_embed_i = paddle.squeeze(
feat_embeddings[0:, fi:fi + 1, 0:], axis=1
) # feat_embeddings: [batch_size, num_fields, sparse_feature_dim]
feat_embed_j = paddle.squeeze(
feat_embeddings[0:, fj:fj + 1, 0:],
axis=1) # [batch_size * sparse_feature_dim]
field_pair_embed_ij = self.field_embeddings[
field_pair_id] # self.field_embeddings [sparse_feature_dim, sparse_feature_dim]
feat_embed_i_tr = paddle.matmul(
feat_embed_i, field_pair_embed_ij +
paddle.transpose(field_pair_embed_ij,
[1, 0])) # [batch_size * embedding_size]
f = batch_dot(feat_embed_i_tr, feat_embed_j,
axes=1) # [batch_size * 1]
pairwise_inner_prods.append(f)
fefm_interaction_embedding = paddle.concat(
pairwise_inner_prods,
axis=1) # [batch_size, num_fields*(num_fields-1)/2]
y_field_emb_second_order = paddle.sum(fefm_interaction_embedding,
axis=1,
keepdim=True)
dnn_input = paddle.reshape(sparse_embeddings, [0, -1])
dnn_input = paddle.concat(
[dnn_input, _dense_emb_one], 1
) # [batch_size, dense_feature_number + sparse_feature_number * sparse_feature_dim]
dnn_input = paddle.concat(
[dnn_input, fefm_interaction_embedding], 1
) # [batch_size, dense_feature_number + sparse_feature_number * sparse_feature_dim + num_fields*(num_fields-1)/2]
return y_first_order, y_field_emb_second_order, dnn_input
def predictEnsembleThree(model,
model_1,
model_crop,
model_path,
model_path_1,
model_path_crop,
transforms,
transforms_crop,
image_list,
image_dir=None,
save_dir='output',
aug_pred=False,
scales=1.0,
flip_horizontal=True,
flip_vertical=False,
is_slide=False,
stride=None,
crop_size=None):
"""
predict and visualize the image_list.
Args:
model (nn.Layer): Used to predict for input image.
model_path (str): The path of pretrained model.
transforms (transform.Compose): Preprocess for input image.
image_list (list): A list of image path to be predicted.
image_dir (str, optional): The root directory of the images predicted. Default: None.
save_dir (str, optional): The directory to save the visualized results. Default: 'output'.
aug_pred (bool, optional): Whether to use mulit-scales and flip augment for predition. Default: False.
scales (list|float, optional): Scales for augment. It is valid when `aug_pred` is True. Default: 1.0.
flip_horizontal (bool, optional): Whether to use flip horizontally augment. It is valid when `aug_pred` is True. Default: True.
flip_vertical (bool, optional): Whether to use flip vertically augment. It is valid when `aug_pred` is True. Default: False.
is_slide (bool, optional): Whether to predict by sliding window. Default: False.
stride (tuple|list, optional): The stride of sliding window, the first is width and the second is height.
It should be provided when `is_slide` is True.
crop_size (tuple|list, optional): The crop size of sliding window, the first is width and the second is height.
It should be provided when `is_slide` is True.
"""
utils.utils.load_entire_model(model, model_path)
model.eval()
utils.utils.load_entire_model(model_1, model_path_1)
model_1.eval()
utils.utils.load_entire_model(model_crop, model_path_crop)
model_crop.eval()
nranks = paddle.distributed.get_world_size()
local_rank = paddle.distributed.get_rank()
if nranks > 1:
img_lists = partition_list(image_list, nranks)
else:
img_lists = [image_list]
added_saved_dir = os.path.join(save_dir, 'added_prediction')
pred_saved_dir = os.path.join(save_dir, 'pseudo_color_prediction')
logger.info("Start to predict...")
progbar_pred = progbar.Progbar(target=len(img_lists[0]), verbose=1)
with paddle.no_grad():
for i, im_path in enumerate(img_lists[local_rank]):
im_origin = cv2.imread(im_path)
ori_shape = im_origin.shape[:2]
im, _ = transforms(im_origin)
im = im[np.newaxis, ...]
im = paddle.to_tensor(im)
ims, _ = transforms_crop(im_origin)
im1 = ims[:, 540:540 + 720, 320:320 + 1280]
im2 = ims[:, 540:540 + 720, 960:960 + 1280]
im3 = ims[:, 540:540 + 720, 1600:1600 + 1280]
im1 = im1[np.newaxis, ...]
im1 = paddle.to_tensor(im1)
im2 = im2[np.newaxis, ...]
im2 = paddle.to_tensor(im2)
im3 = im3[np.newaxis, ...]
im3 = paddle.to_tensor(im3)
ims_ = [im1, im2, im3]
if aug_pred:
pred = infer_ensemble.aug_inference(
model,
model_1,
im,
ori_shape=ori_shape,
transforms=transforms.transforms,
scales=scales,
flip_horizontal=flip_horizontal,
flip_vertical=flip_vertical,
is_slide=is_slide,
stride=stride,
crop_size=crop_size)
else:
pred = infer_ensemble.inference(
model,
model_1,
im,
ori_shape=ori_shape,
transforms=transforms.transforms,
is_slide=is_slide,
stride=stride,
crop_size=crop_size)
preds = []
for ii in range(3):
im_ = ims_[ii]
if aug_pred:
pred_crop = infer_crop.aug_inference(
model,
im_,
ori_shape=ori_shape,
transforms=transforms.transforms,
scales=scales,
flip_horizontal=flip_horizontal,
flip_vertical=flip_vertical,
is_slide=is_slide,
stride=stride,
crop_size=crop_size)
else:
pred_crop = infer_crop.inference(
model,
im_,
ori_shape=ori_shape,
transforms=transforms.transforms,
is_slide=is_slide,
stride=stride,
crop_size=crop_size)
preds.append(pred_crop)
left_ensem = (preds[0][:, :, :, 640:1280] +
preds[1][:, :, :, 0:640]) / 2
right_ensem = (preds[1][:, :, :, 640:1280] +
preds[2][:, :, :, 0:640]) / 2
pred_ensem = paddle.concat([
preds[0][:, :, :, 0:640], left_ensem, right_ensem,
preds[2][:, :, :, 640:1280]
],
axis=3)
logit = F.interpolate(pred_ensem, (432, 768), mode='bilinear')
pred_logit = pred.clone()
pred_logit[:, :, 324:756, 576:1344] = logit
pred = pred + pred_logit
pred = F.interpolate(pred, ori_shape, mode='bilinear')
pred = paddle.argmax(pred, axis=1, keepdim=True, dtype='int32')
pred = paddle.squeeze(pred)
pred = pred.numpy().astype('uint8')
# get the saved name
if image_dir is not None:
im_file = im_path.replace(image_dir, '')
else:
im_file = os.path.basename(im_path)
if im_file[0] == '/':
im_file = im_file[1:]
# save added image
added_image = utils.visualize.visualize(im_path, pred, weight=0.6)
added_image_path = os.path.join(added_saved_dir, im_file)
mkdir(added_image_path)
cv2.imwrite(added_image_path, added_image)
# save pseudo color prediction
pred_mask = utils.visualize.get_pseudo_color_map(pred)
pred_saved_path = os.path.join(pred_saved_dir,
im_file.rsplit(".")[0] + ".png")
mkdir(pred_saved_path)
pred_mask.save(pred_saved_path)
# pred_im = utils.visualize(im_path, pred, weight=0.0)
# pred_saved_path = os.path.join(pred_saved_dir, im_file)
# mkdir(pred_saved_path)
# cv2.imwrite(pred_saved_path, pred_im)
progbar_pred.update(i + 1)
def _mol_encoder(self, graph):
x = self.atom_embedding(graph.node_feat)
x = paddle.squeeze(x, axis=1)
x = paddle.concat([x, graph.node_feat['atom_numeric_feat']], axis=1)
return x
def get_odm_loss(self, odm_target, s2anet_head_out):
(feat_labels, feat_label_weights, feat_bbox_targets, feat_bbox_weights,
pos_inds, neg_inds) = odm_target
odm_cls_score, odm_bbox_pred = s2anet_head_out
# step1: sample count
num_total_samples = len(pos_inds) + len(
neg_inds) if self.sampling else len(pos_inds)
num_total_samples = max(1, num_total_samples)
# step2: calc cls loss
feat_labels = feat_labels.reshape(-1)
feat_label_weights = feat_label_weights.reshape(-1)
odm_cls_score = paddle.squeeze(odm_cls_score, axis=0)
odm_cls_score1 = odm_cls_score
# gt_classes 0~14(data), feat_labels 0~14, sigmoid_focal_loss need class>=1
# for debug 0426
feat_labels = feat_labels + 1
feat_labels = paddle.to_tensor(feat_labels)
feat_labels_one_hot = F.one_hot(feat_labels, self.cls_out_channels + 1)
feat_labels_one_hot = feat_labels_one_hot[:, 1:]
feat_labels_one_hot.stop_gradient = True
num_total_samples = paddle.to_tensor(num_total_samples,
dtype='float32',
stop_gradient=True)
odm_cls = F.sigmoid_focal_loss(odm_cls_score1,
feat_labels_one_hot,
normalizer=num_total_samples,
reduction='none')
feat_label_weights = feat_label_weights.reshape(
feat_label_weights.shape[0], 1)
feat_label_weights = np.repeat(feat_label_weights,
self.cls_out_channels,
axis=1)
feat_label_weights = paddle.to_tensor(feat_label_weights,
stop_gradient=True)
odm_cls = odm_cls * feat_label_weights
odm_cls_total = paddle.sum(odm_cls)
# step3: regression loss
feat_bbox_targets = paddle.to_tensor(feat_bbox_targets,
dtype='float32',
stop_gradient=True)
feat_bbox_targets = paddle.reshape(feat_bbox_targets, [-1, 5])
odm_bbox_pred = paddle.squeeze(odm_bbox_pred, axis=0)
odm_bbox_pred = paddle.reshape(odm_bbox_pred, [-1, 5])
odm_bbox = self.smooth_l1_loss(odm_bbox_pred, feat_bbox_targets)
loss_weight = paddle.to_tensor(self.reg_loss_weight,
dtype='float32',
stop_gradient=True)
odm_bbox = paddle.multiply(odm_bbox, loss_weight)
feat_bbox_weights = paddle.to_tensor(feat_bbox_weights,
stop_gradient=True)
odm_bbox = odm_bbox * feat_bbox_weights
odm_bbox_total = paddle.sum(odm_bbox) / num_total_samples
odm_cls_loss_weight = paddle.to_tensor(self.cls_loss_weight[0],
dtype='float32',
stop_gradient=True)
odm_cls_loss = odm_cls_total * odm_cls_loss_weight
odm_reg_loss = paddle.add_n(odm_bbox_total)
return odm_cls_loss, odm_reg_loss
def forward(self, logits, label, semantic_weights=None):
"""
Forward computation.
Args:
logits (Tensor): Logit tensor, the data type is float32, float64. Shape is
(logit,points). logit'shape: [N, C, point_num]. logit'shape:[N, point_num, 2], where C is number of classes.
label (Tensor): Label tensor, the data type is int64. Shape is (N), where each
value is 0 <= label[i] <= C-1, and if shape is more than 2D, this is
(N, D1, D2,..., Dk), k >= 1.
semantic_weights (Tensor, optional): Weights about loss for each pixels, shape is the same as label. Default: None.
"""
# for loss
logit, points = logits # [N, C, point_num],[N, point_num, 2]
label = label.unsqueeze(1) # [N,1,H,W]
label = point_sample(
label.astype('float32'),
points,
mode='nearest',
align_corners=self.align_corners) # [N, 1, point_num]
label = paddle.squeeze(label, axis=1).astype('int64') # [N, xx]
channel_axis = 1 if self.data_format == 'NCHW' else -1
if self.weight is not None and logit.shape[channel_axis] != len(
self.weight):
raise ValueError(
'The number of weights = {} must be the same as the number of classes = {}.'
.format(len(self.weight), logit.shape[1]))
logit = paddle.transpose(logit, [0, 2, 1])
no_ignore_label = label
#no_ignore_label[label==self.ignore_index] = 0
loss = F.cross_entropy(logit,
no_ignore_label,
ignore_index=self.ignore_index,
reduction='none')
mask = label != self.ignore_index
mask = paddle.cast(mask, 'float32')
loss = loss * mask
if semantic_weights is not None:
loss = loss * semantic_weights
if self.weight is not None:
_one_hot = F.one_hot(label, logit.shape[-1])
_one_hot_weight = _one_hot * self.weight
loss = loss * _one_hot_weight.argmax(-1)
coef = paddle.sum(_one_hot_weight, axis=-1)
#coef = paddle.ones_like(label)
else:
coef = paddle.ones_like(label)
label.stop_gradient = True
mask.stop_gradient = True
if self.top_k_percent_pixels == 1.0:
avg_loss = paddle.mean(loss) / (paddle.mean(mask * coef) +
self.EPS)
return avg_loss
loss = loss.reshape((-1, ))
top_k_pixels = int(self.top_k_percent_pixels * loss.numel())
loss, indices = paddle.topk(loss, top_k_pixels)
coef = coef.reshape((-1, ))
coef = paddle.gather(coef, indices)
coef.stop_gradient = True
return loss.mean() / (paddle.mean(coef) + self.EPS)
def test_tensor_patch_method(self):
paddle.disable_static()
x_np = np.random.uniform(-1, 1, [2, 3]).astype(self.dtype)
y_np = np.random.uniform(-1, 1, [2, 3]).astype(self.dtype)
z_np = np.random.uniform(-1, 1, [6, 9]).astype(self.dtype)
x = paddle.to_tensor(x_np)
y = paddle.to_tensor(y_np)
z = paddle.to_tensor(z_np)
a = paddle.to_tensor([[1, 1], [2, 2], [3, 3]])
b = paddle.to_tensor([[1, 1], [2, 2], [3, 3]])
# 1. Unary operation for Tensor
self.assertEqual(x.dim(), 2)
self.assertEqual(x.ndimension(), 2)
self.assertEqual(x.ndim, 2)
self.assertEqual(x.size(), [2, 3])
self.assertTrue(
np.array_equal(x.sigmoid().numpy(),
fluid.layers.sigmoid(x).numpy()))
self.assertTrue(
np.array_equal(x.logsigmoid().numpy(),
fluid.layers.logsigmoid(x).numpy()))
self.assertTrue(np.array_equal(x.exp().numpy(), paddle.exp(x).numpy()))
self.assertTrue(
np.array_equal(x.tanh().numpy(),
paddle.tanh(x).numpy()))
self.assertTrue(
np.array_equal(x.atan().numpy(),
paddle.atan(x).numpy()))
self.assertTrue(
np.array_equal(x.tanh_shrink().numpy(),
fluid.layers.tanh_shrink(x).numpy()))
self.assertTrue(np.array_equal(x.abs().numpy(), paddle.abs(x).numpy()))
m = x.abs()
self.assertTrue(
np.array_equal(m.sqrt().numpy(),
paddle.sqrt(m).numpy()))
self.assertTrue(
np.array_equal(m.rsqrt().numpy(),
paddle.rsqrt(m).numpy()))
self.assertTrue(
np.array_equal(x.ceil().numpy(),
paddle.ceil(x).numpy()))
self.assertTrue(
np.array_equal(x.floor().numpy(),
paddle.floor(x).numpy()))
self.assertTrue(np.array_equal(x.cos().numpy(), paddle.cos(x).numpy()))
self.assertTrue(
np.array_equal(x.acos().numpy(),
paddle.acos(x).numpy()))
self.assertTrue(
np.array_equal(x.asin().numpy(),
paddle.asin(x).numpy()))
self.assertTrue(np.array_equal(x.sin().numpy(), paddle.sin(x).numpy()))
self.assertTrue(
np.array_equal(x.sinh().numpy(),
paddle.sinh(x).numpy()))
self.assertTrue(
np.array_equal(x.cosh().numpy(),
paddle.cosh(x).numpy()))
self.assertTrue(
np.array_equal(x.round().numpy(),
paddle.round(x).numpy()))
self.assertTrue(
np.array_equal(x.reciprocal().numpy(),
paddle.reciprocal(x).numpy()))
self.assertTrue(
np.array_equal(x.square().numpy(),
paddle.square(x).numpy()))
self.assertTrue(
np.array_equal(x.softplus().numpy(),
fluid.layers.softplus(x).numpy()))
self.assertTrue(
np.array_equal(x.softsign().numpy(),
fluid.layers.softsign(x).numpy()))
self.assertTrue(
np.array_equal(x.rank().numpy(),
paddle.rank(x).numpy()))
self.assertTrue(
np.array_equal(x[0].t().numpy(),
paddle.t(x[0]).numpy()))
m = paddle.to_tensor(np.random.uniform(1, 2, [3, 3]), 'float32')
m = m.matmul(m.t())
self.assertTrue(
np.array_equal(m.cholesky().numpy(),
paddle.cholesky(m).numpy()))
self.assertTrue(
np.array_equal(x.is_empty().numpy(),
paddle.is_empty(x).numpy()))
self.assertTrue(
np.array_equal(x.isfinite().numpy(),
paddle.isfinite(x).numpy()))
self.assertTrue(
np.array_equal(
x.cast('int32').numpy(),
paddle.cast(x, 'int32').numpy()))
self.assertTrue(
np.array_equal(
x.expand([3, 2, 3]).numpy(),
paddle.expand(x, [3, 2, 3]).numpy()))
self.assertTrue(
np.array_equal(
x.tile([2, 2]).numpy(),
paddle.tile(x, [2, 2]).numpy()))
self.assertTrue(
np.array_equal(x.flatten().numpy(),
paddle.flatten(x).numpy()))
index = paddle.to_tensor([0, 1])
self.assertTrue(
np.array_equal(
x.gather(index).numpy(),
paddle.gather(x, index).numpy()))
index = paddle.to_tensor([[0, 1], [1, 2]])
self.assertTrue(
np.array_equal(
x.gather_nd(index).numpy(),
paddle.gather_nd(x, index).numpy()))
self.assertTrue(
np.array_equal(
x.reverse([0, 1]).numpy(),
paddle.reverse(x, [0, 1]).numpy()))
self.assertTrue(
np.array_equal(
a.reshape([3, 2]).numpy(),
paddle.reshape(a, [3, 2]).numpy()))
self.assertTrue(
np.array_equal(
x.slice([0, 1], [0, 0], [1, 2]).numpy(),
paddle.slice(x, [0, 1], [0, 0], [1, 2]).numpy()))
self.assertTrue(
np.array_equal(
x.split(2)[0].numpy(),
paddle.split(x, 2)[0].numpy()))
m = paddle.to_tensor(
np.random.uniform(-1, 1, [1, 6, 1, 1]).astype(self.dtype))
self.assertTrue(
np.array_equal(
m.squeeze([]).numpy(),
paddle.squeeze(m, []).numpy()))
self.assertTrue(
np.array_equal(
m.squeeze([1, 2]).numpy(),
paddle.squeeze(m, [1, 2]).numpy()))
m = paddle.to_tensor([2, 3, 3, 1, 5, 3], 'float32')
self.assertTrue(
np.array_equal(m.unique()[0].numpy(),
paddle.unique(m)[0].numpy()))
self.assertTrue(
np.array_equal(m.unique_with_counts()[2],
paddle.unique_with_counts(m)[2]))
self.assertTrue(np.array_equal(x.flip([0]), paddle.flip(x, [0])))
self.assertTrue(np.array_equal(x.unbind(0), paddle.unbind(x, 0)))
self.assertTrue(np.array_equal(x.roll(1), paddle.roll(x, 1)))
self.assertTrue(np.array_equal(x.cumsum(1), paddle.cumsum(x, 1)))
m = paddle.to_tensor(1)
self.assertTrue(np.array_equal(m.increment(), paddle.increment(m)))
m = x.abs()
self.assertTrue(np.array_equal(m.log(), paddle.log(m)))
self.assertTrue(np.array_equal(x.pow(2), paddle.pow(x, 2)))
self.assertTrue(np.array_equal(x.reciprocal(), paddle.reciprocal(x)))
# 2. Binary operation
self.assertTrue(
np.array_equal(
x.matmul(y, True, False).numpy(),
paddle.matmul(x, y, True, False).numpy()))
self.assertTrue(
np.array_equal(
x.norm(p='fro', axis=[0, 1]).numpy(),
paddle.norm(x, p='fro', axis=[0, 1]).numpy()))
self.assertTrue(
np.array_equal(x.dist(y).numpy(),
paddle.dist(x, y).numpy()))
self.assertTrue(
np.array_equal(x.cross(y).numpy(),
paddle.cross(x, y).numpy()))
m = x.expand([2, 2, 3])
n = y.expand([2, 2, 3]).transpose([0, 2, 1])
self.assertTrue(
np.array_equal(m.bmm(n).numpy(),
paddle.bmm(m, n).numpy()))
self.assertTrue(
np.array_equal(
x.histogram(5, -1, 1).numpy(),
paddle.histogram(x, 5, -1, 1).numpy()))
self.assertTrue(
np.array_equal(x.equal(y).numpy(),
paddle.equal(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.greater_equal(y).numpy(),
paddle.greater_equal(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.greater_than(y).numpy(),
paddle.greater_than(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.less_equal(y).numpy(),
paddle.less_equal(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.less_than(y).numpy(),
paddle.less_than(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.not_equal(y).numpy(),
paddle.not_equal(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.equal_all(y).numpy(),
paddle.equal_all(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.allclose(y).numpy(),
paddle.allclose(x, y).numpy()))
m = x.expand([2, 2, 3])
self.assertTrue(
np.array_equal(
x.expand_as(m).numpy(),
paddle.expand_as(x, m).numpy()))
index = paddle.to_tensor([2, 1, 0])
self.assertTrue(
np.array_equal(
a.scatter(index, b).numpy(),
paddle.scatter(a, index, b).numpy()))
# 3. Bool tensor operation
x = paddle.to_tensor([[True, False], [True, False]])
y = paddle.to_tensor([[False, False], [False, True]])
self.assertTrue(
np.array_equal(x.reduce_all().numpy(),
paddle.reduce_all(x).numpy()))
self.assertTrue(
np.array_equal(x.reduce_any().numpy(),
paddle.reduce_any(x).numpy()))
self.assertTrue(
np.array_equal(
x.logical_and(y).numpy(),
paddle.logical_and(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.logical_not(y).numpy(),
paddle.logical_not(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.logical_or(y).numpy(),
paddle.logical_or(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.logical_xor(y).numpy(),
paddle.logical_xor(x, y).numpy()))
self.assertTrue(
np.array_equal(
x.logical_and(y).numpy(),
paddle.logical_and(x, y).numpy()))
def generate_proposal_target(rpn_rois,
gt_classes,
gt_boxes,
batch_size_per_im,
fg_fraction,
fg_thresh,
bg_thresh,
num_classes,
ignore_thresh=-1.,
is_crowd=None,
use_random=True,
is_cascade=False,
cascade_iou=0.5,
assign_on_cpu=False):
rois_with_gt = []
tgt_labels = []
tgt_bboxes = []
tgt_gt_inds = []
new_rois_num = []
# In cascade rcnn, the threshold for foreground and background
# is used from cascade_iou
fg_thresh = cascade_iou if is_cascade else fg_thresh
bg_thresh = cascade_iou if is_cascade else bg_thresh
for i, rpn_roi in enumerate(rpn_rois):
gt_bbox = gt_boxes[i]
is_crowd_i = is_crowd[i] if is_crowd else None
gt_class = paddle.squeeze(gt_classes[i], axis=-1)
# Concat RoIs and gt boxes except cascade rcnn or none gt
if not is_cascade and gt_bbox.shape[0] > 0:
bbox = paddle.concat([rpn_roi, gt_bbox])
else:
bbox = rpn_roi
# Step1: label bbox
matches, match_labels = label_box(bbox, gt_bbox, fg_thresh, bg_thresh,
False, ignore_thresh, is_crowd_i,
assign_on_cpu)
# Step2: sample bbox
sampled_inds, sampled_gt_classes = sample_bbox(
matches, match_labels, gt_class, batch_size_per_im, fg_fraction,
num_classes, use_random, is_cascade)
# Step3: make output
rois_per_image = bbox if is_cascade else paddle.gather(
bbox, sampled_inds)
sampled_gt_ind = matches if is_cascade else paddle.gather(
matches, sampled_inds)
if gt_bbox.shape[0] > 0:
sampled_bbox = paddle.gather(gt_bbox, sampled_gt_ind)
else:
num = rois_per_image.shape[0]
sampled_bbox = paddle.zeros([num, 4], dtype='float32')
rois_per_image.stop_gradient = True
sampled_gt_ind.stop_gradient = True
sampled_bbox.stop_gradient = True
tgt_labels.append(sampled_gt_classes)
tgt_bboxes.append(sampled_bbox)
rois_with_gt.append(rois_per_image)
tgt_gt_inds.append(sampled_gt_ind)
new_rois_num.append(paddle.shape(sampled_inds)[0])
new_rois_num = paddle.concat(new_rois_num)
return rois_with_gt, tgt_labels, tgt_bboxes, tgt_gt_inds, new_rois_num
def main(args):
"""
Main function
"""
# Basic setting
optimal_mse = 10000
log_iter = 50
log_step = 0
optimal_CI = 0
# Load model config
model_config = json.load(open(args.model_config, 'r'))
model = MolTransModel(model_config)
model = model.cuda()
# Load pretrained model
# params_dict= paddle.load('./pretrained_model/pdb2016_single_tower_1')
# model.set_dict(params_dict)
# Optimizer
# scheduler = paddle.optimizer.lr.LinearWarmup(learning_rate=args.lr, warmup_steps=50, start_lr=0,
# end_lr=args.lr, verbose=False)
optim = utils.Adam(parameters=model.parameters(), learning_rate=args.lr) # Adam
#optim = paddle.optimizer.AdamW(learning_rate=scheduler, parameters=model.parameters(), weight_decay=0.01) # AdamW
# Data Preparation
# Regression Task - Raw Dataset
# X_drugs, X_targets, y = get_raw_db(args.dataset)
# reg_training_set, reg_validation_set, reg_testing_set = data_process(X_drugs, X_targets, y,
# frac=[0.8,0.1,0.1], drug_encoding='Transformer', target_encoding='Transformer',
# split_method='random_split', random_seed=1)
# reg_training_data = DataEncoder(reg_training_set.index.values,
# reg_training_set.Label.values, reg_training_set)
# reg_train_loader = utils.BaseDataLoader(reg_training_data, batch_size=args.batchsize,
# shuffle=True, drop_last=False, num_workers=args.workers)
# reg_validation_data = DataEncoder(reg_validation_set.index.values,
# reg_validation_set.Label.values, reg_validation_set)
# reg_validation_loader = utils.BaseDataLoader(reg_validation_data, batch_size=args.batchsize,
# shuffle=False, drop_last=False, num_workers=args.workers)
# reg_testing_data = DataEncoder(reg_testing_set.index.values,
# reg_testing_set.Label.values, reg_testing_set)
# reg_testing_loader = utils.BaseDataLoader(reg_testing_data, batch_size=args.batchsize,
# shuffle=False, drop_last=False, num_workers=args.workers)
# Regression Task - Benchmark Dataset
trainset, testset = get_reg_db(args.dataset)
trainset_smiles = [d['smiles'] for d in trainset]
trainset_protein = [d['protein'] for d in trainset]
trainset_aff = [d['aff'] for d in trainset]
testset_smiles = [d['smiles'] for d in testset]
testset_protein = [d['protein'] for d in testset]
testset_aff = [d['aff'] for d in testset]
df_data_t = pd.DataFrame(zip(trainset_smiles, trainset_protein, trainset_aff))
df_data_t.rename(columns={0:'SMILES', 1: 'Target Sequence', 2: 'Label'}, inplace=True)
df_data_tt = pd.DataFrame(zip(testset_smiles, testset_protein, testset_aff))
df_data_tt.rename(columns={0:'SMILES', 1: 'Target Sequence', 2: 'Label'}, inplace=True)
reg_training_data = DataEncoder(df_data_t.index.values, df_data_t.Label.values, df_data_t)
reg_train_loader = utils.BaseDataLoader(reg_training_data, batch_size=args.batchsize,
shuffle=True, drop_last=False, num_workers=args.workers)
reg_validation_data = DataEncoder(df_data_tt.index.values, df_data_tt.Label.values, df_data_tt)
reg_validation_loader = utils.BaseDataLoader(reg_validation_data, batch_size=args.batchsize,
shuffle=False, drop_last=False, num_workers=args.workers)
# Initial Testing
print("=====Go for Initial Testing=====")
with paddle.no_grad():
mse, CI, reg_loss = reg_test(reg_validation_loader, model)
print("Testing result: MSE: {}, CI: {}"
.format(mse, CI))
# Training
for epoch in range(args.epochs):
print("=====Go for Training=====")
model.train()
# Regression Task
for batch_id, data in enumerate(reg_train_loader):
d_out, mask_d_out, t_out, mask_t_out, label = data
temp = model(d_out.long().cuda(), t_out.long().cuda(), mask_d_out.long().cuda(), mask_t_out.long().cuda())
label = paddle.cast(label, "float32")
predicts = paddle.squeeze(temp)
loss = reg_loss_fn(predicts, label)
optim.clear_grad()
loss.backward()
optim.step()
#scheduler.step()
if batch_id % log_iter == 0:
print("Training at epoch: {}, step: {}, loss is: {}"
.format(epoch, batch_id, loss.cpu().detach().numpy()))
log_writer.add_scalar(tag="train/loss", step=log_step, value=loss.cpu().detach().numpy())
log_step += 1
# Validation
print("=====Go for Validation=====")
with paddle.no_grad():
mse, CI, reg_loss = reg_test(reg_validation_loader, model)
print("Validation at epoch: {}, MSE: {}, CI: {}, loss is: {}"
.format(epoch, mse, CI, reg_loss))
log_writer.add_scalar(tag="dev/loss", step=log_step, value=reg_loss)
log_writer.add_scalar(tag="dev/mse", step=log_step, value=mse)
log_writer.add_scalar(tag="dev/CI", step=log_step, value=CI)
# Save best model
if mse < optimal_mse:
optimal_mse = mse
print("Saving the best_model with best MSE...")
print("Best MSE: {}".format(optimal_mse))
paddle.save(model.state_dict(), 'DAVIS_bestMSE_model_reg1')
if CI > optimal_CI:
optimal_CI = CI
print("Saving the best_model with best CI...")
print("Best CI: {}".format(optimal_CI))
paddle.save(model.state_dict(), 'DAVIS_bestCI_model_reg1')
# Print final result
print("Best MSE: {}".format(optimal_mse))
print("Best CI: {}".format(optimal_CI))
paddle.save(model.state_dict(), 'DAVIS_final_model_reg1')
def batch_dot(x, y, axes=None):
"""Batchwise dot product.
# >>> x_batch = paddle.ones(shape=(32, 20, 1))
# >>> y_batch = paddle.ones(shape=(32, 30, 20))
# >>> xy_batch_dot = batch_dot(x_batch, y_batch, axes=(1, 2))
# >>> xy_batch_dot.shape
(32, 1, 30)
Shape inference:
Let `x`'s shape be `(100, 20)` and `y`'s shape be `(100, 30, 20)`.
If `axes` is (1, 2), to find the output shape of resultant tensor,
loop through each dimension in `x`'s shape and `y`'s shape:
* `x.shape[0]` : 100 : append to output shape
* `x.shape[1]` : 20 : do not append to output shape,
dimension 1 of `x` has been summed over. (`dot_axes[0]` = 1)
* `y.shape[0]` : 100 : do not append to output shape,
always ignore first dimension of `y`
* `y.shape[1]` : 30 : append to output shape
* `y.shape[2]` : 20 : do not append to output shape,
dimension 2 of `y` has been summed over. (`dot_axes[1]` = 2)
`output_shape` = `(100, 30)`
"""
x_shape = x.shape
y_shape = y.shape
x_ndim = len(x_shape)
y_ndim = len(y_shape)
if x_ndim < 2 or y_ndim < 2:
raise ValueError('Cannot do batch_dot on inputs '
'with rank < 2. '
'Received inputs with shapes ' + str(x_shape) +
' and ' + str(y_shape) + '.')
x_batch_size = x_shape[0]
y_batch_size = y_shape[0]
if x_batch_size is not None and y_batch_size is not None:
if x_batch_size != y_batch_size:
raise ValueError('Cannot do batch_dot on inputs '
'with different batch sizes. '
'Received inputs with shapes ' + str(x_shape) +
' and ' + str(y_shape) + '.')
if isinstance(axes, int):
axes = [axes, axes]
if axes is None:
if y_ndim == 2:
axes = [x_ndim - 1, y_ndim - 1]
else:
axes = [x_ndim - 1, y_ndim - 2]
if any(isinstance(a, (list, tuple)) for a in axes):
raise ValueError('Multiple target dimensions are not supported. ' +
'Expected: None, int, (int, int), ' + 'Provided: ' +
str(axes))
# if tuple, convert to list.
axes = list(axes)
# convert negative indices.
if axes[0] < 0:
axes[0] += x_ndim
if axes[1] < 0:
axes[1] += y_ndim
# sanity checks
if 0 in axes:
raise ValueError('Cannot perform batch_dot over axis 0. '
'If your inputs are not batched, '
'add a dummy batch dimension to your '
'inputs using K.expand_dims(x, 0)')
a0, a1 = axes
d1 = x_shape[a0]
d2 = y_shape[a1]
if d1 is not None and d2 is not None and d1 != d2:
raise ValueError('Cannot do batch_dot on inputs with shapes ' + str(
x_shape) + ' and ' + str(y_shape) + ' with axes=' + str(axes) +
'. x.shape[%d] != '
'y.shape[%d] (%d != %d).' % (axes[0], axes[1], d1, d2
))
# backup ndims. Need them later.
orig_x_ndim = x_ndim
orig_y_ndim = y_ndim
# if rank is 2, expand to 3.
if x_ndim == 2:
x = paddle.unsqueeze(x, 1)
a0 += 1
x_ndim += 1
if y_ndim == 2:
y = paddle.unsqueeze(y, 2)
y_ndim += 1
# bring x's dimension to be reduced to last axis.
if a0 != x_ndim - 1:
pattern = list(range(x_ndim))
for i in range(a0, x_ndim - 1):
pattern[i] = pattern[i + 1]
pattern[-1] = a0
x = paddle.transpose(x, pattern)
# bring y's dimension to be reduced to axis 1.
if a1 != 1:
pattern = list(range(y_ndim))
for i in range(a1, 1, -1):
pattern[i] = pattern[i - 1]
pattern[1] = a1
y = paddle.transpose(y, pattern)
# normalize both inputs to rank 3.
if x_ndim > 3:
# squash middle dimensions of x.
x_shape = x.shape
x_mid_dims = x_shape[1:-1]
x_squashed_shape = paddle.stack([x_shape[0], -1, x_shape[-1]])
x = paddle.reshape(x, x_squashed_shape)
x_squashed = True
else:
x_squashed = False
if y_ndim > 3:
# squash trailing dimensions of y.
y_shape = y.shape
y_trail_dims = y_shape[2:]
y_squashed_shape = paddle.stack([y_shape[0], y_shape[1], -1])
y = paddle.reshape(y, y_squashed_shape)
y_squashed = True
else:
y_squashed = False
result = paddle.matmul(x, y)
# if inputs were squashed, we have to reshape the matmul output.
output_shape = paddle.shape(result)
do_reshape = False
if x_squashed:
output_shape = paddle.concat(
[output_shape[:1], x_mid_dims, output_shape[-1:]], 0)
do_reshape = True
if y_squashed:
output_shape = paddle.concat([output_shape[:-1], y_trail_dims], 0)
do_reshape = True
if do_reshape:
result = paddle.reshape(result, output_shape)
# if the inputs were originally rank 2, we remove the added 1 dim.
if orig_x_ndim == 2:
result = paddle.squeeze(result, 1)
elif orig_y_ndim == 2:
result = paddle.squeeze(result, -1)
return result
def forward(self, inputs):
return paddle.squeeze(inputs, axis=self.axis)
def einsum(equation, *operands):
r"""
Executes the sum of product of provided operands based on the Einstein summation convention.
Einsum can be used to complete a variety of operations, such as sum, transpose,
batch matrix multiplication.
Args:
equation (`str`):
Uses uncased letters to specify the dimension of the operands and result. The input
equation is on the left hand before `->` while the output equation is on the right side.
Einsum can infer the result shape so that the `->` and the result label letters can be omitted.
Operands in the input equation are splited by commas (','), e.g. 'abc,cde' describes two 3D
operands. The dimensions labeled with same letter should be same or be 1. Ellipsis ('...') can
be used to specify the broadcast dimensions.
operands (`Tensor`):
The operands to compute the Einstein sum of. The number of operands should be the same as the
the operands described in input equation.
Returns:
`Tensor`: The result of Einstein sum product.
Example:
.. code-block::
import numpy as np
import paddle
import paddlenlp
np.random.seed(102)
x = paddle.to_tensor(np.random.rand(4))
y = paddle.to_tensor(np.random.rand(5))
# sum
print(paddlenlp.ops.einsum('i->', x))
# Tensor(shape=[], dtype=float64, place=CUDAPlace(0), stop_gradient=True, 2.30369050)
# dot
print(paddlenlp.ops.einsum('i,i->', x, x))
# Tensor(shape=[], dtype=float64, place=CUDAPlace(0), stop_gradient=True, 1.43773247)
# outer
print(paddlenlp.ops.einsum("i,j->ij", x, y)),
# Tensor(shape=[4, 5], dtype=float64, place=CUDAPlace(0), stop_gradient=True,
# [[0.34590188, 0.48353496, 0.09996135, 0.18656330, 0.21392910],
# [0.39122025, 0.54688535, 0.11305780, 0.21100591, 0.24195704],
# [0.17320613, 0.24212422, 0.05005442, 0.09341929, 0.10712238],
# [0.42290818, 0.59118179, 0.12221522, 0.22809690, 0.26155500]])
A = paddle.to_tensor(np.random.rand(2, 3, 2))
B = paddle.to_tensor(np.random.rand(2, 2, 3))
# transpose
print(paddlenlp.ops.einsum('ijk->kji', A))
# Tensor(shape=[2, 3, 2], dtype=float64, place=CUDAPlace(0), stop_gradient=True,
# [[[0.49174730, 0.33344683],
# [0.89440989, 0.26162022],
# [0.36116209, 0.12241719]],
# [[0.49019824, 0.51895050],
# [0.18241053, 0.13092809],
# [0.81059146, 0.55165734]]])
# batch matrix multiplication
print(paddlenlp.ops.einsum('ijk, ikl->ijl', A,B))
# Tensor(shape=[2, 3, 3], dtype=float64, place=CUDAPlace(0), stop_gradient=True,
# [[[0.13654339, 0.39331432, 0.65059661],
# [0.07171420, 0.57518653, 0.77629221],
# [0.21250688, 0.37793541, 0.73643411]],
# [[0.56925339, 0.65859030, 0.57509818],
# [0.30368265, 0.25778348, 0.21630400],
# [0.39587265, 0.58031243, 0.51824755]]])
# Ellipsis transpose
print(paddlenlp.ops.einsum('...jk->...kj', A))
# Tensor(shape=[2, 2, 3], dtype=float64, place=CUDAPlace(0), stop_gradient=True,
# [[[0.49174730, 0.89440989, 0.36116209],
# [0.49019824, 0.18241053, 0.81059146]],
# [[0.33344683, 0.26162022, 0.12241719],
# [0.51895050, 0.13092809, 0.55165734]]])
# Ellipsis batch matrix multiplication
print(paddlenlp.ops.einsum('...jk, ...kl->...jl', A,B))
# Tensor(shape=[2, 3, 3], dtype=float64, place=CUDAPlace(0), stop_gradient=True,
# [[[0.13654339, 0.39331432, 0.65059661],
# [0.07171420, 0.57518653, 0.77629221],
# [0.21250688, 0.37793541, 0.73643411]],
# [[0.56925339, 0.65859030, 0.57509818],
# [0.30368265, 0.25778348, 0.21630400],
# [0.39587265, 0.58031243, 0.51824755]]])
"""
def _mul_sum(left, right, sum_dims):
# assert left.rank() == right.rank(), "number of rank should be equal."
if len(sum_dims) == 0:
return left * right
sum_dims_set = set(sum_dims)
batch_dims = []
left_out_dims = []
right_out_dims = []
batch_size = summed_size = left_size = right_size = 1
dim = len(left.shape)
for i in range(dim):
is_left_summed_dim = left.shape[i] > 1 # not broadcast dim
is_right_summed_dim = right.shape[i] > 1
if i in sum_dims_set:
if is_left_summed_dim and is_right_summed_dim:
assert left.shape[i] == right.shape[
i], "Non-brocast dim should be equal."
summed_size *= left.shape[i]
elif is_left_summed_dim:
left = left.sum(axis=i, keepdim=True)
elif is_right_summed_dim:
right = right.sum(axis=i, keepdim=True)
elif is_left_summed_dim and is_right_summed_dim:
assert left.shape[i] == right.shape[
i], "Non-brocast dim should be equal."
batch_dims.append(i)
batch_size *= left.shape[i]
elif is_left_summed_dim:
left_out_dims.append(i)
left_size *= left.shape[i]
else:
right_out_dims.append(i)
right_size *= right.shape[i]
out_shape = [left.shape[i] for i in batch_dims + left_out_dims]
out_shape.extend([1] * len(sum_dims))
out_shape.extend([right.shape[i] for i in right_out_dims])
left_perm = list(batch_dims)
left_perm.extend(left_out_dims)
left_perm.extend(sum_dims)
left_perm.extend(right_out_dims)
right_perm = list(batch_dims)
right_perm.extend(sum_dims)
right_perm.extend(right_out_dims)
right_perm.extend(left_out_dims)
output_perm = [-1] * (len(batch_dims) + len(left_out_dims) +
len(sum_dims) + len(right_out_dims))
for i, j in enumerate(batch_dims + left_out_dims + sum_dims +
right_out_dims):
output_perm[j] = i
left = paddle.reshape(paddle.transpose(left, perm=left_perm),
(batch_size, left_size, summed_size))
right = paddle.reshape(paddle.transpose(right, perm=right_perm),
(batch_size, summed_size, right_size))
result = paddle.matmul(left, right)
result = paddle.reshape(result, out_shape)
result = paddle.transpose(result, output_perm)
return result
if len(operands) == 1 and isinstance(operands[0], (list, tuple)):
operands = operands[0]
# Equation is case insensitive
num_letters = 26
letters_to_idx = [-1] * num_letters
equation = equation.lower().replace(' ', '')
# 1. Parse the equation
eqns = equation.split("->")
num_eqns_size = len(eqns)
assert num_eqns_size <= 2, "The '->' should exist at most only once"
input_eqn = eqns[0]
output_eqn = None if num_eqns_size <= 1 else eqns[1]
operand_eqns = input_eqn.split(",")
assert len(operand_eqns) == len(
operands
), "Number of operands in equation and the tensors provided should be equal."
# Parse input equation
num_total_idxes = 0
input_operand_idxes = []
letter_frequence = [0] * num_letters
idxes_last_operand = []
num_ell_idxes = -1
first_ell_idx = 0
for i, term in enumerate(operand_eqns):
ell_char_count = 0
operand_rank = int(operands[i].rank().numpy())
curr_num_ell_idxes = operand_rank - len(term) + 3
dims_in_terms = 0
curr_operand_idxes = []
for ch in term:
if ch == '.':
ell_char_count += 1
assert ell_char_count <= 3, "The '.' should only exist in one ellispis '...' in term {}".format(
term)
if ell_char_count == 3:
if num_ell_idxes == -1:
num_ell_idxes = curr_num_ell_idxes
first_ell_idx = num_total_idxes
num_total_idxes += num_ell_idxes
else:
assert curr_num_ell_idxes == num_ell_idxes, "Ellispis in all terms should represent same dimensions ({}).".format(
num_ell_idxes)
for j in range(num_ell_idxes):
curr_operand_idxes.append(j + first_ell_idx)
idxes_last_operand.append(i)
dims_in_terms += num_ell_idxes
else:
assert (
(ell_char_count == 0) or (ell_char_count == 3)
), "'.' must only occur in ellipsis, operand {}".format(term)
assert (ord('a') <= ord(ch) and
ord(ch) <= ord('z')), "only accept alphabet (a-zA-Z)"
letter_num = ord(ch) - ord('a')
if letters_to_idx[letter_num] == -1:
letters_to_idx[letter_num] = num_total_idxes
num_total_idxes += 1
idxes_last_operand.append(i)
else:
idxes_last_operand[letters_to_idx[letter_num]] = i
letter_frequence[letter_num] += 1
curr_operand_idxes.append(letters_to_idx[letter_num])
dims_in_terms += 1
assert dims_in_terms == operand_rank, "Dimension dismatch for operand {}: equation {}, tensor {}".format(
i, dims_in_terms, operand_rank)
input_operand_idxes.append(curr_operand_idxes)
# Parse output equation
idxes_to_output_dims = [-1] * num_total_idxes
num_output_dims = 0
if num_eqns_size == 2:
ell_char_count = 0
for ch in output_eqn:
if ch == '.':
ell_char_count += 1
assert ell_char_count <= 3, "The '.' should only exist in one ellispis '...' in term {}".format(
output_eqn)
if ell_char_count == 3:
assert num_ell_idxes > -1, "Input equation '{}' don't have ellispis.".format(
input_eqn)
for j in range(num_ell_idxes):
idxes_to_output_dims[first_ell_idx +
j] = num_output_dims
num_output_dims += 1
else:
assert (
(ell_char_count == 0) or (ell_char_count == 3)
), "'.' must only occur in ellipsis, operand {}".format(
output_eqn)
assert (ord('a') <= ord(ch) and
ord(ch) <= ord('z')), "only accept alphabet (a-zA-Z)"
letter_num = ord(ch) - ord('a')
assert letters_to_idx[
letter_num] != -1, "character {} doesn't exist in input".format(
ch)
assert idxes_to_output_dims[letters_to_idx[
letter_num]] == -1, "character {} occurs twice in output".format(
ch)
idxes_to_output_dims[
letters_to_idx[letter_num]] = num_output_dims
num_output_dims += 1
else: # num_eqns_size == 1
# Infer the output dims
if num_ell_idxes >= 0:
for j in range(num_ell_idxes):
idxes_to_output_dims[first_ell_idx + j] = num_output_dims
num_output_dims += 1
for j in range(num_letters):
if letter_frequence[j] == 1:
idxes_to_output_dims[letters_to_idx[j]] = num_output_dims
num_output_dims += 1
# Mark sum index
sum_dim = num_output_dims
for i in range(num_total_idxes):
if idxes_to_output_dims[i] == -1:
idxes_to_output_dims[i] = sum_dim
sum_dim += 1
preprocessed_operands = []
size_dims = [-1] * num_total_idxes
for i, preprocessed_operand in enumerate(operands):
idx_to_dims = [-1] * num_total_idxes
curr_operand_idxes = input_operand_idxes[i]
dim = 0
for j, idx in enumerate(curr_operand_idxes):
output_dim = idxes_to_output_dims[idx]
if idx_to_dims[output_dim] == -1:
idx_to_dims[output_dim] = dim
if size_dims[idx] == -1:
size_dims[idx] = preprocessed_operand.shape[dim]
else:
assert size_dims[idx] == preprocessed_operand.shape[
dim], "Dimension size does not match previous size. "
dim += 1
else:
# Diagonal repeated index
# TODO(zhoushunjie): Need to develop a paddle.diagonal api
raise NotImplementedError("Can't support diagonal.")
perm = []
for input_dim in idx_to_dims:
if input_dim > -1:
perm.append(input_dim)
# Transpose the tensor by perm
preprocessed_operand = paddle.transpose(preprocessed_operand,
perm=perm)
for dim, input_dim in enumerate(idx_to_dims):
if input_dim == -1:
preprocessed_operand = paddle.unsqueeze(
preprocessed_operand, dim)
preprocessed_operands.append(preprocessed_operand)
# 2. Execute the mul_sum
sum_dims = []
result = preprocessed_operands[0]
for i in range(num_total_idxes):
if idxes_last_operand[
i] == 0 and idxes_to_output_dims[i] >= num_output_dims:
result = result.sum(axis=idxes_to_output_dims[i], keepdim=True)
for i in range(1, len(preprocessed_operands)):
for j in range(num_total_idxes):
if idxes_last_operand[
j] == i and idxes_to_output_dims[j] >= num_output_dims:
sum_dims.append(idxes_to_output_dims[j])
result = _mul_sum(result, preprocessed_operands[i], sum_dims)
squeeze_dims = [
i for i in range(len(result.shape) - 1, num_output_dims - 1, -1)
]
if len(squeeze_dims) != 0:
result = paddle.squeeze(result, squeeze_dims)
return result
def predict(model,
model_path,
transforms,
image_list,
image_dir=None,
save_dir='output',
aug_pred=False,
scales=1.0,
flip_horizontal=True,
flip_vertical=False,
is_slide=False,
stride=None,
crop_size=None):
"""
predict and visualize the image_list.
Args:
model (nn.Layer): Used to predict for input image.
model_path (str): The path of pretrained model.
transforms (transform.Compose): Preprocess for input image.
image_list (list): A list of images to be predicted.
image_dir (str): The directory of the images to be predicted. Default: None.
save_dir (str): The directory to save the visualized results. Default: 'output'.
"""
para_state_dict = paddle.load(model_path)
model.set_dict(para_state_dict)
model.eval()
added_saved_dir = os.path.join(save_dir, 'added_prediction')
pred_saved_dir = os.path.join(save_dir, 'pseudo_color_prediction')
logger.info("Start to predict...")
progbar_pred = progbar.Progbar(target=len(image_list), verbose=1)
for i, im_path in enumerate(image_list):
im = cv2.imread(im_path)
ori_shape = im.shape[:2]
im, _ = transforms(im)
im = im[np.newaxis, ...]
im = paddle.to_tensor(im)
if aug_pred:
pred = infer.aug_inference(model,
im,
ori_shape=ori_shape,
transforms=transforms.transforms,
scales=scales,
flip_horizontal=flip_horizontal,
flip_vertical=flip_vertical,
is_slide=is_slide,
stride=stride,
crop_size=crop_size)
else:
pred = infer.inference(model,
im,
ori_shape=ori_shape,
transforms=transforms.transforms,
is_slide=is_slide,
stride=stride,
crop_size=crop_size)
pred = paddle.squeeze(pred)
pred = pred.numpy().astype('uint8')
# get the saved name
if image_dir is not None:
im_file = im_path.replace(image_dir, '')
else:
im_file = os.path.basename(im_path)
if im_file[0] == '/':
im_file = im_file[1:]
# save added image
added_image = utils.visualize.visualize(im_path, pred, weight=0.6)
added_image_path = os.path.join(added_saved_dir, im_file)
mkdir(added_image_path)
cv2.imwrite(added_image_path, added_image)
# save pseudo color prediction
pred_mask = utils.visualize.get_pseudo_color_map(pred)
pred_saved_path = os.path.join(pred_saved_dir,
im_file.rsplit(".")[0] + ".png")
mkdir(pred_saved_path)
pred_mask.save(pred_saved_path)
# pred_im = utils.visualize(im_path, pred, weight=0.0)
# pred_saved_path = os.path.join(pred_saved_dir, im_file)
# mkdir(pred_saved_path)
# cv2.imwrite(pred_saved_path, pred_im)
progbar_pred.update(i + 1)
def non_inplace_api_processing(self, var):
return paddle.squeeze(var)
def squeeze(x, axes=[-1]):
return Tensor(paddle.squeeze(x, axes))
def forward(self, inputs):
"""
forward
"""
x = paddle.squeeze(inputs, axis=self.axis)
return x
