def func_sum_op(self): x = np.ones([2, 2], np.float32) with fluid.dygraph.guard(): inputs = [] for _ in range(10): tmp = paddle.to_tensor(x) tmp.stop_gradient = False inputs.append(tmp) ret = paddle.add_n(inputs) loss = fluid.layers.reduce_sum(ret) loss.backward() with fluid.dygraph.guard(): inputs2 = [] for _ in range(10): tmp = paddle.to_tensor(x) tmp.stop_gradient = False inputs2.append(tmp) ret2 = paddle.add_n(inputs2) loss2 = fluid.layers.reduce_sum(ret2) fluid.set_flags({'FLAGS_sort_sum_gradient': True}) loss2.backward() self.assertTrue(np.allclose(ret.numpy(), x * 10)) self.assertTrue(np.allclose(inputs[0].gradient(), x)) self.assertTrue(np.allclose(ret2.numpy(), x * 10)) a = inputs2[0].gradient() self.assertTrue(np.allclose(inputs2[0].gradient(), x))
def forward(self, x): x = self.conv1(x) if self.radix > 1: splited = paddle.split(x, num_or_sections=self.radix, axis=1) gap = paddle.add_n(splited) else: gap = x gap = self.avg_pool2d(gap) gap = self.conv2(gap) atten = self.conv3(gap) atten = self.rsoftmax(atten) if self.radix > 1: attens = paddle.split(atten, num_or_sections=self.radix, axis=1) y = paddle.add_n([ paddle.multiply(split, att) for (att, split) in zip(attens, splited) ]) else: y = paddle.multiply(x, atten) return y
def test_dygraph_final_state_api(self): input_1 = np.random.randint(0, 100, [100, ]).astype('int32') input_2 = np.random.randint(0, 100, [200, ]).astype('int32') with fluid.dygraph.guard(): tensor_1 = fluid.dygraph.to_variable(input_1) tensor_2 = fluid.dygraph.to_variable(input_2) tensor_1.stop_gradient = False tensor_2.stop_gradient = False res_1, res_2 = paddle.tensor.meshgrid((tensor_1, tensor_2)) sum = paddle.add_n([res_1, res_2]) sum.backward() with _test_eager_guard(): tensor_eager_1 = fluid.dygraph.to_variable(input_1) tensor_eager_2 = fluid.dygraph.to_variable(input_2) tensor_eager_1.stop_gradient = False tensor_eager_2.stop_gradient = False res_eager_1, res_eager_2 = paddle.tensor.meshgrid( (tensor_eager_1, tensor_eager_2)) sum_eager = paddle.add_n([res_eager_1, res_eager_2]) sum_eager.backward() self.assertEqual(( tensor_1.grad.numpy() == tensor_eager_1.grad.numpy()).all(), True) self.assertEqual(( tensor_2.grad.numpy() == tensor_eager_2.grad.numpy()).all(), True)
def train_forward(self, dy_model, metrics_list, batch_data, config): np.random.seed(12345) x_spt, y_spt, x_qry, y_qry = self.create_feeds(batch_data, config) update_step = config.get("hyper_parameters.update_step", 5) task_num = x_spt.shape[0] query_size = x_qry.shape[ 1] # 75 = 15 * 5, x_qry.shape = [32,75,1,28,28] loss_list = [] loss_list.clear() correct_list = [] correct_list.clear() task_grad = [[] for _ in range(task_num)] for i in range(task_num): # 外循环 task_net = copy.deepcopy(dy_model) base_lr = config.get( "hyper_parameters.base_optimizer.learning_rate", 0.1) task_optimizer = paddle.optimizer.SGD( learning_rate=base_lr, parameters=task_net.parameters()) for j in range(update_step): #内循环 task_optimizer.clear_grad() # 梯度清零 y_hat = task_net.forward(x_spt[i]) # (setsz, ways) [5,5] loss_spt = F.cross_entropy(y_hat, y_spt[i]) loss_spt.backward() task_optimizer.step() y_hat = task_net.forward(x_qry[i]) loss_qry = F.cross_entropy(y_hat, y_qry[i]) loss_qry.backward() for k in task_net.parameters(): task_grad[i].append(k.grad) loss_list.append(loss_qry) pred_qry = F.softmax(y_hat, axis=1).argmax(axis=1) correct = paddle.equal(pred_qry, y_qry[i]).numpy().sum().item() correct_list.append(correct) loss_average = paddle.add_n(loss_list) / task_num acc = sum(correct_list) / (query_size * task_num) for num, k in enumerate(dy_model.parameters()): tmp_list = [task_grad[i][num] for i in range(task_num)] if tmp_list[0] is not None: k._set_grad_ivar(paddle.add_n(tmp_list) / task_num) acc = paddle.to_tensor(acc) print_dict = {'loss': loss_average, "acc": acc} _ = paddle.ones(shape=[5, 5], dtype="float32") return _, metrics_list, print_dict
def backward_D(self): losses_discriminator = self.nets['Dis'](self.input_data.copy(), self.generated) loss_values = [val.mean() for val in losses_discriminator.values()] loss = paddle.add_n(loss_values) loss.backward() self.losses.update(dict(zip(losses_discriminator.keys(), loss_values)))
def get_loss(self, ): loss = {} # RPN loss rpn_loss_inputs = self.anchor.generate_loss_inputs( self.inputs, self.rpn_head_out, self.anchor_out) loss_rpn = self.rpn_head.get_loss(rpn_loss_inputs) loss.update(loss_rpn) # BBox loss bbox_targets_list = self.proposal.get_targets() loss_bbox = self.bbox_head.get_loss(self.bbox_head_list, bbox_targets_list) loss.update(loss_bbox) if self.with_mask: # Mask loss mask_targets = self.mask.get_targets() loss_mask = self.mask_head.get_loss(self.mask_head_out, mask_targets) loss.update(loss_mask) total_loss = paddle.add_n(list(loss.values())) loss.update({'loss': total_loss}) return loss
def get_loss(self, ): losses = self._forward() losses.update({ 'loss': paddle.add_n([v for k, v in losses.items() if 'log' not in k]) }) return losses
def forward(self, input, mask=None): """ Args: input (obj: `paddle.Tensor`) of shape (batch, seq_len, input_size): Tensor containing the features of the input sequence. mask (obj: `paddle.Tensor`, optional, defaults to `None`) of shape (batch, seq_len) : Tensor is a bool tensor, whose each element identifies whether the input word id is pad token or not. """ forward_input, backward_input = paddle.chunk(input, chunks=2, axis=2) # elementwise-sum forward_x and backward_x # Shape: (batch_size, max_seq_len, hidden_size) h = paddle.add_n([forward_input, backward_input]) # Shape: (batch_size, hidden_size, 1) att_weight = self.att_weight.tile( repeat_times=(paddle.shape(h)[0], 1, 1)) # Shape: (batch_size, max_seq_len, 1) att_score = paddle.bmm(paddle.tanh(h), att_weight) if mask is not None: # mask, remove the effect of 'PAD' mask = paddle.cast(mask, dtype='float32') mask = mask.unsqueeze(axis=-1) inf_tensor = paddle.full( shape=mask.shape, dtype='float32', fill_value=-INF) att_score = paddle.multiply(att_score, mask) + paddle.multiply( inf_tensor, (1 - mask)) # Shape: (batch_size, max_seq_len, 1) att_weight = F.softmax(att_score, axis=1) # Shape: (batch_size, lstm_hidden_size) reps = paddle.bmm(h.transpose(perm=(0, 2, 1)), att_weight).squeeze(axis=-1) reps = paddle.tanh(reps) return reps, att_weight
def _forward(self): body_feats = self.backbone(self.inputs) if self.neck is not None: body_feats = self.neck(body_feats) self.s2anet_head(body_feats) if self.training: loss = self.s2anet_head.get_loss(self.inputs) total_loss = paddle.add_n(list(loss.values())) loss.update({'loss': total_loss}) return loss else: im_shape = self.inputs['im_shape'] scale_factor = self.inputs['scale_factor'] nms_pre = self.s2anet_bbox_post_process.nms_pre pred_scores, pred_bboxes = self.s2anet_head.get_prediction(nms_pre) # post_process pred_bboxes, bbox_num = self.s2anet_bbox_post_process( pred_scores, pred_bboxes) # rescale the prediction back to origin image pred_bboxes = self.s2anet_bbox_post_process.get_pred( pred_bboxes, bbox_num, im_shape, scale_factor) # output output = {'bbox': pred_bboxes, 'bbox_num': bbox_num} return output
def get_loss(self, ): loss = {} # get gt_ins_labels, gt_cate_labels, etc. gt_ins_labels, gt_cate_labels, gt_grid_orders = [], [], [] fg_num = self.inputs['fg_num'] for i in range(len(self.solov2_head.seg_num_grids)): ins_label = 'ins_label{}'.format(i) if ins_label in self.inputs: gt_ins_labels.append(self.inputs[ins_label]) cate_label = 'cate_label{}'.format(i) if cate_label in self.inputs: gt_cate_labels.append(self.inputs[cate_label]) grid_order = 'grid_order{}'.format(i) if grid_order in self.inputs: gt_grid_orders.append(self.inputs[grid_order]) loss_solov2 = self.solov2_head.get_loss(self.cate_pred_list, self.kernel_pred_list, self.seg_pred, gt_ins_labels, gt_cate_labels, gt_grid_orders, fg_num) loss.update(loss_solov2) total_loss = paddle.add_n(list(loss.values())) loss.update({'loss': total_loss}) return loss
def dyfunc_with_if_else_with_list_geneator(x): if 10 > 5: y = paddle.add_n( [paddle.full(shape=[2], fill_value=v) for v in range(5)]) else: y = x return y
def forward(self, inputs1, inputs2): """ forward """ inputs = [inputs1, inputs2] x = paddle.add_n(inputs) return x
def get_loss(self, ): rpn_loss, bbox_loss = self._forward() loss = {} loss.update(rpn_loss) loss.update(bbox_loss) total_loss = paddle.add_n(list(loss.values())) loss.update({'loss': total_loss}) return loss
def forward(self, teacher_model, student_model): teacher_distill_pairs = teacher_model.yolo_head.loss.distill_pairs student_distill_pairs = student_model.yolo_head.loss.distill_pairs distill_reg_loss, distill_cls_loss, distill_obj_loss = [], [], [] for s_pair, t_pair in zip(student_distill_pairs, teacher_distill_pairs): distill_reg_loss.append( self.obj_weighted_reg(s_pair[0], s_pair[1], s_pair[2], s_pair[ 3], t_pair[0], t_pair[1], t_pair[2], t_pair[3], t_pair[4])) distill_cls_loss.append( self.obj_weighted_cls(s_pair[5], t_pair[5], t_pair[4])) distill_obj_loss.append(self.obj_loss(s_pair[4], t_pair[4])) distill_reg_loss = paddle.add_n(distill_reg_loss) distill_cls_loss = paddle.add_n(distill_cls_loss) distill_obj_loss = paddle.add_n(distill_obj_loss) loss = (distill_reg_loss + distill_cls_loss + distill_obj_loss ) * self.weight return loss
def obj_weighted_reg(self, sx, sy, sw, sh, tx, ty, tw, th, tobj): loss_x = ops.sigmoid_cross_entropy_with_logits(sx, F.sigmoid(tx)) loss_y = ops.sigmoid_cross_entropy_with_logits(sy, F.sigmoid(ty)) loss_w = paddle.abs(sw - tw) loss_h = paddle.abs(sh - th) loss = paddle.add_n([loss_x, loss_y, loss_w, loss_h]) weighted_loss = paddle.mean(loss * F.sigmoid(tobj)) return weighted_loss
def get_loss(self, ): loss = {} head_outs, _ = self._forward() loss_gfl = self.head.get_loss(head_outs, self.inputs) loss.update(loss_gfl) total_loss = paddle.add_n(list(loss.values())) loss.update({'loss': total_loss}) return loss
def __call__(self, input, batch, mode="train"): loss_dict = {} for idx, loss_func in enumerate(self.loss_func): loss = loss_func(input, batch, mode=mode) weight = self.loss_weight[idx] loss = {key: loss[key] * weight for key in loss} loss_dict.update(loss) loss_dict["loss"] = paddle.add_n(list(loss_dict.values())) return loss_dict
def test_step(self, data_batch): """Define how the model is going to test, from input to output.""" # NOTE: (shipping) when testing, the net won't call head.loss, we deal with the test processing in /paddlevideo/metrics clips_list = paddle.split( data_batch[0], num_or_sections=3, axis=2) # [N, 3, T, H, W], [N, 3, T, H, W], [N, 3, T, H, W] cls_score = [self.forward_net(imgs) for imgs in clips_list] # [N, C], [N, C], [N, C] cls_score = paddle.add_n(cls_score) # [N, C] in [0,1] return cls_score
def get_prediction(self, head_out_list): """ head_out_list(List[Tensor]): scores, deltas, rois """ pred_list = [] scores_list = [F.softmax(head[0]) for head in head_out_list] scores = paddle.add_n(scores_list) / self.num_cascade_stages # Get deltas and rois from the last stage _, deltas, rois = head_out_list[-1] return scores, deltas, rois
def get_loss(self, ): loss = {} heatmap = self.inputs['ttf_heatmap'] box_target = self.inputs['ttf_box_target'] reg_weight = self.inputs['ttf_reg_weight'] head_loss = self.ttf_head.get_loss(self.hm, self.wh, heatmap, box_target, reg_weight) loss.update(head_loss) total_loss = paddle.add_n(list(loss.values())) loss.update({'loss': total_loss}) return loss
def test_api(self): with fluid.program_guard(fluid.Program(), fluid.Program()): input0 = fluid.layers.fill_constant( shape=[2, 3], dtype='int64', value=5) input1 = fluid.layers.fill_constant( shape=[2, 3], dtype='int64', value=3) expected_result = np.empty((2, 3)) expected_result.fill(8) sum_value = paddle.add_n([input0, input1]) exe = fluid.Executor(fluid.CPUPlace()) result = exe.run(fetch_list=[sum_value]) self.assertEqual((result == expected_result).all(), True) with fluid.dygraph.guard(): input0 = paddle.ones(shape=[2, 3], dtype='float32') expected_result = np.empty((2, 3)) expected_result.fill(2) sum_value = paddle.add_n([input0, input0]) self.assertEqual((sum_value.numpy() == expected_result).all(), True)
def _get_loss_aux(self, boxes, logits, gt_bbox, gt_class, bg_index, num_gts): loss_class = [] loss_bbox = [] loss_giou = [] for aux_boxes, aux_logits in zip(boxes, logits): match_indices = self.matcher(aux_boxes, aux_logits, gt_bbox, gt_class) loss_class.append( self._get_loss_class(aux_logits, gt_class, match_indices, bg_index, num_gts)['loss_class']) loss_ = self._get_loss_bbox(aux_boxes, gt_bbox, match_indices, num_gts) loss_bbox.append(loss_['loss_bbox']) loss_giou.append(loss_['loss_giou']) loss = { 'loss_class_aux': paddle.add_n(loss_class), 'loss_bbox_aux': paddle.add_n(loss_bbox), 'loss_giou_aux': paddle.add_n(loss_giou) } return loss
def forward(self, inputs, training=False): if isinstance(inputs, tuple): experts, routing_inputs = inputs else: experts, routing_inputs = inputs, None if routing_inputs is None: # static gating expert_weights, selector_outputs = self._compute_expert_weights() output = paddle.add_n(inputs=[ expert_weights[i] * experts[i] for i in range(len(experts)) ]) else: # per-example gating expert_weights, selector_outputs = self._compute_example_conditioned_expert_weights( routing_inputs) output = paddle.add_n(inputs=[ paddle.reshape(expert_weights[:, i], [-1, 1]) * experts[i] for i in range(len(experts)) ]) return output
def test_unbind_dygraph(self): with fluid.dygraph.guard(): np_x = np.random.random([2, 3]).astype("float32") x = paddle.to_tensor(np_x) x.stop_gradient = False [res_1, res_2] = paddle.unbind(x, 0) self.assertTrue(np.array_equal(res_1, np_x[0, 0:100])) self.assertTrue(np.array_equal(res_2, np_x[1, 0:100])) out = paddle.add_n([res_1, res_2]) np_grad = np.ones(x.shape, np.float32) out.backward() self.assertTrue(np.array_equal(x.grad.numpy(), np_grad))
def __call__(self, input, batch): loss_dict = {} # just for accelerate classification traing speed if len(self.loss_func) == 1: loss = self.loss_func[0](input, batch) loss_dict.update(loss) loss_dict["loss"] = list(loss.values())[0] else: for idx, loss_func in enumerate(self.loss_func): loss = loss_func(input, batch) weight = self.loss_weight[idx] loss = {key: loss[key] * weight for key in loss} loss_dict.update(loss) loss_dict["loss"] = paddle.add_n(list(loss_dict.values())) return loss_dict
def train(model, data_loader, optimizer, lr_scheduler, epoch, LOG): stages = 4 losses = [AverageMeter() for _ in range(stages)] length_loader = len(data_loader) model.train() for batch_id, data in enumerate(data_loader()): left_img, right_img, gt = data mask = paddle.to_tensor(gt.numpy() > 0) gt_mask = paddle.masked_select(gt, mask) outputs = model(left_img, right_img) outputs = [paddle.squeeze(output) for output in outputs] tem_stage_loss = [] for index in range(stages): temp_loss = args.loss_weights[index] * F.smooth_l1_loss( paddle.masked_select(outputs[index], mask), gt_mask, reduction='mean') tem_stage_loss.append(temp_loss) losses[index].update( float(temp_loss.numpy() / args.loss_weights[index])) sum_loss = paddle.add_n(tem_stage_loss) sum_loss.backward() optimizer.step() optimizer.clear_grad() if batch_id % 5 == 0: info_str = [ 'Stage {} = {:.2f}({:.2f})'.format(x, losses[x].val, losses[x].avg) for x in range(stages) ] info_str = '\t'.join(info_str) info_str = 'Train Epoch{} [{}/{}] lr:{:.5f}\t{}'.format( epoch, batch_id, length_loader, optimizer.get_lr(), info_str) LOG.info(info_str) lr_scheduler.step() info_str = '\t'.join( ['Stage {} = {:.2f}'.format(x, losses[x].avg) for x in range(stages)]) LOG.info('Average train loss: ' + info_str)
def get_loss(self, ): loss = {} # RPN loss rpn_loss_inputs = self.anchor.generate_loss_inputs( self.inputs, self.rpn_head_out, self.anchor_out) loss_rpn = self.rpn_head.get_loss(rpn_loss_inputs) loss.update(loss_rpn) # BBox loss bbox_targets = self.proposal.get_targets() loss_bbox = self.bbox_head.get_loss([self.bbox_head_out], bbox_targets) loss.update(loss_bbox) total_loss = paddle.add_n(list(loss.values())) loss.update({'loss': total_loss}) return loss
def all_gather_tokens(data): """Gathers num of tokens from all nodes. `data` should be a tensor of num of tokens. """ if dist.get_world_size() < 2: return data if not hasattr(all_gather_tokens, '_in_buffer') or all_gather_tokens._in_buffer is None: all_gather_tokens._in_buffer = data all_gather_tokens._out_buffers = [] in_buffer = all_gather_tokens._in_buffer out_buffers = all_gather_tokens._out_buffers dist.all_gather(out_buffers, in_buffer) return paddle.add_n(out_buffers)
def forward(self, input, batch, **kargs): loss_dict = {} loss_all = 0. for idx, loss_func in enumerate(self.loss_func): loss = loss_func(input, batch, **kargs) if isinstance(loss, paddle.Tensor): loss = {"loss_{}_{}".format(str(loss), idx): loss} weight = self.loss_weight[idx] loss = {key: loss[key] * weight for key in loss} if "loss" in loss: loss_all += loss["loss"] else: loss_all += paddle.add_n(list(loss.values())) loss_dict.update(loss) loss_dict["loss"] = loss_all return loss_dict
def forward(self, input, batch, **kargs): loss_dict = {} for idx, loss_func in enumerate(self.loss_func): loss = loss_func(input, batch, **kargs) weight = self.loss_weight[idx] if isinstance(loss, paddle.Tensor): loss = {"loss_{}_{}".format(str(loss), idx): loss * weight} else: loss = { "{}_{}".format(key, idx): loss[key] * weight for key in loss } loss_dict.update(loss) if loss_dict == {}: loss_dict["loss"] = paddle.to_tensor(0.) else: loss_dict["loss"] = paddle.add_n(list(loss_dict.values())) return loss_dict