def compute_scale(self, F, data: Tensor, observed_indicator: Tensor) -> Tensor: """ Parameters ---------- F A module that can either refer to the Symbol API or the NDArray API in MXNet. data tensor containing the data to be scaled. observed_indicator observed_indicator: binary tensor with the same shape as ``data``, that has 1 in correspondence of observed data points, and 0 in correspondence of missing data points. Returns ------- Tensor shape (N, T, C) or (N, C, T) scaled along the specified axis. """ axis_zero = nd.prod( data == data.zeros_like(), self.axis, keepdims=True ) # Along the specified axis, which array are always at zero axis_zero = nd.broadcast_to( axis_zero, shape=data.shape) # Broadcast it to the shape of data min_val = nd.where( 1 - observed_indicator, nd.broadcast_to(data.max(keepdims=True), shape=data.shape), data, ).min( axis=self.axis, keepdims=True ) # return the min value along specified axis while ignoring value according to observed_indicator max_val = nd.where( 1 - observed_indicator, nd.broadcast_to(data.min(keepdims=True), shape=data.shape), data, ).max( axis=self.axis, keepdims=True ) # return the max value along specified axis while ignoring value according to observed_indicator scaled_data = (data - min_val) / (max_val - min_val) # Rescale scaled_data = nd.where( axis_zero, scaled_data.zeros_like(), scaled_data ) # Clip Nan values to zero if the data was equal to zero along specified axis scaled_data = nd.where( scaled_data != scaled_data, scaled_data.ones_like(), scaled_data ) # Clip the Nan values to one. scaled_date!=scaled_data tells us where the Nan values are in scaled_data return nd.where( 1 - observed_indicator, scaled_data.zeros_like(), scaled_data ) # Replace data with zero where observed_indicator tells us to.
def valid(self): loss_sce = gluon.loss.SoftmaxCrossEntropyLoss(from_logits=False) loss_l1 = gluon.loss.L1Loss() obj_loss = LOSS_EVAL('obj_loss') cls_loss = LOSS_EVAL('cls_loss') xywh_loss = LOSS_EVAL('xywh_loss') positive_weight = 5.0 negative_weight = 0.1 class_weight = 1.0 xywh_weight = 5.0 self.validIter.reset() for batchind, batch in enumerate(self.validIter): gtX = batch.data[0].as_in_context(self.ctx) gtY = batch.label[0].as_in_context(self.ctx) prdY = self.net(gtX) prdCls, prdObj, prdXYWH = self.format_net_output(prdY) boxMask, boxCls, boxObj, boxXYWH = self.format_groundtruth(gtY, prdXYWH) lossCls = loss_sce(prdCls, boxCls, boxMask * class_weight) boxWeight = nd.where(boxMask > 0, boxMask * positive_weight, boxMask * negative_weight) lossObj = loss_l1(prdObj, boxObj, boxWeight) lossXYWH = loss_l1(prdXYWH, boxXYWH, boxMask * xywh_weight) obj_loss.update(lossObj) cls_loss.update(lossCls) xywh_loss.update(lossXYWH) print('validation: (%s,%f) (%s,%f) (%s,%f)'%(cls_loss.get()[0], cls_loss.get()[1],\ obj_loss.get()[0], obj_loss.get()[1],\ xywh_loss.get()[0], xywh_loss.get()[1])) return
def update(self, index, weight, grad, state): assert (isinstance(weight, NDArray)) assert (isinstance(grad, NDArray)) self._update_count(index) lr = self._get_lr(index) wd = self._get_wd(index) t = self._index_update_count[index] with bulk(self._bulk): # preprocess grad grad *= self.rescale_grad if self.clip_gradient is not None: grad = clip(grad, -self.clip_gradient, self.clip_gradient) mean, var = state mean *= self.beta1 mean += (1. - self.beta1) * grad var *= self.beta2 var += (1. - self.beta2) * square(grad) r1 = weight.norm() if not self.bias_correction: r1 = minimum(maximum(r1, self.lower_bound), self.upper_bound) sqrt_var = sqrt(var) sqrt_var += self.epsilon g = mean / sqrt_var g += wd * weight else: # apply bias correction mean_hat = mean / (1. - power(self.beta1, t)) var_hat = var / (1. - power(self.beta2, t)) if self._eps_after_sqrt: sqrt(var_hat, out=var_hat) var_hat += self.epsilon else: var_hat += self.epsilon sqrt(var_hat, out=var_hat) mean_hat /= var_hat mean_hat += wd * weight g = mean_hat r2 = g.norm() # calculate lamb_trust_ratio ratio = r1 / r2 # becomes NaN if ratio == NaN or 0, otherwise 0 nan_or_zero = 1 - ratio / ratio r = where(nan_or_zero, ones_like(ratio), ratio) lr *= r # update weight g *= lr weight[:] -= g
def test_program(): import matplotlib.pyplot as plt import matplotlib.patches as patches ssd = ssd_mx.ssd_model() for model in ssd: model.initialize() data = get_iterators(data_shape=304, batch_size=4) batch_data = data.next() # 将-1占位符改为背景标签0,对应坐标框记录为[0,0,0,0] label = nd.where(batch_data.label[0] < 0, nd.zeros_like(batch_data.label[0]), batch_data.label[0]) anchors, class_preds, box_preds = ssd_mx.ssd_forward(batch_data.data[0], ssd) target_labels, target_localizations, target_scores = \ ssd_mx.bboxes_encode(label[:, :, 0], label[:, :, 1:], anchors) # print(label[:, :, 0].shape, label[:, :, 1:].shape) # print('target_labels', [f.shape for f in target_labels]) # print('target_localizations', [f.shape for f in target_localizations]) # print('target_scores', [f.shape for f in target_scores]) print('[*] class_preds', [f.shape for f in class_preds]) print('[*] box_preds', [f.shape for f in box_preds]) # Add loss function. logits, gclasses, no_classes, fpmask, fnmask = \ ssd_mx.data_reshape(class_preds, box_preds, target_labels, target_localizations, target_scores, match_threshold=.5, negative_ratio=3) p_cls_loss = util_mx.FocalLoss() n_cls_loss = util_mx.FocalLoss() print(nd.sum(p_cls_loss(logits, gclasses) * fpmask), nd.sum(n_cls_loss(logits, no_classes) * fnmask)) reg_loss = util_mx.SmoothL1Loss() print(nd.sum(reg_loss(box_preds, target_localizations, fpmask.expand_dims(-1)))) # .asnumpy比使用np.array速度快得多 plt.imshow(batch_data.data[0][2].asnumpy().transpose(1, 2, 0) / 255.) currentAxis = plt.gca() for i in range(6): box = batch_data.label[0][2][i][1:].asnumpy() * 300 if any(box < 0): continue print(int(batch_data.label[0][2][i][0].asscalar())) rect = patches.Rectangle((box[1], box[0]), box[3] - box[1], box[2] - box[0], linewidth=1, edgecolor='g', facecolor='none') currentAxis.add_patch(rect) plt.show()
def forward(self, is_train, req, in_data, out_data, aux): arm_cls_preds = in_data[0] odm_cls_target = in_data[1] odm_loc_target_mask = in_data[2] arm_cls_preds = nd.softmax(data=arm_cls_preds) arm_cls_preds_classes = nd.split(data=arm_cls_preds,axis=1,num_outputs=2) # arm_cls_preds_bg shape : (batch , h*w*num_anchors[:layers]) 负类【0】 arm_cls_preds_bg = nd.reshape(data=arm_cls_preds_classes[0],shape=(0,-1)) prob_temp = nd.ones_like(arm_cls_preds_bg)*0.99 cond1 = arm_cls_preds_bg >= prob_temp # > 0.99 idx is 1 # print('negative cond1 ------- :',heapq.nlargest(2,arm_cls_preds_bg[0])) temp1 = nd.ones_like(odm_cls_target)*(-1) ### TODO: 0 还是-1表示背景?? # 如果ARM分类出的负类的置信度大于0.99,将其在ODM的anchor标号中去掉(-1替代),负类转换为背景 odm_cls_target_mask = nd.where(condition=cond1,x=temp1,y=odm_cls_target) # apply filtering to odm_loc_target_mask # odm_loc_target_mask_shape: (batch, num_anchors, 4) arm_cls_preds_bg = nd.reshape(data=arm_cls_preds_bg,shape=(0,-1,1))#(batch , h*w*num_anchors[:layers],1) # (batch , h*w*num_anchors[:layers] , 4 ) odm_loc_target_mask = nd.reshape(data=odm_loc_target_mask,shape=(0,-1,4)) odm_loc_target_mask = odm_loc_target_mask[:,:,0] #(batch , h*w*num_anchors[:layers]) #(batch , h*w*num_anchors[:layers], 1) ## 取整个batch中 所有行的 第一列,相当于对原来的4个相同label[0 0 0 0 ],[1 1 1 1]变成[0],[1] odm_loc_target_mask = nd.reshape(data=odm_loc_target_mask,shape=(0,-1,1)) loc_temp = nd.ones_like(odm_loc_target_mask)*0.99 cond2 = arm_cls_preds_bg >= loc_temp temp2 = nd.zeros_like(odm_loc_target_mask) # 取0 # 如果ARM分类出的负类的置信度大于0.99,将其在ODM的掩码置0 ## 实际上不管IOU计算的大小,用AMR的分类结果,如果是大于0.99的负类,不管通过IOU判断的正负类结果如何,都设置为背景 odm_loc_target_bg_mask = nd.where(cond2,temp2,odm_loc_target_mask) odm_loc_target_bg_mask = nd.concat(*[odm_loc_target_bg_mask]*4,dim=2) # 还原维度 odm_loc_target_bg_mask = nd.reshape(odm_loc_target_bg_mask,shape=(0,-1)) for ind, val in enumerate([odm_cls_target_mask, odm_loc_target_bg_mask]): self.assign(out_data[ind], req[ind], val)
def hybrid_forward(self, F, fts, ys, ftt, yt): """ Semantic Alignment Loss :param F: Function :param yt: label for the target domain [N] :param ftt: features for the target domain [N, K] :param ys: label for the source domain [M] :param fts: features for the source domain [M, K] :return: """ if self._fn: # Normalize ft fts = F.L2Normalization(fts, mode='instance') ftt = F.L2Normalization(ftt, mode='instance') fts_rpt = F.broadcast_to(fts.expand_dims(axis=0), shape=(self._bs_tgt, self._bs_src, self._embed_size)) ftt_rpt = F.broadcast_to(ftt.expand_dims(axis=1), shape=(self._bs_tgt, self._bs_src, self._embed_size)) dists = F.sum(F.square(ftt_rpt - fts_rpt), axis=2) yt_rpt = F.broadcast_to(yt.expand_dims(axis=1), shape=(self._bs_tgt, self._bs_src)).astype('int32') ys_rpt = F.broadcast_to(ys.expand_dims(axis=0), shape=(self._bs_tgt, self._bs_src)).astype('int32') y_same = F.equal(yt_rpt, ys_rpt).astype('float32') y_diff = F.not_equal(yt_rpt, ys_rpt).astype('float32') intra_cls_dists = dists * y_same inter_cls_dists = dists * y_diff max_dists = F.max(dists, axis=1, keepdims=True) max_dists = F.broadcast_to(max_dists, shape=(self._bs_tgt, self._bs_src)) revised_inter_cls_dists = F.where(y_same, max_dists, inter_cls_dists) max_intra_cls_dist = F.max(intra_cls_dists, axis=1) min_inter_cls_dist = F.min(revised_inter_cls_dists, axis=1) loss = F.relu(max_intra_cls_dist - min_inter_cls_dist + self._margin) return loss
def backward(self, req, out_grad, in_data, out_data, in_grad, aux): # 1.update class center for class_id in xrange(int(self.num_class)): center_diff = nd.where(nd.array( (self.label == class_id).astype(np.int8)), x=self.diff, y=nd.zeros_like(self.diff)) # print nd.where(nd.array(self.label==class_id),diff,nd.zeros(diff)) # print "class id", class_id, self.center[class_id] += self.alpha * nd.sum(center_diff, axis=0) # print self.center # 2. assign grad to former layer self.assign( in_grad[0], req[0], self.diff.reshape(self.diff_reshape).transpose( (0, 3, 1, 2)) * self.grad_scale)
def train(self): loss_sce = gluon.loss.SoftmaxCrossEntropyLoss(from_logits=False) loss_l1 = gluon.loss.L1Loss() obj_loss = LOSS_EVAL('obj_loss') cls_loss = LOSS_EVAL('cls_loss') xywh_loss = LOSS_EVAL('xywh_loss') positive_weight = 5.0 negative_weight = 0.1 class_weight = 1.0 xywh_weight = 5.0 trainer = gluon.Trainer(self.net.collect_params(),"sgd",{"learning_rate":1,"wd":5e-4}) for epoch in range(self.maxEpochNum): self.trainIter.reset() tic = time.time() for batchind, batch in enumerate(self.trainIter): gtY = batch.label[0].as_in_context(self.ctx) gtX = batch.data[0].as_in_context(self.ctx) with autograd.record(): predY = self.net(gtX) predCls, predObj, predXYWH = self.format_net_output(predY) with autograd.pause(): boxMask, boxCls, boxObj, boxXYWH = self.format_groundtruth(gtY,predXYWH) loss0 = loss_sce(predCls, boxCls, boxMask * class_weight) boxWeight = nd.where(boxMask > 0, boxMask * positive_weight, boxMask * negative_weight) loss1 = loss_l1(predObj, boxObj, boxWeight) loss2 = loss_l1(predXYWH, boxXYWH, boxMask * xywh_weight) loss = loss0 + loss1 + loss2 loss.backward() trainer.step(self.batchSize) cls_loss.update(loss0) obj_loss.update(loss1) xywh_loss.update(loss2) print('epoch %d(%.2fsec): (%s,%f) (%s,%f) (%s,%f)'%(epoch,time.time() - tic,cls_loss.get()[0], cls_loss.get()[1],\ obj_loss.get()[0], obj_loss.get()[1],\ xywh_loss.get()[0], xywh_loss.get()[1])) if (1 + epoch) % self.validStep == 0: self.valid() self.net.save_params(os.path.join(self.outRoot,'yolo%.8d.params'%(epoch+1))) return
def hybrid_forward(self, F, pred, label, sample_weight=None): if not self._from_logits: pred = F.log_softmax(pred, axis=self._axis) if self._sparse_label: if self._size_average: valid_label_map = (label != self._ignore_label).astype('float32') loss = -(F.pick(pred, label, axis=self._axis, keepdims=True) * valid_label_map) else: loss = -F.pick(pred, label, axis=self._axis, keepdims=True) loss = F.where( label.expand_dims(axis=self._axis) == self._ignore_label, F.zeros_like(loss), loss) else: label = _reshape_like(F, label, pred) loss = -F.sum(pred * label, axis=self._axis, keepdims=True) loss = _apply_weighting(F, loss, self._weight, sample_weight) if self._size_average: return F.mean(loss, axis=self._batch_axis, exclude=True) * \ valid_label_map.size / F.sum(valid_label_map) else: return F.mean(loss, axis=self._batch_axis, exclude=True)
def margin_loss(self, pick_fc): args = self.args import math m = args.margin_m s = args.margin_s assert s > 0.0 #assert m >= 0.1 assert m < (math.pi / 2) # cos_t * s cos_t = pick_fc / s cos_m = math.cos(m) sin_m = math.sin(m) mm = math.sin(math.pi - m) * m #sin(pi-m)*m=sin(m)*m # threadhold = 0.0 threshold = math.cos(math.pi - m) # threshold < -cos(m) if args.easy_margin: cond = nd.Activation(data=cos_t, act_type='relu') else: cond_v = cos_t - threshold cond = nd.Activation(data=cond_v, act_type='relu') body = cos_t * cos_t body = 1.0 - body sin_t = nd.sqrt(body) #mx.sym.sqrt(body) new_zy = cos_t * cos_m # cos(t+m) = c*c - s*s b = sin_t * sin_m new_zy = new_zy - b new_zy = new_zy * s if args.easy_margin: zy_keep = pick_fc else: zy_keep = pick_fc - s * mm # zy-s*sin(m)*m = s*cos(t) - s*m*sin(m) new_zy = nd.where( cond, new_zy, zy_keep ) # cond < 0, zy_keep= s*cos(theta) or s*cos(theta)-s*m*sin(m) return new_zy
def do_transform_ip(self, buffer: Gst.Buffer) -> Gst.FlowReturn: try: # convert Gst.Buffer to np.ndarray caps = self.sinkpad.get_current_caps() #it's a reference on GstBuffer data #if you modify it output will be modified as well image = gst_buffer_with_caps_for_ndarray(buffer, caps) #c, h, w = image.shape #convert to NDArray t = nd.array(image, dtype=np.float32) #clip low level signals to 0 if self.minimal > 0: if self.zeros_vec is None: #since we don't expect change of frame parametors, # must not make it every time self.zeros_vec = nd.zeros_like(t) t = nd.where(t < self.minimal, self.zeros_vec, t) t = self.apply_conv(t) #clear output image.fill(0) #detach from mxnet context and copy to numpy.nddarray[w,h] conved = t[0, 0, :, :].asnumpy() #simple contrast if self.contrast == True: conved = simpleContrast(conved, 0.0) #copy to top left coner ch, cw = conved.shape image[0, :ch, :cw] = conved except Exception as e: logging.info(conved.shape) logging.error(e) return Gst.FlowReturn.OK
trainer = mx.gluon.Trainer(ssd.collect_params(), 'sgd', {'learning_rate': 0.01, 'wd': 5e-4}) data = get_iterators(data_shape=304, batch_size=batch_size) for epoch in range(30): # reset data iterators and metrics data.reset() cls_metric.reset() # box_metric.reset() tic = time.time() for i, batch in enumerate(data): start_time = time.time() x = batch.data[0].as_in_context(ctx) y = batch.label[0].as_in_context(ctx) # 将-1占位符改为背景标签0,对应坐标框记录为[0,0,0,0] y = nd.where(y < 0, nd.zeros_like(y), y) with mx.autograd.record(): anchors, class_preds, box_preds = ssd(x, False) target_labels, target_localizations, target_scores = \ ssd_mx.bboxes_encode(y[:, :, 0], y[:, :, 1:], anchors) logits, gclasses, no_classes, fpmask, fnmask, localisations, glocalisations = \ ssd_mx.data_reshape(class_preds, box_preds, target_labels, target_localizations, target_scores, match_threshold=.5, negative_ratio=3) p_cls = nd.sum(p_cls_loss(logits, gclasses) * fpmask) n_cls = nd.sum(n_cls_loss(logits, no_classes) * fnmask)
def replace_inf_with_zero(x): return nd.where(nd.abs(x) == np.inf, nd.zeros_like(x), x)
#pdb.set_trace() if a > 0.5: c = boxXYWH[0,y,x,0,:].asnumpy().tolist() c = ['%.2f'%cc for cc in c] c = '-'.join(c) elif b > 0.5: c = boxXYWH[0,y,x,1,:].asnumpy().tolist() c = ['%.2f'%cc for cc in c] c = '-'.join(c) lines.append('[%.3f,%.3f,%s],'%(a,b,c)) lines.append('') with open('dbg.txt','wb') as f: f.write('\r\n'.join(lines)) #pdb.set_trace() loss0 = loss_sce(predCls, boxCls, boxMask * class_weight) boxWeight = nd.where( boxMask > 0, boxMask * positive_weight, boxMask * negative_weight ) loss1 = loss_l1(predObj, boxObj, boxWeight) #pdb.set_trace() loss2 = loss_l1(predXYWH, boxXYWH, xywh_weight * boxMask ) loss = loss0 + loss1 + loss2 loss.backward() trainer.step(batchSize) cls_loss.update(loss0) obj_loss.update(loss1) xywh_loss.update(loss2) print '%d,(%s,%f),(%s,%f),(%s,%f),%f'%(epoch,cls_loss.get()[0], cls_loss.get()[1],\ obj_loss.get()[0], obj_loss.get()[1], xywh_loss.get()[0], xywh_loss.get()[1], time.time() - tic) net.save_params('%d.params'%epoch)
c = '-' #pdb.set_trace() if a > 0.5: c = boxXYWH[0, y, x, 0, :].asnumpy().tolist() c = ['%.2f' % cc for cc in c] c = '-'.join(c) elif b > 0.5: c = boxXYWH[0, y, x, 1, :].asnumpy().tolist() c = ['%.2f' % cc for cc in c] c = '-'.join(c) lines.append('[%.3f,%.3f,%s],' % (a, b, c)) lines.append('') with open('dbg.txt', 'wb') as f: f.write('\r\n'.join(lines)) #pdb.set_trace() loss0 = loss_sce(predCls, boxCls, boxMask * class_weight) boxWeight = nd.where(boxMask > 0, boxMask * positive_weight, boxMask * negative_weight) loss1 = loss_l1(predObj, boxObj, boxWeight) #pdb.set_trace() loss2 = loss_l1(predXYWH, boxXYWH, xywh_weight * boxMask) loss = loss0 + loss1 + loss2 loss.backward() trainer.step(batchSize) cls_loss.update(loss0) obj_loss.update(loss1) xywh_loss.update(loss2) print '%d,(%s,%f),(%s,%f),(%s,%f),%f'%(epoch,cls_loss.get()[0], cls_loss.get()[1],\ obj_loss.get()[0], obj_loss.get()[1], xywh_loss.get()[0], xywh_loss.get()[1], time.time() - tic) net.save_params('%d.params' % epoch)