def tesselate(x, leaf, p_tau): if (len(x) < 2): return add_split(x, leaf, p_tau) split = leaf.parent['decision'] node = leaf.parent['node'] side = nd.sign(split.split(x)) order = nd.argsort(side, axis=None) x = x[order, :] side = side[order, :] orderside = nd.argsort(side, axis=0) * side cutpt = nd.argsort(orderside, axis=None, dtype='int32')[0].asscalar() + 1 x_l = x[0:cutpt] x_r = x[cutpt:None] leaf.parent['side'] = 0 new_leaf = Leaf(layer=tree.layer_initializer(), node=leaf.parent['node'], decision=leaf.parent['decision'], side=1) tree.leaves.add(new_leaf) add_node(x_l, leaf) add_node(x_r, new_leaf) node.child['left'] = leaf.parent['node'] node.child['right'] = new_leaf.parent['node'] tesselate(x_l, leaf, split.tau.data()) tesselate(x_r, new_leaf, split.tau.data())
def sample(match, cls_pred, iou, ratio=3, min_sample=0, threshold=0.5, do=True): if do is False: ones = nd.ones_like(match) sample = nd.where(match > -0.5, ones, ones*-1) return sample sample = nd.zeros_like(match) num_pos = nd.sum(match > -0.5, axis=-1) requre_neg = ratio * num_pos neg_mask = nd.where(match < -0.5, nd.max(iou, axis=-1) < threshold, sample) max_neg = neg_mask.sum(axis=-1) num_neg = nd.minimum(max_neg, nd.maximum(requre_neg, min_sample)).astype('int') neg_prob = cls_pred[:,:,0] max_value = nd.max(cls_pred, axis=-1, keepdims=True) score = max_value[:,:,0] - neg_prob + nd.log( nd.sum( nd.exp(cls_pred-max_value), axis=-1)) score = nd.where(neg_mask, score, nd.zeros_like(score)) argmax = nd.argsort(score, axis=-1, is_ascend=False) sample = nd.where(match > -0.5, nd.ones_like(sample), sample) for i, num in enumerate(num_neg): sample[i, argmax[i,:num.asscalar()]] = -1 return sample
def evaluate_emb(emb, labels): """Evaluate embeddings based on Recall@k.""" d_mat = get_distance_matrix(emb) #d_mat = d_mat.asnumpy() #labels = labels.asnumpy() #directory operate on mxnet.ndarray if convert to numpy,would cause memeory error names = [] accs = [] for i in range(emb.shape[0]): d_mat[i,i]=1e10 index_mat = nd.argsort(d_mat) nd.waitall() if opt.use_viz: viz.log("nd all dist mat") for k in [1, 2, 4, 8, 16]: names.append('Recall@%d' % k) correct, cnt = 0.0, 0.0 index_mat_part = index_mat[:,:k] for i in range(emb.shape[0]): if any(labels[i] == labels[nn] for nn in index_mat_part[i]): correct +=1 cnt +=1 # for i in range(emb.shape[0]): # d_mat[i, i] = 1e10 # nns = argpartition(d_mat[i], k)[:k] # if any(labels[i] == labels[nn] for nn in nns): # correct += 1 # cnt += 1 accs.append(correct/cnt) return names, accs
def prune_weights(sign_to_noise_vec, prediction_vector, percentages): pruning_indices = nd.argsort(sign_to_noise_vec, axis=0) for percentage in percentages: prediction_vector = mus_copy_vec.copy() pruning_indices_percent = pruning_indices[0:int(len(pruning_indices)*percentage)] for pr_ind in pruning_indices_percent: prediction_vector[int(pr_ind.asscalar())] = 0 pruned_weights = restore_weight_structure(prediction_vector) test_accuracy = evaluate_accuracy(test_data, net, pruned_weights) print("%s --> %s" % (percentage, test_accuracy))
def _shard(split, x, l_fn, r_fn): splitsortorder = nd.argsort(split, axis=None) reorderedx = x[splitsortorder, :] reorderedsplit = split[splitsortorder] if (reorderedsplit[0] > 0): r_fn(reorderedx) elif (reorderedsplit[-1] < 0): l_fn(reorderedx) else: splitpt = nd.argsort(reorderedsplit, axis=0) * nd.sign(reorderedsplit) splitpt = nd.argsort(splitpt, axis=None)[0] + 1 lx = nd.slice_axis(reorderedx, 0, 0, int(splitpt.asscalar())) rx = nd.slice_axis(reorderedx, 0, int(splitpt.asscalar()), None) l_fn(lx) r_fn(rx)
def recurse(x, node, p_tau): n = 1 mean = node.center.data() var = (0.5 * node.radius.data())**2 N = x.shape[0] x_mean = nd.mean(x, axis=0) x_var = (N**-1) * nd.sum((x - x_mean)**2, axis=0) z_mean = (n * mean + N * x_mean) / (n + N) z_var = ((n * (mean + var) + N * (x_mean + x_var)) / (n + N)) - z_mean z_radius = 2 * (nd.max(z_var)**0.5) node.center.set_data(z_mean) node.radius.set_data(z_radius) if node.child['decision'] is None: leaf = next(l for l in tree.leaves if l.parent['node'] == node) tesselate(x, leaf, p_tau) return E = nd.random.exponential(z_radius**-1) node.child['decision'].tau.set_data(p_tau + E) split = node.child['decision'] side = nd.sign(split.split(x)) order = nd.argsort(side, axis=None) x = x[order, :] side = side[order, :] if side[0] > 0: recurse(x, node.child['right'], split.tau.data()) elif side[-1] < 0: recurse(x, node.child['left'], split.tau.data()) else: orderside = nd.argsort(side, axis=0) * side cutpt = nd.argsort(orderside, axis=None, dtype='int32')[0].asscalar() + 1 x_l = x[0:cutpt] x_r = x[cutpt:None] recurse(x_l, node.child['left'], split.tau.data()) recurse(x_r, node.child['right'], split.tau.data())
def full_krum(epoch, v, net, f, lr, active, max_flip=1.0): if (f == 0): return v e = 0.00001 / len(v[0]) avg_grads = nd.sum(nd.concat(*v, dim=1), axis=-1, keepdims=True) direction = nd.sign(avg_grads) topk = nd.argsort(nd.abs(avg_grads).reshape(-1)) n_flips = int(max_flip * len(v[0])) current_f = len(np.where(np.where(active < f)[0] < f)[0]) l_max = lambda_max(epoch, v, net, current_f, lr) l = find_lambda(l_max, v, direction, len(v), current_f, lr, topk, max_flip) print(l) if (l > 0 and active[0] < f): v[0][topk[-n_flips:]] = -(direction[topk[-n_flips:]] * l) / lr for i in range(1, f): if (active[i] < f): v[i] = mx.nd.random.uniform(v[0] - e, v[0] + e) return v
def getUniqueMatch(iou, min_threshold=1e-12): N, M = iou.shape iouf = iou.reshape((-1,)) argmax = nd.argsort(iouf, is_ascend=False) argrow = nd.floor(nd.divide(argmax, M)) argcol = nd.modulo(argmax, M) uniquel = set() uniquer = set() match = nd.ones((N,)) * -1 i = 0 while True: if argcol[i].asscalar() not in uniquel and argrow[i].asscalar() not in uniquer: uniquel.add(argcol[i].asscalar()) uniquer.add(argrow[i].asscalar()) if iou[argrow[i], argcol[i]] > min_threshold: match[argrow[i]] = argcol[i] if len(uniquel) == M or len(uniquer) == N: break i += 1 return match.reshape((1,-1))
def validate(net, val_data, val_items, val_shapes, ctx, size, classes): """Test on validation dataset.""" clipper = gcv.nn.bbox.BBoxClipToImage() net.hybridize(static_alloc=True) print("---Detect Total {:d} Image Start.---".format(len(val_items))) result_dict = {} for ib, (batch, item) in enumerate(zip(val_data, val_items)): batch = split_and_load(batch, ctx_list=ctx) for x, y, im_scale in zip(*batch): ids, scores, bboxes = net(x) bboxes = clipper(bboxes, x) im_scale = im_scale.reshape((-1)).asscalar() bboxes *= im_scale inds = nd.argsort(nd.squeeze(ids, axis=(0, 2)), is_ascend=False) ids = nd.squeeze(ids, axis=(0, 2)).asnumpy().astype(np.int8).tolist() valid_ids = [id for id in ids if id is not -1] valid_len = len(valid_ids) if valid_len > 0: # valid_len must > 0 inds = nd.slice_axis(inds, begin=0, end=valid_len, axis=0) scores = nd.take(scores, inds, axis=1) bboxes = nd.take(bboxes, inds, axis=1) scores = scores.asnumpy() bboxes = bboxes.asnumpy() for i, id in enumerate(valid_ids): score = scores[:, i, 0][0] xmin, ymin, xmax, ymax = bboxes[:, i, 0][ 0], bboxes[:, i, 1][0], bboxes[:, i, 2][0], bboxes[:, i, 3][0] result_dict[id] = result_dict.get( id, []) + [[item, score, xmin, ymin, xmax, ymax]] print("Detect Image {:s} Done.".format(item)) print("---Detect Total {:d} Image Done.---".format(len(val_items))) return result_dict
def generate_targets(self, img, boxes): """ img : [H, W, 3] boxes : [N, 5] """ rh, rw, _ = img.shape rh, rw = int(rh/4), int(rw/4) rx = nd.arange(0, rw).reshape((1, -1)) ry = nd.arange(0, rh).reshape((-1, 1)) sx = nd.tile(rx, reps=(rh, 1)) sy = nd.tile(ry, reps=(1, rw)) x0, y0, x1, y1, _ = nd.split(boxes, 5, axis=-1, squeeze_axis=True) areas = (x1 - x0) * (y1 - y0) # areas = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) boxes_id = nd.argsort(areas) boxes_id = nd.concat(nd.array([-1]), boxes_id, dim=0) boxes = nd.take(boxes, nd.argsort(areas)) # min -> max boxes = nd.concat(nd.zeros((1, 5)), boxes, dim=0) # for gt assign confusion x0, y0, x1, y1, cls = nd.split(boxes, num_outputs=5, axis=-1, squeeze_axis=True) n = boxes.shape[0] # [H, W, N] of_l = sx.reshape(-2, 1) - nd.expand_dims(nd.expand_dims(x0/4, axis=0), axis=0) of_t = sy.reshape(-2, 1) - nd.expand_dims(nd.expand_dims(y0/4, axis=0), axis=0) of_r = -(sx.reshape(-2, 1) - nd.expand_dims(nd.expand_dims(x1/4, axis=0), axis=0)) of_b = -(sy.reshape(-2, 1) - nd.expand_dims(nd.expand_dims(y1/4, axis=0), axis=0)) # [H, W, N] eps = 1e-5 # ctr = nd.minimum(of_l, of_r) / (nd.maximum(of_l, of_r) + eps) * \ # nd.minimum(of_t, of_b) / (nd.maximum(of_t, of_b) + eps) # ctr = nd.sqrt(nd.abs(ctr)) # ctr[:, :, 0] = 0 # # flat ctr of_l = of_l * (of_l > 0) of_r = of_r * (of_r > 0) of_t = of_t * (of_t > 0) of_b = of_b * (of_b > 0) # ctr2 = nd.minimum(of_l, of_r) / (nd.maximum(of_l, of_r) + of_l + of_r) * \ # nd.minimum(of_t, of_b) / (nd.maximum(of_t, of_b) + of_t + of_b) # ctr2 = 3 * nd.sqrt(nd.abs(ctr2)) # ctr2[:, :, 0] = 0 # slim ctr # ctr = nd.minimum(of_l, of_r) / (nd.maximum(of_l, of_r) + nd.abs(of_l - of_r) + eps) * \ # nd.minimum(of_t, of_b) / (nd.maximum(of_t, of_b) + nd.abs(of_t - of_b) + eps) ctr = nd.minimum(of_l, of_r) / (nd.maximum(of_l, of_r) + eps) * \ nd.minimum(of_t, of_b) / (nd.maximum(of_t, of_b) + eps) # ctr = nd.power(0.8, 0.1 * nd.sqrt(nd.square(of_l - of_r) + nd.square(of_t - of_b) + eps)) # ctr = nd.power(0.8, nd.sqrt(nd.abs(of_l - of_r) + nd.abs(of_t - of_b) + eps)) ctr = nd.sqrt(nd.abs(ctr)) ctr[:, :, 0] = 0 # [H, W, N, 4] offsets = nd.concat(of_l.reshape(-2, 1), of_t.reshape(-2, 1), of_r.reshape(-2, 1), of_b.reshape(-2, 1), dim=-1) * 4. fh = int(np.ceil(rh / 2)) fw = int(np.ceil(rw / 2)) # fh = int(np.ceil(np.ceil(np.ceil(rh / 2) / 2) / 2)) # fw = int(np.ceil(np.ceil(np.ceil(rw / 2) / 2) / 2)) fm_list = [] for i in range(self._stages): fm_list.append((fh, fw)) fh = int(np.ceil(fh / 2)) fw = int(np.ceil(fw / 2)) fm_list = fm_list[::-1] cls_targets = [] ctr_targets = [] box_targets = [] match_targets = [] stride = int(self._stride/4) for i in range(self._stages): fh, fw = fm_list[i] # cls_target = nd.zeros((fh, fw)) # box_target = nd.zeros((fh, fw, 4)) # ctr_target = nd.zeros((fh, fw)) # match_target = nd.zeros((fh, fw)) cx = nd.arange(0, fw).reshape((1, -1)) cy = nd.arange(0, fh).reshape((-1, 1)) sx = nd.tile(cx, reps=(fh, 1)) sy = nd.tile(cy, reps=(1, fw)) syx = nd.stack(sy.reshape(-1), sx.reshape(-1)).transpose().astype('int32') # bugs in this type # bx = sxy[:, 0] * stride + nd.floor(sxy[:, 0] / 2).astype(np.int32) # by = sxy[:, 1] * stride + nd.floor(sxy[:, 1] / 2).astype(np.int32) by, bx = nd.split(syx*stride, 2, axis=-1, squeeze_axis=True) # by = syx[:, 0] * stride # bx = syx[:, 1] * stride # [FH*FW, N, 4] of_byx = nd.take(offsets.reshape((-1, n, 4)), by*740/4+bx) of_ctr = nd.take(ctr.reshape((-1, n)), by*740/4 + bx) # of_byx = offsets[by, bx] # ctr_aware = ctr[by, bx] # of_byx = nd.gather_nd(offsets, indices=byx.transpose()) min_vr, max_vr = self._valid_range[i] # [FH*FW, N] is_in_box = nd.prod(of_byx > 0, axis=-1) is_valid_area = (of_byx.max(axis=-1) >= min_vr) * (of_byx.max(axis=-1) <= max_vr) # [FH*FW, N] valid_pos = nd.elemwise_mul(is_in_box, is_valid_area) * of_ctr # valid_pos = nd.elemwise_mul(is_in_box, is_valid_area) # of_valid = nd.zeros((fh, fw, n)) # of_valid[syx[:, 0], syx[:, 1], :] = valid_pos * ctr_aware # 1, 0 of_valid = valid_pos.reshape((fh, fw, n)) of_valid[:, :, 0] = 0 # [FH, FW] # gt_inds = nd.argmax(of_valid, axis=-1) gt_inds = nd.argmax(of_valid, axis=-1).reshape(-1) # box targets box_target = nd.take(boxes, gt_inds).slice_axis(begin=0, end=4, axis=-1) # box_target[syx[:, 0], syx[:, 1]] = boxes[gt_inds[syx[:, 0], syx[:, 1]], :4] # box_target = box_target.reshape(-1, 4) # cls targets cls_target = nd.take(cls, gt_inds) # cls_target[syx[:, 0], syx[:, 1]] = cls[gt_inds[syx[:, 0], syx[:, 1]]] # cls_target = cls_target.reshape(-1) # match targets the number of matches less than ctr targets # match_gt_inds = nd.argmax(of_valid * (of_valid > 0.01), axis=-1).reshape(-1) match_target = nd.take(boxes_id, gt_inds) # match_target[syx[:, 0], syx[:, 1]] = boxes_id[match_gt_inds[syx[:,0], syx[:,1]]] # match_target = match_target.reshape(-1) # ctr targets ctr_target = nd.pick(of_ctr, gt_inds) # ctr_target[syx[:, 0], syx[:, 1]] = ctr[by, bx, gt_inds[syx[:, 0], syx[:, 1]]] # ctr_target = ctr_target.reshape(-1) box_targets.append(box_target) cls_targets.append(cls_target) ctr_targets.append(ctr_target) stride = int(stride / 2) match_targets.append(match_target) box_targets = nd.concat(*box_targets, dim=0) cls_targets = nd.concat(*cls_targets, dim=0) ctr_targets = nd.concat(*ctr_targets, dim=0) match_targets = nd.concat(*match_targets, dim=0) return cls_targets, ctr_targets, box_targets, match_targets
def trim(epoch, gradients, net, lr, byz, old_direction, active, blacklist, susp, f=0, cmax=0, utrg=0.0, udet=0.50, urem=3): param_list = [ nd.concat(*[xx.reshape((-1, 1)) for xx in x], dim=0) for x in gradients ] param_list = byz(epoch, param_list, net, f, lr, active) flip_local = nd.zeros(len(param_list)) flip_new = nd.zeros(len(param_list)) penalty = 1.0 - cmax / len(param_list) reward = 1.0 - penalty for i in range(len(param_list)): direction = nd.sign(param_list[i]) flip_local[i] = 0.5 * (mx.nd.sum( direction.reshape(-1) * (direction.reshape(-1) - old_direction.reshape(-1)))).asscalar() #flip = nd.sign(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1))) #flip_new[i] = nd.sum(flip*(param_list[i].reshape(-1)**2)) #flip[param_list[i]<0.0001] = 0 #flip_new[i] = nd.sum(flip).asscalar() #argsorted = nd.argsort(flip_local) argsorted = nd.argsort(flip_local) if (cmax > 0): susp[argsorted[:-cmax]] = susp[argsorted[:-cmax]] + reward susp[argsorted[-cmax:]] = susp[argsorted[-cmax:]] - penalty argsorted = nd.argsort(susp) weights = nd.exp(susp) / nd.sum(nd.exp(susp)) matrix = nd.transpose( nd.transpose(nd.concat(*[ii for ii in param_list], dim=1))) trim_nd = nd.linalg.gemm2(matrix, weights.reshape(-1, 1)) #pdb.set_trace() #print (flip_new, weights) #print (nd.mean(flip_local[:cmax]), nd.mean(flip_new[:cmax]), nd.mean(flip_local[cmax:]), nd.mean(flip_new[cmax:])) '''new_list = [] argsorted = nd.argsort(susp) for i in range(len(param_list)-cmax): new_list.append(param_list[int(argsorted[i].asscalar())]) sorted_array = nd.sort(nd.concat(*new_list, dim=1), axis=-1) trim_nd = nd.mean(sorted_array, axis=-1, keepdims=1)''' global_direction = nd.sign(trim_nd) gfs = 0.5 * (mx.nd.sum( global_direction.reshape(-1) * (global_direction.reshape(-1) - old_direction.reshape(-1))) ).asscalar() '''if (utrg > 0): sorted_array = nd.sort(nd.concat(*param_list, dim=1), axis=-1) n = len(param_list) m = n - f*2 trim_nd = nd.mean(sorted_array[:, f:(f+m)], axis=-1, keepdims=1) direction = nd.sign(trim_nd) gfs = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar() if ((utrg>0 and gfs>=utrg*len(param_list[0])) or (utrg == 0)): flip_score = mx.nd.zeros(len(param_list)) rem = [] for i in range (len(param_list)): direction = nd.sign(param_list[i]) flip_score[i] = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar() flip_local[active[i]] = flip_score[i].asscalar() argsorted = nd.argsort(flip_score) new_list = [] for i in range(len(param_list)-cmax): new_list.append(param_list[int(argsorted[i].asscalar())]) n = len(new_list) f = 0 m = n - f*2 sorted_array = nd.sort(nd.concat(*new_list, dim=1), axis=-1) trim_nd = nd.mean(sorted_array[:, f:(f+m)], axis=-1, keepdims=1) for i in range(len(new_list), len(param_list)): index = int(argsorted[i].asscalar()) if (flip_score[index] >= udet*len(param_list[0])): susp[active[index]] = susp[active[index]] + 1 if (susp[active[index]] >= urem): blacklist[active[index]] = 1 rem.append(active[index]) active = removearr(active, sorted(rem), len(param_list)) direction = nd.sign(trim_nd) gfs = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar() cmax = cmax - len(rem) ''' idx = 0 for j, (param) in enumerate(net.collect_params().values()): if param.grad_req == 'null': continue param.set_data( param.data() - lr * trim_nd[idx:(idx + param.data().size)].reshape(param.data().shape)) idx += param.data().size return trim_nd, direction, cmax, gfs, flip_local, flip_new
def test_argsort(): b = create_2d_tensor(rows=LARGE_X, columns=SMALL_Y) s = nd.argsort(b, axis=0, is_ascend=False, dtype=np.int64) mx.nd.waitall() assert (s[0].asnumpy() == (LARGE_X - 1)).all()
def test_argsort(): a = create_vector(size=LARGE_X) s = nd.argsort(a, axis=0, is_ascend=False, dtype=np.int64) assert s[0] == (LARGE_X - 1)
def MultiBoxTarget(anchors, class_predictions, labels, hard_neg_ratio=3, ctx=gpu(), verbose=False): '''將真實方框(ground truth boxes)和預設錨框(default anchor boxes)做配對。 labels: 真實方框 (ground truth boxes)。 anchors: 預設錨框 (default anchor boxes)。 hard_neg_ratio: 負樣本(背景)和正樣本(有物體的錨框數)的比例。預設是3:1。 ''' if verbose: print("anchors shape=", anchors.shape) assert anchors.shape[0] == 1 batch_size = len(labels) num_priors = anchors.shape[1] if verbose: print("batch size=\t", batch_size) print("num priors=\t", num_priors) anchor_shifts = nd.zeros((batch_size, anchors.shape[1] * 4), ctx=ctx) box_mask = nd.zeros((batch_size, anchors.shape[1] * 4), ctx=ctx) anchor_classes = nd.zeros((batch_size, num_priors), ctx=ctx) - 1 anchor_indices = nd.zeros((batch_size, num_priors), ctx=ctx) - 1 classes_mask = nd.zeros((batch_size, anchors.shape[1]), ctx=ctx) shifts_tmp = nd.zeros_like(anchors[0]) mask_tmp = nd.zeros_like(anchors[0]) for i in range(batch_size): label_locs = labels[i][:, 1:] label_classes = labels[i][:, 0] class_preds = class_predictions[i] # obtain IoU ious = iou(label_locs, anchors[0]) # identify how many ground truth objects are there in this batch if -1 in label_classes: num_obj = label_classes.argmin(axis=0).astype("int32").asscalar() else: num_obj = label_classes.size if num_obj == 0: continue # matching anchor boxes with ground truth boxes ious_flag = ious > 0.5 # find locations of the best priors best_prior_idx = ious[0:num_obj, :].argmax(axis=1) idx_row = [*range(best_prior_idx.size)] ious_flag[idx_row, best_prior_idx] += 1 if_negative = label_classes != -1 label_classes = label_classes + if_negative * 1 # add the -1 class to the end label_classes = nd.concat(label_classes, nd.array([-1], ctx=label_classes.context), dim=0) if verbose: print("label classes=\t", label_classes) if_matched = ious_flag.sum(axis=0) != 0 label_classes_last_idx = len(label_classes) - 1 anchor_indices[i] = (ious_flag.argmax(axis=0) - label_classes_last_idx ) * if_matched + label_classes_last_idx anchor_classes[i] = label_classes[anchor_indices[i]] # find indices of the negative anchors neg_indices, = np.where(anchor_classes[i].asnumpy() == -1) # count number of positive/negative/hard negative anchors num_neg_anchors = len(neg_indices) num_pos_anchors = num_priors - num_neg_anchors num_hard_neg_anchors = num_pos_anchors * hard_neg_ratio # hard negative mining #conf_loss_indices = nd.argsort( (-nd.softmax( class_preds[neg_indices] ) )[:,0], is_ascend=False ).astype("int32") conf_loss_indices = nd.argsort((-class_preds[neg_indices])[:, 0], is_ascend=False).astype("int32") neg_indices_sorted = nd.array(neg_indices, ctx=class_preds.context, dtype="int32")[conf_loss_indices] hard_neg_indices = neg_indices_sorted[:num_hard_neg_anchors] anchor_classes[i][hard_neg_indices] = 0 # find indices of the positive anchors pos_indices, = np.where( anchor_indices[i].asnumpy() < label_classes_last_idx) pos_indices = nd.array(pos_indices, ctx=hard_neg_indices.context).astype("int32") cls_indices = nd.concatenate([hard_neg_indices, pos_indices ]) # store indices for classification. classes_mask[i][cls_indices] = 1 if verbose: print("========================================") display( pd.DataFrame(anchor_classes[i].asnumpy()).groupby(0).indices) df = pd.DataFrame( anchor_classes[i].asnumpy()).reset_index().groupby(0).count() df.index.name = "class" df.index = df.index.astype("int32") df.columns = ["number of default anchor boxes"] display(df) print("========================================") # obtain locations of the positve anchors pos_anchors_loc = anchors[0][pos_indices] # obtain indices of the ground truth labels idx_gt = anchor_indices[i][pos_indices] # obtain locations of the ground truth labels labels_loc = label_locs[idx_gt] assert len(pos_anchors_loc) == len(labels_loc) # calculate location differences between ground truth labels and positive anchors shifts_tmp[pos_indices] = loc_difference_calculator( pos_anchors_loc, labels_loc) mask_tmp[pos_indices] = 1. anchor_shifts[i] = shifts_tmp.reshape(-1) box_mask[i] = mask_tmp.reshape(-1) return anchor_shifts, box_mask, anchor_classes, classes_mask
def generate_targets(self, img, boxes): """ img : [H, W, 3] boxes : [N, 5] """ rh, rw, _ = img.shape rx = nd.arange(0, rw).reshape((1, -1)) ry = nd.arange(0, rh).reshape((-1, 1)) sx = nd.tile(rx, reps=(rh, 1)) sy = nd.tile(ry, reps=(1, rw)) areas = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) boxes = boxes[nd.argsort(areas)] boxes = nd.concat(nd.zeros((1, 5)), boxes, dim=0) # for gt assign confusion x0, y0, x1, y1, cls = nd.split(boxes, num_outputs=5, axis=-1, squeeze_axis=True) n = boxes.shape[0] # [H, W, N] of_l = sx.reshape(-2, 1) - nd.expand_dims(nd.expand_dims(x0, axis=0), axis=0) of_t = sy.reshape(-2, 1) - nd.expand_dims(nd.expand_dims(y0, axis=0), axis=0) of_r = -(sx.reshape(-2, 1) - nd.expand_dims(nd.expand_dims(x1, axis=0), axis=0)) of_b = -(sy.reshape(-2, 1) - nd.expand_dims(nd.expand_dims(y1, axis=0), axis=0)) # [H, W, N] eps = 1e-5 ctr =(nd.minimum(of_l, of_r) / nd.maximum(of_l, of_r)) * \ (nd.minimum(of_t, of_b) / nd.maximum(of_t, of_b) + eps) ctr = nd.sqrt(nd.abs(ctr)) ctr[:, :, 0] = 0 # [H, W, N, 4] offsets = nd.concat(of_l.reshape(-2, 1), of_t.reshape(-2, 1), of_r.reshape(-2, 1), of_b.reshape(-2, 1), dim=-1) # fh = int(np.ceil(((rh + 1) / 2) // 2 / 2)) # fw = int(np.ceil(((rw + 1) / 2) // 2 / 2)) fh = int(np.ceil(np.ceil(np.ceil(rh / 2) / 2) / 2)) fw = int(np.ceil(np.ceil(np.ceil(rw / 2) / 2) / 2)) fm_list = [] for i in range(self._stages): fm_list.append((fh, fw)) fh = int(np.ceil(fh / 2)) fw = int(np.ceil(fw / 2)) fm_list = fm_list[::-1] cls_targets = [] ctr_targets = [] box_targets = [] cor_targets = [] stride = self._stride for i in range(self._stages): fh, fw = fm_list[i] cls_target = nd.zeros((fh, fw)) box_target = nd.zeros((fh, fw, 4)) ctr_target = nd.zeros((fh, fw)) cx = nd.arange(0, fw).reshape((1, -1)) cy = nd.arange(0, fh).reshape((-1, 1)) sx = nd.tile(cx, reps=(fh, 1)) sy = nd.tile(cy, reps=(1, fw)) syx = nd.stack(sy.reshape(-1), sx.reshape(-1)).transpose().astype('int32') # bugs in this type # bx = sxy[:, 0] * stride + nd.floor(sxy[:, 0] / 2).astype(np.int32) # by = sxy[:, 1] * stride + nd.floor(sxy[:, 1] / 2).astype(np.int32) by = syx[:, 0] * stride bx = syx[:, 1] * stride cor_targets.append(nd.stack(bx, by, axis=1)) # [FH*FW, N, 4] of_byx = offsets[by, bx] # of_byx = nd.gather_nd(offsets, indices=byx.transpose()) min_vr, max_vr = self._valid_range[i] # [FH*FW, N] is_in_box = nd.prod(of_byx > 0, axis=-1) is_valid_area = (of_byx.max(axis=-1) >= min_vr) * (of_byx.max(axis=-1) <= max_vr) # [FH*FW, N] valid_pos = nd.elemwise_mul(is_in_box, is_valid_area) of_valid = nd.zeros((fh, fw, n)) of_valid[syx[:, 0], syx[:, 1], :] = valid_pos # 1, 0 of_valid[:, :, 0] = 0 # [FH, FW] gt_inds = nd.argmax(of_valid, axis=-1) # box targets box_target[syx[:, 0], syx[:, 1]] = boxes[gt_inds[syx[:, 0], syx[:, 1]], :4] box_target = box_target.reshape(-1, 4) # cls targets cls_target[syx[:, 0], syx[:, 1]] = cls[gt_inds[syx[:, 0], syx[:, 1]]] cls_target = cls_target.reshape(-1) # ctr targets ctr_target[syx[:, 0], syx[:, 1]] = ctr[by, bx, gt_inds[syx[:, 0], syx[:, 1]]] ctr_target = ctr_target.reshape(-1) box_targets.append(box_target) cls_targets.append(cls_target) ctr_targets.append(ctr_target) stride = int(stride / 2) box_targets = nd.concat(*box_targets, dim=0) cls_targets = nd.concat(*cls_targets, dim=0) ctr_targets = nd.concat(*ctr_targets, dim=0) cor_targets = nd.concat(*cor_targets, dim=0) cor_targets = cor_targets.astype('float32') return cls_targets, ctr_targets, box_targets, cor_targets
def forward(self, features, image_shape, labels=None, bboxes=None): """ :param features: OrderedDict, each features: (B, C, H, W) :param image_shape: :param labels: :param bboxes: :return: """ anchors = OrderedDict() # each anchors:(B, H, W, num_anchors, 4) pred_logits = OrderedDict() # each pred_logits:(B, H, W, num_anchors) pred_bbox_deltas = OrderedDict( ) # each pred_bbox_deltas:(B, H, W, num_anchors, 4) B = 0 device = cpu() for k, feature in features.items(): if k in self.scales.keys(): B, C, H, W = feature.shape feature = self.head(feature) pred_logits[k] = self.object_cls(feature).transpose(axes=(0, 2, 3, 1)) device = pred_logits[k].context pred_bbox_deltas[k] = self.object_reg(feature).transpose( axes=(0, 2, 3, 1)).reshape(B, H, W, -1, 4) anchors_per_sample = nd.array(self.generate_anchors( feature.shape[-2:], image_shape, k), ctx=device) anchors[k] = nd.stack(*([ anchors_per_sample, ] * B), axis=0) pred_logits = split_batch(pred_logits, B) pred_bbox_deltas = split_batch(pred_bbox_deltas, B) anchors = split_batch(anchors, B) pred_logits = [_combine_keys(i) for i in pred_logits ] # list each pred_logits:(B*H*W*num_anchors,) pred_bbox_deltas = [ _combine_keys(i) for i in pred_bbox_deltas ] # list each pred_bbox_deltas:(B*H*W*num_anchors,4) anchors = [_combine_keys(i) for i in anchors ] # list each anchors:(B*H*W*num_anchors, 4) object_bboxes_out = [] rpn_cls_losses = [] rpn_reg_losses = [] if not autograd.is_training(): for pred_logits_per_sample, pred_bbox_deltas_per_sample, anchors_per_sample in zip( pred_logits, pred_bbox_deltas, anchors): sorted_indices = nd.argsort(pred_logits_per_sample, axis=0, is_ascend=False) sorted_indices = sorted_indices[ 0:min(len(sorted_indices), self.pre_nms_top_n_in_test)] pred_logits_per_sample = pred_logits_per_sample[sorted_indices] pred_bbox_deltas_per_sample = pred_bbox_deltas_per_sample[ sorted_indices] anchors_per_sample = anchors_per_sample[sorted_indices] object_bboxes = bbox_decode(pred_bbox_deltas_per_sample, anchors_per_sample) object_bboxes = clip_bbox(object_bboxes, image_shape) nms_input = nd.concat( nd.arange(len(pred_bbox_deltas_per_sample), ctx=device).reshape(-1, 1), # index of object bboxes nd.sigmoid(pred_logits_per_sample).reshape(-1, 1), object_bboxes, dim=1) outs = nd.contrib.box_nms(nms_input, self.nms_thresh, valid_thresh=0.05) outs = outs[:min(len(outs), self.post_nms_top_n_in_test)] indices, object_bboxes = outs[:, 0], outs[:, 2:6] object_bboxes = nd.contrib.boolean_mask( object_bboxes, indices != -1) object_bboxes_out.append(object_bboxes) return object_bboxes_out, None for pred_logits_per_sample, pred_bbox_deltas_per_sample, anchors_per_sample, labels_per_sample, bboxes_per_sample in zip( pred_logits, pred_bbox_deltas, anchors, labels, bboxes): gt_mask = labels_per_sample != -1 labels_per_sample = nd.contrib.boolean_mask( labels_per_sample, gt_mask) bboxes_per_sample = nd.contrib.boolean_mask( bboxes_per_sample, gt_mask) sorted_indices = nd.argsort(pred_logits_per_sample, axis=0, is_ascend=False) sorted_indices = sorted_indices[0:min(len(sorted_indices), self. pre_nms_top_n_in_train)] pred_logits_per_sample = pred_logits_per_sample[sorted_indices] pred_bbox_deltas_per_sample = pred_bbox_deltas_per_sample[ sorted_indices] anchors_per_sample = anchors_per_sample[sorted_indices] object_bboxes = bbox_decode(pred_bbox_deltas_per_sample, anchors_per_sample) object_bboxes = clip_bbox(object_bboxes, image_shape) nms_input = nd.concat( nd.arange(len(pred_bbox_deltas_per_sample), ctx=device).reshape(-1, 1), # index of object bboxes nd.sigmoid(pred_logits_per_sample).reshape(-1, 1), object_bboxes, dim=1) outs = nd.contrib.box_nms(nms_input, self.nms_thresh, valid_thresh=0.05) outs = outs[:min(len(outs), self.post_nms_top_n_in_train)] indices, object_bboxes = outs[:, 0], outs[:, 2:6] valid_mask = indices != -1 indices = nd.contrib.boolean_mask(indices, valid_mask) object_bboxes = nd.contrib.boolean_mask(object_bboxes, valid_mask) object_bboxes_out.append(object_bboxes) pred_logits_per_sample = pred_logits_per_sample[indices] pred_bbox_deltas_per_sample = pred_bbox_deltas_per_sample[indices] anchors_per_sample = anchors_per_sample[indices] iou_matrix = nd.contrib.box_iou(anchors_per_sample, bboxes_per_sample) matched_indices = match_target(iou_matrix, self.fg_threshold) object_bboxes_gt = bboxes_per_sample[matched_indices] indices = np.arange(len(anchors_per_sample)) fg_mask = matched_indices >= 0 fg_indices = indices[fg_mask] matched_indices = match_target(iou_matrix, self.bg_threshold) bg_mask = matched_indices < 0 bg_indices = indices[bg_mask] num_pos_samples = len(fg_indices) batch_size = min(int(num_pos_samples / self.positive_fraction), self.batch_size_per_image) num_neg_samples = min(len(bg_indices), batch_size - num_pos_samples) bg_indices = bg_indices[:num_neg_samples] assert len(fg_indices) > 0 pred_logits_batch = nd.concat(pred_logits_per_sample[fg_indices], pred_logits_per_sample[bg_indices], dim=0) pred_labels_batch = nd.concat(nd.ones(len(fg_indices), ctx=device), nd.zeros(len(bg_indices), ctx=device), dim=0) rpn_cls_loss = self.object_cls_loss( pred_logits_batch, pred_labels_batch).sum() / batch_size rpn_cls_losses.append(rpn_cls_loss) pred_bbox_deltas = pred_bbox_deltas_per_sample[fg_indices] bbox_deltas = bbox_encode(object_bboxes_gt[fg_indices], anchors_per_sample[fg_indices]) rpn_reg_loss = self.object_reg_loss(pred_bbox_deltas, bbox_deltas).sum() / batch_size rpn_reg_losses.append(rpn_reg_loss) rpn_cls_loss = sum(rpn_cls_losses) / B rpn_reg_loss = sum(rpn_reg_losses) / B return object_bboxes_out, (rpn_cls_loss, rpn_reg_loss)
def test_argsort(): b = create_vector(size=LARGE_X) s = nd.argsort(b, axis=0, is_ascend=False, dtype=np.int64) assert (s[0].asnumpy() == (LARGE_X - 1)).all()
def krum(epoch, gradients, net, lr, byz, old_direction, active, blacklist, susp, f=0, cmax=0, utrg=0, udet=0.50, urem=3, max_flip=1.0): param_list = [ nd.concat(*[xx.reshape((-1, 1)) for xx in x], dim=0) for x in gradients ] param_list = byz(epoch, param_list, net, f, lr, active, max_flip) flip_local = nd.zeros(len(param_list)) penalty = 1.0 - cmax / len(param_list) reward = 1.0 - penalty for i in range(len(param_list)): direction = nd.sign(param_list[i]) flip_local[i] = 0.5 * (mx.nd.sum( direction.reshape(-1) * (direction.reshape(-1) - old_direction.reshape(-1)))).asscalar() argsorted = nd.argsort(flip_local) susp[argsorted[:-cmax]] = susp[argsorted[:-cmax]] - reward susp[argsorted[-cmax:]] = susp[argsorted[-cmax:]] + penalty new_list = [] argsorted = nd.argsort(susp) for i in range(len(param_list) - cmax): new_list.append(param_list[int(argsorted[i].asscalar())]) k = len(new_list) - 0 - 2 dist = mx.nd.zeros((len(new_list), len(new_list))) for i in range(0, len(new_list)): for j in range(0, i): dist[i][j] = nd.norm(new_list[i] - new_list[j]) dist[j][i] = dist[i][j] sorted_dist = mx.nd.sort(dist) sum_dist = mx.nd.sum(sorted_dist[:, :k + 1], axis=1) model_selected = argsorted[mx.nd.argmin(sum_dist).asscalar().astype( int)].asscalar().astype(int) global_direction = nd.sign(param_list[model_selected]) gfs = 0.5 * (mx.nd.sum( global_direction.reshape(-1) * (global_direction.reshape(-1) - old_direction.reshape(-1))) ).asscalar() '''if (utrg > 0): k = len(param_list) - f - 2 dist = mx.nd.zeros((len(param_list),len(param_list))) for i in range (0, len(param_list)): for j in range(0, i): dist[i][j] = nd.norm(param_list[i] - param_list[j]) dist[j][i] = dist[i][j] sorted_dist = mx.nd.sort(dist) sum_dist = mx.nd.sum(sorted_dist[:,:k+1], axis=1) model_selected = mx.nd.argmin(sum_dist).asscalar().astype(int) direction = nd.sign(param_list[model_selected]) gfs = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar() for i in range (len(param_list)): direction = nd.sign(param_list[i]) flip_score = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar() flip_local[active[i]] = flip_score if ((utrg>0 and gfs>=utrg*len(param_list[0])) or (utrg == 0)): flip_score = mx.nd.zeros(len(param_list)) rem = [] for i in range (len(param_list)): direction = nd.sign(param_list[i]) flip_score[i] = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar() flip_local[active[i]] = flip_score[i].asscalar() argsorted = nd.argsort(flip_score) new_list = [] for i in range(len(param_list)-cmax): new_list.append(param_list[int(argsorted[i].asscalar())]) k = len(new_list) - 0 - 2 dist = mx.nd.zeros((len(new_list),len(new_list))) for i in range (0, len(new_list)): for j in range(0, i): dist[i][j] = nd.norm(new_list[i] - new_list[j]) dist[j][i] = dist[i][j] for i in range(len(new_list), len(param_list)): index = int(argsorted[i].asscalar()) if (flip_score[index] >= udet*len(param_list[0])): susp[active[index]] = susp[active[index]] + 1 if (susp[active[index]] >= urem): blacklist[active[index]] = 1 rem.append(active[index]) active = removearr(active, sorted(rem), len(param_list)) sorted_dist = mx.nd.sort(dist) sum_dist = mx.nd.sum(sorted_dist[:,:k+1], axis=1) model_selected = argsorted[mx.nd.argmin(sum_dist).asscalar().astype(int)].asscalar().astype(int) direction = nd.sign(param_list[model_selected]) gfs = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar() cmax = cmax - len(rem) ''' idx = 0 for j, (param) in enumerate(net.collect_params().values()): if param.grad_req == 'null': continue param.set_data(param.data() - lr * param_list[model_selected][idx:( idx + param.data().size)].reshape(param.data().shape)) idx += param.data().size return model_selected, direction, cmax, gfs, flip_local, 1.0 #flip_score[len(param_list)-cmax-1].asscalar()/len(param_list[0])
def forward(self, is_train, req, in_data, out_data, aux): nms_start_time = time.time() #inputs cls_score = in_data[0] bbox_pred = in_data[1] rois = in_data[2] im_info = in_data[3] fc_all_2_relu = in_data[4] nms_rank_weight = in_data[5] nms_rank_bias = in_data[6] roi_feat_embedding_weight = in_data[7] roi_feat_embedding_bias = in_data[8] nms_pair_pos_fc1_1_weight = in_data[9] nms_pair_pos_fc1_1_bias = in_data[10] nms_query_1_weight = in_data[11] nms_query_1_bias = in_data[12] nms_key_1_weight = in_data[13] nms_key_1_bias = in_data[14] nms_linear_out_1_weight = in_data[15] nms_linear_out_1_bias = in_data[16] nms_logit_weight = in_data[17] nms_logit_bias = in_data[18] if self.has_non_gt_index: non_gt_index = in_data[19] else: non_gt_index = None if self.nongt_dim is not None: cls_score_nongt = nd.slice_axis(data=cls_score, axis=0, begin=0, end=self.nongt_dim) # cls_score_nongt = monitor_wrapper(cls_score_nongt, 'cls_score_nongt') bbox_pred_nongt = nd.slice_axis(data=bbox_pred, axis=0, begin=0, end=self.nongt_dim) elif non_gt_index is not None: cls_score_nongt = nd.take(a=cls_score, indices=non_gt_index) bbox_pred_nongt = nd.take(a=bbox_pred, indices=non_gt_index) else: cls_score_nongt = cls_score bbox_pred_nongt = bbox_pred bbox_pred_nongt = nd.BlockGrad(bbox_pred_nongt) # remove batch idx and gt roi sliced_rois = nd.slice_axis(data=rois, axis=1, begin=1, end=None) if self.nongt_dim is not None: sliced_rois = nd.slice_axis(data=sliced_rois, axis=0, begin=0, end=self.nongt_dim) elif non_gt_index is not None: sliced_rois = nd.take(a=sliced_rois, indices=non_gt_index) # bbox_pred_nobg, [num_rois, 4*(num_reg_classes-1)] bbox_pred_nobg = nd.slice_axis(data=bbox_pred_nongt, axis=1, begin=4, end=None) # [num_boxes, 4, num_reg_classes-1] refined_bbox = refine_bbox_nd(sliced_rois, bbox_pred_nobg, im_info, means=self.bbox_means, stds=self.bbox_stds) # softmax cls_score to cls_prob, [num_rois, num_classes] cls_prob = nd.softmax(data=cls_score_nongt, axis=-1) cls_prob_nobg = nd.slice_axis(cls_prob, axis=1, begin=1, end=None) sorted_cls_prob_nobg = nd.sort(data=cls_prob_nobg, axis=0, is_ascend=False) # sorted_score, [first_n, num_fg_classes] sorted_score = nd.slice_axis(sorted_cls_prob_nobg, axis=0, begin=0, end=self.first_n, name='sorted_score') max_score_per_class = sorted_score.max(axis=0) max_score_per_class_numpy = max_score_per_class.asnumpy() valid_class_thresh = self.class_thresh valid_class_thresh = np.minimum(valid_class_thresh, max_score_per_class_numpy.max()) valid_class_indices = np.where( max_score_per_class_numpy >= valid_class_thresh)[0] invalid_class_indices = np.where( max_score_per_class_numpy < valid_class_thresh)[0] num_valid_classes = len(valid_class_indices) valid_class_indices_nd = nd.array(valid_class_indices, ctx=sorted_score.context) # sort by score rank_indices = nd.argsort(data=cls_prob_nobg, axis=0, is_ascend=False) # first_rank_indices, [first_n, num_fg_classes] first_rank_indices = nd.slice_axis(rank_indices, axis=0, begin=0, end=self.first_n) valid_first_rank_indices = first_rank_indices.transpose().take( valid_class_indices_nd).transpose() # sorted_bbox, [first_n, num_fg_classes, 4, num_reg_classes-1] sorted_bbox = nd.take(a=refined_bbox, indices=first_rank_indices) if self.class_agnostic: # sorted_bbox, [first_n, num_fg_classes, 4] sorted_bbox = nd.Reshape(sorted_bbox, shape=(0, 0, 0), name='sorted_bbox') else: cls_mask = nd.arange(0, self.num_fg_classes) cls_mask = nd.Reshape(cls_mask, shape=(1, -1, 1)) cls_mask = nd.broadcast_to(cls_mask, shape=(self.first_n, 0, 4)) # sorted_bbox, [first_n, num_fg_classes, 4] sorted_bbox = nd.pick(data=sorted_bbox, name='sorted_bbox', index=cls_mask, axis=3) valid_sorted_bbox = sorted_bbox.transpose( (1, 0, 2)).take(valid_class_indices_nd).transpose((1, 0, 2)) # sorted_bbox = monitor_wrapper(sorted_bbox, 'sorted_bbox') # nms_rank_embedding, [first_n, 1024] nms_rank_embedding = extract_rank_embedding_nd(self.first_n, 1024) # nms_rank_feat, [first_n, 1024] nms_rank_feat = nd.FullyConnected(name='nms_rank', data=nms_rank_embedding, num_hidden=128, weight=nms_rank_weight, bias=nms_rank_bias) # nms_position_matrix, [num_valid_classes, first_n, first_n, 4] nms_position_matrix = extract_multi_position_matrix_nd( valid_sorted_bbox) # roi_feature_embedding, [num_rois, 1024] # fc_all_2_relu = monitor_wrapper(fc_all_2_relu, 'fc_all_2_relu') roi_feat_embedding = nd.FullyConnected( name='roi_feat_embedding', data=fc_all_2_relu, num_hidden=128, weight=roi_feat_embedding_weight, bias=roi_feat_embedding_bias) # sorted_roi_feat, [first_n, num_valid_classes, 128] sorted_roi_feat = nd.take(a=roi_feat_embedding, indices=valid_first_rank_indices) # vectorized nms # nms_embedding_feat, [first_n, num_valid_classes, 128] nms_embedding_feat = nd.broadcast_add(lhs=sorted_roi_feat, rhs=nd.expand_dims(nms_rank_feat, axis=1)) # nms_attention_1, [first_n, num_valid_classes, 1024] nms_attention_1 = nms_attention_nd( nms_embedding_feat, nms_position_matrix, nms_pair_pos_fc1_1_weight, nms_pair_pos_fc1_1_bias, nms_query_1_weight, nms_query_1_bias, nms_key_1_weight, nms_key_1_bias, nms_linear_out_1_weight, nms_linear_out_1_bias, num_rois=self.first_n, index=1, group=self.nms_attention_group, dim=self.nms_attention_dim, fc_dim=self.nms_attention_fc_dim, feat_dim=self.nms_attention_feat_dim) nms_all_feat_1 = nms_embedding_feat + nms_attention_1 nms_all_feat_1_relu = nd.Activation(data=nms_all_feat_1, act_type='relu', name='nms_all_feat_1_relu') # [first_n * num_valid_classes, 1024] nms_all_feat_1_relu_reshape = nd.Reshape(nms_all_feat_1_relu, shape=(-3, -2)) # logit, [first_n * num_valid_classes, num_thresh] nms_conditional_logit = nd.FullyConnected( name='nms_logit', data=nms_all_feat_1_relu_reshape, num_hidden=self.num_thresh, weight=nms_logit_weight, bias=nms_logit_bias) # logit_reshape, [first_n, num_valid_classes, num_thresh] nms_conditional_logit_reshape = nd.Reshape(nms_conditional_logit, shape=(self.first_n, num_valid_classes, self.num_thresh)) nms_conditional_score = nd.Activation( data=nms_conditional_logit_reshape, act_type='sigmoid', name='nms_conditional_score') if num_valid_classes == self.num_fg_classes: full_nms_conditional_score = nms_conditional_score else: full_nms_conditional_score = nd.concat( nms_conditional_score, nd.zeros( (self.first_n, self.num_fg_classes - num_valid_classes, self.num_thresh), ctx=nms_conditional_score.context), dim=1) all_indexes = np.concatenate( (valid_class_indices, invalid_class_indices)) restore_indexes = np.zeros((self.num_fg_classes)) restore_indexes[all_indexes] = np.arange(self.num_fg_classes) restore_indexes = nd.array(restore_indexes, ctx=nms_conditional_score.context) full_nms_conditional_score = full_nms_conditional_score.transpose( (1, 0, 2)).take(restore_indexes).transpose((1, 0, 2)) sorted_score_reshape = nd.expand_dims(sorted_score, axis=2) # sorted_score_reshape = nd.BlockGrad(sorted_score_reshape) nms_multi_score = nd.broadcast_mul(lhs=sorted_score_reshape, rhs=full_nms_conditional_score) _ = nms_multi_score.mean().asnumpy() all_time = time.time() - nms_start_time if 'learn_nms_time' not in globals().keys( ) or 'learn_nms_count' not in globals().keys(): globals()['learn_nms_time'] = [] globals()['learn_nms_count'] = 0 if globals()['learn_nms_count'] >= 1000: globals()['learn_nms_time'].pop(0) globals()['learn_nms_time'].append(all_time) else: globals()['learn_nms_time'].append(all_time) globals()['learn_nms_count'] += 1 if globals()['learn_nms_count'] % 250 == 0: print("--->> learn nms running average time cost: {}".format( float(sum(globals()['learn_nms_time'])) / (1000 if globals()['learn_nms_count'] > 1000 else globals()['learn_nms_count']))) self.assign(out_data[0], req[0], nms_multi_score) self.assign(out_data[1], req[1], sorted_bbox) self.assign(out_data[2], req[2], sorted_score)
def median(epoch, gradients, net, lr, byz, old_direction, active, blacklist, susp, f=0, cmax=0, utrg=0.0, udet=0.50, urem=3): param_list = [ nd.concat(*[xx.reshape((-1, 1)) for xx in x], dim=0) for x in gradients ] param_list = byz(epoch, param_list, net, f, lr, active) flip_local = nd.zeros(len(param_list)) penalty = 1.0 - cmax / len(param_list) reward = 1.0 - penalty for i in range(len(param_list)): direction = nd.sign(param_list[i]) flip_local[i] = 0.5 * (mx.nd.sum( direction.reshape(-1) * (direction.reshape(-1) - old_direction.reshape(-1)))).asscalar() argsorted = nd.argsort(flip_local) susp[argsorted[:-cmax]] = susp[argsorted[:-cmax]] - reward susp[argsorted[-cmax:]] = susp[argsorted[-cmax:]] + penalty new_list = [] argsorted = nd.argsort(susp) for i in range(len(param_list) - cmax): new_list.append(param_list[int(argsorted[i].asscalar())]) sorted_array = nd.sort(nd.concat(*new_list, dim=1), axis=-1) if (len(new_list) % 2 == 1): trim_nd = sorted_array[:, int(len(new_list) / 2)] else: trim_nd = (sorted_array[:, int(len(new_list) / 2) - 1] + sorted_array[:, int(len(new_list) / 2)]) / 2 global_direction = nd.sign(trim_nd) gfs = 0.5 * (mx.nd.sum( global_direction.reshape(-1) * (global_direction.reshape(-1) - old_direction.reshape(-1))) ).asscalar() '''if (utrg > 0): sorted_array = nd.sort(nd.concat(*param_list, dim=1), axis=-1) n = len(param_list) m = n - f*2 if (len(param_list)%2 == 1): trim_nd = sorted_array[:, int(len(param_list)/2)] else: trim_nd = (sorted_array[:, int(len(param_list)/2)-1] + sorted_array[:, int(len(param_list)/2)])/2 direction = nd.sign(trim_nd) gfs = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar() if ((utrg>0 and gfs>=utrg*len(param_list[0])) or (utrg == 0)): flip_score = mx.nd.zeros(len(param_list)) rem = [] for i in range (len(param_list)): direction = nd.sign(param_list[i]) flip_score[i] = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar() flip_local[active[i]] = flip_score[i].asscalar() argsorted = nd.argsort(flip_score) new_list = [] for i in range(len(param_list)-cmax): new_list.append(param_list[int(argsorted[i].asscalar())]) n = len(new_list) f = 0 sorted_array = nd.sort(nd.concat(*new_list, dim=1), axis=-1) if (len(new_list)%2 == 1): trim_nd = sorted_array[:, int(len(new_list)/2)] else: trim_nd = (sorted_array[:, int(len(new_list)/2)-1] + sorted_array[:, int(len(new_list)/2)])/2 for i in range(len(new_list), len(param_list)): index = int(argsorted[i].asscalar()) if (flip_score[index] >= udet*len(param_list[0])): susp[active[index]] = susp[active[index]] + 1 if (susp[active[index]] >= urem): blacklist[active[index]] = 1 rem.append(active[index]) active = removearr(active, sorted(rem), len(param_list)) direction = nd.sign(trim_nd) gfs = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar() cmax = cmax - len(rem) ''' idx = 0 for j, (param) in enumerate(net.collect_params().values()): if param.grad_req == 'null': continue param.set_data( param.data() - lr * trim_nd[idx:(idx + param.data().size)].reshape(param.data().shape)) idx += param.data().size return trim_nd, direction, cmax, gfs, flip_local
def prune_by_percent(self, p): criterion = self._compute_criterion() th_idx = nd.argsort(criterion)[int(p * self._channels)] self.mask = (criterion >= th_idx).reshape(1, -1, 1, 1)
def score(x, y): _sim = nd.dot(x, y, transpose_b=True) _indices = nd.argsort(-_sim, axis=1) return _sim, _indices[:, 1: 100]