def tesselate(x, leaf, p_tau):
if (len(x) < 2):
return
add_split(x, leaf, p_tau)
split = leaf.parent['decision']
node = leaf.parent['node']
side = nd.sign(split.split(x))
order = nd.argsort(side, axis=None)
x = x[order, :]
side = side[order, :]
orderside = nd.argsort(side, axis=0) * side
cutpt = nd.argsort(orderside, axis=None,
dtype='int32')[0].asscalar() + 1
x_l = x[0:cutpt]
x_r = x[cutpt:None]
leaf.parent['side'] = 0
new_leaf = Leaf(layer=tree.layer_initializer(),
node=leaf.parent['node'],
decision=leaf.parent['decision'],
side=1)
tree.leaves.add(new_leaf)
add_node(x_l, leaf)
add_node(x_r, new_leaf)
node.child['left'] = leaf.parent['node']
node.child['right'] = new_leaf.parent['node']
tesselate(x_l, leaf, split.tau.data())
tesselate(x_r, new_leaf, split.tau.data())
def sample(match, cls_pred, iou, ratio=3, min_sample=0, threshold=0.5, do=True):
if do is False:
ones = nd.ones_like(match)
sample = nd.where(match > -0.5, ones, ones*-1)
return sample
sample = nd.zeros_like(match)
num_pos = nd.sum(match > -0.5, axis=-1)
requre_neg = ratio * num_pos
neg_mask = nd.where(match < -0.5, nd.max(iou, axis=-1) < threshold, sample)
max_neg = neg_mask.sum(axis=-1)
num_neg = nd.minimum(max_neg, nd.maximum(requre_neg, min_sample)).astype('int')
neg_prob = cls_pred[:,:,0]
max_value = nd.max(cls_pred, axis=-1, keepdims=True)
score = max_value[:,:,0] - neg_prob + nd.log(
nd.sum(
nd.exp(cls_pred-max_value), axis=-1))
score = nd.where(neg_mask, score, nd.zeros_like(score))
argmax = nd.argsort(score, axis=-1, is_ascend=False)
sample = nd.where(match > -0.5, nd.ones_like(sample), sample)
for i, num in enumerate(num_neg):
sample[i, argmax[i,:num.asscalar()]] = -1
return sample
def evaluate_emb(emb, labels):
"""Evaluate embeddings based on Recall@k."""
d_mat = get_distance_matrix(emb)
#d_mat = d_mat.asnumpy()
#labels = labels.asnumpy() #directory operate on mxnet.ndarray if convert to numpy,would cause memeory error
names = []
accs = []
for i in range(emb.shape[0]):
d_mat[i,i]=1e10
index_mat = nd.argsort(d_mat)
nd.waitall()
if opt.use_viz:
viz.log("nd all dist mat")
for k in [1, 2, 4, 8, 16]:
names.append('Recall@%d' % k)
correct, cnt = 0.0, 0.0
index_mat_part = index_mat[:,:k]
for i in range(emb.shape[0]):
if any(labels[i] == labels[nn] for nn in index_mat_part[i]):
correct +=1
cnt +=1
# for i in range(emb.shape[0]):
# d_mat[i, i] = 1e10
# nns = argpartition(d_mat[i], k)[:k]
# if any(labels[i] == labels[nn] for nn in nns):
# correct += 1
# cnt += 1
accs.append(correct/cnt)
return names, accs
def prune_weights(sign_to_noise_vec, prediction_vector, percentages):
pruning_indices = nd.argsort(sign_to_noise_vec, axis=0)
for percentage in percentages:
prediction_vector = mus_copy_vec.copy()
pruning_indices_percent = pruning_indices[0:int(len(pruning_indices)*percentage)]
for pr_ind in pruning_indices_percent:
prediction_vector[int(pr_ind.asscalar())] = 0
pruned_weights = restore_weight_structure(prediction_vector)
test_accuracy = evaluate_accuracy(test_data, net, pruned_weights)
print("%s --> %s" % (percentage, test_accuracy))
def _shard(split, x, l_fn, r_fn):
splitsortorder = nd.argsort(split, axis=None)
reorderedx = x[splitsortorder, :]
reorderedsplit = split[splitsortorder]
if (reorderedsplit[0] > 0):
r_fn(reorderedx)
elif (reorderedsplit[-1] < 0):
l_fn(reorderedx)
else:
splitpt = nd.argsort(reorderedsplit,
axis=0) * nd.sign(reorderedsplit)
splitpt = nd.argsort(splitpt, axis=None)[0] + 1
lx = nd.slice_axis(reorderedx, 0, 0, int(splitpt.asscalar()))
rx = nd.slice_axis(reorderedx, 0, int(splitpt.asscalar()),
None)
l_fn(lx)
r_fn(rx)
def recurse(x, node, p_tau):
n = 1
mean = node.center.data()
var = (0.5 * node.radius.data())**2
N = x.shape[0]
x_mean = nd.mean(x, axis=0)
x_var = (N**-1) * nd.sum((x - x_mean)**2, axis=0)
z_mean = (n * mean + N * x_mean) / (n + N)
z_var = ((n * (mean + var) + N * (x_mean + x_var)) / (n + N)) - z_mean
z_radius = 2 * (nd.max(z_var)**0.5)
node.center.set_data(z_mean)
node.radius.set_data(z_radius)
if node.child['decision'] is None:
leaf = next(l for l in tree.leaves if l.parent['node'] == node)
tesselate(x, leaf, p_tau)
return
E = nd.random.exponential(z_radius**-1)
node.child['decision'].tau.set_data(p_tau + E)
split = node.child['decision']
side = nd.sign(split.split(x))
order = nd.argsort(side, axis=None)
x = x[order, :]
side = side[order, :]
if side[0] > 0:
recurse(x, node.child['right'], split.tau.data())
elif side[-1] < 0:
recurse(x, node.child['left'], split.tau.data())
else:
orderside = nd.argsort(side, axis=0) * side
cutpt = nd.argsort(orderside, axis=None,
dtype='int32')[0].asscalar() + 1
x_l = x[0:cutpt]
x_r = x[cutpt:None]
recurse(x_l, node.child['left'], split.tau.data())
recurse(x_r, node.child['right'], split.tau.data())
def full_krum(epoch, v, net, f, lr, active, max_flip=1.0):
if (f == 0):
return v
e = 0.00001 / len(v[0])
avg_grads = nd.sum(nd.concat(*v, dim=1), axis=-1, keepdims=True)
direction = nd.sign(avg_grads)
topk = nd.argsort(nd.abs(avg_grads).reshape(-1))
n_flips = int(max_flip * len(v[0]))
current_f = len(np.where(np.where(active < f)[0] < f)[0])
l_max = lambda_max(epoch, v, net, current_f, lr)
l = find_lambda(l_max, v, direction, len(v), current_f, lr, topk, max_flip)
print(l)
if (l > 0 and active[0] < f):
v[0][topk[-n_flips:]] = -(direction[topk[-n_flips:]] * l) / lr
for i in range(1, f):
if (active[i] < f):
v[i] = mx.nd.random.uniform(v[0] - e, v[0] + e)
return v
def getUniqueMatch(iou, min_threshold=1e-12):
N, M = iou.shape
iouf = iou.reshape((-1,))
argmax = nd.argsort(iouf, is_ascend=False)
argrow = nd.floor(nd.divide(argmax, M))
argcol = nd.modulo(argmax, M)
uniquel = set()
uniquer = set()
match = nd.ones((N,)) * -1
i = 0
while True:
if argcol[i].asscalar() not in uniquel and argrow[i].asscalar() not in uniquer:
uniquel.add(argcol[i].asscalar())
uniquer.add(argrow[i].asscalar())
if iou[argrow[i], argcol[i]] > min_threshold:
match[argrow[i]] = argcol[i]
if len(uniquel) == M or len(uniquer) == N:
break
i += 1
return match.reshape((1,-1))
def validate(net, val_data, val_items, val_shapes, ctx, size, classes):
"""Test on validation dataset."""
clipper = gcv.nn.bbox.BBoxClipToImage()
net.hybridize(static_alloc=True)
print("---Detect Total {:d} Image Start.---".format(len(val_items)))
result_dict = {}
for ib, (batch, item) in enumerate(zip(val_data, val_items)):
batch = split_and_load(batch, ctx_list=ctx)
for x, y, im_scale in zip(*batch):
ids, scores, bboxes = net(x)
bboxes = clipper(bboxes, x)
im_scale = im_scale.reshape((-1)).asscalar()
bboxes *= im_scale
inds = nd.argsort(nd.squeeze(ids, axis=(0, 2)), is_ascend=False)
ids = nd.squeeze(ids,
axis=(0, 2)).asnumpy().astype(np.int8).tolist()
valid_ids = [id for id in ids if id is not -1]
valid_len = len(valid_ids)
if valid_len > 0: # valid_len must > 0
inds = nd.slice_axis(inds, begin=0, end=valid_len, axis=0)
scores = nd.take(scores, inds, axis=1)
bboxes = nd.take(bboxes, inds, axis=1)
scores = scores.asnumpy()
bboxes = bboxes.asnumpy()
for i, id in enumerate(valid_ids):
score = scores[:, i, 0][0]
xmin, ymin, xmax, ymax = bboxes[:, i, 0][
0], bboxes[:, i, 1][0], bboxes[:, i,
2][0], bboxes[:, i,
3][0]
result_dict[id] = result_dict.get(
id, []) + [[item, score, xmin, ymin, xmax, ymax]]
print("Detect Image {:s} Done.".format(item))
print("---Detect Total {:d} Image Done.---".format(len(val_items)))
return result_dict
def generate_targets(self, img, boxes):
"""
img : [H, W, 3]
boxes : [N, 5]
"""
rh, rw, _ = img.shape
rh, rw = int(rh/4), int(rw/4)
rx = nd.arange(0, rw).reshape((1, -1))
ry = nd.arange(0, rh).reshape((-1, 1))
sx = nd.tile(rx, reps=(rh, 1))
sy = nd.tile(ry, reps=(1, rw))
x0, y0, x1, y1, _ = nd.split(boxes, 5, axis=-1, squeeze_axis=True)
areas = (x1 - x0) * (y1 - y0)
# areas = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])
boxes_id = nd.argsort(areas)
boxes_id = nd.concat(nd.array([-1]), boxes_id, dim=0)
boxes = nd.take(boxes, nd.argsort(areas)) # min -> max
boxes = nd.concat(nd.zeros((1, 5)), boxes, dim=0) # for gt assign confusion
x0, y0, x1, y1, cls = nd.split(boxes, num_outputs=5, axis=-1, squeeze_axis=True)
n = boxes.shape[0]
# [H, W, N]
of_l = sx.reshape(-2, 1) - nd.expand_dims(nd.expand_dims(x0/4, axis=0), axis=0)
of_t = sy.reshape(-2, 1) - nd.expand_dims(nd.expand_dims(y0/4, axis=0), axis=0)
of_r = -(sx.reshape(-2, 1) - nd.expand_dims(nd.expand_dims(x1/4, axis=0), axis=0))
of_b = -(sy.reshape(-2, 1) - nd.expand_dims(nd.expand_dims(y1/4, axis=0), axis=0))
# [H, W, N]
eps = 1e-5
# ctr = nd.minimum(of_l, of_r) / (nd.maximum(of_l, of_r) + eps) * \
# nd.minimum(of_t, of_b) / (nd.maximum(of_t, of_b) + eps)
# ctr = nd.sqrt(nd.abs(ctr))
# ctr[:, :, 0] = 0
# # flat ctr
of_l = of_l * (of_l > 0)
of_r = of_r * (of_r > 0)
of_t = of_t * (of_t > 0)
of_b = of_b * (of_b > 0)
# ctr2 = nd.minimum(of_l, of_r) / (nd.maximum(of_l, of_r) + of_l + of_r) * \
# nd.minimum(of_t, of_b) / (nd.maximum(of_t, of_b) + of_t + of_b)
# ctr2 = 3 * nd.sqrt(nd.abs(ctr2))
# ctr2[:, :, 0] = 0
# slim ctr
# ctr = nd.minimum(of_l, of_r) / (nd.maximum(of_l, of_r) + nd.abs(of_l - of_r) + eps) * \
# nd.minimum(of_t, of_b) / (nd.maximum(of_t, of_b) + nd.abs(of_t - of_b) + eps)
ctr = nd.minimum(of_l, of_r) / (nd.maximum(of_l, of_r) + eps) * \
nd.minimum(of_t, of_b) / (nd.maximum(of_t, of_b) + eps)
# ctr = nd.power(0.8, 0.1 * nd.sqrt(nd.square(of_l - of_r) + nd.square(of_t - of_b) + eps))
# ctr = nd.power(0.8, nd.sqrt(nd.abs(of_l - of_r) + nd.abs(of_t - of_b) + eps))
ctr = nd.sqrt(nd.abs(ctr))
ctr[:, :, 0] = 0
# [H, W, N, 4]
offsets = nd.concat(of_l.reshape(-2, 1), of_t.reshape(-2, 1),
of_r.reshape(-2, 1), of_b.reshape(-2, 1), dim=-1) * 4.
fh = int(np.ceil(rh / 2))
fw = int(np.ceil(rw / 2))
# fh = int(np.ceil(np.ceil(np.ceil(rh / 2) / 2) / 2))
# fw = int(np.ceil(np.ceil(np.ceil(rw / 2) / 2) / 2))
fm_list = []
for i in range(self._stages):
fm_list.append((fh, fw))
fh = int(np.ceil(fh / 2))
fw = int(np.ceil(fw / 2))
fm_list = fm_list[::-1]
cls_targets = []
ctr_targets = []
box_targets = []
match_targets = []
stride = int(self._stride/4)
for i in range(self._stages):
fh, fw = fm_list[i]
# cls_target = nd.zeros((fh, fw))
# box_target = nd.zeros((fh, fw, 4))
# ctr_target = nd.zeros((fh, fw))
# match_target = nd.zeros((fh, fw))
cx = nd.arange(0, fw).reshape((1, -1))
cy = nd.arange(0, fh).reshape((-1, 1))
sx = nd.tile(cx, reps=(fh, 1))
sy = nd.tile(cy, reps=(1, fw))
syx = nd.stack(sy.reshape(-1), sx.reshape(-1)).transpose().astype('int32')
# bugs in this type
# bx = sxy[:, 0] * stride + nd.floor(sxy[:, 0] / 2).astype(np.int32)
# by = sxy[:, 1] * stride + nd.floor(sxy[:, 1] / 2).astype(np.int32)
by, bx = nd.split(syx*stride, 2, axis=-1, squeeze_axis=True)
# by = syx[:, 0] * stride
# bx = syx[:, 1] * stride
# [FH*FW, N, 4]
of_byx = nd.take(offsets.reshape((-1, n, 4)), by*740/4+bx)
of_ctr = nd.take(ctr.reshape((-1, n)), by*740/4 + bx)
# of_byx = offsets[by, bx]
# ctr_aware = ctr[by, bx]
# of_byx = nd.gather_nd(offsets, indices=byx.transpose())
min_vr, max_vr = self._valid_range[i]
# [FH*FW, N]
is_in_box = nd.prod(of_byx > 0, axis=-1)
is_valid_area = (of_byx.max(axis=-1) >= min_vr) * (of_byx.max(axis=-1) <= max_vr)
# [FH*FW, N]
valid_pos = nd.elemwise_mul(is_in_box, is_valid_area) * of_ctr
# valid_pos = nd.elemwise_mul(is_in_box, is_valid_area)
# of_valid = nd.zeros((fh, fw, n))
# of_valid[syx[:, 0], syx[:, 1], :] = valid_pos * ctr_aware # 1, 0
of_valid = valid_pos.reshape((fh, fw, n))
of_valid[:, :, 0] = 0
# [FH, FW]
# gt_inds = nd.argmax(of_valid, axis=-1)
gt_inds = nd.argmax(of_valid, axis=-1).reshape(-1)
# box targets
box_target = nd.take(boxes, gt_inds).slice_axis(begin=0, end=4, axis=-1)
# box_target[syx[:, 0], syx[:, 1]] = boxes[gt_inds[syx[:, 0], syx[:, 1]], :4]
# box_target = box_target.reshape(-1, 4)
# cls targets
cls_target = nd.take(cls, gt_inds)
# cls_target[syx[:, 0], syx[:, 1]] = cls[gt_inds[syx[:, 0], syx[:, 1]]]
# cls_target = cls_target.reshape(-1)
# match targets the number of matches less than ctr targets
# match_gt_inds = nd.argmax(of_valid * (of_valid > 0.01), axis=-1).reshape(-1)
match_target = nd.take(boxes_id, gt_inds)
# match_target[syx[:, 0], syx[:, 1]] = boxes_id[match_gt_inds[syx[:,0], syx[:,1]]]
# match_target = match_target.reshape(-1)
# ctr targets
ctr_target = nd.pick(of_ctr, gt_inds)
# ctr_target[syx[:, 0], syx[:, 1]] = ctr[by, bx, gt_inds[syx[:, 0], syx[:, 1]]]
# ctr_target = ctr_target.reshape(-1)
box_targets.append(box_target)
cls_targets.append(cls_target)
ctr_targets.append(ctr_target)
stride = int(stride / 2)
match_targets.append(match_target)
box_targets = nd.concat(*box_targets, dim=0)
cls_targets = nd.concat(*cls_targets, dim=0)
ctr_targets = nd.concat(*ctr_targets, dim=0)
match_targets = nd.concat(*match_targets, dim=0)
return cls_targets, ctr_targets, box_targets, match_targets
def trim(epoch,
gradients,
net,
lr,
byz,
old_direction,
active,
blacklist,
susp,
f=0,
cmax=0,
utrg=0.0,
udet=0.50,
urem=3):
param_list = [
nd.concat(*[xx.reshape((-1, 1)) for xx in x], dim=0) for x in gradients
]
param_list = byz(epoch, param_list, net, f, lr, active)
flip_local = nd.zeros(len(param_list))
flip_new = nd.zeros(len(param_list))
penalty = 1.0 - cmax / len(param_list)
reward = 1.0 - penalty
for i in range(len(param_list)):
direction = nd.sign(param_list[i])
flip_local[i] = 0.5 * (mx.nd.sum(
direction.reshape(-1) *
(direction.reshape(-1) - old_direction.reshape(-1)))).asscalar()
#flip = nd.sign(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))
#flip_new[i] = nd.sum(flip*(param_list[i].reshape(-1)**2))
#flip[param_list[i]<0.0001] = 0
#flip_new[i] = nd.sum(flip).asscalar()
#argsorted = nd.argsort(flip_local)
argsorted = nd.argsort(flip_local)
if (cmax > 0):
susp[argsorted[:-cmax]] = susp[argsorted[:-cmax]] + reward
susp[argsorted[-cmax:]] = susp[argsorted[-cmax:]] - penalty
argsorted = nd.argsort(susp)
weights = nd.exp(susp) / nd.sum(nd.exp(susp))
matrix = nd.transpose(
nd.transpose(nd.concat(*[ii for ii in param_list], dim=1)))
trim_nd = nd.linalg.gemm2(matrix, weights.reshape(-1, 1))
#pdb.set_trace()
#print (flip_new, weights)
#print (nd.mean(flip_local[:cmax]), nd.mean(flip_new[:cmax]), nd.mean(flip_local[cmax:]), nd.mean(flip_new[cmax:]))
'''new_list = []
argsorted = nd.argsort(susp)
for i in range(len(param_list)-cmax):
new_list.append(param_list[int(argsorted[i].asscalar())])
sorted_array = nd.sort(nd.concat(*new_list, dim=1), axis=-1)
trim_nd = nd.mean(sorted_array, axis=-1, keepdims=1)'''
global_direction = nd.sign(trim_nd)
gfs = 0.5 * (mx.nd.sum(
global_direction.reshape(-1) *
(global_direction.reshape(-1) - old_direction.reshape(-1)))
).asscalar()
'''if (utrg > 0):
sorted_array = nd.sort(nd.concat(*param_list, dim=1), axis=-1)
n = len(param_list)
m = n - f*2
trim_nd = nd.mean(sorted_array[:, f:(f+m)], axis=-1, keepdims=1)
direction = nd.sign(trim_nd)
gfs = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar()
if ((utrg>0 and gfs>=utrg*len(param_list[0])) or (utrg == 0)):
flip_score = mx.nd.zeros(len(param_list))
rem = []
for i in range (len(param_list)):
direction = nd.sign(param_list[i])
flip_score[i] = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar()
flip_local[active[i]] = flip_score[i].asscalar()
argsorted = nd.argsort(flip_score)
new_list = []
for i in range(len(param_list)-cmax):
new_list.append(param_list[int(argsorted[i].asscalar())])
n = len(new_list)
f = 0
m = n - f*2
sorted_array = nd.sort(nd.concat(*new_list, dim=1), axis=-1)
trim_nd = nd.mean(sorted_array[:, f:(f+m)], axis=-1, keepdims=1)
for i in range(len(new_list), len(param_list)):
index = int(argsorted[i].asscalar())
if (flip_score[index] >= udet*len(param_list[0])):
susp[active[index]] = susp[active[index]] + 1
if (susp[active[index]] >= urem):
blacklist[active[index]] = 1
rem.append(active[index])
active = removearr(active, sorted(rem), len(param_list))
direction = nd.sign(trim_nd)
gfs = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar()
cmax = cmax - len(rem) '''
idx = 0
for j, (param) in enumerate(net.collect_params().values()):
if param.grad_req == 'null':
continue
param.set_data(
param.data() - lr *
trim_nd[idx:(idx + param.data().size)].reshape(param.data().shape))
idx += param.data().size
return trim_nd, direction, cmax, gfs, flip_local, flip_new
def test_argsort():
b = create_2d_tensor(rows=LARGE_X, columns=SMALL_Y)
s = nd.argsort(b, axis=0, is_ascend=False, dtype=np.int64)
mx.nd.waitall()
assert (s[0].asnumpy() == (LARGE_X - 1)).all()
def test_argsort():
a = create_vector(size=LARGE_X)
s = nd.argsort(a, axis=0, is_ascend=False, dtype=np.int64)
assert s[0] == (LARGE_X - 1)
def MultiBoxTarget(anchors,
class_predictions,
labels,
hard_neg_ratio=3,
ctx=gpu(),
verbose=False):
'''將真實方框(ground truth boxes)和預設錨框(default anchor boxes)做配對。
labels: 真實方框 (ground truth boxes)。
anchors: 預設錨框 (default anchor boxes)。
hard_neg_ratio: 負樣本(背景)和正樣本(有物體的錨框數)的比例。預設是3:1。
'''
if verbose:
print("anchors shape=", anchors.shape)
assert anchors.shape[0] == 1
batch_size = len(labels)
num_priors = anchors.shape[1]
if verbose:
print("batch size=\t", batch_size)
print("num priors=\t", num_priors)
anchor_shifts = nd.zeros((batch_size, anchors.shape[1] * 4), ctx=ctx)
box_mask = nd.zeros((batch_size, anchors.shape[1] * 4), ctx=ctx)
anchor_classes = nd.zeros((batch_size, num_priors), ctx=ctx) - 1
anchor_indices = nd.zeros((batch_size, num_priors), ctx=ctx) - 1
classes_mask = nd.zeros((batch_size, anchors.shape[1]), ctx=ctx)
shifts_tmp = nd.zeros_like(anchors[0])
mask_tmp = nd.zeros_like(anchors[0])
for i in range(batch_size):
label_locs = labels[i][:, 1:]
label_classes = labels[i][:, 0]
class_preds = class_predictions[i]
# obtain IoU
ious = iou(label_locs, anchors[0])
# identify how many ground truth objects are there in this batch
if -1 in label_classes:
num_obj = label_classes.argmin(axis=0).astype("int32").asscalar()
else:
num_obj = label_classes.size
if num_obj == 0:
continue
# matching anchor boxes with ground truth boxes
ious_flag = ious > 0.5
# find locations of the best priors
best_prior_idx = ious[0:num_obj, :].argmax(axis=1)
idx_row = [*range(best_prior_idx.size)]
ious_flag[idx_row, best_prior_idx] += 1
if_negative = label_classes != -1
label_classes = label_classes + if_negative * 1
# add the -1 class to the end
label_classes = nd.concat(label_classes,
nd.array([-1], ctx=label_classes.context),
dim=0)
if verbose:
print("label classes=\t", label_classes)
if_matched = ious_flag.sum(axis=0) != 0
label_classes_last_idx = len(label_classes) - 1
anchor_indices[i] = (ious_flag.argmax(axis=0) - label_classes_last_idx
) * if_matched + label_classes_last_idx
anchor_classes[i] = label_classes[anchor_indices[i]]
# find indices of the negative anchors
neg_indices, = np.where(anchor_classes[i].asnumpy() == -1)
# count number of positive/negative/hard negative anchors
num_neg_anchors = len(neg_indices)
num_pos_anchors = num_priors - num_neg_anchors
num_hard_neg_anchors = num_pos_anchors * hard_neg_ratio
# hard negative mining
#conf_loss_indices = nd.argsort( (-nd.softmax( class_preds[neg_indices] ) )[:,0], is_ascend=False ).astype("int32")
conf_loss_indices = nd.argsort((-class_preds[neg_indices])[:, 0],
is_ascend=False).astype("int32")
neg_indices_sorted = nd.array(neg_indices,
ctx=class_preds.context,
dtype="int32")[conf_loss_indices]
hard_neg_indices = neg_indices_sorted[:num_hard_neg_anchors]
anchor_classes[i][hard_neg_indices] = 0
# find indices of the positive anchors
pos_indices, = np.where(
anchor_indices[i].asnumpy() < label_classes_last_idx)
pos_indices = nd.array(pos_indices,
ctx=hard_neg_indices.context).astype("int32")
cls_indices = nd.concatenate([hard_neg_indices, pos_indices
]) # store indices for classification.
classes_mask[i][cls_indices] = 1
if verbose:
print("========================================")
display(
pd.DataFrame(anchor_classes[i].asnumpy()).groupby(0).indices)
df = pd.DataFrame(
anchor_classes[i].asnumpy()).reset_index().groupby(0).count()
df.index.name = "class"
df.index = df.index.astype("int32")
df.columns = ["number of default anchor boxes"]
display(df)
print("========================================")
# obtain locations of the positve anchors
pos_anchors_loc = anchors[0][pos_indices]
# obtain indices of the ground truth labels
idx_gt = anchor_indices[i][pos_indices]
# obtain locations of the ground truth labels
labels_loc = label_locs[idx_gt]
assert len(pos_anchors_loc) == len(labels_loc)
# calculate location differences between ground truth labels and positive anchors
shifts_tmp[pos_indices] = loc_difference_calculator(
pos_anchors_loc, labels_loc)
mask_tmp[pos_indices] = 1.
anchor_shifts[i] = shifts_tmp.reshape(-1)
box_mask[i] = mask_tmp.reshape(-1)
return anchor_shifts, box_mask, anchor_classes, classes_mask
def generate_targets(self, img, boxes):
"""
img : [H, W, 3]
boxes : [N, 5]
"""
rh, rw, _ = img.shape
rx = nd.arange(0, rw).reshape((1, -1))
ry = nd.arange(0, rh).reshape((-1, 1))
sx = nd.tile(rx, reps=(rh, 1))
sy = nd.tile(ry, reps=(1, rw))
areas = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])
boxes = boxes[nd.argsort(areas)]
boxes = nd.concat(nd.zeros((1, 5)), boxes, dim=0) # for gt assign confusion
x0, y0, x1, y1, cls = nd.split(boxes, num_outputs=5, axis=-1, squeeze_axis=True)
n = boxes.shape[0]
# [H, W, N]
of_l = sx.reshape(-2, 1) - nd.expand_dims(nd.expand_dims(x0, axis=0), axis=0)
of_t = sy.reshape(-2, 1) - nd.expand_dims(nd.expand_dims(y0, axis=0), axis=0)
of_r = -(sx.reshape(-2, 1) - nd.expand_dims(nd.expand_dims(x1, axis=0), axis=0))
of_b = -(sy.reshape(-2, 1) - nd.expand_dims(nd.expand_dims(y1, axis=0), axis=0))
# [H, W, N]
eps = 1e-5
ctr =(nd.minimum(of_l, of_r) / nd.maximum(of_l, of_r)) * \
(nd.minimum(of_t, of_b) / nd.maximum(of_t, of_b) + eps)
ctr = nd.sqrt(nd.abs(ctr))
ctr[:, :, 0] = 0
# [H, W, N, 4]
offsets = nd.concat(of_l.reshape(-2, 1), of_t.reshape(-2, 1),
of_r.reshape(-2, 1), of_b.reshape(-2, 1), dim=-1)
# fh = int(np.ceil(((rh + 1) / 2) // 2 / 2))
# fw = int(np.ceil(((rw + 1) / 2) // 2 / 2))
fh = int(np.ceil(np.ceil(np.ceil(rh / 2) / 2) / 2))
fw = int(np.ceil(np.ceil(np.ceil(rw / 2) / 2) / 2))
fm_list = []
for i in range(self._stages):
fm_list.append((fh, fw))
fh = int(np.ceil(fh / 2))
fw = int(np.ceil(fw / 2))
fm_list = fm_list[::-1]
cls_targets = []
ctr_targets = []
box_targets = []
cor_targets = []
stride = self._stride
for i in range(self._stages):
fh, fw = fm_list[i]
cls_target = nd.zeros((fh, fw))
box_target = nd.zeros((fh, fw, 4))
ctr_target = nd.zeros((fh, fw))
cx = nd.arange(0, fw).reshape((1, -1))
cy = nd.arange(0, fh).reshape((-1, 1))
sx = nd.tile(cx, reps=(fh, 1))
sy = nd.tile(cy, reps=(1, fw))
syx = nd.stack(sy.reshape(-1), sx.reshape(-1)).transpose().astype('int32')
# bugs in this type
# bx = sxy[:, 0] * stride + nd.floor(sxy[:, 0] / 2).astype(np.int32)
# by = sxy[:, 1] * stride + nd.floor(sxy[:, 1] / 2).astype(np.int32)
by = syx[:, 0] * stride
bx = syx[:, 1] * stride
cor_targets.append(nd.stack(bx, by, axis=1))
# [FH*FW, N, 4]
of_byx = offsets[by, bx]
# of_byx = nd.gather_nd(offsets, indices=byx.transpose())
min_vr, max_vr = self._valid_range[i]
# [FH*FW, N]
is_in_box = nd.prod(of_byx > 0, axis=-1)
is_valid_area = (of_byx.max(axis=-1) >= min_vr) * (of_byx.max(axis=-1) <= max_vr)
# [FH*FW, N]
valid_pos = nd.elemwise_mul(is_in_box, is_valid_area)
of_valid = nd.zeros((fh, fw, n))
of_valid[syx[:, 0], syx[:, 1], :] = valid_pos # 1, 0
of_valid[:, :, 0] = 0
# [FH, FW]
gt_inds = nd.argmax(of_valid, axis=-1)
# box targets
box_target[syx[:, 0], syx[:, 1]] = boxes[gt_inds[syx[:, 0], syx[:, 1]], :4]
box_target = box_target.reshape(-1, 4)
# cls targets
cls_target[syx[:, 0], syx[:, 1]] = cls[gt_inds[syx[:, 0], syx[:, 1]]]
cls_target = cls_target.reshape(-1)
# ctr targets
ctr_target[syx[:, 0], syx[:, 1]] = ctr[by, bx, gt_inds[syx[:, 0], syx[:, 1]]]
ctr_target = ctr_target.reshape(-1)
box_targets.append(box_target)
cls_targets.append(cls_target)
ctr_targets.append(ctr_target)
stride = int(stride / 2)
box_targets = nd.concat(*box_targets, dim=0)
cls_targets = nd.concat(*cls_targets, dim=0)
ctr_targets = nd.concat(*ctr_targets, dim=0)
cor_targets = nd.concat(*cor_targets, dim=0)
cor_targets = cor_targets.astype('float32')
return cls_targets, ctr_targets, box_targets, cor_targets
def forward(self, features, image_shape, labels=None, bboxes=None):
"""
:param features: OrderedDict, each features: (B, C, H, W)
:param image_shape:
:param labels:
:param bboxes:
:return:
"""
anchors = OrderedDict() # each anchors:(B, H, W, num_anchors, 4)
pred_logits = OrderedDict() # each pred_logits:(B, H, W, num_anchors)
pred_bbox_deltas = OrderedDict(
) # each pred_bbox_deltas:(B, H, W, num_anchors, 4)
B = 0
device = cpu()
for k, feature in features.items():
if k in self.scales.keys():
B, C, H, W = feature.shape
feature = self.head(feature)
pred_logits[k] = self.object_cls(feature).transpose(axes=(0, 2,
3,
1))
device = pred_logits[k].context
pred_bbox_deltas[k] = self.object_reg(feature).transpose(
axes=(0, 2, 3, 1)).reshape(B, H, W, -1, 4)
anchors_per_sample = nd.array(self.generate_anchors(
feature.shape[-2:], image_shape, k),
ctx=device)
anchors[k] = nd.stack(*([
anchors_per_sample,
] * B), axis=0)
pred_logits = split_batch(pred_logits, B)
pred_bbox_deltas = split_batch(pred_bbox_deltas, B)
anchors = split_batch(anchors, B)
pred_logits = [_combine_keys(i) for i in pred_logits
] # list each pred_logits:(B*H*W*num_anchors,)
pred_bbox_deltas = [
_combine_keys(i) for i in pred_bbox_deltas
] # list each pred_bbox_deltas:(B*H*W*num_anchors,4)
anchors = [_combine_keys(i) for i in anchors
] # list each anchors:(B*H*W*num_anchors, 4)
object_bboxes_out = []
rpn_cls_losses = []
rpn_reg_losses = []
if not autograd.is_training():
for pred_logits_per_sample, pred_bbox_deltas_per_sample, anchors_per_sample in zip(
pred_logits, pred_bbox_deltas, anchors):
sorted_indices = nd.argsort(pred_logits_per_sample,
axis=0,
is_ascend=False)
sorted_indices = sorted_indices[
0:min(len(sorted_indices), self.pre_nms_top_n_in_test)]
pred_logits_per_sample = pred_logits_per_sample[sorted_indices]
pred_bbox_deltas_per_sample = pred_bbox_deltas_per_sample[
sorted_indices]
anchors_per_sample = anchors_per_sample[sorted_indices]
object_bboxes = bbox_decode(pred_bbox_deltas_per_sample,
anchors_per_sample)
object_bboxes = clip_bbox(object_bboxes, image_shape)
nms_input = nd.concat(
nd.arange(len(pred_bbox_deltas_per_sample),
ctx=device).reshape(-1,
1), # index of object bboxes
nd.sigmoid(pred_logits_per_sample).reshape(-1, 1),
object_bboxes,
dim=1)
outs = nd.contrib.box_nms(nms_input,
self.nms_thresh,
valid_thresh=0.05)
outs = outs[:min(len(outs), self.post_nms_top_n_in_test)]
indices, object_bboxes = outs[:, 0], outs[:, 2:6]
object_bboxes = nd.contrib.boolean_mask(
object_bboxes, indices != -1)
object_bboxes_out.append(object_bboxes)
return object_bboxes_out, None
for pred_logits_per_sample, pred_bbox_deltas_per_sample, anchors_per_sample, labels_per_sample, bboxes_per_sample in zip(
pred_logits, pred_bbox_deltas, anchors, labels, bboxes):
gt_mask = labels_per_sample != -1
labels_per_sample = nd.contrib.boolean_mask(
labels_per_sample, gt_mask)
bboxes_per_sample = nd.contrib.boolean_mask(
bboxes_per_sample, gt_mask)
sorted_indices = nd.argsort(pred_logits_per_sample,
axis=0,
is_ascend=False)
sorted_indices = sorted_indices[0:min(len(sorted_indices), self.
pre_nms_top_n_in_train)]
pred_logits_per_sample = pred_logits_per_sample[sorted_indices]
pred_bbox_deltas_per_sample = pred_bbox_deltas_per_sample[
sorted_indices]
anchors_per_sample = anchors_per_sample[sorted_indices]
object_bboxes = bbox_decode(pred_bbox_deltas_per_sample,
anchors_per_sample)
object_bboxes = clip_bbox(object_bboxes, image_shape)
nms_input = nd.concat(
nd.arange(len(pred_bbox_deltas_per_sample),
ctx=device).reshape(-1, 1), # index of object bboxes
nd.sigmoid(pred_logits_per_sample).reshape(-1, 1),
object_bboxes,
dim=1)
outs = nd.contrib.box_nms(nms_input,
self.nms_thresh,
valid_thresh=0.05)
outs = outs[:min(len(outs), self.post_nms_top_n_in_train)]
indices, object_bboxes = outs[:, 0], outs[:, 2:6]
valid_mask = indices != -1
indices = nd.contrib.boolean_mask(indices, valid_mask)
object_bboxes = nd.contrib.boolean_mask(object_bboxes, valid_mask)
object_bboxes_out.append(object_bboxes)
pred_logits_per_sample = pred_logits_per_sample[indices]
pred_bbox_deltas_per_sample = pred_bbox_deltas_per_sample[indices]
anchors_per_sample = anchors_per_sample[indices]
iou_matrix = nd.contrib.box_iou(anchors_per_sample,
bboxes_per_sample)
matched_indices = match_target(iou_matrix, self.fg_threshold)
object_bboxes_gt = bboxes_per_sample[matched_indices]
indices = np.arange(len(anchors_per_sample))
fg_mask = matched_indices >= 0
fg_indices = indices[fg_mask]
matched_indices = match_target(iou_matrix, self.bg_threshold)
bg_mask = matched_indices < 0
bg_indices = indices[bg_mask]
num_pos_samples = len(fg_indices)
batch_size = min(int(num_pos_samples / self.positive_fraction),
self.batch_size_per_image)
num_neg_samples = min(len(bg_indices),
batch_size - num_pos_samples)
bg_indices = bg_indices[:num_neg_samples]
assert len(fg_indices) > 0
pred_logits_batch = nd.concat(pred_logits_per_sample[fg_indices],
pred_logits_per_sample[bg_indices],
dim=0)
pred_labels_batch = nd.concat(nd.ones(len(fg_indices), ctx=device),
nd.zeros(len(bg_indices),
ctx=device),
dim=0)
rpn_cls_loss = self.object_cls_loss(
pred_logits_batch, pred_labels_batch).sum() / batch_size
rpn_cls_losses.append(rpn_cls_loss)
pred_bbox_deltas = pred_bbox_deltas_per_sample[fg_indices]
bbox_deltas = bbox_encode(object_bboxes_gt[fg_indices],
anchors_per_sample[fg_indices])
rpn_reg_loss = self.object_reg_loss(pred_bbox_deltas,
bbox_deltas).sum() / batch_size
rpn_reg_losses.append(rpn_reg_loss)
rpn_cls_loss = sum(rpn_cls_losses) / B
rpn_reg_loss = sum(rpn_reg_losses) / B
return object_bboxes_out, (rpn_cls_loss, rpn_reg_loss)
def test_argsort():
b = create_vector(size=LARGE_X)
s = nd.argsort(b, axis=0, is_ascend=False, dtype=np.int64)
assert (s[0].asnumpy() == (LARGE_X - 1)).all()
def krum(epoch,
gradients,
net,
lr,
byz,
old_direction,
active,
blacklist,
susp,
f=0,
cmax=0,
utrg=0,
udet=0.50,
urem=3,
max_flip=1.0):
param_list = [
nd.concat(*[xx.reshape((-1, 1)) for xx in x], dim=0) for x in gradients
]
param_list = byz(epoch, param_list, net, f, lr, active, max_flip)
flip_local = nd.zeros(len(param_list))
penalty = 1.0 - cmax / len(param_list)
reward = 1.0 - penalty
for i in range(len(param_list)):
direction = nd.sign(param_list[i])
flip_local[i] = 0.5 * (mx.nd.sum(
direction.reshape(-1) *
(direction.reshape(-1) - old_direction.reshape(-1)))).asscalar()
argsorted = nd.argsort(flip_local)
susp[argsorted[:-cmax]] = susp[argsorted[:-cmax]] - reward
susp[argsorted[-cmax:]] = susp[argsorted[-cmax:]] + penalty
new_list = []
argsorted = nd.argsort(susp)
for i in range(len(param_list) - cmax):
new_list.append(param_list[int(argsorted[i].asscalar())])
k = len(new_list) - 0 - 2
dist = mx.nd.zeros((len(new_list), len(new_list)))
for i in range(0, len(new_list)):
for j in range(0, i):
dist[i][j] = nd.norm(new_list[i] - new_list[j])
dist[j][i] = dist[i][j]
sorted_dist = mx.nd.sort(dist)
sum_dist = mx.nd.sum(sorted_dist[:, :k + 1], axis=1)
model_selected = argsorted[mx.nd.argmin(sum_dist).asscalar().astype(
int)].asscalar().astype(int)
global_direction = nd.sign(param_list[model_selected])
gfs = 0.5 * (mx.nd.sum(
global_direction.reshape(-1) *
(global_direction.reshape(-1) - old_direction.reshape(-1)))
).asscalar()
'''if (utrg > 0):
k = len(param_list) - f - 2
dist = mx.nd.zeros((len(param_list),len(param_list)))
for i in range (0, len(param_list)):
for j in range(0, i):
dist[i][j] = nd.norm(param_list[i] - param_list[j])
dist[j][i] = dist[i][j]
sorted_dist = mx.nd.sort(dist)
sum_dist = mx.nd.sum(sorted_dist[:,:k+1], axis=1)
model_selected = mx.nd.argmin(sum_dist).asscalar().astype(int)
direction = nd.sign(param_list[model_selected])
gfs = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar()
for i in range (len(param_list)):
direction = nd.sign(param_list[i])
flip_score = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar()
flip_local[active[i]] = flip_score
if ((utrg>0 and gfs>=utrg*len(param_list[0])) or (utrg == 0)):
flip_score = mx.nd.zeros(len(param_list))
rem = []
for i in range (len(param_list)):
direction = nd.sign(param_list[i])
flip_score[i] = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar()
flip_local[active[i]] = flip_score[i].asscalar()
argsorted = nd.argsort(flip_score)
new_list = []
for i in range(len(param_list)-cmax):
new_list.append(param_list[int(argsorted[i].asscalar())])
k = len(new_list) - 0 - 2
dist = mx.nd.zeros((len(new_list),len(new_list)))
for i in range (0, len(new_list)):
for j in range(0, i):
dist[i][j] = nd.norm(new_list[i] - new_list[j])
dist[j][i] = dist[i][j]
for i in range(len(new_list), len(param_list)):
index = int(argsorted[i].asscalar())
if (flip_score[index] >= udet*len(param_list[0])):
susp[active[index]] = susp[active[index]] + 1
if (susp[active[index]] >= urem):
blacklist[active[index]] = 1
rem.append(active[index])
active = removearr(active, sorted(rem), len(param_list))
sorted_dist = mx.nd.sort(dist)
sum_dist = mx.nd.sum(sorted_dist[:,:k+1], axis=1)
model_selected = argsorted[mx.nd.argmin(sum_dist).asscalar().astype(int)].asscalar().astype(int)
direction = nd.sign(param_list[model_selected])
gfs = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar()
cmax = cmax - len(rem) '''
idx = 0
for j, (param) in enumerate(net.collect_params().values()):
if param.grad_req == 'null':
continue
param.set_data(param.data() - lr * param_list[model_selected][idx:(
idx + param.data().size)].reshape(param.data().shape))
idx += param.data().size
return model_selected, direction, cmax, gfs, flip_local, 1.0 #flip_score[len(param_list)-cmax-1].asscalar()/len(param_list[0])
def forward(self, is_train, req, in_data, out_data, aux):
nms_start_time = time.time()
#inputs
cls_score = in_data[0]
bbox_pred = in_data[1]
rois = in_data[2]
im_info = in_data[3]
fc_all_2_relu = in_data[4]
nms_rank_weight = in_data[5]
nms_rank_bias = in_data[6]
roi_feat_embedding_weight = in_data[7]
roi_feat_embedding_bias = in_data[8]
nms_pair_pos_fc1_1_weight = in_data[9]
nms_pair_pos_fc1_1_bias = in_data[10]
nms_query_1_weight = in_data[11]
nms_query_1_bias = in_data[12]
nms_key_1_weight = in_data[13]
nms_key_1_bias = in_data[14]
nms_linear_out_1_weight = in_data[15]
nms_linear_out_1_bias = in_data[16]
nms_logit_weight = in_data[17]
nms_logit_bias = in_data[18]
if self.has_non_gt_index:
non_gt_index = in_data[19]
else:
non_gt_index = None
if self.nongt_dim is not None:
cls_score_nongt = nd.slice_axis(data=cls_score,
axis=0,
begin=0,
end=self.nongt_dim)
# cls_score_nongt = monitor_wrapper(cls_score_nongt, 'cls_score_nongt')
bbox_pred_nongt = nd.slice_axis(data=bbox_pred,
axis=0,
begin=0,
end=self.nongt_dim)
elif non_gt_index is not None:
cls_score_nongt = nd.take(a=cls_score, indices=non_gt_index)
bbox_pred_nongt = nd.take(a=bbox_pred, indices=non_gt_index)
else:
cls_score_nongt = cls_score
bbox_pred_nongt = bbox_pred
bbox_pred_nongt = nd.BlockGrad(bbox_pred_nongt)
# remove batch idx and gt roi
sliced_rois = nd.slice_axis(data=rois, axis=1, begin=1, end=None)
if self.nongt_dim is not None:
sliced_rois = nd.slice_axis(data=sliced_rois,
axis=0,
begin=0,
end=self.nongt_dim)
elif non_gt_index is not None:
sliced_rois = nd.take(a=sliced_rois, indices=non_gt_index)
# bbox_pred_nobg, [num_rois, 4*(num_reg_classes-1)]
bbox_pred_nobg = nd.slice_axis(data=bbox_pred_nongt,
axis=1,
begin=4,
end=None)
# [num_boxes, 4, num_reg_classes-1]
refined_bbox = refine_bbox_nd(sliced_rois,
bbox_pred_nobg,
im_info,
means=self.bbox_means,
stds=self.bbox_stds)
# softmax cls_score to cls_prob, [num_rois, num_classes]
cls_prob = nd.softmax(data=cls_score_nongt, axis=-1)
cls_prob_nobg = nd.slice_axis(cls_prob, axis=1, begin=1, end=None)
sorted_cls_prob_nobg = nd.sort(data=cls_prob_nobg,
axis=0,
is_ascend=False)
# sorted_score, [first_n, num_fg_classes]
sorted_score = nd.slice_axis(sorted_cls_prob_nobg,
axis=0,
begin=0,
end=self.first_n,
name='sorted_score')
max_score_per_class = sorted_score.max(axis=0)
max_score_per_class_numpy = max_score_per_class.asnumpy()
valid_class_thresh = self.class_thresh
valid_class_thresh = np.minimum(valid_class_thresh,
max_score_per_class_numpy.max())
valid_class_indices = np.where(
max_score_per_class_numpy >= valid_class_thresh)[0]
invalid_class_indices = np.where(
max_score_per_class_numpy < valid_class_thresh)[0]
num_valid_classes = len(valid_class_indices)
valid_class_indices_nd = nd.array(valid_class_indices,
ctx=sorted_score.context)
# sort by score
rank_indices = nd.argsort(data=cls_prob_nobg, axis=0, is_ascend=False)
# first_rank_indices, [first_n, num_fg_classes]
first_rank_indices = nd.slice_axis(rank_indices,
axis=0,
begin=0,
end=self.first_n)
valid_first_rank_indices = first_rank_indices.transpose().take(
valid_class_indices_nd).transpose()
# sorted_bbox, [first_n, num_fg_classes, 4, num_reg_classes-1]
sorted_bbox = nd.take(a=refined_bbox, indices=first_rank_indices)
if self.class_agnostic:
# sorted_bbox, [first_n, num_fg_classes, 4]
sorted_bbox = nd.Reshape(sorted_bbox,
shape=(0, 0, 0),
name='sorted_bbox')
else:
cls_mask = nd.arange(0, self.num_fg_classes)
cls_mask = nd.Reshape(cls_mask, shape=(1, -1, 1))
cls_mask = nd.broadcast_to(cls_mask, shape=(self.first_n, 0, 4))
# sorted_bbox, [first_n, num_fg_classes, 4]
sorted_bbox = nd.pick(data=sorted_bbox,
name='sorted_bbox',
index=cls_mask,
axis=3)
valid_sorted_bbox = sorted_bbox.transpose(
(1, 0, 2)).take(valid_class_indices_nd).transpose((1, 0, 2))
# sorted_bbox = monitor_wrapper(sorted_bbox, 'sorted_bbox')
# nms_rank_embedding, [first_n, 1024]
nms_rank_embedding = extract_rank_embedding_nd(self.first_n, 1024)
# nms_rank_feat, [first_n, 1024]
nms_rank_feat = nd.FullyConnected(name='nms_rank',
data=nms_rank_embedding,
num_hidden=128,
weight=nms_rank_weight,
bias=nms_rank_bias)
# nms_position_matrix, [num_valid_classes, first_n, first_n, 4]
nms_position_matrix = extract_multi_position_matrix_nd(
valid_sorted_bbox)
# roi_feature_embedding, [num_rois, 1024]
# fc_all_2_relu = monitor_wrapper(fc_all_2_relu, 'fc_all_2_relu')
roi_feat_embedding = nd.FullyConnected(
name='roi_feat_embedding',
data=fc_all_2_relu,
num_hidden=128,
weight=roi_feat_embedding_weight,
bias=roi_feat_embedding_bias)
# sorted_roi_feat, [first_n, num_valid_classes, 128]
sorted_roi_feat = nd.take(a=roi_feat_embedding,
indices=valid_first_rank_indices)
# vectorized nms
# nms_embedding_feat, [first_n, num_valid_classes, 128]
nms_embedding_feat = nd.broadcast_add(lhs=sorted_roi_feat,
rhs=nd.expand_dims(nms_rank_feat,
axis=1))
# nms_attention_1, [first_n, num_valid_classes, 1024]
nms_attention_1 = nms_attention_nd(
nms_embedding_feat,
nms_position_matrix,
nms_pair_pos_fc1_1_weight,
nms_pair_pos_fc1_1_bias,
nms_query_1_weight,
nms_query_1_bias,
nms_key_1_weight,
nms_key_1_bias,
nms_linear_out_1_weight,
nms_linear_out_1_bias,
num_rois=self.first_n,
index=1,
group=self.nms_attention_group,
dim=self.nms_attention_dim,
fc_dim=self.nms_attention_fc_dim,
feat_dim=self.nms_attention_feat_dim)
nms_all_feat_1 = nms_embedding_feat + nms_attention_1
nms_all_feat_1_relu = nd.Activation(data=nms_all_feat_1,
act_type='relu',
name='nms_all_feat_1_relu')
# [first_n * num_valid_classes, 1024]
nms_all_feat_1_relu_reshape = nd.Reshape(nms_all_feat_1_relu,
shape=(-3, -2))
# logit, [first_n * num_valid_classes, num_thresh]
nms_conditional_logit = nd.FullyConnected(
name='nms_logit',
data=nms_all_feat_1_relu_reshape,
num_hidden=self.num_thresh,
weight=nms_logit_weight,
bias=nms_logit_bias)
# logit_reshape, [first_n, num_valid_classes, num_thresh]
nms_conditional_logit_reshape = nd.Reshape(nms_conditional_logit,
shape=(self.first_n,
num_valid_classes,
self.num_thresh))
nms_conditional_score = nd.Activation(
data=nms_conditional_logit_reshape,
act_type='sigmoid',
name='nms_conditional_score')
if num_valid_classes == self.num_fg_classes:
full_nms_conditional_score = nms_conditional_score
else:
full_nms_conditional_score = nd.concat(
nms_conditional_score,
nd.zeros(
(self.first_n, self.num_fg_classes - num_valid_classes,
self.num_thresh),
ctx=nms_conditional_score.context),
dim=1)
all_indexes = np.concatenate(
(valid_class_indices, invalid_class_indices))
restore_indexes = np.zeros((self.num_fg_classes))
restore_indexes[all_indexes] = np.arange(self.num_fg_classes)
restore_indexes = nd.array(restore_indexes,
ctx=nms_conditional_score.context)
full_nms_conditional_score = full_nms_conditional_score.transpose(
(1, 0, 2)).take(restore_indexes).transpose((1, 0, 2))
sorted_score_reshape = nd.expand_dims(sorted_score, axis=2)
# sorted_score_reshape = nd.BlockGrad(sorted_score_reshape)
nms_multi_score = nd.broadcast_mul(lhs=sorted_score_reshape,
rhs=full_nms_conditional_score)
_ = nms_multi_score.mean().asnumpy()
all_time = time.time() - nms_start_time
if 'learn_nms_time' not in globals().keys(
) or 'learn_nms_count' not in globals().keys():
globals()['learn_nms_time'] = []
globals()['learn_nms_count'] = 0
if globals()['learn_nms_count'] >= 1000:
globals()['learn_nms_time'].pop(0)
globals()['learn_nms_time'].append(all_time)
else:
globals()['learn_nms_time'].append(all_time)
globals()['learn_nms_count'] += 1
if globals()['learn_nms_count'] % 250 == 0:
print("--->> learn nms running average time cost: {}".format(
float(sum(globals()['learn_nms_time'])) /
(1000 if globals()['learn_nms_count'] > 1000 else
globals()['learn_nms_count'])))
self.assign(out_data[0], req[0], nms_multi_score)
self.assign(out_data[1], req[1], sorted_bbox)
self.assign(out_data[2], req[2], sorted_score)
def median(epoch,
gradients,
net,
lr,
byz,
old_direction,
active,
blacklist,
susp,
f=0,
cmax=0,
utrg=0.0,
udet=0.50,
urem=3):
param_list = [
nd.concat(*[xx.reshape((-1, 1)) for xx in x], dim=0) for x in gradients
]
param_list = byz(epoch, param_list, net, f, lr, active)
flip_local = nd.zeros(len(param_list))
penalty = 1.0 - cmax / len(param_list)
reward = 1.0 - penalty
for i in range(len(param_list)):
direction = nd.sign(param_list[i])
flip_local[i] = 0.5 * (mx.nd.sum(
direction.reshape(-1) *
(direction.reshape(-1) - old_direction.reshape(-1)))).asscalar()
argsorted = nd.argsort(flip_local)
susp[argsorted[:-cmax]] = susp[argsorted[:-cmax]] - reward
susp[argsorted[-cmax:]] = susp[argsorted[-cmax:]] + penalty
new_list = []
argsorted = nd.argsort(susp)
for i in range(len(param_list) - cmax):
new_list.append(param_list[int(argsorted[i].asscalar())])
sorted_array = nd.sort(nd.concat(*new_list, dim=1), axis=-1)
if (len(new_list) % 2 == 1):
trim_nd = sorted_array[:, int(len(new_list) / 2)]
else:
trim_nd = (sorted_array[:, int(len(new_list) / 2) - 1] +
sorted_array[:, int(len(new_list) / 2)]) / 2
global_direction = nd.sign(trim_nd)
gfs = 0.5 * (mx.nd.sum(
global_direction.reshape(-1) *
(global_direction.reshape(-1) - old_direction.reshape(-1)))
).asscalar()
'''if (utrg > 0):
sorted_array = nd.sort(nd.concat(*param_list, dim=1), axis=-1)
n = len(param_list)
m = n - f*2
if (len(param_list)%2 == 1):
trim_nd = sorted_array[:, int(len(param_list)/2)]
else:
trim_nd = (sorted_array[:, int(len(param_list)/2)-1] + sorted_array[:, int(len(param_list)/2)])/2
direction = nd.sign(trim_nd)
gfs = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar()
if ((utrg>0 and gfs>=utrg*len(param_list[0])) or (utrg == 0)):
flip_score = mx.nd.zeros(len(param_list))
rem = []
for i in range (len(param_list)):
direction = nd.sign(param_list[i])
flip_score[i] = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar()
flip_local[active[i]] = flip_score[i].asscalar()
argsorted = nd.argsort(flip_score)
new_list = []
for i in range(len(param_list)-cmax):
new_list.append(param_list[int(argsorted[i].asscalar())])
n = len(new_list)
f = 0
sorted_array = nd.sort(nd.concat(*new_list, dim=1), axis=-1)
if (len(new_list)%2 == 1):
trim_nd = sorted_array[:, int(len(new_list)/2)]
else:
trim_nd = (sorted_array[:, int(len(new_list)/2)-1] + sorted_array[:, int(len(new_list)/2)])/2
for i in range(len(new_list), len(param_list)):
index = int(argsorted[i].asscalar())
if (flip_score[index] >= udet*len(param_list[0])):
susp[active[index]] = susp[active[index]] + 1
if (susp[active[index]] >= urem):
blacklist[active[index]] = 1
rem.append(active[index])
active = removearr(active, sorted(rem), len(param_list))
direction = nd.sign(trim_nd)
gfs = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar()
cmax = cmax - len(rem) '''
idx = 0
for j, (param) in enumerate(net.collect_params().values()):
if param.grad_req == 'null':
continue
param.set_data(
param.data() - lr *
trim_nd[idx:(idx + param.data().size)].reshape(param.data().shape))
idx += param.data().size
return trim_nd, direction, cmax, gfs, flip_local
def prune_by_percent(self, p):
criterion = self._compute_criterion()
th_idx = nd.argsort(criterion)[int(p * self._channels)]
self.mask = (criterion >= th_idx).reshape(1, -1, 1, 1)
def score(x, y):
_sim = nd.dot(x, y, transpose_b=True)
_indices = nd.argsort(-_sim, axis=1)
return _sim, _indices[:, 1: 100]
