def _ohem_single(self, score_gt, score_pred, training_masks):
if self.debug:
print("score_gt_shape:", score_gt.shape, "score_pred_shape:", score_pred.shape, \
"train_mask_shape:", training_masks.shape)
pos_gt_thres = F.where(score_gt > 0.5, F.ones_like(score_gt), F.zeros_like(score_gt))
pos_num = F.sum(pos_gt_thres) - F.sum(pos_gt_thres * training_masks)
if pos_num == 0:
selected_mask = training_masks
return selected_mask
neg_lt_thres = F.where(score_gt <= 0.5, F.ones_like(score_gt), F.zeros_like(score_gt))
neg_num = F.sum(neg_lt_thres)
neg_num = min(pos_num * 3, neg_num)
if neg_num == 0:
selected_mask = training_masks
return training_masks
neg_score = neg_lt_thres * score_pred
neg_score_sorted = F.sort(neg_score.reshape(-1), is_ascend=0, axis=None)
threshold = neg_score_sorted[neg_num - 1]
score_gt_thres = F.where(score_pred >= threshold, F.ones_like(score_pred), F.zeros_like(score_pred))
trained_sample_mask = F.logical_or(score_gt_thres, pos_gt_thres)
selected_mask = F.logical_and(trained_sample_mask, training_masks)
return selected_mask
def predict_and_render_radiance(ray_batch, model_coarse, model_fine, options, mode="train",
encode_position_fn=None, encode_direction_fn=None):
# TESTED
num_rays = ray_batch.shape[0]
ro, rd = ray_batch[..., :3], ray_batch[..., 3:6]
bounds = ray_batch[..., 6:8].reshape((-1, 1, 2))
near, far = bounds[..., 0], bounds[..., 1]
# TODO: Use actual values for "near" and "far" (instead of 0. and 1.) when not enabling "ndc".
t_vals = nd.linspace(0.0, 1.0, getattr(options.nerf, mode).num_coarse, dtype=ro.dtype, ctx=ro.context)
if not getattr(options.nerf, mode).lindisp:
z_vals = near * (1.0 - t_vals) + far * t_vals
else:
z_vals = 1.0 / (1.0 / near * (1.0 - t_vals) + 1.0 / far * t_vals)
z_vals = z_vals.broadcast_to((num_rays, getattr(options.nerf, mode).num_coarse))
if getattr(options.nerf, mode).perturb:
# Get intervals between samples.
mids = 0.5 * (z_vals[..., 1:] + z_vals[..., :-1])
upper = nd.concat(*[mids, z_vals[..., -1:]], dim=-1)
lower = nd.concat(*[z_vals[..., :1], mids], dim=-1)
# Stratified samples in those intervals.
t_rand = nd.random.uniform(0.0, 1.0, z_vals.shape, dtype=ro.dtype, ctx=ro.context)
z_vals = lower + (upper - lower) * t_rand
# pts -> (num_rays, N_samples, 3)
pts = ro[..., None, :] + rd[..., None, :] * z_vals[..., :, None]
radiance_field = run_network(model_coarse, pts, ray_batch, getattr(options.nerf, mode).chunksize,
encode_position_fn, encode_direction_fn)
rfns = getattr(options.nerf, mode).radiance_field_noise_std
wb = getattr(options.nerf, mode).white_background
rgb_coarse, disp_coarse, acc_coarse, weights, depth_coarse, = volume_render_radiance_field(radiance_field,
z_vals, rd,
radiance_field_noise_std=rfns,
white_background=wb)
rgb_fine, disp_fine, acc_fine = None, None, None
if getattr(options.nerf, mode).num_fine > 0:
# rgb_map_0, disp_map_0, acc_map_0 = rgb_map, disp_map, acc_map
z_vals_mid = 0.5 * (z_vals[..., 1:] + z_vals[..., :-1])
z_samples = sample_pdf(z_vals_mid.asnumpy(), weights[..., 1:-1].asnumpy(), getattr(options.nerf, mode).num_fine,
det=(getattr(options.nerf, mode).perturb == 0.0))
z_samples = nd.array(z_samples, ctx=z_vals.context)
z_vals = nd.sort(nd.concat(*[z_vals, z_samples], dim=-1), axis=-1)
# pts -> (N_rays, N_samples + N_importance, 3)
pts = ro[..., None, :] + rd[..., None, :] * z_vals[..., :, None]
radiance_field = run_network(model_fine, pts, ray_batch, getattr(options.nerf, mode).chunksize,
encode_position_fn, encode_direction_fn)
rfns = getattr(options.nerf, mode).radiance_field_noise_std
wb = getattr(options.nerf, mode).white_background
rgb_fine, disp_fine, acc_fine, _, _ = volume_render_radiance_field(radiance_field, z_vals, rd,
radiance_field_noise_std=rfns,
white_background=wb)
return rgb_coarse, disp_coarse, acc_coarse, rgb_fine, disp_fine, acc_fine
def test_index(array, idx, i):
sim = nd.dot(array[i], array, transpose_b=True)
index = idx[i]
index = index[:30]
print('===============================')
print(sim[index])
sim = nd.sort(sim, is_ascend=0)
print("mean all", nd.mean(sim))
print("mean 100", nd.mean(sim[:100]))
print("mean 500", nd.mean(sim[:500]))
print("mean 1000", nd.mean(sim[:1000]))
print("mean 10000", nd.mean(sim[:10000]))
print("mean 20000", nd.mean(sim[:20000]))
print("mean -1000", nd.mean(sim[len(sim) - 1000:]))
def marginal_median(gradients, net, lr, f = 0, byz = no_byz, factor = 0):
# X is a 2d list of nd array
param_list = [nd.concat(*[xx.reshape((-1, 1)) for xx in x], dim=0) for x in gradients]
byz(param_list, f, factor)
sorted_array = nd.sort(nd.concat(*param_list, dim=1), axis=-1)
if sorted_array.shape[-1] % 2 == 1:
median_nd = sorted_array[:, sorted_array.shape[-1]/2]
else:
median_nd = (sorted_array[:, (sorted_array.shape[-1]/2-1)] + sorted_array[:, (sorted_array.shape[-1]/2)]) / 2.
# np_array = nd.concat(*param_list, dim=1).asnumpy()
# median_nd = nd.array(np.median(np_array, axis=1))
idx = 0
for j, (param) in enumerate(net.collect_params().values()):
if param.grad_req != 'null':
# mapping back to the collection of ndarray
param.set_data(param.data() - lr * median_nd[idx:(idx+param.data().size)].reshape(param.data().shape))
idx += param.data().size
def trim(epoch, gradients, net, lr, f, byz, b=20):
param_list = [
nd.concat(*[xx.reshape((-1, 1)) for xx in x], dim=0) for x in gradients
]
param_list = byz(epoch, param_list, net, lr, f)
sorted_array = nd.sort(nd.concat(*param_list, dim=1), axis=-1)
n = len(param_list)
q = f
m = n - b * 2
trim_nd = nd.mean(sorted_array[:, b:(b + m)], axis=-1, keepdims=1)
idx = 0
for j, (param) in enumerate(net.collect_params().values()):
if param.grad_req == 'null':
continue
param.set_data(
param.data() - lr *
trim_nd[idx:(idx + param.data().size)].reshape(param.data().shape))
idx += param.data().size
def median(epoch, gradients, net, lr, f, byz):
param_list = [
nd.concat(*[xx.reshape((-1, 1)) for xx in x], dim=0) for x in gradients
]
param_list = byz(epoch, param_list, net, lr, f)
sorted_array = nd.sort(nd.concat(*param_list, dim=1), axis=-1)
#print(nd.norm(sorted_array))
if sorted_array.shape[-1] % 2 == 1:
median_nd = sorted_array[:, int(sorted_array.shape[-1] / 2)]
else:
median_nd = (sorted_array[:, int((sorted_array.shape[-1] / 2 - 1))] +
sorted_array[:, int((sorted_array.shape[-1] / 2))]) / 2
idx = 0
for j, (param) in enumerate(net.collect_params().values()):
if param.grad_req == 'null':
continue
param.set_data(param.data() - lr * median_nd[idx:(
idx + param.data().size)].reshape(param.data().shape))
idx += param.data().size
def evaluate_accuracy(data_iterator, net):
# num_access = 0
# num_predict = 0
# softmax_cross_entropy = gluon.loss.L1Loss()
# output = nd.rint(nd.maximum(0, output))
# accuracies.append(nd.mean(nd.abs(output-label)).asscalar())
# loss = softmax_cross_entropy(output, label)
# accuracies.append(nd.sum(loss).asscalar())
# return sum(accuracies)/float(len(accuracies))
# num_access += nd.sum(label).asscalar()
# num_access += nd.sum(nd.maximum(0, output)).asscalar()
# num_predict += nd.sum(nd.minimum(label, output)).asscalar()
# return 0 if num_access == 0 else num_predict*1.0/num_access
accuracies = []
for i, (data, label) in enumerate(data_iterator):
data = data.as_in_context(model_ctx)
label = label.as_in_context(model_ctx)
output = net(data)
batch_size = data.shape[0]
num_sets = data.shape[1]
num_access = 0
for j in range(num_sets):
if label[0][j].asscalar() > 0:
num_access += 1
predict_index = nd.topk(output, k=num_access)
predict_index = nd.sort(predict_index, axis=1)
correct = 0.0
for j in range(batch_size):
for k in range(num_access):
if label[j][int(
predict_index[j][k].asscalar())].asscalar() == k + 1:
correct += 1.0
accuracies.append(correct / batch_size / num_access)
return sum(accuracies) / len(accuracies)
def bulyan(epoch, gradients, net, lr, byz, f=0):
param_list = [
nd.concat(*[xx.reshape((-1, 1)) for xx in x], dim=0) for x in gradients
]
param_list = byz(epoch, param_list, net, f, lr, np.arange(len(param_list)))
k = len(param_list) - f - 2
dist = mx.nd.zeros((len(param_list), len(param_list)))
for i in range(0, len(param_list)):
for j in range(0, i):
dist[i][j] = nd.norm(param_list[i] - param_list[j])
dist[j][i] = dist[i][j]
sorted_dist = mx.nd.sort(dist)
sum_dist = mx.nd.sum(sorted_dist[:, :k + 1], axis=1)
bulyan_list = []
bul_client_list = np.ones(len(param_list)) * (-1)
for i in range(len(param_list) - 2 * f):
chosen = int(nd.argmin(sum_dist).asscalar())
sum_dist[chosen] = 10**8
bul_client_list[i] = chosen
bulyan_list.append(param_list[chosen])
for j in range(len(sum_dist)):
sum_dist[j] = sum_dist[j] - dist[j][chosen]
sorted_array = nd.sort(nd.concat(*bulyan_list, dim=1), axis=-1)
trim_nd = nd.mean(sorted_array[:, f:(len(bulyan_list) - f)],
axis=-1,
keepdims=1)
idx = 0
for j, (param) in enumerate(net.collect_params().values()):
if param.grad_req == 'null':
continue
param.set_data(
param.data() - lr *
trim_nd[idx:(idx + param.data().size)].reshape(param.data().shape))
idx += param.data().size
return trim_nd, bul_client_list
def test_descend(x):
s = nd.sort(x, axis=0, is_ascend=False)
assert s[-1] == 0
def check_ascend(x):
s = nd.sort(x, is_ascend=True)
assert s[0] == 0
# a = nd.random.normal(scale = 2, shape=3)
# print("a= ", a)
# a.attach_grad()
# with autograd.record():
# d = f(a)
# d.backward()
# print("d = ", d)
# print("a.grad == (d / a) ?", a.grad == (d / a))
# print("grad= ", a.grad )
# print("d / a = ", d / a )
elim = nd.zeros(10)
elim[8] = 1
def second_bid(a):
b = a.copy()
s = b * elim
z = s
print("res= ", z)
return z
x = nd.sort(nd.random.randn(1, 10, loc=20, scale=5)[0])
print("init x = ", x)
x.attach_grad()
with autograd.record():
d = second_bid(x)
d.backward()
print("grad= ", x.grad)
def test_sort():
b = create_vector(size=LARGE_X)
s = nd.sort(b, axis=0, is_ascend=False)
assert np.sum(s[-1].asnumpy() == 0).all()
s = nd.sort(b, is_ascend=True)
assert np.sum(s[0].asnumpy() == 0).all()
def median(epoch,
gradients,
net,
lr,
byz,
old_direction,
active,
blacklist,
susp,
f=0,
cmax=0,
utrg=0.0,
udet=0.50,
urem=3):
param_list = [
nd.concat(*[xx.reshape((-1, 1)) for xx in x], dim=0) for x in gradients
]
param_list = byz(epoch, param_list, net, f, lr, active)
flip_local = nd.zeros(len(param_list))
penalty = 1.0 - cmax / len(param_list)
reward = 1.0 - penalty
for i in range(len(param_list)):
direction = nd.sign(param_list[i])
flip_local[i] = 0.5 * (mx.nd.sum(
direction.reshape(-1) *
(direction.reshape(-1) - old_direction.reshape(-1)))).asscalar()
argsorted = nd.argsort(flip_local)
susp[argsorted[:-cmax]] = susp[argsorted[:-cmax]] - reward
susp[argsorted[-cmax:]] = susp[argsorted[-cmax:]] + penalty
new_list = []
argsorted = nd.argsort(susp)
for i in range(len(param_list) - cmax):
new_list.append(param_list[int(argsorted[i].asscalar())])
sorted_array = nd.sort(nd.concat(*new_list, dim=1), axis=-1)
if (len(new_list) % 2 == 1):
trim_nd = sorted_array[:, int(len(new_list) / 2)]
else:
trim_nd = (sorted_array[:, int(len(new_list) / 2) - 1] +
sorted_array[:, int(len(new_list) / 2)]) / 2
global_direction = nd.sign(trim_nd)
gfs = 0.5 * (mx.nd.sum(
global_direction.reshape(-1) *
(global_direction.reshape(-1) - old_direction.reshape(-1)))
).asscalar()
'''if (utrg > 0):
sorted_array = nd.sort(nd.concat(*param_list, dim=1), axis=-1)
n = len(param_list)
m = n - f*2
if (len(param_list)%2 == 1):
trim_nd = sorted_array[:, int(len(param_list)/2)]
else:
trim_nd = (sorted_array[:, int(len(param_list)/2)-1] + sorted_array[:, int(len(param_list)/2)])/2
direction = nd.sign(trim_nd)
gfs = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar()
if ((utrg>0 and gfs>=utrg*len(param_list[0])) or (utrg == 0)):
flip_score = mx.nd.zeros(len(param_list))
rem = []
for i in range (len(param_list)):
direction = nd.sign(param_list[i])
flip_score[i] = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar()
flip_local[active[i]] = flip_score[i].asscalar()
argsorted = nd.argsort(flip_score)
new_list = []
for i in range(len(param_list)-cmax):
new_list.append(param_list[int(argsorted[i].asscalar())])
n = len(new_list)
f = 0
sorted_array = nd.sort(nd.concat(*new_list, dim=1), axis=-1)
if (len(new_list)%2 == 1):
trim_nd = sorted_array[:, int(len(new_list)/2)]
else:
trim_nd = (sorted_array[:, int(len(new_list)/2)-1] + sorted_array[:, int(len(new_list)/2)])/2
for i in range(len(new_list), len(param_list)):
index = int(argsorted[i].asscalar())
if (flip_score[index] >= udet*len(param_list[0])):
susp[active[index]] = susp[active[index]] + 1
if (susp[active[index]] >= urem):
blacklist[active[index]] = 1
rem.append(active[index])
active = removearr(active, sorted(rem), len(param_list))
direction = nd.sign(trim_nd)
gfs = 0.5*(mx.nd.sum(direction.reshape(-1)*(direction.reshape(-1)-old_direction.reshape(-1)))).asscalar()
cmax = cmax - len(rem) '''
idx = 0
for j, (param) in enumerate(net.collect_params().values()):
if param.grad_req == 'null':
continue
param.set_data(
param.data() - lr *
trim_nd[idx:(idx + param.data().size)].reshape(param.data().shape))
idx += param.data().size
return trim_nd, direction, cmax, gfs, flip_local
def EULmedian(epoch,
gradients,
net,
lr,
byz,
test_data,
net_name,
f=0,
gpu=-1):
param_list = [
nd.concat(*[xx.reshape((-1, 1)) for xx in x], dim=0) for x in gradients
]
param_list = byz(epoch, param_list, net, f, lr, np.arange(len(param_list)))
sorted_array = nd.sort(nd.concat(*param_list, dim=1), axis=-1)
if (len(param_list) % 2 == 1):
trim_nd = sorted_array[:, int(len(param_list) / 2)]
else:
trim_nd = (sorted_array[:, int(len(param_list) / 2) - 1] +
sorted_array[:, int(len(param_list) / 2)]) / 2
idx = 0
for j, (param) in enumerate(net.collect_params().values()):
if param.grad_req == 'null':
continue
param.set_data(
param.data() - lr *
trim_nd[idx:(idx + param.data().size)].reshape(param.data().shape))
idx += param.data().size
accuracy_all, loss_all = evaluate_accuracy(test_data, net, net_name, gpu)
prev_trim = deepcopy(trim_nd)
accuracy_np = np.zeros(len(param_list))
loss_np = np.zeros(len(param_list))
for i in range(len(param_list)):
param_eul = deepcopy(param_list)
param_eul[i] = (mx.nd.ones(len(param_list[0])) * (-10000)).reshape(
(-1, 1))
sorted_array = nd.sort(nd.concat(*param_eul, dim=1), axis=-1)
if (len(param_eul) % 2 == 1):
trim_nd = sorted_array[:, int(len(param_eul) / 2)]
else:
trim_nd = (sorted_array[:, int(len(param_eul) / 2) - 1] +
sorted_array[:, int(len(param_eul) / 2)]) / 2
idx = 0
for j, (param) in enumerate(net.collect_params().values()):
if param.grad_req == 'null':
continue
param.set_data(param.data() - lr * (trim_nd[idx:(
idx + param.data().size)].reshape(param.data().shape)) + lr *
(prev_trim[idx:(idx + param.data().size
)].reshape(param.data().shape)))
idx += param.data().size
accuracy_eul, loss_eul = evaluate_accuracy(test_data, net, net_name,
gpu)
accuracy_np[i] = accuracy_all - accuracy_eul
loss_np[i] = loss_all - loss_eul
prev_trim = deepcopy(trim_nd)
intersection = np.intersect1d(
np.argsort(accuracy_np)[:-f],
np.argsort(loss_np)[f:])
eul_client_list = np.ones(len(param_list)) * (-1)
eul_client_list[:len(intersection)] = intersection
removed = np.setdiff1d(np.arange(len(param_list)), intersection)
for i in range(len(removed)):
param_list[int(removed[i])] = (mx.nd.ones(len(param_list[0])) *
(-10000)).reshape((-1, 1))
sorted_array = nd.sort(nd.concat(*param_list, dim=1), axis=-1)
if (len(param_list) % 2 == 1):
trim_nd = sorted_array[:, int(len(param_list) / 2)]
else:
trim_nd = (sorted_array[:, int(len(param_list) / 2) - 1] +
sorted_array[:, int(len(param_list) / 2)]) / 2
idx = 0
for j, (param) in enumerate(net.collect_params().values()):
if param.grad_req == 'null':
continue
param.set_data(
param.data() + lr * prev_trim[idx:(
idx + param.data().size)].reshape(param.data().shape) - lr *
(trim_nd[idx:
(idx + param.data().size)].reshape(param.data().shape)))
idx += param.data().size
del trim_nd
del prev_trim
del param_list
del param_eul
return net, eul_client_list
def test_ascend(x):
s = nd.sort(x, is_ascend=True)
assert s[0] == 0
def forward(self, is_train, req, in_data, out_data, aux):
nms_start_time = time.time()
#inputs
cls_score = in_data[0]
bbox_pred = in_data[1]
rois = in_data[2]
im_info = in_data[3]
fc_all_2_relu = in_data[4]
nms_rank_weight = in_data[5]
nms_rank_bias = in_data[6]
roi_feat_embedding_weight = in_data[7]
roi_feat_embedding_bias = in_data[8]
nms_pair_pos_fc1_1_weight = in_data[9]
nms_pair_pos_fc1_1_bias = in_data[10]
nms_query_1_weight = in_data[11]
nms_query_1_bias = in_data[12]
nms_key_1_weight = in_data[13]
nms_key_1_bias = in_data[14]
nms_linear_out_1_weight = in_data[15]
nms_linear_out_1_bias = in_data[16]
nms_logit_weight = in_data[17]
nms_logit_bias = in_data[18]
if self.has_non_gt_index:
non_gt_index = in_data[19]
else:
non_gt_index = None
if self.nongt_dim is not None:
cls_score_nongt = nd.slice_axis(data=cls_score,
axis=0,
begin=0,
end=self.nongt_dim)
# cls_score_nongt = monitor_wrapper(cls_score_nongt, 'cls_score_nongt')
bbox_pred_nongt = nd.slice_axis(data=bbox_pred,
axis=0,
begin=0,
end=self.nongt_dim)
elif non_gt_index is not None:
cls_score_nongt = nd.take(a=cls_score, indices=non_gt_index)
bbox_pred_nongt = nd.take(a=bbox_pred, indices=non_gt_index)
else:
cls_score_nongt = cls_score
bbox_pred_nongt = bbox_pred
bbox_pred_nongt = nd.BlockGrad(bbox_pred_nongt)
# remove batch idx and gt roi
sliced_rois = nd.slice_axis(data=rois, axis=1, begin=1, end=None)
if self.nongt_dim is not None:
sliced_rois = nd.slice_axis(data=sliced_rois,
axis=0,
begin=0,
end=self.nongt_dim)
elif non_gt_index is not None:
sliced_rois = nd.take(a=sliced_rois, indices=non_gt_index)
# bbox_pred_nobg, [num_rois, 4*(num_reg_classes-1)]
bbox_pred_nobg = nd.slice_axis(data=bbox_pred_nongt,
axis=1,
begin=4,
end=None)
# [num_boxes, 4, num_reg_classes-1]
refined_bbox = refine_bbox_nd(sliced_rois,
bbox_pred_nobg,
im_info,
means=self.bbox_means,
stds=self.bbox_stds)
# softmax cls_score to cls_prob, [num_rois, num_classes]
cls_prob = nd.softmax(data=cls_score_nongt, axis=-1)
cls_prob_nobg = nd.slice_axis(cls_prob, axis=1, begin=1, end=None)
sorted_cls_prob_nobg = nd.sort(data=cls_prob_nobg,
axis=0,
is_ascend=False)
# sorted_score, [first_n, num_fg_classes]
sorted_score = nd.slice_axis(sorted_cls_prob_nobg,
axis=0,
begin=0,
end=self.first_n,
name='sorted_score')
max_score_per_class = sorted_score.max(axis=0)
max_score_per_class_numpy = max_score_per_class.asnumpy()
valid_class_thresh = self.class_thresh
valid_class_thresh = np.minimum(valid_class_thresh,
max_score_per_class_numpy.max())
valid_class_indices = np.where(
max_score_per_class_numpy >= valid_class_thresh)[0]
invalid_class_indices = np.where(
max_score_per_class_numpy < valid_class_thresh)[0]
num_valid_classes = len(valid_class_indices)
valid_class_indices_nd = nd.array(valid_class_indices,
ctx=sorted_score.context)
# sort by score
rank_indices = nd.argsort(data=cls_prob_nobg, axis=0, is_ascend=False)
# first_rank_indices, [first_n, num_fg_classes]
first_rank_indices = nd.slice_axis(rank_indices,
axis=0,
begin=0,
end=self.first_n)
valid_first_rank_indices = first_rank_indices.transpose().take(
valid_class_indices_nd).transpose()
# sorted_bbox, [first_n, num_fg_classes, 4, num_reg_classes-1]
sorted_bbox = nd.take(a=refined_bbox, indices=first_rank_indices)
if self.class_agnostic:
# sorted_bbox, [first_n, num_fg_classes, 4]
sorted_bbox = nd.Reshape(sorted_bbox,
shape=(0, 0, 0),
name='sorted_bbox')
else:
cls_mask = nd.arange(0, self.num_fg_classes)
cls_mask = nd.Reshape(cls_mask, shape=(1, -1, 1))
cls_mask = nd.broadcast_to(cls_mask, shape=(self.first_n, 0, 4))
# sorted_bbox, [first_n, num_fg_classes, 4]
sorted_bbox = nd.pick(data=sorted_bbox,
name='sorted_bbox',
index=cls_mask,
axis=3)
valid_sorted_bbox = sorted_bbox.transpose(
(1, 0, 2)).take(valid_class_indices_nd).transpose((1, 0, 2))
# sorted_bbox = monitor_wrapper(sorted_bbox, 'sorted_bbox')
# nms_rank_embedding, [first_n, 1024]
nms_rank_embedding = extract_rank_embedding_nd(self.first_n, 1024)
# nms_rank_feat, [first_n, 1024]
nms_rank_feat = nd.FullyConnected(name='nms_rank',
data=nms_rank_embedding,
num_hidden=128,
weight=nms_rank_weight,
bias=nms_rank_bias)
# nms_position_matrix, [num_valid_classes, first_n, first_n, 4]
nms_position_matrix = extract_multi_position_matrix_nd(
valid_sorted_bbox)
# roi_feature_embedding, [num_rois, 1024]
# fc_all_2_relu = monitor_wrapper(fc_all_2_relu, 'fc_all_2_relu')
roi_feat_embedding = nd.FullyConnected(
name='roi_feat_embedding',
data=fc_all_2_relu,
num_hidden=128,
weight=roi_feat_embedding_weight,
bias=roi_feat_embedding_bias)
# sorted_roi_feat, [first_n, num_valid_classes, 128]
sorted_roi_feat = nd.take(a=roi_feat_embedding,
indices=valid_first_rank_indices)
# vectorized nms
# nms_embedding_feat, [first_n, num_valid_classes, 128]
nms_embedding_feat = nd.broadcast_add(lhs=sorted_roi_feat,
rhs=nd.expand_dims(nms_rank_feat,
axis=1))
# nms_attention_1, [first_n, num_valid_classes, 1024]
nms_attention_1 = nms_attention_nd(
nms_embedding_feat,
nms_position_matrix,
nms_pair_pos_fc1_1_weight,
nms_pair_pos_fc1_1_bias,
nms_query_1_weight,
nms_query_1_bias,
nms_key_1_weight,
nms_key_1_bias,
nms_linear_out_1_weight,
nms_linear_out_1_bias,
num_rois=self.first_n,
index=1,
group=self.nms_attention_group,
dim=self.nms_attention_dim,
fc_dim=self.nms_attention_fc_dim,
feat_dim=self.nms_attention_feat_dim)
nms_all_feat_1 = nms_embedding_feat + nms_attention_1
nms_all_feat_1_relu = nd.Activation(data=nms_all_feat_1,
act_type='relu',
name='nms_all_feat_1_relu')
# [first_n * num_valid_classes, 1024]
nms_all_feat_1_relu_reshape = nd.Reshape(nms_all_feat_1_relu,
shape=(-3, -2))
# logit, [first_n * num_valid_classes, num_thresh]
nms_conditional_logit = nd.FullyConnected(
name='nms_logit',
data=nms_all_feat_1_relu_reshape,
num_hidden=self.num_thresh,
weight=nms_logit_weight,
bias=nms_logit_bias)
# logit_reshape, [first_n, num_valid_classes, num_thresh]
nms_conditional_logit_reshape = nd.Reshape(nms_conditional_logit,
shape=(self.first_n,
num_valid_classes,
self.num_thresh))
nms_conditional_score = nd.Activation(
data=nms_conditional_logit_reshape,
act_type='sigmoid',
name='nms_conditional_score')
if num_valid_classes == self.num_fg_classes:
full_nms_conditional_score = nms_conditional_score
else:
full_nms_conditional_score = nd.concat(
nms_conditional_score,
nd.zeros(
(self.first_n, self.num_fg_classes - num_valid_classes,
self.num_thresh),
ctx=nms_conditional_score.context),
dim=1)
all_indexes = np.concatenate(
(valid_class_indices, invalid_class_indices))
restore_indexes = np.zeros((self.num_fg_classes))
restore_indexes[all_indexes] = np.arange(self.num_fg_classes)
restore_indexes = nd.array(restore_indexes,
ctx=nms_conditional_score.context)
full_nms_conditional_score = full_nms_conditional_score.transpose(
(1, 0, 2)).take(restore_indexes).transpose((1, 0, 2))
sorted_score_reshape = nd.expand_dims(sorted_score, axis=2)
# sorted_score_reshape = nd.BlockGrad(sorted_score_reshape)
nms_multi_score = nd.broadcast_mul(lhs=sorted_score_reshape,
rhs=full_nms_conditional_score)
_ = nms_multi_score.mean().asnumpy()
all_time = time.time() - nms_start_time
if 'learn_nms_time' not in globals().keys(
) or 'learn_nms_count' not in globals().keys():
globals()['learn_nms_time'] = []
globals()['learn_nms_count'] = 0
if globals()['learn_nms_count'] >= 1000:
globals()['learn_nms_time'].pop(0)
globals()['learn_nms_time'].append(all_time)
else:
globals()['learn_nms_time'].append(all_time)
globals()['learn_nms_count'] += 1
if globals()['learn_nms_count'] % 250 == 0:
print("--->> learn nms running average time cost: {}".format(
float(sum(globals()['learn_nms_time'])) /
(1000 if globals()['learn_nms_count'] > 1000 else
globals()['learn_nms_count'])))
self.assign(out_data[0], req[0], nms_multi_score)
self.assign(out_data[1], req[1], sorted_bbox)
self.assign(out_data[2], req[2], sorted_score)
def test_sort():
b = create_2d_tensor(rows=LARGE_X, columns=SMALL_Y)
s = nd.sort(b, axis=0, is_ascend=False)
assert np.sum(s[-1][SMALL_Y//2:SMALL_Y].asnumpy() == 0).all()
s = nd.sort(b, is_ascend=False)
assert np.sum(s[0].asnumpy() == 0).all()
out2 = nd.abs(loaded[re_n_k[j]]) / thr - 1
temp = nd.zeros_like(out2)
out2 = nd.where(nd.abs(out2) > 0.1, temp, out2)
for i in range(2):
if lm: out2 = nd.sum(out2, axis=-1)
out = nd.sum(out, axis=-1)
ax = plt.subplot(rows, cols, idx)
# ratio of 1/3 [numbers in kernel]
ratio1 = nd.sum(nd.where(out >= 2, nd.where(out <= 4, nd.ones_like(out),
nd.zeros_like(out)), nd.zeros_like(out)))
ratio = (ratio1 / nd.sum(nd.ones_like(out))).asscalar()
# plot
out = nd.sort(out.reshape(-1)).asnumpy()
x = range(len(out))
l1, = ax.plot(x, out, 'r', label=tag + '_' + repr(ratio))
print 'len kenels', tag, len(out)
if lm:
out2 = out2 * 10 + 15
out2 = nd.sort(out2.reshape(-1)).asnumpy()
l2, = ax.plot(x, out2, 'b', label='X20plus5')
ax.set_yticks(range(1, 1 + k * k))
if lm:
ax.legend([l1, l2], loc='best')
else:
ax.legend(loc='best')
# histogram
# out = out.reshape(-1).asnumpy()
