def clip_grad(grads: Union[Generator[NDArray, NDArray, NDArray], List[NDArray],
Tuple[NDArray]],
clip_method: GradientClippingMethod,
clip_val: float,
inplace=True) -> List[NDArray]:
"""
Clip gradient values inplace
:param grads: gradients to be clipped
:param clip_method: clipping method
:param clip_val: clipping value. Interpreted differently depending on clipping method.
:param inplace: modify grads if True, otherwise create NDArrays
:return: clipped gradients
"""
output = list(grads) if inplace else list(nd.empty(g.shape) for g in grads)
if clip_method == GradientClippingMethod.ClipByGlobalNorm:
norm_unclipped_grads = global_norm(grads)
scale = clip_val / (norm_unclipped_grads.asscalar() + 1e-8
) # todo: use branching operators?
if scale < 1.0:
for g, o in zip(grads, output):
nd.broadcast_mul(g, nd.array([scale]), out=o)
elif clip_method == GradientClippingMethod.ClipByValue:
for g, o in zip(grads, output):
g.clip(-clip_val, clip_val, out=o)
elif clip_method == GradientClippingMethod.ClipByNorm:
for g, o in zip(grads, output):
nd.broadcast_mul(g,
nd.minimum(1.0, clip_val / (g.norm() + 1e-8)),
out=o)
else:
raise KeyError('Unsupported gradient clipping method')
return output
def forward(self, pred, target):
batch, dim = pred.shape
# pos = mxnet.nd.greater(target,0).reshape(batch,dim,1)
# neg = mxnet.nd.equal(target,0).reshape(batch,1,dim)
#construct the weight Li_j
L = mxnet.nd.zeros(shape=(batch, dim), ctx=pred.context)
for i in range(batch):
for j in range(dim):
if target[i, j] == 1:
sample_score_margin = -1
num_trails = 0
while ((sample_score_margin < 0)
and (num_trails < self.max_num_trails)):
neg_labels_idx = np.array([
idx for idx, v in enumerate(target[i, :]) if v == 0
])
if len(neg_labels_idx) > 0:
neg_idx = np.random.choice(neg_labels_idx,
replace=False)
sample_score_margin = pred[i, neg_idx] - pred[i, j]
num_trails += 1
else:
num_trails = 1
pass
r_j = int(np.floor(self.max_num_trails / num_trails))
#print("the rank of r_j is ",r_j,"the length of self.rank_weights is ",len(self.rank_weights))
#print(L.shape,L[i,j],self.rank_weights[r_j])
L[i, j] = self.rank_weights[r_j]
#finish approximate weight
L.detach()
dist = pred.reshape(batch, dim, 1) - pred.reshape(batch, 1, dim)
pos = mxnet.nd.greater(target, 0).reshape(batch, dim, 1)
neg = mxnet.nd.equal(target, 0).reshape(batch, 1, dim)
# pos_matrix = mxnet.nd.concat(*([pos] * dim), dim=2)
# neg_matrix = mxnet.nd.concat(*([neg] * dim), dim=1)
# print(L.shape,pos_matrix.shape,neg_matrix.shape,dist.shape)
#print("L",L)
# loss_matrix = L*nd.sum(nd.relu(1-pos_matrix*neg_matrix*dist),axis=1)
# print(loss_matrix.shape)
#print("weight L shape",L.shape)# namely (batch,dim)
filter_matrix = nd.broadcast_mul(pos, neg)
#loss_element = 1+filter_matrix*(-dist)
loss_element = nd.relu(1 + filter_matrix * (-dist))
# print(L.shape,loss_element.shape)
# L= nd.array([[3.0000, 5.8333, 0.0000, 0.0000],[0.0000, 3.0000, 0.0000, 3.0000]])
L = L.reshape(batch, dim, 1)
# print("L",L)
# print("loss_element",loss_element)
# print(nd.broadcast_mul(L,loss_element))
loss_matrix = nd.sum(nd.broadcast_mul(L, loss_element))
return nd.sum(loss_matrix)
def get_pred_result(hm_pred, offset_pred, wh_pred, k=100):
ctx = hm_pred.context
batch_size, num_classes, _, _ = hm_pred.shape
topk_cat_x_idx, topk_cat_y_idx, cls_id = topk(hm_pred, k=k)
batch_index = nd.arange(batch_size)
batch_indices = nd.repeat(batch_index, repeats=num_classes)
batch_indices = nd.reshape(batch_indices, (1, batch_size*k))
batch_indices = batch_indices.as_in_context(ctx)
cls_id = nd.reshape(cls_id, (1, batch_size*k))
topk_cat_y_idx = nd.reshape(topk_cat_y_idx, (1, batch_size*k))
topk_cat_x_idx = nd.reshape(topk_cat_x_idx, (1, batch_size*k))
score_indices = nd.concat(batch_indices, cls_id, topk_cat_y_idx, topk_cat_x_idx, dim=0)
scores = nd.gather_nd(hm_pred, score_indices)
fake_idx_0 = nd.zeros_like(nd.arange(batch_size*k)).reshape((1, -1))
fake_idx_0 = fake_idx_0.as_in_context(ctx)
fake_idx_1 = nd.ones((1, batch_size*k))
fake_idx_1 = fake_idx_1.as_in_context(ctx)
fake_indices_0 = nd.concat(batch_indices, fake_idx_0, topk_cat_y_idx, topk_cat_x_idx, dim=0)
fake_indices_1 = nd.concat(batch_indices, fake_idx_1, topk_cat_y_idx, topk_cat_x_idx, dim=0)
x_offset = nd.gather_nd(offset_pred, fake_indices_0)
y_offset = nd.gather_nd(offset_pred, fake_indices_1)
h = nd.gather_nd(wh_pred, fake_indices_0)
w = nd.gather_nd(wh_pred, fake_indices_1)
x_offset_ = nd.broadcast_mul(topk_cat_x_idx, x_offset)
y_offset_ = nd.broadcast_mul(topk_cat_y_idx, y_offset)
topk_cat_x_idx = nd.broadcast_add(topk_cat_x_idx, x_offset_)
topk_cat_y_idx = nd.broadcast_add(topk_cat_y_idx, y_offset_)
xmin = topk_cat_x_idx - w/2
ymin = topk_cat_y_idx - h/2
xmax = topk_cat_x_idx + w/2
ymax = topk_cat_y_idx + h/2
xmin = nd.reshape(xmin, (batch_size, k)).expand_dims(axis=-1)
ymin = nd.reshape(ymin, (batch_size, k)).expand_dims(axis=-1)
xmax = nd.reshape(xmax, (batch_size, k)).expand_dims(axis=-1)
ymax = nd.reshape(ymax, (batch_size, k)).expand_dims(axis=-1)
cls_id = nd.reshape(cls_id, (batch_size, k)).expand_dims(axis=-1)
scores = nd.reshape(scores, (batch_size, k)).expand_dims(axis=-1)
results = nd.concat(xmin, ymin, xmax, ymax, cls_id, scores, dim=-1)
return results
def refine_bbox_nd(bbox, bbox_delta, im_info=None, means=None, stds=None):
xmin, ymin, xmax, ymax = nd.split(data=bbox, num_outputs=4, axis=1)
bbox_width = xmax - xmin + 1.
bbox_height = ymax - ymin + 1.
center_x = 0.5 * (xmin + xmax)
center_y = 0.5 * (ymin + ymax)
bbox_delta_reshape = nd.Reshape(data=bbox_delta, shape=(0, -1, 4))
dx, dy, dw, dh = nd.split(data=bbox_delta_reshape,
num_outputs=4,
axis=2,
squeeze_axis=1)
if (means is not None) and (stds is not None):
dx = dx * stds[0] + means[0]
dy = dy * stds[1] + means[1]
dw = dw * stds[2] + means[2]
dh = dh * stds[3] + means[3]
refine_center_x = nd.broadcast_add(lhs=center_x,
rhs=nd.broadcast_mul(lhs=bbox_width,
rhs=dx))
refine_center_y = nd.broadcast_add(lhs=center_y,
rhs=nd.broadcast_mul(lhs=bbox_height,
rhs=dy))
refined_width = nd.broadcast_mul(lhs=bbox_width, rhs=nd.exp(dw))
refined_height = nd.broadcast_mul(lhs=bbox_height, rhs=nd.exp(dh))
w_offset = 0.5 * (refined_width - 1.)
h_offset = 0.5 * (refined_height - 1.)
refined_xmin = nd.expand_dims(refine_center_x - w_offset, axis=1)
refined_ymin = nd.expand_dims(refine_center_y - h_offset, axis=1)
refined_xmax = nd.expand_dims(refine_center_x + w_offset, axis=1)
refined_ymax = nd.expand_dims(refine_center_y + h_offset, axis=1)
refined_bbox = nd.concat(refined_xmin,
refined_ymin,
refined_xmax,
refined_ymax,
dim=1)
if im_info is not None:
# assume im_info [[height, width, scale]] with shape (1,3)
im_hw = nd.slice_axis(im_info, axis=1, begin=0, end=2)
im_wh = nd.reverse(im_hw, axis=1)
im_wh = im_wh - 1.
im_wh = nd.tile(data=im_wh, reps=(1, 2))
im_wh = nd.Reshape(im_wh, shape=(1, 4, 1))
refined_bbox = nd.broadcast_minimum(lhs=refined_bbox, rhs=im_wh)
refined_bbox = nd.broadcast_maximum(lhs=refined_bbox,
rhs=nd.zeros_like(refined_bbox))
# print refined_bbox.debug_str()
return refined_bbox
def forward(self, pred, target):
"""
pred is the output prob,target the multi-class set label
"""
batch, dim = pred.shape
dist = nd.broadcast_minus(pred.reshape(batch, dim, 1),
pred.reshape(batch, 1, dim))
pos = mxnet.nd.greater(target, 0).reshape(batch, dim, 1)
neg = mxnet.nd.equal(target, 0).reshape(batch, 1, dim)
pos.detach()
neg.detach()
# pos_matrix = mxnet.nd.concat(*([pos]*dim),dim=2)
# neg_matrix = mxnet.nd.concat(*([neg]*dim),dim=1)
#print(pos_matrix.shape,neg_matrix.shape,dist.shape)
#loss_matrix = nd.log(1+nd.sum(pos_matrix*neg_matrix*nd.exp(-dist)))
# print("----------------------")
# print("pos is ",pos)
# print("neg is ",neg)
# print("multiply is ",nd.broadcast_mul(pos,neg))
# print("the distance is ",dist)
# print("the mat mul is ",nd.broadcast_mul(pos,neg)*dist)
# print("-----------------------")
loss_matrix = nd.log(
1 + nd.sum(nd.broadcast_mul(pos, neg) * nd.exp(-dist)))
return loss_matrix
def outer(self, tensor_in_1, tensor_in_2):
"""
The outer product of two tensors.
Args:
tensor_in_1 (Tensor): Tensor object
tensor_in_2 (Tensor): Tensor object
Returns:
MXNet NDArray: The outer product.
"""
tensor_in_1 = self.astensor(tensor_in_1)
tensor_in_2 = self.astensor(tensor_in_2)
tensor_1_shape = tensor_in_1.shape
tensor_2_shape = tensor_in_2.shape
if len(tensor_1_shape) == 1:
tensor_1_shape = (tensor_1_shape[0], 1)
if len(tensor_2_shape) == 1:
tensor_2_shape = (tensor_2_shape[0], 1)
rows1, cols1 = tensor_1_shape
rows2, cols2 = tensor_2_shape
return nd.reshape(
nd.broadcast_mul(
tensor_in_1.reshape((rows1, 1, cols1, 1)),
tensor_in_2.reshape((1, rows2, 1, cols2)),
),
(rows1 * cols1, rows2 * cols2),
)
def forward(self, data, hidden_state, mask):
features, _ = self.elmo_container[0](data, hidden_state, mask)
combined_features = nd.concat(*[nd.expand_dims(f, axis=0) for f in features], dim=0)
scaled_elmo_layers = nd.broadcast_mul(lhs=combined_features,
rhs=self.elmo_s.data().softmax(axis=0))
scaled_elmo_embedding = nd.broadcast_mul(lhs=nd.sum(scaled_elmo_layers, axis=0),
rhs=self.gamma.data())
highway_output = self.highway(scaled_elmo_embedding.reshape(shape=(-1,
self._embedding_size)))
highway_output = highway_output.reshape(shape=scaled_elmo_embedding.shape)
encoded_data = self.encoder(highway_output.transpose(axes=(1, 0, 2)))
intents = self.intent_dense(encoded_data)
slots = self.slot_dense(highway_output)
return intents, slots
def forward(self, input_sample):
embedding_part_sparse = self.get_embedding_array(
input_sample) #(?,n1,e)
# time_start = time.time()
linear_dense_input = self.get_linear_dense_input(input_sample)
linear_logit = self.get_linear_logit(embedding_part_sparse,
linear_dense_input)
# print('linear_logit_time_cost:[%d]s' % np.int(time.time() - time_start))
# time_start = time.time()
# print("linear_logit:")
# print(linear_logit.mean().asscalar())
dense_part_dense = self.get_dense_array(input_sample) # (?,n2,e)
merge_sparse_dense = nd.concat(embedding_part_sparse,
dense_part_dense,
dim=1) # ?,f,e
xv = nd.broadcast_mul(merge_sparse_dense, self.params.get('v').data())
fm_embedding_part = nd.square(
xv.sum(axis=1)) - nd.square(xv).sum(axis=1)
fm_embedding_part = fm_embedding_part.sum(axis=1).reshape(
(-1, 1)) / 2 # (?,1)
# print(fm_embedding_part)
net_embedding = nn.Sequential()
net_embedding.add(self.bn_embedding)
fm_embedding_part = net_embedding(fm_embedding_part)
deep_input = merge_sparse_dense.flatten() # ?,f*e
cin_output = self.cin(merge_sparse_dense)
# print('cin_time_cost:[%d]s' % np.int(time.time() - time_start))
# time_start = time.time()
# print("cin_output:")
# print(cin_output.mean().asscalar())
net = nn.Sequential()
for i in range(len(self.dense_list)):
net.add(self.dense_list[i])
net.add(self.bn_list[i])
net.add(self.activation_list[i])
net.add(self.dropout_list[i])
net.add(self.dnn_out)
deep_output = net(deep_input)
# print('dnn_time_cost:[%d]s' % np.int(time.time() - time_start))
# print('deep_output:')
# print(deep_output.mean().asscalar())
# print('----')
# print(deep_output)
deep_fm = nd.sigmoid(linear_logit + deep_output + cin_output)
# print('==========')
# print('deep_fm:')
# print(deep_fm.mean().asscalar())
# print('---------------------')
return deep_fm
def accuracy_metric(gallery_features, gallery_label, query_features, query_label):
B1 = nd.sum(nd.square(gallery_features), axis=1, keepdims=True)
B2 = nd.sum(nd.square(query_features), axis=1, keepdims=True)
dist_mat = nd.broadcast_add(B2, B1.T) - 2 * nd.dot(query_features, gallery_features.T)
label_mask = nd.broadcast_equal(dist_mat, nd.min(dist_mat, axis=1, keepdims=True)).astype('float32')
pre_label_mat = nd.broadcast_mul(label_mask, gallery_label.reshape(1, -1).astype('float32'))
pre_label_list = nd.max(pre_label_mat, axis=1)
cor_num = nd.sum(nd.equal(pre_label_list, query_label.astype('float32')))
return cor_num.asnumpy()[0] / len(query_label)
def forward(self, is_train, req, in_data, out_data, aux):
x = in_data[0]
if is_train:
self._spatial_dropout_mask = nd.broadcast_greater(
nd.random_uniform(low=0, high=1, shape=(1, self._num_filters, 1, 1), ctx=self._ctx),
nd.ones(shape=(1, self._num_filters, 1, 1), ctx=self._ctx) * self._p,
ctx=self._ctx
)
y = nd.broadcast_mul(x, self._spatial_dropout_mask, ctx=self._ctx) / (1 - self._p)
self.assign(out_data[0], req[0], y)
else:
self.assign(out_data[0], req[0], x)
def apply_attention(self, F, inputs, hidden, encoder_outputs):
#inputs : decoder input의미
concated = F.concat(inputs, hidden, dim=1)
#(,max_seq_length) : max_seq_length 개별 시퀀스의 중요도
attn_weights = F.softmax(self.attdense(concated), axis=1)
# (N,max_seq_length,n_hidden) x (N,max_seq_length) = (N, max_seq_length, n_hidden)
#attn_weigths 가중치를 인코더 출력값에 곱해줌
w_encoder_outputs = F.broadcast_mul(encoder_outputs,
attn_weights.expand_dims(2))
#(N, vocab_size * max_seq_length), (N, max_seq_length * n_hidden) = (N, ...)
output = F.concat(inputs.flatten(), w_encoder_outputs.flatten(), dim=1)
output = self.dropout(output)
#(N, vocab_size)
output = self.attn_combine(output)
#attention weight은 시각화를 위해 뽑아둔다.
return (output, attn_weights)
def forward(self, words, wordsmask=None):
"""Compute embedding of words in batch.
Parameters
----------
words : mx.nd.NDArray
Array of token indices.
wordsmask : mx.nd.NDArray
Mask for embeddings returned by the word level embedding operator.
"""
#pylint: disable=arguments-differ
if wordsmask is not None:
wordsmask = nd.expand_dims(wordsmask, axis=-1)
return nd.broadcast_mul(self.embedding(words), wordsmask)
else:
return self.embedding(words)
def forward(self, x, y, x_mask, y_mask):
l_x = get_length(x_mask)
l_y = get_length(y_mask)
h_H, hidden_H= self.title_lstm(x, l_x)
h_S, hidden_S = self.abstract_lstm(y, l_y)
x_mask_ = return_mask(x_mask ,y_mask)
y_mask_ = return_mask(y_mask ,x_mask)
u_H,_ = self.ta_mutal(h_H, h_S, h_S, x_mask_)
h_S_hat,_ = self.at_mutal(h_S, h_H, h_H, y_mask_)
u_H = self.ffn1(u_H)
G_t = nd.sigmoid(self.W_G(nd.concat(h_S, h_S_hat, dim = -1))).squeeze()
h_S_ = nd.stack(*[nd.broadcast_mul(i,j.expand_dims(1)) for (i,j) in zip(h_S, G_t)])
u_S = self.ffn2(h_S_hat + h_S_)
u_X = nd.concat(u_H, u_S, dim =1)
mask_u = nd.concat(x_mask,y_mask, dim = -1)
mask_u = return_mask(mask_u, mask_u)
u_X, weight = self.self_attn(u_X, u_X, u_X, mask_u)
u_X = self.ffn3(u_X)
s = self.final_linear(self.title_linear(hidden_H)+ self.abstract_linear(hidden_S))
return s, u_X, weight # s refers to deocder intial hidden state, u_X refers to memory
def forward(self, input_sample):
embedding_part_sparse = self.get_embedding_array(input_sample) #(?,n1,e)
linear_dense_input = self.get_linear_dense_input(input_sample)
linear_logit = self.get_linear_logit(embedding_part_sparse,linear_dense_input)
dense_part_dense = self.get_dense_array(input_sample) # (?,n2,e)
merge_sparse_dense = nd.concat(embedding_part_sparse,dense_part_dense,dim=1) # ?,f,e
xv = nd.broadcast_mul(merge_sparse_dense,self.params.get('v').data())
fm_embedding_part = nd.square(xv.sum(axis=1)) - nd.square(xv).sum(axis=1)
fm_embedding_part = fm_embedding_part.sum(axis=1).reshape((-1,1)) / 2 # (?,1)
net_embedding = nn.Sequential()
net_embedding.add(self.bn_embedding)
fm_embedding_part = net_embedding(fm_embedding_part)
deep_input = merge_sparse_dense.flatten() # ?,f*e
net = nn.Sequential()
for i in range(len(self.dense_list)):
net.add(self.dense_list[i])
net.add(self.bn_list[i])
net.add(self.activation_list[i])
net.add(self.dropout_list[i])
net.add(self.dnn_out)
deep_output = net(deep_input)
deep_fm = nd.sigmoid(linear_logit + fm_embedding_part + deep_output)
return deep_fm
def forward(self,
words,
subwords,
wordsmask=None,
subwordsmask=None,
words_to_unique_subwords_indices=None):
"""Compute embedding of words in batch.
Parameters
----------
words : mx.nd.NDArray
Array of token indices.
subwords : mx.nd.NDArray
The subwords associated with the tokens in `words`. If
words_to_unique_subwords_indices is specified may contain the
subwords of the unique tokens in `words` with
`words_to_unique_subwords_indices` containing the reverse mapping.
wordsmask : mx.nd.NDArray, optional
Mask for embeddings returned by the word level embedding operator.
subwordsmask : mx.nd.NDArray, optional
A mask for the subword embeddings looked up from `subwords`.
Applied before sum reducing the subword embeddings.
words_to_unique_subwords_indices : mx.nd.NDArray, optional
Mapping from the position in the `words` array to the position in
the words_to_unique_subwords_indices` array.
"""
#pylint: disable=arguments-differ
embeddings = self.embedding(words)
if wordsmask is not None:
wordsmask = nd.expand_dims(wordsmask, axis=-1)
embeddings = nd.broadcast_mul(embeddings, wordsmask)
else:
wordsmask = 1
if words_to_unique_subwords_indices is None:
assert words.shape[0] == subwords.shape[0]
if subwordsmask is None:
subwordsmask = nd.ones_like(subwords)
num_embeddings = \
nd.sum(subwordsmask, axis=-1, keepdims=True) + wordsmask
subword_embeddings = self.subword_embedding(subwords, subwordsmask)
return nd.broadcast_div(embeddings + subword_embeddings,
num_embeddings)
else:
if subwordsmask is None:
subwordsmask = nd.ones_like(subwords)
subword_embedding_weights = self.subword_embedding(
subwords, subwordsmask)
words_to_unique_subwords_indices = \
words_to_unique_subwords_indices.reshape(words.shape)
subword_embeddings = nd.Embedding(
data=words_to_unique_subwords_indices,
weight=subword_embedding_weights,
input_dim=subword_embedding_weights.shape[0],
output_dim=self.embedding_size)
num_embeddings = nd.Embedding(
data=words_to_unique_subwords_indices,
weight=nd.sum(subwordsmask, axis=-1, keepdims=True),
input_dim=subword_embedding_weights.shape[0],
output_dim=1).reshape(words.shape).expand_dims(-1) + wordsmask
return nd.broadcast_div(embeddings + subword_embeddings,
num_embeddings)
def forward(self, is_train, req, in_data, out_data, aux):
nms_start_time = time.time()
#inputs
cls_score = in_data[0]
bbox_pred = in_data[1]
rois = in_data[2]
im_info = in_data[3]
fc_all_2_relu = in_data[4]
nms_rank_weight = in_data[5]
nms_rank_bias = in_data[6]
roi_feat_embedding_weight = in_data[7]
roi_feat_embedding_bias = in_data[8]
nms_pair_pos_fc1_1_weight = in_data[9]
nms_pair_pos_fc1_1_bias = in_data[10]
nms_query_1_weight = in_data[11]
nms_query_1_bias = in_data[12]
nms_key_1_weight = in_data[13]
nms_key_1_bias = in_data[14]
nms_linear_out_1_weight = in_data[15]
nms_linear_out_1_bias = in_data[16]
nms_logit_weight = in_data[17]
nms_logit_bias = in_data[18]
if self.has_non_gt_index:
non_gt_index = in_data[19]
else:
non_gt_index = None
if self.nongt_dim is not None:
cls_score_nongt = nd.slice_axis(data=cls_score,
axis=0,
begin=0,
end=self.nongt_dim)
# cls_score_nongt = monitor_wrapper(cls_score_nongt, 'cls_score_nongt')
bbox_pred_nongt = nd.slice_axis(data=bbox_pred,
axis=0,
begin=0,
end=self.nongt_dim)
elif non_gt_index is not None:
cls_score_nongt = nd.take(a=cls_score, indices=non_gt_index)
bbox_pred_nongt = nd.take(a=bbox_pred, indices=non_gt_index)
else:
cls_score_nongt = cls_score
bbox_pred_nongt = bbox_pred
bbox_pred_nongt = nd.BlockGrad(bbox_pred_nongt)
# remove batch idx and gt roi
sliced_rois = nd.slice_axis(data=rois, axis=1, begin=1, end=None)
if self.nongt_dim is not None:
sliced_rois = nd.slice_axis(data=sliced_rois,
axis=0,
begin=0,
end=self.nongt_dim)
elif non_gt_index is not None:
sliced_rois = nd.take(a=sliced_rois, indices=non_gt_index)
# bbox_pred_nobg, [num_rois, 4*(num_reg_classes-1)]
bbox_pred_nobg = nd.slice_axis(data=bbox_pred_nongt,
axis=1,
begin=4,
end=None)
# [num_boxes, 4, num_reg_classes-1]
refined_bbox = refine_bbox_nd(sliced_rois,
bbox_pred_nobg,
im_info,
means=self.bbox_means,
stds=self.bbox_stds)
# softmax cls_score to cls_prob, [num_rois, num_classes]
cls_prob = nd.softmax(data=cls_score_nongt, axis=-1)
cls_prob_nobg = nd.slice_axis(cls_prob, axis=1, begin=1, end=None)
sorted_cls_prob_nobg = nd.sort(data=cls_prob_nobg,
axis=0,
is_ascend=False)
# sorted_score, [first_n, num_fg_classes]
sorted_score = nd.slice_axis(sorted_cls_prob_nobg,
axis=0,
begin=0,
end=self.first_n,
name='sorted_score')
max_score_per_class = sorted_score.max(axis=0)
max_score_per_class_numpy = max_score_per_class.asnumpy()
valid_class_thresh = self.class_thresh
valid_class_thresh = np.minimum(valid_class_thresh,
max_score_per_class_numpy.max())
valid_class_indices = np.where(
max_score_per_class_numpy >= valid_class_thresh)[0]
invalid_class_indices = np.where(
max_score_per_class_numpy < valid_class_thresh)[0]
num_valid_classes = len(valid_class_indices)
valid_class_indices_nd = nd.array(valid_class_indices,
ctx=sorted_score.context)
# sort by score
rank_indices = nd.argsort(data=cls_prob_nobg, axis=0, is_ascend=False)
# first_rank_indices, [first_n, num_fg_classes]
first_rank_indices = nd.slice_axis(rank_indices,
axis=0,
begin=0,
end=self.first_n)
valid_first_rank_indices = first_rank_indices.transpose().take(
valid_class_indices_nd).transpose()
# sorted_bbox, [first_n, num_fg_classes, 4, num_reg_classes-1]
sorted_bbox = nd.take(a=refined_bbox, indices=first_rank_indices)
if self.class_agnostic:
# sorted_bbox, [first_n, num_fg_classes, 4]
sorted_bbox = nd.Reshape(sorted_bbox,
shape=(0, 0, 0),
name='sorted_bbox')
else:
cls_mask = nd.arange(0, self.num_fg_classes)
cls_mask = nd.Reshape(cls_mask, shape=(1, -1, 1))
cls_mask = nd.broadcast_to(cls_mask, shape=(self.first_n, 0, 4))
# sorted_bbox, [first_n, num_fg_classes, 4]
sorted_bbox = nd.pick(data=sorted_bbox,
name='sorted_bbox',
index=cls_mask,
axis=3)
valid_sorted_bbox = sorted_bbox.transpose(
(1, 0, 2)).take(valid_class_indices_nd).transpose((1, 0, 2))
# sorted_bbox = monitor_wrapper(sorted_bbox, 'sorted_bbox')
# nms_rank_embedding, [first_n, 1024]
nms_rank_embedding = extract_rank_embedding_nd(self.first_n, 1024)
# nms_rank_feat, [first_n, 1024]
nms_rank_feat = nd.FullyConnected(name='nms_rank',
data=nms_rank_embedding,
num_hidden=128,
weight=nms_rank_weight,
bias=nms_rank_bias)
# nms_position_matrix, [num_valid_classes, first_n, first_n, 4]
nms_position_matrix = extract_multi_position_matrix_nd(
valid_sorted_bbox)
# roi_feature_embedding, [num_rois, 1024]
# fc_all_2_relu = monitor_wrapper(fc_all_2_relu, 'fc_all_2_relu')
roi_feat_embedding = nd.FullyConnected(
name='roi_feat_embedding',
data=fc_all_2_relu,
num_hidden=128,
weight=roi_feat_embedding_weight,
bias=roi_feat_embedding_bias)
# sorted_roi_feat, [first_n, num_valid_classes, 128]
sorted_roi_feat = nd.take(a=roi_feat_embedding,
indices=valid_first_rank_indices)
# vectorized nms
# nms_embedding_feat, [first_n, num_valid_classes, 128]
nms_embedding_feat = nd.broadcast_add(lhs=sorted_roi_feat,
rhs=nd.expand_dims(nms_rank_feat,
axis=1))
# nms_attention_1, [first_n, num_valid_classes, 1024]
nms_attention_1 = nms_attention_nd(
nms_embedding_feat,
nms_position_matrix,
nms_pair_pos_fc1_1_weight,
nms_pair_pos_fc1_1_bias,
nms_query_1_weight,
nms_query_1_bias,
nms_key_1_weight,
nms_key_1_bias,
nms_linear_out_1_weight,
nms_linear_out_1_bias,
num_rois=self.first_n,
index=1,
group=self.nms_attention_group,
dim=self.nms_attention_dim,
fc_dim=self.nms_attention_fc_dim,
feat_dim=self.nms_attention_feat_dim)
nms_all_feat_1 = nms_embedding_feat + nms_attention_1
nms_all_feat_1_relu = nd.Activation(data=nms_all_feat_1,
act_type='relu',
name='nms_all_feat_1_relu')
# [first_n * num_valid_classes, 1024]
nms_all_feat_1_relu_reshape = nd.Reshape(nms_all_feat_1_relu,
shape=(-3, -2))
# logit, [first_n * num_valid_classes, num_thresh]
nms_conditional_logit = nd.FullyConnected(
name='nms_logit',
data=nms_all_feat_1_relu_reshape,
num_hidden=self.num_thresh,
weight=nms_logit_weight,
bias=nms_logit_bias)
# logit_reshape, [first_n, num_valid_classes, num_thresh]
nms_conditional_logit_reshape = nd.Reshape(nms_conditional_logit,
shape=(self.first_n,
num_valid_classes,
self.num_thresh))
nms_conditional_score = nd.Activation(
data=nms_conditional_logit_reshape,
act_type='sigmoid',
name='nms_conditional_score')
if num_valid_classes == self.num_fg_classes:
full_nms_conditional_score = nms_conditional_score
else:
full_nms_conditional_score = nd.concat(
nms_conditional_score,
nd.zeros(
(self.first_n, self.num_fg_classes - num_valid_classes,
self.num_thresh),
ctx=nms_conditional_score.context),
dim=1)
all_indexes = np.concatenate(
(valid_class_indices, invalid_class_indices))
restore_indexes = np.zeros((self.num_fg_classes))
restore_indexes[all_indexes] = np.arange(self.num_fg_classes)
restore_indexes = nd.array(restore_indexes,
ctx=nms_conditional_score.context)
full_nms_conditional_score = full_nms_conditional_score.transpose(
(1, 0, 2)).take(restore_indexes).transpose((1, 0, 2))
sorted_score_reshape = nd.expand_dims(sorted_score, axis=2)
# sorted_score_reshape = nd.BlockGrad(sorted_score_reshape)
nms_multi_score = nd.broadcast_mul(lhs=sorted_score_reshape,
rhs=full_nms_conditional_score)
_ = nms_multi_score.mean().asnumpy()
all_time = time.time() - nms_start_time
if 'learn_nms_time' not in globals().keys(
) or 'learn_nms_count' not in globals().keys():
globals()['learn_nms_time'] = []
globals()['learn_nms_count'] = 0
if globals()['learn_nms_count'] >= 1000:
globals()['learn_nms_time'].pop(0)
globals()['learn_nms_time'].append(all_time)
else:
globals()['learn_nms_time'].append(all_time)
globals()['learn_nms_count'] += 1
if globals()['learn_nms_count'] % 250 == 0:
print("--->> learn nms running average time cost: {}".format(
float(sum(globals()['learn_nms_time'])) /
(1000 if globals()['learn_nms_count'] > 1000 else
globals()['learn_nms_count'])))
self.assign(out_data[0], req[0], nms_multi_score)
self.assign(out_data[1], req[1], sorted_bbox)
self.assign(out_data[2], req[2], sorted_score)
def backward(self, req, out_grad, in_data, out_data, in_grad, aux):
dy = out_grad[0]
dx = nd.broadcast_mul(self._spatial_dropout_mask, dy)
self.assign(in_grad[0], req[0], dx)
# %%
# Define input vector.
x = nd.array([[1, 2, 3]])
# %%
# Compute forward and gradient
with autograd.record():
y = fc1(x)
y.backward()
# %%
# Verify gradient:
# y = x * W.T + b.T
# dy/dW = x * dW/dW + dx/dW * W, where dx/dW = 0
dydw = fc1.w.grad()
dydw_ = nd.broadcast_mul(x, nd.ones_like(fc1.w.data()))
print("\nx", x)
print("\nw", fc1.w.data())
print("\ndydw", dydw)
if nd.sum(dydw).asscalar() > 1 and np.allclose(dydw.asnumpy(), dydw_.asnumpy()):
print("Success!")
print("================================================================================")
print("= Test normalized FullyConnected")
print("================================================================================")
# %%
# Define network, containing the DenseNormalized only.
fc1 = DenseNormalized(3, 2, with_norm=True)
def nms(hm):
hm_max = nd.Pooling(data=hm, kernel=(3, 3), pool_type='max',
stride=(1, 1), pad=(1, 1))
max_idx = (hm_max == hm)
hm = nd.broadcast_mul(hm, max_idx)
return hm
