def lenet(X, params):
h1_conv = nd.Convolution(X, weight=params[0], bias=params[1], kernel=(3, 3), num_filter=20)
h1_activation = nd.Activation(h1_conv, act_type='relu')
h1 = nd.Pooling(data=h1_activation, pool_type='avg', kernel=(2, 2), stride=(2, 2))
h2_conv = nd.Convolution(h1, weight=params[2], bias=params[3], kernel=(5, 5), num_filter=50)
h2_activation = nd.relu(h2_conv)
h2 = nd.Pooling(data=h2_activation, pool_type='avg', kernel=(2, 2), stride=(2, 2))
h2 = nd.flatten(h2)
h3_linear = nd.dot(h2, params[4]) + params[5]
h3 = nd.relu(h3_linear)
yhat = nd.dot(h3, params[6]) + params[7]
return yhat
def _sample_visible_from_hidden(self, hidden):
""" Sample the visible units from the hidden units.
This is the Negative phase of the 1-step Contrastive Divergence algorithm.
:param hidden: activations of the hidden units
:return: dreamed visible state probability
"""
batch_size = hidden.shape[0]
activations = self._transpose_batch(
nd.batch_dot(self._broadcast_to_batch(self.weights, batch_size),
self._transpose_batch(hidden)))
if self.visible_bias is not None:
activations += self._broadcast_to_batch(self.visible_bias,
batch_size)
dreamed_visible = nd.Activation(activations, act_type="sigmoid")
return dreamed_visible
def _sample_hidden_from_visible(self, visible):
""" Sample the hidden units from the visible units.
This is the Positive phase of the 1-step Contrastive Divergence algorithm.
:param visible: activations of the visible units
:return: tuple(hidden state probability, hidden state)
"""
batch_size = visible.shape[0]
activations = nd.batch_dot(
visible, self._broadcast_to_batch(self.weights, batch_size))
if self.hidden_bias is not None:
activations += self._broadcast_to_batch(self.hidden_bias,
batch_size)
hidden_pr = nd.Activation(activations, act_type="sigmoid")
hidden = self._sample_bernoulli(hidden_pr)
return hidden_pr, hidden
def forward(self, data):
""" Forward process of MFDense layer
Parameters
----------
data: NDArray with shape [n, b, in_dim]
Returns
-------
output: NDArray with shape [n, b, out_dim]
"""
ctx = data.context
weight = nd.dot(self.w1.data(ctx), self.w2.data(ctx)).reshape(
(-1, self.in_dim, self.out_dim)) # [n, in_dim, out_dim]
bias = nd.dot(self.b1.data(ctx), self.b2.data(ctx)).reshape(
(-1, 1, self.out_dim)) # [n, 1, out_dim]
output = nd.batch_dot(data, weight) + bias
if self.activation is not None:
output = nd.Activation(output, act_type=self.activation)
return output
def Net(X, is_training=False, verbose=False):
# 第一层卷积
h1_conv = nd.Convolution(X,
weight=W1,
bias=b1,
kernel=W1.shape[2:],
num_filter=c1)
h1_bn = batch_norm(h1_conv, gamma1, beta1, is_training, moving_mean1,
moving_variance1)
h1_activation = nd.Activation(h1_bn, act_type='relu')
h1 = nd.Pooling(h1_activation,
pool_type='max',
kernel=(2, 2),
stride=(2, 2))
# second Convolution
h2_conv = nd.Convolution(h1,
weight=W2,
bias=b2,
kernel=W2.shape[2:],
num_filter=c2)
h2_bn = batch_norm(h2_conv, gamma2, beta2, is_training, moving_mean2,
moving_variance2)
h2_activation = nd.relu(h2_bn)
h2 = nd.Pooling(h2_activation,
pool_type='max',
kernel=(2, 2),
stride=(2, 2))
h2 = nd.flatten(h2)
h3_linear = nd.dot(h2, W3) + b3
h3 = nd.relu(h3_linear)
h4_linear = nd.dot(h3, W4) + b4
if verbose:
print('1st conv block:', h1.shape)
print('2nd conv block:', h2.shape)
print('1st dense block:', h3.shape)
print('2nd dense block:', h4_linear.shape)
print('output:', h4_linear)
return h4_linear
def sigmoid(x):
return nd.Activation(x, act_type='sigmoid')
def __call__(self, x: nd.NDArray) -> nd.NDArray:
return nd.Activation(x, act_type='softrelu')
def softplus(x):
return nd.Activation(x, act_type='softrelu')
def nms_attention_nd(roi_feat,
position_mat,
nms_pair_pos_fc1_1_weight,
nms_pair_pos_fc1_1_bias,
nms_query_1_weight,
nms_query_1_bias,
nms_key_1_weight,
nms_key_1_bias,
nms_linear_out_1_weight,
nms_linear_out_1_bias,
num_rois,
dim=(1024, 1024, 1024),
fc_dim=(64, 16),
feat_dim=1024,
group=16,
index=1):
""" Attetion module with vectorized version
Args:
roi_feat: [num_rois, num_fg_classes, feat_dim]
position_mat: [num_fg_classes, num_rois, num_rois, 4]
num_rois: number of rois
dim: key, query and linear_out dim
fc_dim:
feat_dim:
group:
index:
Returns:
output: [num_rois, num_fg_classes, fc_dim]
"""
dim_group = (dim[0] / group, dim[1] / group, dim[2] / group)
roi_feat = nd.transpose(roi_feat, axes=(1, 0, 2))
# roi_feat_reshape, [num_fg_classes*num_rois, feat_dim]
roi_feat_reshape = nd.Reshape(roi_feat, shape=(-3, -2))
# position_embedding, [num_fg_classes, num_rois, num_rois, fc_dim[0]]
position_embedding = extract_pairwise_multi_position_embedding_nd(
position_mat, fc_dim[0])
# [num_fg_classes * num_rois * num_rois, fc_dim[0]]
position_embedding_reshape = nd.Reshape(position_embedding,
shape=(-1, fc_dim[0]))
# position_feat_1, [num_fg_classes * num_rois * num_rois, fc_dim[1]]
position_feat_1 = nd.FullyConnected(name='nms_pair_pos_fc1_' + str(index),
data=position_embedding_reshape,
weight=nms_pair_pos_fc1_1_weight,
bias=nms_pair_pos_fc1_1_bias,
num_hidden=fc_dim[1])
# position_feat_1, [num_fg_classes, num_rois, num_rois, fc_dim[1]]
position_feat_1 = nd.Reshape(position_feat_1,
shape=(-1, num_rois, num_rois, fc_dim[1]))
aff_weight = nd.Activation(data=position_feat_1, act_type='relu')
# aff_weight, [num_fg_classes, fc_dim[1], num_rois, num_rois]
aff_weight = nd.transpose(aff_weight, axes=(0, 3, 1, 2))
####################### multi head in batch###########################
assert dim[0] == dim[1], 'Matrix multi requires the same dims!'
# q_data, [num_fg_classes * num_rois, dim[0]]
q_data = nd.FullyConnected(name='nms_query_' + str(index),
data=roi_feat_reshape,
weight=nms_query_1_weight,
bias=nms_query_1_bias,
num_hidden=dim[0])
# q_data, [num_fg_classes, num_rois, group, dim_group[0]]
q_data_batch = nd.Reshape(q_data,
shape=(-1, num_rois, group, dim_group[0]))
q_data_batch = nd.transpose(q_data_batch, axes=(0, 2, 1, 3))
# q_data_batch, [num_fg_classes * group, num_rois, dim_group[0]]
q_data_batch = nd.Reshape(q_data_batch, shape=(-3, -2))
k_data = nd.FullyConnected(name='nms_key_' + str(index),
data=roi_feat_reshape,
weight=nms_key_1_weight,
bias=nms_key_1_bias,
num_hidden=dim[1])
# k_data, [num_fg_classes, num_rois, group, dim_group[1]]
k_data_batch = nd.Reshape(k_data,
shape=(-1, num_rois, group, dim_group[1]))
k_data_batch = nd.transpose(k_data_batch, axes=(0, 2, 1, 3))
# k_data_batch, [num_fg_classes * group, num_rois, dim_group[1]]
k_data_batch = nd.Reshape(k_data_batch, shape=(-3, -2))
v_data = roi_feat
aff = nd.batch_dot(lhs=q_data_batch,
rhs=k_data_batch,
transpose_a=False,
transpose_b=True)
# aff_scale, [num_fg_classes * group, num_rois, num_rois]
aff_scale = (1.0 / math.sqrt(float(dim_group[1]))) * aff
assert fc_dim[1] == group, 'Check the dimensions in attention!'
# [num_fg_classes * fc_dim[1], num_rois, num_rois]
aff_weight_reshape = nd.Reshape(aff_weight, shape=(-3, -2))
# weighted_aff, [num_fg_classes * fc_dim[1], num_rois, num_rois]
weighted_aff = nd.log(nd.maximum(aff_weight_reshape, 1e-6)) + aff_scale
# aff_softmax, [num_fg_classes * fc_dim[1], num_rois, num_rois]
aff_softmax = nd.softmax(data=weighted_aff,
axis=2,
name='nms_softmax_' + str(index))
aff_softmax_reshape = nd.Reshape(aff_softmax,
shape=(-1, fc_dim[1] * num_rois, 0))
# output_t, [num_fg_classes, fc_dim[1] * num_rois, feat_dim]
output_t = nd.batch_dot(lhs=aff_softmax_reshape, rhs=v_data)
# output_t_reshape, [num_fg_classes, fc_dim[1], num_rois, feat_dim]
output_t_reshape = nd.Reshape(output_t,
shape=(-1, fc_dim[1], num_rois, feat_dim))
# output_t_reshape, [fc_dim[1], feat_dim, num_rois, num_fg_classes]
output_t_reshape = nd.transpose(output_t_reshape, axes=(1, 3, 2, 0))
# output_t_reshape, [1, fc_dim[1] * feat_dim, num_rois, num_fg_classes]
output_t_reshape = nd.Reshape(output_t_reshape,
shape=(1, fc_dim[1] * feat_dim, num_rois,
-1))
linear_out = nd.Convolution(name='nms_linear_out_' + str(index),
data=output_t_reshape,
weight=nms_linear_out_1_weight,
bias=nms_linear_out_1_bias,
kernel=(1, 1),
num_filter=dim[2],
num_group=fc_dim[1])
# [dim[2], num_rois, num_fg_classes]
linear_out_reshape = nd.Reshape(linear_out, shape=(dim[2], num_rois, -1))
# [num_rois, num_fg_classes, dim[2]]
output = nd.transpose(linear_out_reshape, axes=(1, 2, 0))
return output
def forward(self, is_train, req, in_data, out_data, aux):
nms_start_time = time.time()
#inputs
cls_score = in_data[0]
bbox_pred = in_data[1]
rois = in_data[2]
im_info = in_data[3]
fc_all_2_relu = in_data[4]
nms_rank_weight = in_data[5]
nms_rank_bias = in_data[6]
roi_feat_embedding_weight = in_data[7]
roi_feat_embedding_bias = in_data[8]
nms_pair_pos_fc1_1_weight = in_data[9]
nms_pair_pos_fc1_1_bias = in_data[10]
nms_query_1_weight = in_data[11]
nms_query_1_bias = in_data[12]
nms_key_1_weight = in_data[13]
nms_key_1_bias = in_data[14]
nms_linear_out_1_weight = in_data[15]
nms_linear_out_1_bias = in_data[16]
nms_logit_weight = in_data[17]
nms_logit_bias = in_data[18]
if self.has_non_gt_index:
non_gt_index = in_data[19]
else:
non_gt_index = None
if self.nongt_dim is not None:
cls_score_nongt = nd.slice_axis(data=cls_score,
axis=0,
begin=0,
end=self.nongt_dim)
# cls_score_nongt = monitor_wrapper(cls_score_nongt, 'cls_score_nongt')
bbox_pred_nongt = nd.slice_axis(data=bbox_pred,
axis=0,
begin=0,
end=self.nongt_dim)
elif non_gt_index is not None:
cls_score_nongt = nd.take(a=cls_score, indices=non_gt_index)
bbox_pred_nongt = nd.take(a=bbox_pred, indices=non_gt_index)
else:
cls_score_nongt = cls_score
bbox_pred_nongt = bbox_pred
bbox_pred_nongt = nd.BlockGrad(bbox_pred_nongt)
# remove batch idx and gt roi
sliced_rois = nd.slice_axis(data=rois, axis=1, begin=1, end=None)
if self.nongt_dim is not None:
sliced_rois = nd.slice_axis(data=sliced_rois,
axis=0,
begin=0,
end=self.nongt_dim)
elif non_gt_index is not None:
sliced_rois = nd.take(a=sliced_rois, indices=non_gt_index)
# bbox_pred_nobg, [num_rois, 4*(num_reg_classes-1)]
bbox_pred_nobg = nd.slice_axis(data=bbox_pred_nongt,
axis=1,
begin=4,
end=None)
# [num_boxes, 4, num_reg_classes-1]
refined_bbox = refine_bbox_nd(sliced_rois,
bbox_pred_nobg,
im_info,
means=self.bbox_means,
stds=self.bbox_stds)
# softmax cls_score to cls_prob, [num_rois, num_classes]
cls_prob = nd.softmax(data=cls_score_nongt, axis=-1)
cls_prob_nobg = nd.slice_axis(cls_prob, axis=1, begin=1, end=None)
sorted_cls_prob_nobg = nd.sort(data=cls_prob_nobg,
axis=0,
is_ascend=False)
# sorted_score, [first_n, num_fg_classes]
sorted_score = nd.slice_axis(sorted_cls_prob_nobg,
axis=0,
begin=0,
end=self.first_n,
name='sorted_score')
max_score_per_class = sorted_score.max(axis=0)
max_score_per_class_numpy = max_score_per_class.asnumpy()
valid_class_thresh = self.class_thresh
valid_class_thresh = np.minimum(valid_class_thresh,
max_score_per_class_numpy.max())
valid_class_indices = np.where(
max_score_per_class_numpy >= valid_class_thresh)[0]
invalid_class_indices = np.where(
max_score_per_class_numpy < valid_class_thresh)[0]
num_valid_classes = len(valid_class_indices)
valid_class_indices_nd = nd.array(valid_class_indices,
ctx=sorted_score.context)
# sort by score
rank_indices = nd.argsort(data=cls_prob_nobg, axis=0, is_ascend=False)
# first_rank_indices, [first_n, num_fg_classes]
first_rank_indices = nd.slice_axis(rank_indices,
axis=0,
begin=0,
end=self.first_n)
valid_first_rank_indices = first_rank_indices.transpose().take(
valid_class_indices_nd).transpose()
# sorted_bbox, [first_n, num_fg_classes, 4, num_reg_classes-1]
sorted_bbox = nd.take(a=refined_bbox, indices=first_rank_indices)
if self.class_agnostic:
# sorted_bbox, [first_n, num_fg_classes, 4]
sorted_bbox = nd.Reshape(sorted_bbox,
shape=(0, 0, 0),
name='sorted_bbox')
else:
cls_mask = nd.arange(0, self.num_fg_classes)
cls_mask = nd.Reshape(cls_mask, shape=(1, -1, 1))
cls_mask = nd.broadcast_to(cls_mask, shape=(self.first_n, 0, 4))
# sorted_bbox, [first_n, num_fg_classes, 4]
sorted_bbox = nd.pick(data=sorted_bbox,
name='sorted_bbox',
index=cls_mask,
axis=3)
valid_sorted_bbox = sorted_bbox.transpose(
(1, 0, 2)).take(valid_class_indices_nd).transpose((1, 0, 2))
# sorted_bbox = monitor_wrapper(sorted_bbox, 'sorted_bbox')
# nms_rank_embedding, [first_n, 1024]
nms_rank_embedding = extract_rank_embedding_nd(self.first_n, 1024)
# nms_rank_feat, [first_n, 1024]
nms_rank_feat = nd.FullyConnected(name='nms_rank',
data=nms_rank_embedding,
num_hidden=128,
weight=nms_rank_weight,
bias=nms_rank_bias)
# nms_position_matrix, [num_valid_classes, first_n, first_n, 4]
nms_position_matrix = extract_multi_position_matrix_nd(
valid_sorted_bbox)
# roi_feature_embedding, [num_rois, 1024]
# fc_all_2_relu = monitor_wrapper(fc_all_2_relu, 'fc_all_2_relu')
roi_feat_embedding = nd.FullyConnected(
name='roi_feat_embedding',
data=fc_all_2_relu,
num_hidden=128,
weight=roi_feat_embedding_weight,
bias=roi_feat_embedding_bias)
# sorted_roi_feat, [first_n, num_valid_classes, 128]
sorted_roi_feat = nd.take(a=roi_feat_embedding,
indices=valid_first_rank_indices)
# vectorized nms
# nms_embedding_feat, [first_n, num_valid_classes, 128]
nms_embedding_feat = nd.broadcast_add(lhs=sorted_roi_feat,
rhs=nd.expand_dims(nms_rank_feat,
axis=1))
# nms_attention_1, [first_n, num_valid_classes, 1024]
nms_attention_1 = nms_attention_nd(
nms_embedding_feat,
nms_position_matrix,
nms_pair_pos_fc1_1_weight,
nms_pair_pos_fc1_1_bias,
nms_query_1_weight,
nms_query_1_bias,
nms_key_1_weight,
nms_key_1_bias,
nms_linear_out_1_weight,
nms_linear_out_1_bias,
num_rois=self.first_n,
index=1,
group=self.nms_attention_group,
dim=self.nms_attention_dim,
fc_dim=self.nms_attention_fc_dim,
feat_dim=self.nms_attention_feat_dim)
nms_all_feat_1 = nms_embedding_feat + nms_attention_1
nms_all_feat_1_relu = nd.Activation(data=nms_all_feat_1,
act_type='relu',
name='nms_all_feat_1_relu')
# [first_n * num_valid_classes, 1024]
nms_all_feat_1_relu_reshape = nd.Reshape(nms_all_feat_1_relu,
shape=(-3, -2))
# logit, [first_n * num_valid_classes, num_thresh]
nms_conditional_logit = nd.FullyConnected(
name='nms_logit',
data=nms_all_feat_1_relu_reshape,
num_hidden=self.num_thresh,
weight=nms_logit_weight,
bias=nms_logit_bias)
# logit_reshape, [first_n, num_valid_classes, num_thresh]
nms_conditional_logit_reshape = nd.Reshape(nms_conditional_logit,
shape=(self.first_n,
num_valid_classes,
self.num_thresh))
nms_conditional_score = nd.Activation(
data=nms_conditional_logit_reshape,
act_type='sigmoid',
name='nms_conditional_score')
if num_valid_classes == self.num_fg_classes:
full_nms_conditional_score = nms_conditional_score
else:
full_nms_conditional_score = nd.concat(
nms_conditional_score,
nd.zeros(
(self.first_n, self.num_fg_classes - num_valid_classes,
self.num_thresh),
ctx=nms_conditional_score.context),
dim=1)
all_indexes = np.concatenate(
(valid_class_indices, invalid_class_indices))
restore_indexes = np.zeros((self.num_fg_classes))
restore_indexes[all_indexes] = np.arange(self.num_fg_classes)
restore_indexes = nd.array(restore_indexes,
ctx=nms_conditional_score.context)
full_nms_conditional_score = full_nms_conditional_score.transpose(
(1, 0, 2)).take(restore_indexes).transpose((1, 0, 2))
sorted_score_reshape = nd.expand_dims(sorted_score, axis=2)
# sorted_score_reshape = nd.BlockGrad(sorted_score_reshape)
nms_multi_score = nd.broadcast_mul(lhs=sorted_score_reshape,
rhs=full_nms_conditional_score)
_ = nms_multi_score.mean().asnumpy()
all_time = time.time() - nms_start_time
if 'learn_nms_time' not in globals().keys(
) or 'learn_nms_count' not in globals().keys():
globals()['learn_nms_time'] = []
globals()['learn_nms_count'] = 0
if globals()['learn_nms_count'] >= 1000:
globals()['learn_nms_time'].pop(0)
globals()['learn_nms_time'].append(all_time)
else:
globals()['learn_nms_time'].append(all_time)
globals()['learn_nms_count'] += 1
if globals()['learn_nms_count'] % 250 == 0:
print("--->> learn nms running average time cost: {}".format(
float(sum(globals()['learn_nms_time'])) /
(1000 if globals()['learn_nms_count'] > 1000 else
globals()['learn_nms_count'])))
self.assign(out_data[0], req[0], nms_multi_score)
self.assign(out_data[1], req[1], sorted_bbox)
self.assign(out_data[2], req[2], sorted_score)
def forward(self, x):
new_x = self.noise(x)
self.singure = self.weight_first.data() * self.position
self.first_layer = nd.Activation(nd.dot(new_x, self.singure), act_type='relu')
self.output = self.second_layer(self.noise2(self.first_layer))
return self.output