def compute_scale(self, F, data: Tensor,
observed_indicator: Tensor) -> Tensor:
"""
Parameters
----------
F
A module that can either refer to the Symbol API or the NDArray
API in MXNet.
data
tensor containing the data to be scaled.
observed_indicator
observed_indicator: binary tensor with the same shape as
``data``, that has 1 in correspondence of observed data points,
and 0 in correspondence of missing data points.
Returns
-------
Tensor
shape (N, T, C) or (N, C, T) scaled along the specified axis.
"""
axis_zero = nd.prod(
data == data.zeros_like(), self.axis, keepdims=True
) # Along the specified axis, which array are always at zero
axis_zero = nd.broadcast_to(
axis_zero, shape=data.shape) # Broadcast it to the shape of data
min_val = nd.where(
1 - observed_indicator,
nd.broadcast_to(data.max(keepdims=True), shape=data.shape),
data,
).min(
axis=self.axis, keepdims=True
) # return the min value along specified axis while ignoring value according to observed_indicator
max_val = nd.where(
1 - observed_indicator,
nd.broadcast_to(data.min(keepdims=True), shape=data.shape),
data,
).max(
axis=self.axis, keepdims=True
) # return the max value along specified axis while ignoring value according to observed_indicator
scaled_data = (data - min_val) / (max_val - min_val) # Rescale
scaled_data = nd.where(
axis_zero, scaled_data.zeros_like(), scaled_data
) # Clip Nan values to zero if the data was equal to zero along specified axis
scaled_data = nd.where(
scaled_data != scaled_data, scaled_data.ones_like(), scaled_data
) # Clip the Nan values to one. scaled_date!=scaled_data tells us where the Nan values are in scaled_data
return nd.where(
1 - observed_indicator, scaled_data.zeros_like(), scaled_data
) # Replace data with zero where observed_indicator tells us to.
def valid(self):
loss_sce = gluon.loss.SoftmaxCrossEntropyLoss(from_logits=False)
loss_l1 = gluon.loss.L1Loss()
obj_loss = LOSS_EVAL('obj_loss')
cls_loss = LOSS_EVAL('cls_loss')
xywh_loss = LOSS_EVAL('xywh_loss')
positive_weight = 5.0
negative_weight = 0.1
class_weight = 1.0
xywh_weight = 5.0
self.validIter.reset()
for batchind, batch in enumerate(self.validIter):
gtX = batch.data[0].as_in_context(self.ctx)
gtY = batch.label[0].as_in_context(self.ctx)
prdY = self.net(gtX)
prdCls, prdObj, prdXYWH = self.format_net_output(prdY)
boxMask, boxCls, boxObj, boxXYWH = self.format_groundtruth(gtY, prdXYWH)
lossCls = loss_sce(prdCls, boxCls, boxMask * class_weight)
boxWeight = nd.where(boxMask > 0, boxMask * positive_weight, boxMask * negative_weight)
lossObj = loss_l1(prdObj, boxObj, boxWeight)
lossXYWH = loss_l1(prdXYWH, boxXYWH, boxMask * xywh_weight)
obj_loss.update(lossObj)
cls_loss.update(lossCls)
xywh_loss.update(lossXYWH)
print('validation: (%s,%f) (%s,%f) (%s,%f)'%(cls_loss.get()[0], cls_loss.get()[1],\
obj_loss.get()[0], obj_loss.get()[1],\
xywh_loss.get()[0], xywh_loss.get()[1]))
return
def update(self, index, weight, grad, state):
assert (isinstance(weight, NDArray))
assert (isinstance(grad, NDArray))
self._update_count(index)
lr = self._get_lr(index)
wd = self._get_wd(index)
t = self._index_update_count[index]
with bulk(self._bulk):
# preprocess grad
grad *= self.rescale_grad
if self.clip_gradient is not None:
grad = clip(grad, -self.clip_gradient, self.clip_gradient)
mean, var = state
mean *= self.beta1
mean += (1. - self.beta1) * grad
var *= self.beta2
var += (1. - self.beta2) * square(grad)
r1 = weight.norm()
if not self.bias_correction:
r1 = minimum(maximum(r1, self.lower_bound), self.upper_bound)
sqrt_var = sqrt(var)
sqrt_var += self.epsilon
g = mean / sqrt_var
g += wd * weight
else:
# apply bias correction
mean_hat = mean / (1. - power(self.beta1, t))
var_hat = var / (1. - power(self.beta2, t))
if self._eps_after_sqrt:
sqrt(var_hat, out=var_hat)
var_hat += self.epsilon
else:
var_hat += self.epsilon
sqrt(var_hat, out=var_hat)
mean_hat /= var_hat
mean_hat += wd * weight
g = mean_hat
r2 = g.norm()
# calculate lamb_trust_ratio
ratio = r1 / r2
# becomes NaN if ratio == NaN or 0, otherwise 0
nan_or_zero = 1 - ratio / ratio
r = where(nan_or_zero, ones_like(ratio), ratio)
lr *= r
# update weight
g *= lr
weight[:] -= g
def test_program():
import matplotlib.pyplot as plt
import matplotlib.patches as patches
ssd = ssd_mx.ssd_model()
for model in ssd:
model.initialize()
data = get_iterators(data_shape=304, batch_size=4)
batch_data = data.next()
# 将-1占位符改为背景标签0，对应坐标框记录为[0,0,0,0]
label = nd.where(batch_data.label[0] < 0,
nd.zeros_like(batch_data.label[0]), batch_data.label[0])
anchors, class_preds, box_preds = ssd_mx.ssd_forward(batch_data.data[0], ssd)
target_labels, target_localizations, target_scores = \
ssd_mx.bboxes_encode(label[:, :, 0],
label[:, :, 1:],
anchors)
# print(label[:, :, 0].shape, label[:, :, 1:].shape)
# print('target_labels', [f.shape for f in target_labels])
# print('target_localizations', [f.shape for f in target_localizations])
# print('target_scores', [f.shape for f in target_scores])
print('[*] class_preds', [f.shape for f in class_preds])
print('[*] box_preds', [f.shape for f in box_preds])
# Add loss function.
logits, gclasses, no_classes, fpmask, fnmask = \
ssd_mx.data_reshape(class_preds, box_preds,
target_labels, target_localizations, target_scores,
match_threshold=.5,
negative_ratio=3)
p_cls_loss = util_mx.FocalLoss()
n_cls_loss = util_mx.FocalLoss()
print(nd.sum(p_cls_loss(logits, gclasses) * fpmask),
nd.sum(n_cls_loss(logits, no_classes) * fnmask))
reg_loss = util_mx.SmoothL1Loss()
print(nd.sum(reg_loss(box_preds, target_localizations, fpmask.expand_dims(-1))))
# .asnumpy比使用np.array速度快得多
plt.imshow(batch_data.data[0][2].asnumpy().transpose(1, 2, 0) / 255.)
currentAxis = plt.gca()
for i in range(6):
box = batch_data.label[0][2][i][1:].asnumpy() * 300
if any(box < 0):
continue
print(int(batch_data.label[0][2][i][0].asscalar()))
rect = patches.Rectangle((box[1], box[0]), box[3] - box[1], box[2] - box[0],
linewidth=1, edgecolor='g', facecolor='none')
currentAxis.add_patch(rect)
plt.show()
def forward(self, is_train, req, in_data, out_data, aux):
arm_cls_preds = in_data[0]
odm_cls_target = in_data[1]
odm_loc_target_mask = in_data[2]
arm_cls_preds = nd.softmax(data=arm_cls_preds)
arm_cls_preds_classes = nd.split(data=arm_cls_preds,axis=1,num_outputs=2)
# arm_cls_preds_bg shape : (batch , h*w*num_anchors[:layers]) 负类【0】
arm_cls_preds_bg = nd.reshape(data=arm_cls_preds_classes[0],shape=(0,-1))
prob_temp = nd.ones_like(arm_cls_preds_bg)*0.99
cond1 = arm_cls_preds_bg >= prob_temp # > 0.99 idx is 1
# print('negative cond1 ------- :',heapq.nlargest(2,arm_cls_preds_bg[0]))
temp1 = nd.ones_like(odm_cls_target)*(-1) ### TODO： 0 还是-1表示背景??
# 如果ARM分类出的负类的置信度大于0.99，将其在ODM的anchor标号中去掉（-1替代），负类转换为背景
odm_cls_target_mask = nd.where(condition=cond1,x=temp1,y=odm_cls_target)
# apply filtering to odm_loc_target_mask
# odm_loc_target_mask_shape: (batch, num_anchors, 4)
arm_cls_preds_bg = nd.reshape(data=arm_cls_preds_bg,shape=(0,-1,1))#(batch , h*w*num_anchors[:layers],1)
# (batch , h*w*num_anchors[:layers] , 4 )
odm_loc_target_mask = nd.reshape(data=odm_loc_target_mask,shape=(0,-1,4))
odm_loc_target_mask = odm_loc_target_mask[:,:,0] #(batch , h*w*num_anchors[:layers])
#(batch , h*w*num_anchors[:layers], 1)
## 取整个batch中 所有行的 第一列，相当于对原来的4个相同label[0 0 0 0 ],[1 1 1 1]变成[0],[1]
odm_loc_target_mask = nd.reshape(data=odm_loc_target_mask,shape=(0,-1,1))
loc_temp = nd.ones_like(odm_loc_target_mask)*0.99
cond2 = arm_cls_preds_bg >= loc_temp
temp2 = nd.zeros_like(odm_loc_target_mask) # 取0
# 如果ARM分类出的负类的置信度大于0.99，将其在ODM的掩码置0
## 实际上不管IOU计算的大小，用AMR的分类结果，如果是大于0.99的负类，不管通过IOU判断的正负类结果如何，都设置为背景
odm_loc_target_bg_mask = nd.where(cond2,temp2,odm_loc_target_mask)
odm_loc_target_bg_mask = nd.concat(*[odm_loc_target_bg_mask]*4,dim=2)
# 还原维度
odm_loc_target_bg_mask = nd.reshape(odm_loc_target_bg_mask,shape=(0,-1))
for ind, val in enumerate([odm_cls_target_mask, odm_loc_target_bg_mask]):
self.assign(out_data[ind], req[ind], val)
def hybrid_forward(self, F, fts, ys, ftt, yt):
"""
Semantic Alignment Loss
:param F: Function
:param yt: label for the target domain [N]
:param ftt: features for the target domain [N, K]
:param ys: label for the source domain [M]
:param fts: features for the source domain [M, K]
:return:
"""
if self._fn:
# Normalize ft
fts = F.L2Normalization(fts, mode='instance')
ftt = F.L2Normalization(ftt, mode='instance')
fts_rpt = F.broadcast_to(fts.expand_dims(axis=0),
shape=(self._bs_tgt, self._bs_src,
self._embed_size))
ftt_rpt = F.broadcast_to(ftt.expand_dims(axis=1),
shape=(self._bs_tgt, self._bs_src,
self._embed_size))
dists = F.sum(F.square(ftt_rpt - fts_rpt), axis=2)
yt_rpt = F.broadcast_to(yt.expand_dims(axis=1),
shape=(self._bs_tgt,
self._bs_src)).astype('int32')
ys_rpt = F.broadcast_to(ys.expand_dims(axis=0),
shape=(self._bs_tgt,
self._bs_src)).astype('int32')
y_same = F.equal(yt_rpt, ys_rpt).astype('float32')
y_diff = F.not_equal(yt_rpt, ys_rpt).astype('float32')
intra_cls_dists = dists * y_same
inter_cls_dists = dists * y_diff
max_dists = F.max(dists, axis=1, keepdims=True)
max_dists = F.broadcast_to(max_dists,
shape=(self._bs_tgt, self._bs_src))
revised_inter_cls_dists = F.where(y_same, max_dists, inter_cls_dists)
max_intra_cls_dist = F.max(intra_cls_dists, axis=1)
min_inter_cls_dist = F.min(revised_inter_cls_dists, axis=1)
loss = F.relu(max_intra_cls_dist - min_inter_cls_dist + self._margin)
return loss
def backward(self, req, out_grad, in_data, out_data, in_grad, aux):
# 1.update class center
for class_id in xrange(int(self.num_class)):
center_diff = nd.where(nd.array(
(self.label == class_id).astype(np.int8)),
x=self.diff,
y=nd.zeros_like(self.diff))
# print nd.where(nd.array(self.label==class_id),diff,nd.zeros(diff))
# print "class id", class_id,
self.center[class_id] += self.alpha * nd.sum(center_diff, axis=0)
# print self.center
# 2. assign grad to former layer
self.assign(
in_grad[0], req[0],
self.diff.reshape(self.diff_reshape).transpose(
(0, 3, 1, 2)) * self.grad_scale)
def train(self):
loss_sce = gluon.loss.SoftmaxCrossEntropyLoss(from_logits=False)
loss_l1 = gluon.loss.L1Loss()
obj_loss = LOSS_EVAL('obj_loss')
cls_loss = LOSS_EVAL('cls_loss')
xywh_loss = LOSS_EVAL('xywh_loss')
positive_weight = 5.0
negative_weight = 0.1
class_weight = 1.0
xywh_weight = 5.0
trainer = gluon.Trainer(self.net.collect_params(),"sgd",{"learning_rate":1,"wd":5e-4})
for epoch in range(self.maxEpochNum):
self.trainIter.reset()
tic = time.time()
for batchind, batch in enumerate(self.trainIter):
gtY = batch.label[0].as_in_context(self.ctx)
gtX = batch.data[0].as_in_context(self.ctx)
with autograd.record():
predY = self.net(gtX)
predCls, predObj, predXYWH = self.format_net_output(predY)
with autograd.pause():
boxMask, boxCls, boxObj, boxXYWH = self.format_groundtruth(gtY,predXYWH)
loss0 = loss_sce(predCls, boxCls, boxMask * class_weight)
boxWeight = nd.where(boxMask > 0, boxMask * positive_weight, boxMask * negative_weight)
loss1 = loss_l1(predObj, boxObj, boxWeight)
loss2 = loss_l1(predXYWH, boxXYWH, boxMask * xywh_weight)
loss = loss0 + loss1 + loss2
loss.backward()
trainer.step(self.batchSize)
cls_loss.update(loss0)
obj_loss.update(loss1)
xywh_loss.update(loss2)
print('epoch %d(%.2fsec): (%s,%f) (%s,%f) (%s,%f)'%(epoch,time.time() - tic,cls_loss.get()[0], cls_loss.get()[1],\
obj_loss.get()[0], obj_loss.get()[1],\
xywh_loss.get()[0], xywh_loss.get()[1]))
if (1 + epoch) % self.validStep == 0:
self.valid()
self.net.save_params(os.path.join(self.outRoot,'yolo%.8d.params'%(epoch+1)))
return
def hybrid_forward(self, F, pred, label, sample_weight=None):
if not self._from_logits:
pred = F.log_softmax(pred, axis=self._axis)
if self._sparse_label:
if self._size_average:
valid_label_map = (label !=
self._ignore_label).astype('float32')
loss = -(F.pick(pred, label, axis=self._axis, keepdims=True) *
valid_label_map)
else:
loss = -F.pick(pred, label, axis=self._axis, keepdims=True)
loss = F.where(
label.expand_dims(axis=self._axis) == self._ignore_label,
F.zeros_like(loss), loss)
else:
label = _reshape_like(F, label, pred)
loss = -F.sum(pred * label, axis=self._axis, keepdims=True)
loss = _apply_weighting(F, loss, self._weight, sample_weight)
if self._size_average:
return F.mean(loss, axis=self._batch_axis, exclude=True) * \
valid_label_map.size / F.sum(valid_label_map)
else:
return F.mean(loss, axis=self._batch_axis, exclude=True)
def margin_loss(self, pick_fc):
args = self.args
import math
m = args.margin_m
s = args.margin_s
assert s > 0.0
#assert m >= 0.1
assert m < (math.pi / 2)
# cos_t * s
cos_t = pick_fc / s
cos_m = math.cos(m)
sin_m = math.sin(m)
mm = math.sin(math.pi - m) * m #sin(pi-m)*m=sin(m)*m
# threadhold = 0.0
threshold = math.cos(math.pi - m) # threshold < -cos(m)
if args.easy_margin:
cond = nd.Activation(data=cos_t, act_type='relu')
else:
cond_v = cos_t - threshold
cond = nd.Activation(data=cond_v, act_type='relu')
body = cos_t * cos_t
body = 1.0 - body
sin_t = nd.sqrt(body) #mx.sym.sqrt(body)
new_zy = cos_t * cos_m # cos(t+m) = c*c - s*s
b = sin_t * sin_m
new_zy = new_zy - b
new_zy = new_zy * s
if args.easy_margin:
zy_keep = pick_fc
else:
zy_keep = pick_fc - s * mm # zy-s*sin(m)*m = s*cos(t) - s*m*sin(m)
new_zy = nd.where(
cond, new_zy, zy_keep
) # cond < 0, zy_keep= s*cos(theta) or s*cos(theta)-s*m*sin(m)
return new_zy
def do_transform_ip(self, buffer: Gst.Buffer) -> Gst.FlowReturn:
try:
# convert Gst.Buffer to np.ndarray
caps = self.sinkpad.get_current_caps()
#it's a reference on GstBuffer data
#if you modify it output will be modified as well
image = gst_buffer_with_caps_for_ndarray(buffer, caps)
#c, h, w = image.shape
#convert to NDArray
t = nd.array(image, dtype=np.float32)
#clip low level signals to 0
if self.minimal > 0:
if self.zeros_vec is None:
#since we don't expect change of frame parametors,
# must not make it every time
self.zeros_vec = nd.zeros_like(t)
t = nd.where(t < self.minimal, self.zeros_vec, t)
t = self.apply_conv(t)
#clear output
image.fill(0)
#detach from mxnet context and copy to numpy.nddarray[w,h]
conved = t[0, 0, :, :].asnumpy()
#simple contrast
if self.contrast == True:
conved = simpleContrast(conved, 0.0)
#copy to top left coner
ch, cw = conved.shape
image[0, :ch, :cw] = conved
except Exception as e:
logging.info(conved.shape)
logging.error(e)
return Gst.FlowReturn.OK
trainer = mx.gluon.Trainer(ssd.collect_params(),
'sgd', {'learning_rate': 0.01, 'wd': 5e-4})
data = get_iterators(data_shape=304, batch_size=batch_size)
for epoch in range(30):
# reset data iterators and metrics
data.reset()
cls_metric.reset()
# box_metric.reset()
tic = time.time()
for i, batch in enumerate(data):
start_time = time.time()
x = batch.data[0].as_in_context(ctx)
y = batch.label[0].as_in_context(ctx)
# 将-1占位符改为背景标签0，对应坐标框记录为[0,0,0,0]
y = nd.where(y < 0, nd.zeros_like(y), y)
with mx.autograd.record():
anchors, class_preds, box_preds = ssd(x, False)
target_labels, target_localizations, target_scores = \
ssd_mx.bboxes_encode(y[:, :, 0],
y[:, :, 1:],
anchors)
logits, gclasses, no_classes, fpmask, fnmask, localisations, glocalisations = \
ssd_mx.data_reshape(class_preds, box_preds,
target_labels, target_localizations, target_scores,
match_threshold=.5,
negative_ratio=3)
p_cls = nd.sum(p_cls_loss(logits, gclasses) * fpmask)
n_cls = nd.sum(n_cls_loss(logits, no_classes) * fnmask)
def replace_inf_with_zero(x):
return nd.where(nd.abs(x) == np.inf, nd.zeros_like(x), x)
#pdb.set_trace()
if a > 0.5:
c = boxXYWH[0,y,x,0,:].asnumpy().tolist()
c = ['%.2f'%cc for cc in c]
c = '-'.join(c)
elif b > 0.5:
c = boxXYWH[0,y,x,1,:].asnumpy().tolist()
c = ['%.2f'%cc for cc in c]
c = '-'.join(c)
lines.append('[%.3f,%.3f,%s],'%(a,b,c))
lines.append('')
with open('dbg.txt','wb') as f:
f.write('\r\n'.join(lines))
#pdb.set_trace()
loss0 = loss_sce(predCls, boxCls, boxMask * class_weight)
boxWeight = nd.where( boxMask > 0, boxMask * positive_weight, boxMask * negative_weight )
loss1 = loss_l1(predObj, boxObj, boxWeight)
#pdb.set_trace()
loss2 = loss_l1(predXYWH, boxXYWH, xywh_weight * boxMask )
loss = loss0 + loss1 + loss2
loss.backward()
trainer.step(batchSize)
cls_loss.update(loss0)
obj_loss.update(loss1)
xywh_loss.update(loss2)
print '%d,(%s,%f),(%s,%f),(%s,%f),%f'%(epoch,cls_loss.get()[0], cls_loss.get()[1],\
obj_loss.get()[0], obj_loss.get()[1], xywh_loss.get()[0], xywh_loss.get()[1], time.time() - tic)
net.save_params('%d.params'%epoch)
c = '-'
#pdb.set_trace()
if a > 0.5:
c = boxXYWH[0, y, x, 0, :].asnumpy().tolist()
c = ['%.2f' % cc for cc in c]
c = '-'.join(c)
elif b > 0.5:
c = boxXYWH[0, y, x, 1, :].asnumpy().tolist()
c = ['%.2f' % cc for cc in c]
c = '-'.join(c)
lines.append('[%.3f,%.3f,%s],' % (a, b, c))
lines.append('')
with open('dbg.txt', 'wb') as f:
f.write('\r\n'.join(lines))
#pdb.set_trace()
loss0 = loss_sce(predCls, boxCls, boxMask * class_weight)
boxWeight = nd.where(boxMask > 0, boxMask * positive_weight,
boxMask * negative_weight)
loss1 = loss_l1(predObj, boxObj, boxWeight)
#pdb.set_trace()
loss2 = loss_l1(predXYWH, boxXYWH, xywh_weight * boxMask)
loss = loss0 + loss1 + loss2
loss.backward()
trainer.step(batchSize)
cls_loss.update(loss0)
obj_loss.update(loss1)
xywh_loss.update(loss2)
print '%d,(%s,%f),(%s,%f),(%s,%f),%f'%(epoch,cls_loss.get()[0], cls_loss.get()[1],\
obj_loss.get()[0], obj_loss.get()[1], xywh_loss.get()[0], xywh_loss.get()[1], time.time() - tic)
net.save_params('%d.params' % epoch)
