def predict(self, imgs, sizes=None, visualize=False):
"""
在计算mAP的时候使用
:param imgs: 一个batch的图片
:param sizes: batch中每张图片的输入尺寸
:return: 返回所有一个batch中所有图片的坐标,类,类概率值 三个值都是list型数据,里面包含的是numpy数据
"""
self.eval()
if visualize:
self.use_preset('visualize')
prepared_imgs = imgs
bboxes = list()
labels = list()
scores = list()
for img, size in zip(prepared_imgs, sizes):
img = at.totensor(img[None]).float()
scale = img.shape[3] / size[1]
roi_cls_loc, roi_scores, rois = self(img, scale=scale)
roi_score = roi_scores.data
roi_cls_loc = roi_cls_loc.data
roi = at.totensor(rois) / scale
mean = torch.Tensor(self.loc_normalize_mean).repeat(self.n_class)[None]
std = torch.Tensor(self.loc_normalize_std).repeat(self.n_class)[None]
roi_cls_loc = (roi_cls_loc * std + mean)
roi_cls_loc = roi_cls_loc.view(-1, self.n_class, 4)
roi = roi.view(-1, 1, 4).expand_as(roi_cls_loc)
# 从这里其实可以看出，roi_cls_loc是位置偏差,是用来计算更接近的位置,而不是位置,
cls_bbox = loc2bbox(at.tonumpy(roi).reshape((-1, 4)),
at.tonumpy(roi_cls_loc).reshape((-1, 4)))
cls_bbox = at.totensor(cls_bbox)
cls_bbox = cls_bbox.view(-1, self.n_class * 4)
# 这里应该是控制cls_bbox的边界,在min和max之间
cls_bbox[:, 0::2] = (cls_bbox[:, 0::2]).clamp(min=0, max=size[0])
cls_bbox[:, 1::2] = (cls_bbox[:, 1::2]).clamp(min=0, max=size[1])
prob = at.tonumpy(F.softmax(at.totensor(roi_score), dim=1))
raw_cls_bbox = at.tonumpy(cls_bbox)
raw_prob = at.tonumpy(prob)
bbox, label, score = self._suppress(raw_cls_bbox, raw_prob)
bboxes.append(bbox)
labels.append(label)
scores.append(score)
self.use_preset('evaluate')
self.train()
# 这里返回的是预测图片的bbox，labels，scores
return bboxes, labels, scores
def clustering(self, mymodel, outdim):
mymodel.eval()
features = np.zeros([self.lendata, outdim])
with torch.no_grad():
for i, outputt in enumerate(self.dataloader):
image, label, path, id = outputt
out, feature = mymodel(image)
#features[i, :] = tonumpy(feature['lastlinear'])
features[i, :] = tonumpy(feature)
'''feature=mymodel(image)
features[i,:]=tonumpy(feature)'''
self.ids.append(id)
self.filenames.append(path)
if self.use_pca:
model2 = PCA(n_components=500, random_state=728)
model2.fit(np.transpose(features))
temp = np.transpose(model2.components_)
self.images_new = temp
else:
self.images_new = features
model = KMeans(n_clusters=self.n_clusters)
s = model.fit(self.images_new)
print('Kmeans-Clustering..........')
iddict = {}
dict = {}
center_mean = {}
globalmean = 0.
centers = model.cluster_centers_
labels = model.labels_
l, f = centers.shape
for i in range(l):
dict[str(i)] = {'center': centers[i, :].tolist(), 'features': []}
for i in range(self.lendata):
dict[str(labels[i])]['features'].append(
self.images_new[i].tolist())
iddict[str(scalar(self.ids[i]))] = {}
iddict[str(scalar(self.ids[i]))]['label'] = labels[i]
iddict[str(scalar(self.ids[i]))]['feature'] = self.images_new[i]
for d in dict:
ft = np.asarray(dict[d]['features'])
temp = pow(ft, 2).sum(1).reshape([ft.shape[0], -1])
dis = temp - 2 * np.dot(ft, np.transpose(ft)) + np.transpose(temp)
dis = np.around(dis, decimals=5).sum()
n = ft.shape[0]
if n != 1:
dis /= n * (n - 1)
center_mean[d] = dis
globalmean += float(dis) / l
return iddict, center_mean, globalmean
def forward(self, x):
y = x.view(-1, x.size(1), x.size(2) * x.size(3)) # b x c x hw
vladres = np.zeros([y.size(0), self.k, y.size(-1)])
for i in range(y.size(0)):
feature = tonumpy(y[i])
c, hw = feature.shape
self.model.fit(feature)
clusters = self.model.cluster_centers_
labels = self.model.labels_
for j in range(c):
label = labels[j]
vladres[i, label, :] += feature[j] - clusters[label]
vladres = totensor(vladres)
return vladres
def train(**kwargs):
opt._parse(kwargs)
dataset = Dataset(opt)
print('load data')
dataloader = data_.DataLoader(dataset, \
batch_size=1, \
shuffle=True, \
# pin_memory=True,
num_workers=opt.num_workers)
testset = TestDataset(opt)
test_dataloader = data_.DataLoader(testset,
batch_size=1,
num_workers=opt.test_num_workers,
shuffle=False, \
pin_memory=True
)
faster_rcnn = FasterRCNNVGG16()
print('model construct completed')
trainer = FasterRCNNTrainer(faster_rcnn).cuda()
if opt.load_path:
trainer.load(opt.load_path)
print('load pretrained model from %s' % opt.load_path)
trainer.vis.text(dataset.db.label_names, win='labels')
best_map = 0
lr_ = opt.lr
for epoch in range(opt.epoch):
trainer.reset_meters()
for ii, (img, bbox_, label_, scale) in tqdm(enumerate(dataloader)):
scale = at.scalar(scale)
img, bbox, label = img.cuda().float(), bbox_.cuda(), label_.cuda()
trainer.train_step(img, bbox, label, scale)
if (ii + 1) % opt.plot_every == 0:
if os.path.exists(opt.debug_file):
ipdb.set_trace()
# plot loss
trainer.vis.plot_many(trainer.get_meter_data())
# plot groud truth bboxes
ori_img_ = inverse_normalize(at.tonumpy(img[0]))
# plot predicti bboxes
_bboxes, _labels, _scores = trainer.faster_rcnn.predict([ori_img_], visualize=True)
# rpn confusion matrix(meter)
trainer.vis.text(str(trainer.rpn_cm.value().tolist()), win='rpn_cm')
# roi confusion matrix
trainer.vis.img('roi_cm', at.totensor(trainer.roi_cm.conf, False).float())
eval_result = eval(test_dataloader, faster_rcnn, test_num=opt.test_num)
trainer.vis.plot('test_map', eval_result['map'])
lr_ = trainer.faster_rcnn.optimizer.param_groups[0]['lr']
log_info = 'lr:{}, map:{},loss:{}'.format(str(lr_),
str(eval_result['map']),
str(trainer.get_meter_data()))
trainer.vis.log(log_info)
if eval_result['map'] > best_map:
best_map = eval_result['map']
best_path = trainer.save(best_map=best_map)
if epoch == 9:
trainer.load(best_path)
trainer.faster_rcnn.scale_lr(opt.lr_decay)
lr_ = lr_ * opt.lr_decay
if epoch == 13:
break
def forward(self, imgs, bboxes, labels, scale):
# 每次进来一张图,对应的bbox，对应的label
n = bboxes.shape[0]
if n != 1:
raise ValueError('Currently only batch size 1 is supported.')
_, _, H, W = imgs.shape
img_size = (H, W)
# 求出特征图
features = self.faster_rcnn.extractor(imgs)
# 求出rpn网络输出的损失，roi，所有anchor
# rpn_locs(N, H W A, 4)
rpn_locs, rpn_scores, rois, anchor = self.faster_rcnn.rpn(
features, img_size, scale)
# bbox(R, 4) label(R, 1)
bbox = bboxes[0]
label = labels[0]
rpn_score = rpn_scores[0]
rpn_loc = rpn_locs[0]
roi = rois
# bbox和label是真实的bbox和label
# roi是预测的bbox
# 返回的是选择后的roi以及roi对应真实bbox的位置偏差，以及roi所给的label
sample_roi, gt_roi_loc, gt_roi_label = self.proposal_target_creator(
roi, at.tonumpy(bbox), at.tonumpy(label), self.loc_normalize_mean,
self.loc_normalize_std)
# 应该是返回在特征图上的roi全连接之后的位置和概率
roi_cls_loc, roi_score = self.faster_rcnn.head(features, sample_roi)
# rpn loss
# 给anchor标记标签，loc上默认值是0，正样本为位移偏移量
# label上默认值为-1，正样本为1
gt_rpn_loc, gt_rpn_label = self.anchor_target_creator(
at.tonumpy(bbox), anchor, img_size)
gt_rpn_label = at.totensor(gt_rpn_label).long()
gt_rpn_loc = at.totensor(gt_rpn_loc)
# rpn_loc(H * W * A, 4)
rpn_loc_loss = _fast_rcnn_loc_loss(rpn_loc, gt_rpn_loc,
gt_rpn_label.data, self.rpn_sigma)
# 计算rpn网络的分类损失，忽略label = -1的
rpn_cls_loss = F.cross_entropy(rpn_score,
gt_rpn_label,
ignore_index=-1)
# ROI loss
n_sample = roi_cls_loc.shape[0]
roi_cls_loc = roi_cls_loc.view(n_sample, -1, 4)
roi_loc = roi_cls_loc[torch.arange(0, n_sample).long(),
at.totensor(gt_roi_label).long()]
gt_roi_label = at.totensor(gt_roi_label).long()
gt_roi_loc = at.totensor(gt_roi_loc)
roi_loc_loss = _fast_rcnn_loc_loss(roi_loc, gt_roi_loc,
gt_roi_label.data, self.roi_sigma)
roi_cls_loss = nn.CrossEntropyLoss()(roi_score, gt_roi_label)
losses = {
'rpn_loc_loss': rpn_loc_loss,
'rpn_cls_loss': rpn_cls_loss,
'roi_loc_loss': roi_loc_loss,
'roi_cls_loss': roi_cls_loss
}
losses['total_loss'] = sum(losses.values())
return losses
def forward(self, imgs, bboxes, labels, scale):
"""Forward Faster R-CNN and calculate losses.
Here are notations used.
* :math:`N` is the batch size.
* :math:`R` is the number of bounding boxes per image.
Currently, only :math:`N=1` is supported.
Args:
imgs (~torch.autograd.Variable): A variable with a batch of images.
bboxes (~torch.autograd.Variable): A batch of bounding boxes.
Its shape is :math:`(N, R, 4)`.
labels (~torch.autograd..Variable): A batch of labels.
Its shape is :math:`(N, R)`. The background is excluded from
the definition, which means that the range of the value
is :math:`[0, L - 1]`. :math:`L` is the number of foreground
classes.
scale (float): Amount of scaling applied to
the raw image during preprocessing.
Returns:
namedtuple of 5 losses
"""
n = bboxes.shape[0]
if n != 1:
raise ValueError('Currently only batch size 1 is supported.')
_, _, H, W = imgs.shape
img_size = (H, W)
features = self.faster_rcnn.extractor(imgs)
rpn_locs, rpn_scores, rois, roi_indices, anchor = \
self.faster_rcnn.rpn(features, img_size, scale)
# Since batch size is one, convert variables to singular form
bbox = bboxes[0]
label = labels[0]
rpn_score = rpn_scores[0]
rpn_loc = rpn_locs[0]
roi = rois
# Sample RoIs and forward
# it's fine to break the computation graph of rois,
# consider them as constant input
sample_roi, gt_roi_loc, gt_roi_label = self.proposal_target_creator(
roi, at.tonumpy(bbox), at.tonumpy(label), self.loc_normalize_mean,
self.loc_normalize_std)
# NOTE it's all zero because now it only support for batch=1 now
sample_roi_index = t.zeros(len(sample_roi))
roi_cls_loc, roi_score = self.faster_rcnn.head(features, sample_roi,
sample_roi_index)
# ------------------ RPN losses -------------------#
gt_rpn_loc, gt_rpn_label = self.anchor_target_creator(
at.tonumpy(bbox), anchor, img_size)
gt_rpn_label = at.totensor(gt_rpn_label).long()
gt_rpn_loc = at.totensor(gt_rpn_loc)
rpn_loc_loss = _fast_rcnn_loc_loss(rpn_loc, gt_rpn_loc,
gt_rpn_label.data, self.rpn_sigma)
# NOTE: default value of ignore_index is -100 ...
rpn_cls_loss = F.cross_entropy(rpn_score,
gt_rpn_label.cuda(),
ignore_index=-1)
_gt_rpn_label = gt_rpn_label[gt_rpn_label > -1]
_rpn_score = at.tonumpy(rpn_score)[at.tonumpy(gt_rpn_label) > -1]
self.rpn_cm.add(at.totensor(_rpn_score, False),
_gt_rpn_label.data.long())
# ------------------ ROI losses (fast rcnn loss) -------------------#
n_sample = roi_cls_loc.shape[0]
roi_cls_loc = roi_cls_loc.view(n_sample, -1, 4)
roi_loc = roi_cls_loc[t.arange(0, n_sample).long().cuda(), \
at.totensor(gt_roi_label).long()]
gt_roi_label = at.totensor(gt_roi_label).long()
gt_roi_loc = at.totensor(gt_roi_loc)
roi_loc_loss = _fast_rcnn_loc_loss(roi_loc.contiguous(), gt_roi_loc,
gt_roi_label.data, self.roi_sigma)
roi_cls_loss = nn.CrossEntropyLoss()(roi_score, gt_roi_label.cuda())
self.roi_cm.add(at.totensor(roi_score, False),
gt_roi_label.data.long())
losses = [rpn_loc_loss, rpn_cls_loss, roi_loc_loss, roi_cls_loss]
losses = losses + [sum(losses)]
return LossTuple(*losses)