def eval_dataset_cls(cfg_path, device=None):
"""分类问题的eval dataset:
等效于runner中的load_from + val，但可用来脱离runner进行独立的数据集验证
"""
# 准备验证所用的对象
cfg = get_config(cfg_path)
dataset = get_dataset(cfg.valset, cfg.transform_val)
dataloader = get_dataloader(dataset, cfg.valloader)
model = get_model(cfg)
if device is None:
device = torch.device(cfg.load_device)
# TODO: 如下两句的顺序
load_checkpoint(model, cfg.load_from, device)
model = model.to(device)
# 开始验证
buffer = {'acc': []}
n_correct = 0
model.eval()
for c_iter, data_batch in enumerate(dataloader):
with torch.no_grad(): # 停止反向传播，只进行前向计算
img = to_device(data_batch['img'], device)
label = to_device(data_batch['gt_labels'], device)
y_pred = model(img)
label = torch.cat(label, dim=0)
acc1 = accuracy(y_pred, label, topk=1)
buffer['acc'].append(acc1)
# 计算总体精度
n_correct += buffer['acc'][-1] * len(data_batch['gt_labels'])
vis_loss_acc(buffer, title='eval dataset')
print('ACC on dataset: %.3f', n_correct / len(dataset))
def vis_dataset_bbox_area(cfg_path):
"""用于统计一个数据集的所有bbox的面积值，以及对bbox的w,h进行：
"""
from utils.prepare_training import get_config, get_dataset
cfg = get_config(cfg_path)
cfg.trainset.params.ann_file = [cfg.trainset.params.ann_file[0]
] # 先只用voc07
trainset = get_dataset(cfg.trainset, cfg.transform)
class_names = trainset.CLASSES
ws = []
hs = []
areas = []
labels = []
for data in tqdm(trainset):
img_meta = data['img_meta']
gt_labels = data['gt_labels']
gt_bboxes = data['gt_bboxes']
w = gt_bboxes[:, 2] - gt_bboxes[:, 0]
h = gt_bboxes[:, 3] - gt_bboxes[:, 1]
area = w * h
ws.extend(w)
hs.extend(h)
areas.extend(area)
labels.extend(gt_labels)
ws = np.array([w.item() for w in ws]) # (k,)
hs = np.array([h.item() for h in hs]) # (k,)
areas = np.array([area.item() for area in areas]) # (k,)
labels = np.array([label.item() for label in labels]) # (k,)
# 先绘制总的分布图
plt.figure()
plt.title('all')
plt.hist(areas, 30, range=(0, 90000))
plt.show()
# 再分别绘制每个类的hist
plt.figure()
for class_id in range(1, 21): # 假定20类
inds = labels == class_id
class_areas = areas[inds]
plt.subplot(4, 5, class_id)
plt.title(class_names[class_id - 1])
plt.hist(class_areas, 30, range=(0, 90000))
plt.show()
# 然后计算size = sqrt(area), 绘制size的scatter
plt.figure()
plt.title('w and h scatter')
plt.scatter(ws, hs)
# 然后对横坐标w,纵坐标h的size尺寸做聚类
data = np.concatenate([ws[:, None], hs[:, None]], axis=1)
centers = kmean(data, k=5)
plt.scatter(centers[:, 0], centers[:, 1], s=50, c='r')
plt.show()
def train(cfg_path):
"""训练demo"""
# 获得配置信息
cfg = get_config(cfg_path)
# 创建logger
logger = get_logger(cfg.log_level)
logger.info("start training:")
# 创建模型
model = OneStageDetector(cfg)
model.to(cfg.device)
# 创建数据
dataset = get_dataset(cfg.data.train)
dataloader = DataLoader(
dataset,
batch_size=cfg.batch_size,
sampler=cfg.sampler,
num_workers=cfg.num_workers,
collate_fn=partial(collate, samples_per_gpu=cfg.data.imgs_per_gpu),
pin_memory=False)
# 创建训练器并开始训练
runner = Runner(model, batch_processor, cfg.optimizer, cfg.work_dir,
cfg.log_level)
runner.register_hooks()
runner.run(dataloader)
def __init__(self, cfg_path, load_from=None, load_device=None):
self.type = 'det' # 用来判断是什么类型的预测器
# 准备验证所用的对象
self.cfg = get_config(cfg_path)
# 为了便于eval，不必常去修改cfg里边的设置，直接在func里边添加2个参数即可
if load_from is not None:
self.cfg.load_from = load_from
if load_device is not None:
self.cfg.load_device = load_device
self.model = get_model(self.cfg)
self.device = torch.device(self.cfg.load_device)
load_checkpoint(self.model, self.cfg.load_from, self.device)
self.model = self.model.to(self.device)
def eval_dataset_det(cfg_path,
load_from=None,
load_device=None,
resume_from=None,
result_file=None):
"""检测问题的eval dataset:
为了便于eval，添加2个形参参数，不必常去修改cfg里边的设置
"""
# 准备验证所用的对象
cfg = get_config(cfg_path)
cfg.valloader.params.batch_size = 1 # 强制固定验证时batch_size=1
# 为了便于eval，不必常去修改cfg里边的设置，直接在func里边添加几个参数即可
if load_from is not None:
cfg.load_from = load_from
if load_device is not None:
cfg.load_device = load_device
if resume_from is not None:
cfg.resume_from = resume_from
dataset = get_dataset(cfg.valset, cfg.transform_val)
dataloader = get_dataloader(dataset, cfg.valloader, len(cfg.gpus))
model = get_model(cfg)
device = torch.device(cfg.load_device)
load_checkpoint(model, cfg.load_from, device)
model = model.to(device)
# 如果没有验证过
if result_file is None:
# 开始验证
model.eval()
all_bbox_cls = []
for c_iter, data_batch in enumerate(dataloader):
with torch.no_grad(): # 停止反向传播，只进行前向计算
bbox_det = batch_detector(
model, data_batch, device,
return_loss=False)['bboxes'] # 提取bbox即可(n_cls,)(m,5)
# 显示进度
if c_iter % 100 == 0:
print('%d / %d finished predict.' % (c_iter, len(dataset)))
all_bbox_cls.append(bbox_det) # (n_img,)(n_class,)(k,5)
# 保存预测结果到文件
filename = get_time_str() + '_eval_result.pkl'
save2pkl(all_bbox_cls, cfg.work_dir + filename)
# 如果有现成验证文件
else:
all_bbox_cls = loadvar(result_file)
# 评估
voc_eval(all_bbox_cls, dataset, iou_thr=0.5)
def __init__(self, cfg_path, load_from=None, load_device=None):
super().__init__()
self.type = 'cls'
# 准备验证所用的对象
self.cfg = get_config(cfg_path)
# 为了便于eval，不必常去修改cfg里边的设置，直接在func里边添加2个参数即可
if load_from is not None:
self.cfg.load_from = load_from
if load_device is not None:
self.cfg.load_device = load_device
self.model = get_model(self.cfg)
self.device = torch.device(self.cfg.load_device)
if self.cfg.load_from is not None or self.cfg.resume_from is not None:
load_checkpoint(self.model, self.cfg.load_from, self.device)
self.model = self.model.to(self.device)
def load_dataset(cfg_path):
"""加载经过变换之后的数据集，返回实际训练时的img尺寸和bbox尺寸
"""
cfg = get_config(cfg_path)
dataset = get_dataset(cfg.trainset, cfg.transform)
ww = []
hh = []
# data = dataset[0]
for data in tqdm(dataset):
bbox = data['gt_bboxes']
w = (bbox[:, 2] - bbox[:, 0]).numpy()
h = (bbox[:, 3] - bbox[:, 1]).numpy()
ww.append(w)
hh.append(h)
ww = np.concatenate(ww, axis=0) # (m,)
hh = np.concatenate(hh, axis=0)
bboxes = np.concatenate([ww[:, None], hh[:, None]], axis=1)
return bboxes # (m, 2)
from utils.tools import parse_log
from utils.dataset_classes import get_classes
from utils.visualization import vis_all_opencv, vis_all_pyplot, vis_cam
def train_ssd(cfg_path, resume_from=None):
runner = Runner(cfg, resume_from)
runner.train()
if __name__ == "__main__":
task = 'train'
cfg_path = './cfg_detector_ssdvgg16_voc.py'
cfg = get_config(cfg_path)
if task == 'train': # 模型训练
train_ssd(cfg, resume_from=None)
if task == 'log':
parse_log(
paths=['/home/ubuntu/mytrain/ssd_vgg_voc/20191025_182352.log'])
if task == 'eval': # 数据集评估
eval_dataset_det(
cfg_path=cfg_path,
load_from='/home/ubuntu/mytrain/ssd_vgg_voc/epoch_11.pth',
load_device='cuda')
if task == 'load': # 已有数据集评估文件，重新载入进行评估
