def eval_dataset_cls(cfg_path, device=None): """分类问题的eval dataset: 等效于runner中的load_from + val,但可用来脱离runner进行独立的数据集验证 """ # 准备验证所用的对象 cfg = get_config(cfg_path) dataset = get_dataset(cfg.valset, cfg.transform_val) dataloader = get_dataloader(dataset, cfg.valloader) model = get_model(cfg) if device is None: device = torch.device(cfg.load_device) # TODO: 如下两句的顺序 load_checkpoint(model, cfg.load_from, device) model = model.to(device) # 开始验证 buffer = {'acc': []} n_correct = 0 model.eval() for c_iter, data_batch in enumerate(dataloader): with torch.no_grad(): # 停止反向传播,只进行前向计算 img = to_device(data_batch['img'], device) label = to_device(data_batch['gt_labels'], device) y_pred = model(img) label = torch.cat(label, dim=0) acc1 = accuracy(y_pred, label, topk=1) buffer['acc'].append(acc1) # 计算总体精度 n_correct += buffer['acc'][-1] * len(data_batch['gt_labels']) vis_loss_acc(buffer, title='eval dataset') print('ACC on dataset: %.3f', n_correct / len(dataset))
def vis_dataset_bbox_area(cfg_path): """用于统计一个数据集的所有bbox的面积值,以及对bbox的w,h进行: """ from utils.prepare_training import get_config, get_dataset cfg = get_config(cfg_path) cfg.trainset.params.ann_file = [cfg.trainset.params.ann_file[0] ] # 先只用voc07 trainset = get_dataset(cfg.trainset, cfg.transform) class_names = trainset.CLASSES ws = [] hs = [] areas = [] labels = [] for data in tqdm(trainset): img_meta = data['img_meta'] gt_labels = data['gt_labels'] gt_bboxes = data['gt_bboxes'] w = gt_bboxes[:, 2] - gt_bboxes[:, 0] h = gt_bboxes[:, 3] - gt_bboxes[:, 1] area = w * h ws.extend(w) hs.extend(h) areas.extend(area) labels.extend(gt_labels) ws = np.array([w.item() for w in ws]) # (k,) hs = np.array([h.item() for h in hs]) # (k,) areas = np.array([area.item() for area in areas]) # (k,) labels = np.array([label.item() for label in labels]) # (k,) # 先绘制总的分布图 plt.figure() plt.title('all') plt.hist(areas, 30, range=(0, 90000)) plt.show() # 再分别绘制每个类的hist plt.figure() for class_id in range(1, 21): # 假定20类 inds = labels == class_id class_areas = areas[inds] plt.subplot(4, 5, class_id) plt.title(class_names[class_id - 1]) plt.hist(class_areas, 30, range=(0, 90000)) plt.show() # 然后计算size = sqrt(area), 绘制size的scatter plt.figure() plt.title('w and h scatter') plt.scatter(ws, hs) # 然后对横坐标w,纵坐标h的size尺寸做聚类 data = np.concatenate([ws[:, None], hs[:, None]], axis=1) centers = kmean(data, k=5) plt.scatter(centers[:, 0], centers[:, 1], s=50, c='r') plt.show()
def train(cfg_path): """训练demo""" # 获得配置信息 cfg = get_config(cfg_path) # 创建logger logger = get_logger(cfg.log_level) logger.info("start training:") # 创建模型 model = OneStageDetector(cfg) model.to(cfg.device) # 创建数据 dataset = get_dataset(cfg.data.train) dataloader = DataLoader( dataset, batch_size=cfg.batch_size, sampler=cfg.sampler, num_workers=cfg.num_workers, collate_fn=partial(collate, samples_per_gpu=cfg.data.imgs_per_gpu), pin_memory=False) # 创建训练器并开始训练 runner = Runner(model, batch_processor, cfg.optimizer, cfg.work_dir, cfg.log_level) runner.register_hooks() runner.run(dataloader)
def __init__(self, cfg_path, load_from=None, load_device=None): self.type = 'det' # 用来判断是什么类型的预测器 # 准备验证所用的对象 self.cfg = get_config(cfg_path) # 为了便于eval,不必常去修改cfg里边的设置,直接在func里边添加2个参数即可 if load_from is not None: self.cfg.load_from = load_from if load_device is not None: self.cfg.load_device = load_device self.model = get_model(self.cfg) self.device = torch.device(self.cfg.load_device) load_checkpoint(self.model, self.cfg.load_from, self.device) self.model = self.model.to(self.device)
def eval_dataset_det(cfg_path, load_from=None, load_device=None, resume_from=None, result_file=None): """检测问题的eval dataset: 为了便于eval,添加2个形参参数,不必常去修改cfg里边的设置 """ # 准备验证所用的对象 cfg = get_config(cfg_path) cfg.valloader.params.batch_size = 1 # 强制固定验证时batch_size=1 # 为了便于eval,不必常去修改cfg里边的设置,直接在func里边添加几个参数即可 if load_from is not None: cfg.load_from = load_from if load_device is not None: cfg.load_device = load_device if resume_from is not None: cfg.resume_from = resume_from dataset = get_dataset(cfg.valset, cfg.transform_val) dataloader = get_dataloader(dataset, cfg.valloader, len(cfg.gpus)) model = get_model(cfg) device = torch.device(cfg.load_device) load_checkpoint(model, cfg.load_from, device) model = model.to(device) # 如果没有验证过 if result_file is None: # 开始验证 model.eval() all_bbox_cls = [] for c_iter, data_batch in enumerate(dataloader): with torch.no_grad(): # 停止反向传播,只进行前向计算 bbox_det = batch_detector( model, data_batch, device, return_loss=False)['bboxes'] # 提取bbox即可(n_cls,)(m,5) # 显示进度 if c_iter % 100 == 0: print('%d / %d finished predict.' % (c_iter, len(dataset))) all_bbox_cls.append(bbox_det) # (n_img,)(n_class,)(k,5) # 保存预测结果到文件 filename = get_time_str() + '_eval_result.pkl' save2pkl(all_bbox_cls, cfg.work_dir + filename) # 如果有现成验证文件 else: all_bbox_cls = loadvar(result_file) # 评估 voc_eval(all_bbox_cls, dataset, iou_thr=0.5)
def __init__(self, cfg_path, load_from=None, load_device=None): super().__init__() self.type = 'cls' # 准备验证所用的对象 self.cfg = get_config(cfg_path) # 为了便于eval,不必常去修改cfg里边的设置,直接在func里边添加2个参数即可 if load_from is not None: self.cfg.load_from = load_from if load_device is not None: self.cfg.load_device = load_device self.model = get_model(self.cfg) self.device = torch.device(self.cfg.load_device) if self.cfg.load_from is not None or self.cfg.resume_from is not None: load_checkpoint(self.model, self.cfg.load_from, self.device) self.model = self.model.to(self.device)
def load_dataset(cfg_path): """加载经过变换之后的数据集,返回实际训练时的img尺寸和bbox尺寸 """ cfg = get_config(cfg_path) dataset = get_dataset(cfg.trainset, cfg.transform) ww = [] hh = [] # data = dataset[0] for data in tqdm(dataset): bbox = data['gt_bboxes'] w = (bbox[:, 2] - bbox[:, 0]).numpy() h = (bbox[:, 3] - bbox[:, 1]).numpy() ww.append(w) hh.append(h) ww = np.concatenate(ww, axis=0) # (m,) hh = np.concatenate(hh, axis=0) bboxes = np.concatenate([ww[:, None], hh[:, None]], axis=1) return bboxes # (m, 2)
from utils.tools import parse_log from utils.dataset_classes import get_classes from utils.visualization import vis_all_opencv, vis_all_pyplot, vis_cam def train_ssd(cfg_path, resume_from=None): runner = Runner(cfg, resume_from) runner.train() if __name__ == "__main__": task = 'train' cfg_path = './cfg_detector_ssdvgg16_voc.py' cfg = get_config(cfg_path) if task == 'train': # 模型训练 train_ssd(cfg, resume_from=None) if task == 'log': parse_log( paths=['/home/ubuntu/mytrain/ssd_vgg_voc/20191025_182352.log']) if task == 'eval': # 数据集评估 eval_dataset_det( cfg_path=cfg_path, load_from='/home/ubuntu/mytrain/ssd_vgg_voc/epoch_11.pth', load_device='cuda') if task == 'load': # 已有数据集评估文件,重新载入进行评估