def eval(self): self.metric.reset() self.model.eval() if self.args.distributed: model = self.model.module else: model = self.model logging.info("Start validation, Total sample: {:d}".format( len(self.val_loader))) import time time_start = time.time() for i, (image, target, filename) in enumerate(self.val_loader): image = image.to(self.device) target = target.to(self.device) with torch.no_grad(): output = model.evaluate(image) self.metric.update(output, target) pixAcc, mIoU = self.metric.get() logging.info( "Sample: {:d}, validation pixAcc: {:.3f}, mIoU: {:.3f}".format( i + 1, pixAcc * 100, mIoU * 100)) synchronize() logging.info('Eval use time: {}'.format(time.time() - time_start))
def eval(self): self.metric.reset() self.model.eval() if self.args.distributed: model = self.model.module else: model = self.model logging.info("Using Val/Test img scale : {}".format(cfg.TEST.IMG_SCALE)) logging.info("Start validation, Total sample: {:d}".format(len(self.val_loader))) import time time_start = time.time() pbar = tqdm(self.val_loader) for image, target, filename in pbar: image = image.to(self.device) target = target.to(self.device) assert image.shape[0] == 1, "Only batch-size 1 allowed when evaluating on test/val images" with torch.no_grad(): output = mmseg_evaluate(model, image, target) self.metric.update(output, target) pixAcc, mIoU = self.metric.get() pbar.set_postfix_str("pixAcc: {:.3f}, mIoU: {:.3f}".format(pixAcc * 100, mIoU * 100)) synchronize() pixAcc, mIoU, category_iou = self.metric.get(return_category_iou=True) logging.info('Eval use time: {:.3f} second'.format(time.time() - time_start)) logging.info('End validation pixAcc: {:.3f}, mIoU: {:.3f}'.format(pixAcc * 100, mIoU * 100))
def eval(self): self.metric.reset() self.model.eval() if self.args.distributed: model = self.model.module else: model = self.model logging.info("Start validation, Total sample: {:d}".format(len(self.val_loader))) import time time_start = time.time() for i, (image, target, filename) in enumerate(self.val_loader): image = image.to(self.device) target = target.to(self.device) with torch.no_grad(): output = model.evaluate(image) self.metric.update(output, target) pixAcc, mIoU = self.metric.get() logging.info("Sample: {:d}, validation pixAcc: {:.3f}, mIoU: {:.3f}".format( i + 1, pixAcc * 100, mIoU * 100)) synchronize() pixAcc, mIoU, category_iou = self.metric.get(return_category_iou=True) logging.info('Eval use time: {:.3f} second'.format(time.time() - time_start)) logging.info('End validation pixAcc: {:.3f}, mIoU: {:.3f}'.format( pixAcc * 100, mIoU * 100)) headers = ['class id', 'class name', 'iou'] table = [] for i, cls_name in enumerate(self.classes): table.append([cls_name, category_iou[i]]) logging.info('Category iou: \n {}'.format(tabulate(table, headers, tablefmt='grid', showindex="always", numalign='center', stralign='center')))
def eval(self): self.metric.reset() self.model.eval() if self.args.distributed: model = self.model.module else: model = self.model logging.info("Start validation, Total sample: {:d}".format( len(self.val_loader))) import time time_start = time.time() for i, (image, target, filename) in enumerate(self.val_loader): image = image.to(self.device) target = target.to(self.device) with torch.no_grad(): output = model.evaluate(image) # import pdb; pdb.set_trace() # do operations here, NOTE : We are saving with batch size of 1 # np.save('npy_files_voc/' + os.path.basename(filename[0]).strip('.jpg'), output[0].cpu().numpy()) output = F.interpolate(output, (image.shape[2], image.shape[3]), mode='bilinear', align_corners=True) output = torch.argmax(output, 1) self.metric.update(output, target) pixAcc, mIoU = self.metric.get() logging.info( "Sample: {:d}, validation pixAcc: {:.3f}, mIoU: {:.3f}".format( i + 1, pixAcc * 100, mIoU * 100)) synchronize() pixAcc, mIoU, category_iou = self.metric.get(return_category_iou=True) logging.info('Eval use time: {:.3f} second'.format(time.time() - time_start)) logging.info('End validation pixAcc: {:.3f}, mIoU: {:.3f}'.format( pixAcc * 100, mIoU * 100)) headers = ['class id', 'class name', 'iou'] table = [] for i, cls_name in enumerate(self.classes): table.append([cls_name, category_iou[i]]) logging.info('Category iou: \n {}'.format( tabulate(table, headers, tablefmt='grid', showindex="always", numalign='center', stralign='center')))
def eval(self): self.metric.reset() self.model.eval() if self.args.distributed: model = self.model.module else: model = self.model logging.info("Start validation, Total sample: {:d}".format( len(self.val_loader))) for i, (image, target, filename) in enumerate(self.val_loader): image = image.to(self.device) target = target.to(self.device) with torch.no_grad(): size = image.size()[2:] if size[0] < cfg.TEST.CROP_SIZE[0] and size[ 1] < cfg.TEST.CROP_SIZE[1]: pad_height = cfg.TEST.CROP_SIZE[0] - size[0] pad_width = cfg.TEST.CROP_SIZE[1] - size[1] image = F.pad(image, (0, pad_height, 0, pad_width)) output = model(image)[0] output = output[..., :size[0], :size[1]] else: output = model(image)[0] self.metric.update(output, target) pixAcc, mIoU = self.metric.get() logging.info( "Sample: {:d}, validation pixAcc: {:.3f}, mIoU: {:.3f}".format( i + 1, pixAcc * 100, mIoU * 100)) # Todo # if self.args.save_pred: # pred = torch.argmax(output, 1) # pred = pred.cpu().data.numpy() # # predict = pred.squeeze(0) # mask = get_color_pallete(predict, self.args.dataset) # mask.save(os.path.join(outdir, os.path.splitext(filename[0])[0] + '.png')) synchronize()
def eval(self): self.metric.reset() self.model.eval() if self.args.distributed: model = self.model.module else: model = self.model logging.info("Start validation, Total sample: {:d}".format( len(self.val_loader))) import time time_start = time.time() tot_conf = torch.Tensor([]).reshape(-1, 1) tot_obj = torch.Tensor([]).reshape(-1, 1) tot_label_for_image = torch.Tensor([]).reshape(-1, 1) for i, (image, target, filename) in enumerate(self.val_loader): image = image.to(self.device) target = target.to(self.device) with torch.no_grad(): output = model.evaluate(image) # import pdb; pdb.set_trace() doingCali = True usingCRF = True if (doingCali): # predetermined # temp=1.6127 temp = 2.8 else: temp = 1 output = output / temp # to be removed for temp scaling if (not usingCRF): output_post = output else: output_post = [] output = F.softmax(output, dim=1) output_numpy = output.cpu().numpy() def get_raw_image(file_location): # load in bgr in H W C format raw_image = cv2.imread(file_location, cv2.IMREAD_COLOR).astype(np.float32) mean_bgr = np.array([103.53, 116.28, 123.675]) # Do some subtraction raw_image -= mean_bgr # converted to C H W raw_image = raw_image.transpose(2, 0, 1) raw_image = raw_image.astype(np.uint8) raw_image = raw_image.transpose(1, 2, 0) return raw_image for j, image_file_loc in enumerate(filename): prob_to_use = output_numpy[j] if (usingCRF): raw_image = get_raw_image(image_file_loc) prob_post = self.postprocessor(raw_image, prob_to_use) prob_to_use = prob_post output_post.append(prob_post) # import pdb;pdb.set_trace() prob_to_use = torch.tensor(prob_to_use) # Neels implementation labels = torch.argmax(prob_to_use, dim=0) conf = torch.max(prob_to_use, dim=0)[0].cpu() obj = labels.cpu().float() label_for_image = target[j].view(-1, 1).cpu().float() sel = (label_for_image >= 0) tot_conf = torch.cat( [tot_conf, conf.view(-1, 1)[sel].view(-1, 1)], dim=0) tot_obj = torch.cat( [tot_obj, obj.view(-1, 1)[sel].view(-1, 1)], dim=0) tot_label_for_image = torch.cat([ tot_label_for_image, label_for_image.view(-1, 1)[sel].view(-1, 1) ], dim=0) if (usingCRF): output_post = np.array(output_post) output_post = torch.tensor(output_post) output_post = output_post.to(self.device) self.metric.update(output_post, target) # self.metric.update(output, target) pixAcc, mIoU = self.metric.get() logging.info( "Sample: {:d}, validation pixAcc: {:.3f}, mIoU: {:.3f}".format( i + 1, pixAcc * 100, mIoU * 100)) print(tot_conf.shape, tot_obj.shape, tot_label_for_image.shape) import pickle ece_folder = "eceData" makedirs(ece_folder) # postfix="DLV2_UnCal" postfix = "Foggy_Calibrated_DLV3Plus" saveDir = os.path.join(ece_folder, postfix) makedirs(saveDir) file = open(os.path.join(saveDir, "conf.pickle"), "wb") pickle.dump(tot_conf, file) file.close() file = open(os.path.join(saveDir, "obj.pickle"), "wb") pickle.dump(tot_obj, file) file.close() file = open(os.path.join(saveDir, "gt.pickle"), "wb") pickle.dump(tot_label_for_image, file) file.close() synchronize() pixAcc, mIoU, category_iou = self.metric.get(return_category_iou=True) logging.info('Eval use time: {:.3f} second'.format(time.time() - time_start)) logging.info('End validation pixAcc: {:.3f}, mIoU: {:.3f}'.format( pixAcc * 100, mIoU * 100)) headers = ['class id', 'class name', 'iou'] table = [] for i, cls_name in enumerate(self.classes): table.append([cls_name, category_iou[i]]) logging.info('Category iou: \n {}'.format( tabulate(table, headers, tablefmt='grid', showindex="always", numalign='center', stralign='center')))
def eval(self): self.metric.reset() self.model.eval() if self.args.distributed: model = self.model.module else: model = self.model logging.info("Start validation, Total sample: {:d}".format( len(self.val_loader))) import time time_start = time.time() widgets = [ 'Inference: ', Percentage(), ' ', Bar('#'), ' ', Timer(), ' ', ETA(), ' ', FileTransferSpeed() ] pbar = ProgressBar(widgets=widgets, maxval=10 * len(self.val_loader)).start() for i, (image, target, boundary, filename) in enumerate(self.val_loader): image = image.to(self.device) target = target.to(self.device) boundary = boundary.to(self.device) filename = filename[0] with torch.no_grad(): output, output_boundary = model.evaluate(image) if 'hard' in filename: self.metric_hard.update(output, target) self.count_hard += 1 elif 'easy' in filename: self.metric_easy.update(output, target) self.count_easy += 1 else: print(filename) continue self.metric.update(output, target) pbar.update(10 * i + 1) pbar.finish() synchronize() pixAcc, mIoU, category_iou, mae, mBer, category_Ber = self.metric.get( return_category_iou=True) pixAcc_e, mIoU_e, category_iou_e, mae_e, mBer_e, category_Ber_e = self.metric_easy.get( return_category_iou=True) pixAcc_h, mIoU_h, category_iou_h, mae_h, mBer_h, category_Ber_h = self.metric_hard.get( return_category_iou=True) logging.info('Eval use time: {:.3f} second'.format(time.time() - time_start)) logging.info( 'End validation pixAcc: {:.2f}, mIoU: {:.2f}, mae: {:.3f}, mBer: {:.2f}' .format(pixAcc * 100, mIoU * 100, mae, mBer)) logging.info( 'End validation easy pixAcc: {:.2f}, mIoU: {:.2f}, mae: {:.3f}, mBer: {:.2f}' .format(pixAcc_e * 100, mIoU_e * 100, mae_e, mBer_e)) logging.info( 'End validation hard pixAcc: {:.2f}, mIoU: {:.2f}, mae: {:.3f}, mBer: {:.2f}' .format(pixAcc_h * 100, mIoU_h * 100, mae_h, mBer_h)) headers = [ 'class id', 'class name', 'iou', 'iou_easy', 'iou_hard', 'ber', 'ber_easy', 'ber_hard' ] table = [] for i, cls_name in enumerate(self.classes): table.append([ cls_name, category_iou[i], category_iou_e[i], category_iou_h[i], category_Ber[i], category_Ber_e[i], category_Ber_h[i] ]) logging.info('Category iou: \n {}'.format( tabulate(table, headers, tablefmt='grid', showindex="always", numalign='center', stralign='center'))) logging.info('easy images: {}, hard images: {}'.format( self.count_easy, self.count_hard))
def eval(self): self.metric.reset() self.model.eval() if self.args.distributed: model = self.model.module else: model = self.model one_five = torch.ones(1) * 1.5 one_five = one_five.to(self.device) temp = torch.nn.Parameter(one_five) print(temp) criterion = torch.nn.CrossEntropyLoss( ignore_index=cfg.DATASET.IGNORE_INDEX).to(self.device) optimizer = torch.optim.SGD([temp], lr=1) logging.info("Start validation, Total sample: {:d}".format( len(self.val_loader))) import time time_start = time.time() loss_series = list() temp_series = list() for epoch in range(10): logging.info("Epoch Started {}".format(epoch)) loss_epoch = 0.0 for i, (image, target, filename) in enumerate(self.val_loader): optimizer.zero_grad() image = image.to(self.device) target = target.to(self.device) with torch.no_grad(): output = model.evaluate(image) # output = output.cpu() output = output / temp # print(output.shape) # print(target.shape) loss = criterion(output, target) loss_epoch += loss.item() loss.backward() optimizer.step() logging.info("Batch {} loss for Temp Scaling : {}".format( i, loss)) logging.info("Epoch {} loss for Temp Scaling : {}".format( epoch, loss_epoch / (len(self.val_loader)))) logging.info("Epoch {} Temp Scaling factor is : {}".format( epoch, temp.item())) loss_series.append(loss_epoch) temp_series.append(temp.item()) print(loss_series) print(temp_series) synchronize() print('Final scaled temp : {}'.format(temp))