def eval(model, data_loader, device): model.eval() coco_evaluator = CocoEvaluator(coco, iou_types) data_iter = iter(data_loader) for data in data_iter: images, guide_images, targets = data images = list(image.to(device) for image in images) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] guide_images = guide_images_processing(guide_images, device) torch.cuda.synchronize() outputs = model(images, guide_images) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] res = {target["image_id"].item(): output for target, output in zip(targets,outputs)} coco_evaluator.update(res) coco_evaluator.synchronize_between_processes() coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return
def evaluate(model, data_loader, device): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) for images, targets in metric_logger.log_every(data_loader, 100, header): fine_targets = [] for t in targets: fine_targets.append({ 'boxes': t['boxes'].to(device), 'labels': torch.tensor([l[0] for l in t['labels']]).to(device), 'image_id': t['image_id'].to(device), 'area': t['area'].to(device), 'iscrowd': t['iscrowd'].to(device) }) targets = fine_targets images = list(img.to(device) for img in images) torch.cuda.synchronize() model_time = time.time() outputs = model(images) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] model_time = time.time() - model_time res = { target["image_id"].item(): output for target, output in zip(targets, outputs) } evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return coco_evaluator
def evaluate(model, data_loader, device, coco_evaluator=None, coco_api=None): if coco_evaluator and coco_api: raise ValueError("Either coco_evaluator or coco_api or neither "\ "should be set. No other combination should be used") n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' if not coco_evaluator and not coco_api: # Build the COCO API and pickle it for future use coco_api = get_coco_api_from_dataset(data_loader.dataset) with open(f"./{type(data_loader.dataset).__name__}_coco.pkl", "wb") as f: pickle.dump(coco_api, f) if not coco_evaluator and coco_api: # Only build the COCO evaluator if coco_api is passed # as this means it did not exist iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco_api, iou_types) for image, targets in metric_logger.log_every(data_loader, 100, header): image = list(img.to(device) for img in image) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] torch.cuda.synchronize() model_time = time.time() outputs = model(image) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] model_time = time.time() - model_time res = { target["image_id"].item(): output for target, output in zip(targets, outputs) } evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return coco_evaluator
def evaluate_withpq(model, data_loader, device): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) test_data_pkl = h5py.File( 'resnet_imagenet_features/backbone.7.0_test_reconstructed.h5', 'r') #test_data_pkl = h5py.File('resnet_imagenet_features/backbone.7.0_test.h5', 'r') for images, targets in tqdm(data_loader, desc=header): images = list(image for image in images) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] image_id = targets[0]['image_id'].item() quantized_x = test_data_pkl[str(image_id)][()] quantized_x = torch.from_numpy(quantized_x) imagepq = quantized_x.to(device) torch.cuda.synchronize() model_time = time.time() #print ("----",image_id,"----",imagepq.shape) outputs = model(images, imagepq, targets) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] model_time = time.time() - model_time res = { target["image_id"].item(): output for target, output in zip(targets, outputs) } evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() coco_evaluator.summarize_per_category() torch.set_num_threads(n_threads) test_data_pkl.close() return coco_evaluator
def evaluate(model, data_loader, device, coco=None): iou_types = ["bbox"] # coco = get_coco_api_from_dataset(data_loader.dataset) # with open('coco_dfg.pickle', 'wb') as f: # pickle.dump(coco, f) if not coco: coco = get_coco_api_from_dataset(data_loader.dataset) # with open('coco_dfg.pickle', 'rb') as f: # coco = pickle.load(f) n_threads = torch.get_num_threads() torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = "Test:" model.to(device) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) to_tensor = torchvision.transforms.ToTensor() # model = nn.DataParallel(model) # model = torch.jit.script(model) for image, targets in metric_logger.log_every(data_loader, 50, header): image = list(to_tensor(img).to(device) for img in image) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] torch.cuda.synchronize() model_time = time.time() outputs = model(image) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] model_time = time.time() - model_time res = { target["image_id"].item(): output for target, output in zip(targets, outputs) } evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return coco_evaluator
def evaluate(model, data_loader, device, box_threshold=0.001): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset, box_threshold) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) for images, targets in metric_logger.log_every(data_loader, 100, header): images = list(img.to(device) for img in images) targets = [{ k: v.to(device) if torch.is_tensor(v) else v for k, v in t.items() } for t in targets] torch.cuda.synchronize() model_time = time.time() if box_threshold is None: outputs = model(images) else: outputs = model(images, box_threshold) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] model_time = time.time() - model_time res = { target["image_id"]: output for target, output in zip(targets, outputs) } # ofekp: this used to be target["image_id"].item() evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return coco_evaluator
def evaluate(model, data_loader, epoch, device): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) total_loss = 0.0 for image, targets in metric_logger.log_every(data_loader, 100, header): image = list(img.to(device) for img in image) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] model.train() loss_dict = model(image, targets) # import ipdb; ipdb.set_trace() losses = sum(loss for loss in loss_dict.values()) # import ipdb; ipdb.set_trace() # #option total_loss = total_loss + losses model.eval() torch.cuda.synchronize() model_time = time.time() outputs = model(image) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] model_time = time.time() - model_time res = {target["image_id"].item(): output for target, output in zip(targets, outputs)} evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) tb.add_scalar('Val Loss', total_loss, epoch) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return coco_evaluator
def evaluate(model, data_loader, device, epoch, test_writer): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) for image, targets in metric_logger.log_every(data_loader, 100, header): image = list(img.to(device) for img in image) targets = [{k: v.to(device) for k, v in t.items() if k != 'resizes'} for t in targets] torch.cuda.synchronize() model_time = time.time() outputs = model(image) outputs = [{ k: v.to(cpu_device) for k, v in t.items() if k != 'resizes' } for t in outputs] model_time = time.time() - model_time res = { target["image_id"].item(): output for target, output in zip(targets, outputs) } evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() test_writer.add_scalar('IoU', coco_evaluator.coco_eval['segm'].stats[0], global_step=epoch) torch.set_num_threads(n_threads) return coco_evaluator
def evaluate(model, data_loader, device): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) print_freq = 50 for ref_images, guide_images, targets in metric_logger.log_every( data_loader, print_freq, header): ref_images = list(image.to(device) for image in ref_images) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] guide_images = guide_images_processing(guide_images, device) torch.cuda.synchronize() model_time = time.time() outputs = model(ref_images, guide_images) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] model_time = time.time() - model_time res = { target["image_id"].item(): output for target, output in zip(targets, outputs) } evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return coco_evaluator
def evaluate(model, data_loader, device): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' model.cuda() #coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) for image, targets in metric_logger.log_every(data_loader, 100, header): #print(image) #image = torchvision.transforms.ToTensor()(image[0]) # Returns a scaler tuple #print(image.shape) # dim of image 1080x1920 image = torchvision.transforms.ToTensor()(image[0]).to(device) #image = img.to(device) for img in image targets = [{k: v.to(device) for k, v in t.items()} for t in targets] torch.cuda.synchronize() model_time = time.time() outputs = model([image]) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] model_time = time.time() - model_time res = { target["image_id"].item(): output for target, output in zip(targets, outputs) } evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return coco_evaluator
def evaluate(model, data_loader, device): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) print(coco_evaluator.coco_eval['bbox'].params) coco_evaluator.coco_eval['bbox'].params.iouThrs = np.linspace( 0.05, 0.95, int(np.round((0.95 - .5) / .05)) + 1, endpoint=True) for data in metric_logger.log_every(data_loader, 100, header): images = list(img.to(device) for img in data['images']) targets = data['targets'] if torch.cuda.is_available(): torch.cuda.synchronize() model_time = time.time() outputs = model(images) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] model_time = time.time() - model_time res = { target["image_id"].item(): output for target, output in zip(targets, outputs) } evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return coco_evaluator
def evaluate_ms(model, data_loader, device): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) idx = 0 for images, targets in metric_logger.log_every(data_loader, 100, header): images = list(img.to(device) for img in images) # c,h,w = images[0].shape torch.cuda.synchronize() model_time = time.time() scales = [1.1] for scale in scales: images_reshaped = F.interpolate(images[0].unsqueeze(0), scale_factor=scale, mode='bilinear', align_corners=True) # _,c, re_h, re_w = images_reshaped.shape # ration_h, ration_w = re_h/h, re_w/w outputs = model([images_reshaped.squeeze(0)]) outputs[0]['boxes'] = outputs[0]['boxes']/scale outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] model_time = time.time() - model_time res = {target["image_id"].item(): output for target, output in zip(targets, outputs)} evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return coco_evaluator
def evaluate_coco(model, data_loader): """ """ import torchvision def _get_iou_types(model): model_without_ddp = model if isinstance(model, torch.nn.parallel.DistributedDataParallel): model_without_ddp = model.module iou_types = ["bbox"] if isinstance(model_without_ddp, torchvision.models.detection.MaskRCNN): iou_types.append("segm") if isinstance(model_without_ddp, torchvision.models.detection.KeypointRCNN): iou_types.append("keypoints") return iou_types device = model_device(model) cpu_device = torch.device("cpu") n_threads = torch.get_num_threads() torch.set_num_threads(1) model.eval() coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) for images, targets in tqdm(data_loader): images = list(img.to(device) for img in images) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] outputs = model(images) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] res = { target["image_id"].item(): output for target, output in zip(targets, outputs) } coco_evaluator.update(res) # accumulate predictions from all images coco_evaluator.synchronize_between_processes() coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return coco_evaluator
def evaluate(model, data_loader, device): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) for index, (image, targets) in enumerate(metric_logger.log_every(data_loader, 100, header)): image = list(img.to(device) for img in image) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] ts = copy.deepcopy(targets) torch.cuda.synchronize() model_time = time.time() outputs = model(image) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] # print(outputs) pred_mask = outputs[0]['masks'] # print(pred_mask.shape) gt = ts[0]['masks'] if index > 1: fig = visualizer(pred_mask,image[0],gt,epoch=0) plt.show() model_time = time.time() - model_time res = {target["image_id"].item(): output for target, output in zip(targets, outputs)} evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return coco_evaluator
def evaluate(model, data_loader, device): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) for image, targets in metric_logger.log_every(data_loader, 100, header): image = list(img.to(device) for img in image) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] torch.cuda.synchronize() model_time = time.time() outputs = model(image) # 可视化 # for img in image: # Image.fromarray(img.mul(255).permute(1, 2, 0).byte().cpu().numpy()) # Image.fromarray(outputs[0]['masks'][0, 0].mul(255).byte().cpu().numpy()) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] model_time = time.time() - model_time res = {target["image_id"].item(): output for target, output in zip(targets, outputs)} evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return coco_evaluator
def evaluate(model, data_loader, device): n_threads = torch.get_num_threads() torch.set_num_threads(1) model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = data_loader.dataset.coco iou_types = ["bbox"] coco_evaluator = CocoEvaluator(coco, iou_types) for image, targets in metric_logger.log_every(data_loader, 100, header): image = list(img.to(device) for img in image) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] torch.cuda.synchronize() model_time = time.time() outputs = model(image) outputs = [{k: v for k, v in t.items()} for t in outputs] model_time = time.time() - model_time res = {target["image_id"].item(): output for target, output in zip(targets, outputs)} evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return coco_evaluator
def evaluate(model, data_loader, device): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) count = 0 for images, targets, valids, _ in metric_logger.log_every( data_loader, 100, header): if False in valids: continue images = list(img.to(device) for img in images) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] torch.cuda.synchronize() model_time = time.time() outputs = model(images) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] img = images[0].cpu().numpy().transpose(1, 2, 0) ax = plt.subplot(1, 1, 1) ax.set_axis_off() ax.imshow(img) count += 1 mask_count = 3 for box, mask_tensor in zip( outputs[0]['boxes'], outputs[0]['masks'] ): # Wont work when not using gt_boxes because we can have less boxes than masks box = box.int().tolist() mask = ((mask_tensor[0].cpu().numpy().astype(np.float)) >= 0.5).astype(np.float) full_mask = np.expand_dims(mask, axis=-1).repeat(4, axis=-1) full_mask[:, :, 0] = 0. full_mask[:, :, 1] = 1 full_mask[:, :, 2] = 0. ax.imshow(full_mask, alpha=0.3) mask_count -= 1 if mask_count == 0: break plt.savefig(f'data/{count}.png') plt.clf() model_time = time.time() - model_time res = { target["image_id"].item(): output for target, output in zip(targets, outputs) } evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return coco_evaluator
def evaluate_yolo_2017(model, data_loader, device): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) transform = GeneralizedRCNNTransform(416, 416, [0, 0, 0], [1, 1, 1]) transform.eval() for image, targets in metric_logger.log_every(data_loader, 100, header): image = list(img.to(device) for img in image) original_image_sizes = [img.shape[-2:] for img in image] targets = [{k: v.to(device) for k, v in t.items()} for t in targets] torch.cuda.synchronize() model_time = time.time() transformed_img = transform(image) transformed_shape = transformed_img[0].tensors.shape[-2:] inf_out, _ = model(transformed_img[0].tensors) # Run NMS output = non_max_suppression(inf_out, conf_thres=0.001, iou_thres=0.6) # Statistics per image predictions = [] for si, pred in enumerate(output): prediction = {'boxes': [], 'labels': [], 'scores': []} if pred is None: continue # Append to text file # with open('test.txt', 'a') as file: # [file.write('%11.5g' * 7 % tuple(x) + '\n') for x in pred] # Clip boxes to image bounds clip_coords(pred, transformed_shape) # Append to pycocotools JSON dictionary # [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ... image_id = int(targets[si]['image_id']) box = pred[:, :4].clone() # xyxy # scale_coords(transformed_shape, box, shapes[si][0], shapes[si][1]) # to original shape # box = xyxy2xywh(box) # xywh # box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner for di, d in enumerate(pred): box_T = [floatn(x, 3) for x in box[di]] label = coco91class[int(d[5])] score = floatn(d[4], 5) prediction['boxes'].append(box_T) prediction['labels'].append(label) prediction['scores'].append(score) prediction['boxes'] = torch.tensor(prediction['boxes']) prediction['labels'] = torch.tensor(prediction['labels']) prediction['scores'] = torch.tensor(prediction['scores']) predictions.append(prediction) outputs = transform.postprocess(predictions, transformed_img[0].image_sizes, original_image_sizes) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in predictions] model_time = time.time() - model_time res = { target["image_id"].item(): output for target, output in zip(targets, outputs) } evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return coco_evaluator
def evaluate(model, data_loader, device, box_threshold=0.001): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset, box_threshold) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) for images, targets in metric_logger.log_every(data_loader, 100, header): # images = list(img.to(device) for img in images) # targets = [{k: v.to(device) if torch.is_tensor(v) else v for k, v in t.items()} for t in targets] torch.cuda.synchronize() model_time = time.time() # if box_threshold is None: # outputs = model(images) # else: # outputs = model(images, box_threshold) outputs = model(images, targets) # outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] predictions = outputs['detections'].to(cpu_device) batch_predictions = [] batch_size = predictions.shape[0] for i in range(batch_size): num_of_detections = len( torch.where(predictions[i][:, 4] > 0.0001)[0]) batch_predictions.insert( i, { 'boxes': predictions[i][:num_of_detections, 0:4], 'scores': predictions[i][:num_of_detections, 4], 'labels': predictions[i][:num_of_detections, 5], }) if num_of_detections > 0: try: print("max score was [{}]".format( batch_predictions[0]['scores'][0])) except: print("exception when using batch_predictions during eval") print("batch_size [{}]".format(batch_size)) print(batch_predictions) model_time = time.time() - model_time # vis = visualize.Visualize('.', targets['img_size'][0][0]) # num_of_detections = len(torch.where(targets['cls'][0] > -1)[0]) # vis.show_image_data(images[0], targets['cls'][0,:num_of_detections].int(), None, targets['bbox'][0,:num_of_detections,[1,0,3,2]]) # print("img ids: [{}]".format(targets['image_id'].to(cpu_device).tolist())) res = { image_id: output for image_id, output in zip( targets['image_id'].to(cpu_device).tolist(), batch_predictions) } # ofekp: this used to be target["image_id"].item() evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return coco_evaluator
def evaluate(model, data_loader, device): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) # changing these two lines a bit to have iteration number and to keep image tensor for i, (images, targets) in enumerate(metric_logger.log_every(data_loader, 100, header)): img = images[0] images = list(img.to(device) for img in images) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] torch.cuda.synchronize() model_time = time.time() outputs = model(images) # ### OUR CODE ### # let's track bounding box and labels predictions for the first 50 images # as we hardly want to track all validation images # but want to see how the predicted bounding boxes and labels are changing during the process if i < 50: # let's add tracking images with predicted bounding boxes logger.add_image_with_boxes( # adding pred_images tag to combine images in one subgroup "pred_images/PD-{}".format(i), # passing image tensor img, # passing predicted bounding boxes outputs[0]["boxes"].cpu(), # mapping & passing predicted labels labels=[ COCO_INSTANCE_CATEGORY_NAMES[i] for i in outputs[0]["labels"].cpu().numpy() ], ) # ### END OUR CODE ### outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] model_time = time.time() - model_time res = {target["image_id"].item(): output for target, output in zip(targets, outputs)} evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return coco_evaluator
def evaluate(model, data_loader, device): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) idx = 0 results = [] for image, targets in metric_logger.log_every(data_loader, 100, header): image = list(img.to(device) for img in image) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] torch.cuda.synchronize() model_time = time.time() outputs = model(image) ########################################### results = model.tracker.run(predicted=outputs, image=image, idx_frame=idx, results=results) ########################################### outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] model_time = time.time() - model_time res = { target["image_id"].item(): output for target, output in zip(targets, outputs) } evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) img = transforms.ToPILImage()(image[0].detach().cpu()).convert("RGB") draw = ImageDraw.Draw(img) for k in range(min(len(targets[0]['boxes']), len(outputs[0]['boxes']))): draw.rectangle(targets[0]['boxes'][k].detach().cpu().numpy(), outline="red") draw.rectangle(outputs[0]['boxes'][k].detach().cpu().numpy(), outline="blue") # write_results("./results/mot.txt", results, 'mot') # img.show() img.save("./results/%4d" % int(idx + 1) + ".png") idx += 1 # del img # time.sleep(10) # import psutil # for proc in psutil.process_iter(): # if proc.name() == "display": # proc.kill() # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return coco_evaluator
def evaluate(model, data_loader, device): hookF = {} for name, module in model.named_modules(): if isinstance(module, torch.jit.ScriptModule): continue else: hookF[name] = statistics(module) n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) for image, targets in metric_logger.log_every(data_loader, 100, header): image = list(img.to(device) for img in image) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] torch.cuda.synchronize() model_time = time.time() outputs = model(image) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] model_time = time.time() - model_time res = { target["image_id"].item(): output for target, output in zip(targets, outputs) } evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) stat = {} for key, statObj in hookF.items(): stat[key] = [{ "min_input": float(statObj.input_stat.min), "max_input": float(statObj.input_stat.max), "min_output": float(statObj.output_stat.min), "max_output": float(statObj.output_stat.max), "avg_min_input": float(statObj.input_stat.avg_min), "avg_max_input": float(statObj.input_stat.avg_max), "avg_min_output": float(statObj.output_stat.avg_min), "avg_max_output": float(statObj.output_stat.avg_max) }] del stat[''] #save the dictionary as a json file with open('Pytorch_Obj_Det_Stat.json', 'w') as fp: for k, v in stat.items(): json.dump(stat, fp, indent=0) return coco_evaluator
data_iter = iter(data_loader) for data in data_iter: images, guide_images, targets = data images = list(image.to(device) for image in images) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] guide_images = guide_images_processing(guide_images, device) torch.cuda.synchronize() outputs = model(images, guide_images) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] res = { target["image_id"].item(): output for target, output in zip(targets, outputs) } coco_evaluator.update(res) coco_evaluator.synchronize_between_processes() coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) print("That's it!")
def evaluate(model, data_loader, device, epoch_num=None, check_num=200): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) idx = 0 for images, targets in metric_logger.log_every(data_loader, 100, header): images = list(img.to(device) for img in images) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] torch.cuda.synchronize() model_time = time.time() outputs_set = model(images) # 输出:对应a validate batch里面的每一个输出组成的list outputs_list = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs_set] # 对于minibatch里面的每个output # outputs_list包含一个个 t 是 {'boxes','labels','scores','masks'},每个的值都是一个tensor model_time = time.time() - model_time res = { target["image_id"].item(): output for target, output in zip(targets, outputs_list) } # 构建一个dict,每个键为target["image_id"].item() 即imageid # 值为对应数据在模型预测的时候的输出t, 是 {'boxes','labels','scores','masks'}字典, # 其内每个的值都是一个tensor,长度=预测目标数 idx += 1 if idx - idx // check_num * check_num == 0: # 每100次记录一次 if epoch_num is not None: coco_a_result_check(images, targets, res, 'E' + str(epoch_num) + '_' + str(idx)) else: coco_a_result_check(images, targets, res) ''' for key in res: print(len(res[key]['boxes'])) # 一开始mask rcnn网络输出是100个框(detr 200),后续学好了之后框的数量会大大下降。 ''' evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return coco_evaluator
def evaluate(model, data_loader, device): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset2(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) for images, targets in data_loader: images = list(img.to(device) for img in images) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] torch.cuda.synchronize() model_time = time.time() predictions = model(images) predictions = [{k: v.to(cpu_device) for k, v in t.items()} for t in predictions ] #[{'boxes':,'labels':,'scores':}] batch_size = 1 outputs = [] count = 0 for image in predictions: boxes = image['boxes'] scores = image['scores'] lables = image['labels'] indices = torch.ops.torchvision.nms(boxes, scores, 0.1) count += 1 boxes = torch.stack([boxes[i] for i in indices]) scores = torch.stack([scores[i] for i in indices]) lables = torch.stack([lables[i] for i in indices]) outputs.append({ 'boxes': boxes, 'labels': lables, 'scores': scores }) model_time = time.time() - model_time res = { target["image_id"].item(): output for target, output in zip(targets, outputs) } evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return coco_evaluator
def evaluate(model, data_loader, device, writer, epoch): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) for image, targets in metric_logger.log_every(data_loader, 100, header): image = list(img.to(device) for img in image) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] torch.cuda.synchronize() model_time = time.time() outputs = model(image) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] model_time = time.time() - model_time res = { target["image_id"].item(): output for target, output in zip(targets, outputs) } evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) # TB Logs (remember, when not specified MaxDet = 100) writer.add_scalar('mAP @ [.5:.0.5:.95] @ all_size', coco_evaluator.coco_eval['bbox'].stats[0], epoch) writer.add_scalar('mAP @ 0.5 @ all_size', coco_evaluator.coco_eval['bbox'].stats[1], epoch) writer.add_scalar('mAP @ 0.75 @ all_size', coco_evaluator.coco_eval['bbox'].stats[2], epoch) writer.add_scalar('mAP @ [.5:.0.5:.95] @ SMALL', coco_evaluator.coco_eval['bbox'].stats[3], epoch) writer.add_scalar('mAP @ [.5:.0.5:.95] @ MEDIUM', coco_evaluator.coco_eval['bbox'].stats[4], epoch) writer.add_scalar('mAP @ [.5:.0.5:.95] @ LARGE', coco_evaluator.coco_eval['bbox'].stats[5], epoch) writer.add_scalar('mAR @ [.5:.0.5:.95] @ all_size @ MaxDet=1', coco_evaluator.coco_eval['bbox'].stats[6], epoch) writer.add_scalar('mAR @ [.5:.0.5:.95] @ all_size @ MaxDet=10', coco_evaluator.coco_eval['bbox'].stats[7], epoch) writer.add_scalar('mAR @ [.5:.0.5:.95] @ all_size @ MaxDet=100', coco_evaluator.coco_eval['bbox'].stats[8], epoch) writer.add_scalar('mAR @ [.5:.0.5:.95] @ SMALL', coco_evaluator.coco_eval['bbox'].stats[9], epoch) writer.add_scalar('mAR @ [.5:.0.5:.95] @ MEDIUM', coco_evaluator.coco_eval['bbox'].stats[10], epoch) writer.add_scalar('mAR @ [.5:.0.5:.95] @ LARGE', coco_evaluator.coco_eval['bbox'].stats[11], epoch) return coco_evaluator, coco_evaluator.coco_eval['bbox'].stats[1]
def main(): # train on the GPU or on the CPU, if a GPU is not available device = torch.device( 'cuda') if torch.cuda.is_available() else torch.device('cpu') # our dataset has two classes only - background and person num_classes = 2 # get the model using our helper function model = get_model_instance_segmentation(num_classes) # load pretrain_dict pretrain_dict = torch.load( os.path.join("C:\\zhulei\\maskRcnn\\models", "_epoch-9.pth")) model.load_state_dict(pretrain_dict) # move model to the right device model.to(device) # use our dataset and defined transformations dataset_test = PennFudanDataset('C:\\zhulei\\maskRcnn\\data\\test', get_transform(train=False)) data_loader_test = torch.utils.data.DataLoader(dataset_test, batch_size=1, shuffle=False, num_workers=1, collate_fn=utils.collate_fn) n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader_test.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) for image, targets in metric_logger.log_every(data_loader_test, 100, header): image = list(img.to(device) for img in image) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] torch.cuda.synchronize() model_time = time.time() outputs = model(image) instance_segmentation_api(image[0], outputs) # 可视化 # for img in image: # Image.fromarray((img.mul(255).permute(1, 2, 0).byte().cpu().numpy())[0]) # print(outputs[0]['masks'].shape) # for i in range(99): # result = Image.fromarray(outputs[0]['masks'][i, 0].mul(255).byte().cpu().numpy()) # result.show() outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] model_time = time.time() - model_time res = { target["image_id"].item(): output for target, output in zip(targets, outputs) } evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads)
def evaluate(model, data_loader, device, is_vis=False, draw_bbox=False, vis_dir='./vis_results'): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) res_list = [] for images, targets in metric_logger.log_every(data_loader, 100, header): images = list(img.to(device) for img in images) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] torch.cuda.synchronize() model_time = time.time() outputs = model(images) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] model_time = time.time() - model_time res = { target["image_id"].item(): output for target, output in zip(targets, outputs) } evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) res_list.append(res) # visualization if is_vis: if os.path.exists(vis_dir) == False: os.makedirs(vis_dir) coco_evaluator.visualize(res, images, vis_dir, draw_bbox) with open(os.path.join(vis_dir, 'pred_res.pkl'), 'wb') as pkl_file: pickle.dump(res_list, pkl_file) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() mAP_scores = coco_evaluator.summarize() torch.set_num_threads(n_threads) return mAP_scores
def inference(model, dataloader, datatype, args): batch_time = AverageMeter('Time', ':6.3f') batch_size = args.batch_size warmup_iters = args.warmup_iterations max_iters = args.max_iterations if dataloader is None else len(dataloader) model.eval() coco = get_coco_api_from_dataset(dataloader.dataset) iou_types = ["bbox"] iou_types.append("segm") coco_evaluator = CocoEvaluator(coco, iou_types) if args.ipex: import intel_extension_for_pytorch as ipex model = model.to(memory_format=torch.channels_last) model = ipex.optimize(model, dtype=datatype, level="O1", conv_bn_folding=False, replace_dropout_with_identity=False) model.backbone = ipex.optimize(model.backbone, dtype=datatype, level="O1") else: if args.jit: model = model.to(memory_format=torch.channels_last) else: from torch.utils import mkldnn as mkldnn_utils model = mkldnn_utils.to_mkldnn(model, dtype=datatype) if args.jit: x = torch.randn(batch_size, 3, 1200, 1200).to(memory_format=torch.channels_last) if args.precision == "bf16": with torch.cpu.amp.autocast(), torch.no_grad(): model.backbone = torch.jit.trace(model.backbone, x, strict=False) model.backbone = torch.jit.freeze(model.backbone) else: with torch.no_grad(): model.backbone = torch.jit.trace(model.backbone, x, strict=False) model.backbone = torch.jit.freeze(model.backbone) with torch.no_grad(): if dataloader is None: print( "Models for detection tasks need to use real dataset. You need to specify coco dataset. " ) exit(1) else: for i, batch in enumerate(dataloader): images = batch[0] if not args.ipex and not args.jit: images = list(img.to(datatype) for img in images) if args.ipex and args.precision == "bf16": with torch.cpu.amp.autocast(): if i == warmup_iters: with profile( activities=[ProfilerActivity.CPU], record_shapes=True ) as prof, record_function("model_inference"): output = model(images) else: output = model(images) else: if i == warmup_iters: with profile( activities=[ProfilerActivity.CPU], record_shapes=True) as prof, record_function( "model_inference"): output = model(images) else: output = model(images) if i > warmup_iters: break for i, batch in enumerate(dataloader): images = batch[0] end = time.time() if not args.ipex and not args.jit: images = list(img.to(datatype) for img in images) if args.ipex and args.precision == "bf16": with torch.cpu.amp.autocast(): output = model(images) else: output = model(images) batch_time.update(time.time() - end) output = [{k: v.to(torch.float32) for k, v in t.items()} for t in output] res = { target["image_id"].item(): output for target, output in zip(batch[1], output) } coco_evaluator.update(res) if max_iters != -1 and i >= max_iters: break print(prof.key_averages().table(sort_by="cpu_time_total", row_limit=-1)) latency = batch_time.avg / batch_size * 1000 perf = batch_size / batch_time.avg coco_evaluator.synchronize_between_processes() coco_evaluator.accumulate() coco_evaluator.summarize() print("Bbox AP: {:.5f} ".format(coco_evaluator.coco_eval['bbox'].stats[0])) print("Segm AP: {:.5f} ".format(coco_evaluator.coco_eval['segm'].stats[0])) print('Latency: %.3f ms' % latency) print("Throughput: {:.3f} fps".format(perf))
def evaluate_bin(model, data_loader, device, bin_folder): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) bin_folder = bin_folder jsonPath = os.path.join(args.output_dir, 'images_shape.json') with open(jsonPath) as json_file: shape_dict = json.load(json_file) # model.transform = IdentityTransform(model.transform.min_size, model.transform.max_size, model.transform.image_mean, model.transform.image_std) model.eval() for image, targets in metric_logger.log_every(data_loader, 100, header): image = list(img.to(device) for img in image) original_image_sizes = [img.shape[-2:] for img in image] targets = [{k: v.to(device) for k, v in t.items()} for t in targets] img_id = targets[0]['image_id'].cpu().numpy()[0] path = os.path.join(bin_folder, str(img_id) + '.bin') f = open(path, 'rb') transformed_img = np.fromfile(f, np.float32) transformed_img = np.reshape(transformed_img, shape_dict[str(img_id)][0][0]) image_sizes_not_devisible = np.asarray(shape_dict[str(img_id)][1][0]) image_sizes_not_devisible = torch.from_numpy(image_sizes_not_devisible) transformed_img_T = torch.from_numpy(transformed_img) transformed_img_T = transformed_img_T.to(device) torch.cuda.synchronize() model_time = time.time() outputs = model(transformed_img_T) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] outputs = model.transform.postprocess(outputs, [image_sizes_not_devisible], original_image_sizes) model_time = time.time() - model_time res = { target["image_id"].item(): output for target, output in zip(targets, outputs) } evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads)