def benchmark(model: nn.Module, transform, batch_size=64, device=device, fast: bool = False): valid_dataset = ImageNet(root="/home/zuppif/Downloads/ImageNet", split="val", transform=transform) valid_loader = torch.utils.data.DataLoader( valid_dataset, batch_size=batch_size, shuffle=False, num_workers=12, pin_memory=True, ) evaluator = ImageNetEvaluator(model_name="test", paper_arxiv_id="1905.11946") model.eval().to(device) num_batches = int( math.ceil(len(valid_loader.dataset) / float(valid_loader.batch_size))) start = time.time() with torch.no_grad(): pbar = tqdm(np.arange(num_batches), leave=False) for i_val, (images, labels) in enumerate(valid_loader): images = images.to(device) labels = torch.squeeze(labels.to(device)) net_out = model(images) image_ids = [ get_img_id(img[0]) for img in valid_loader.dataset.imgs[i_val * valid_loader.batch_size:(i_val + 1) * valid_loader.batch_size] ] evaluator.add(dict(zip(image_ids, list(net_out.cpu().numpy())))) pbar.set_description(f"f1={evaluator.top1.avg:.2f}") pbar.update(1) if fast: break pbar.close() stop = time.time() if fast: return evaluator.top1.avg, None, None else: res = evaluator.get_results() return res["Top 1 Accuracy"], res["Top 5 Accuracy"], stop - start
def run_eval(model, data, loader, name): evaluator = ImageNetEvaluator( model_name=name, paper_arxiv_id='1912.11370', paper_results=PAPER_RESULTS.get(name), ) with torch.no_grad(): for i, (x, _) in enumerate(loader): print(f'\rEvaluating batch {i}/{len(loader)}', flush=True, end='') x = x.to(device='cuda', non_blocking=True) y = model(x).cpu().numpy() bs = loader.batch_size evaluator.add({ get_img_id(data, i * bs + j): list(logits) for j, logits in enumerate(y) }) if evaluator.cache_exists: break for k, v in evaluator.save().to_dict().items(): print(f"{k}: {v}")
model.cuda() model.eval() with torch.no_grad(): # warmup input = torch.randn((batch_size,) + data_config['input_size']).cuda() model(input) bar = tqdm(desc="Evaluation", mininterval=5, total=50000) evaluator.reset_time() sample_count = 0 for input, target in loader: output = model(input) num_samples = len(output) image_ids = [filenames[i] for i in range(sample_count, sample_count + num_samples)] output = output.cpu().numpy() evaluator.add(dict(zip(image_ids, list(output)))) sample_count += num_samples bar.update(num_samples) if evaluator.cache_exists: break bar.close() evaluator.save() for k, v in evaluator.results.items(): print(k, v) for k, v in evaluator.speed_mem_metrics.items(): print(k, v) torch.cuda.empty_cache()
from sotabencheval.image_classification import ImageNetEvaluator evaluator = ImageNetEvaluator( # automatically compare to this paper model_name='ResNeXt-101-32x8d', paper_arxiv_id='1611.05431' ) predictions = ... # use your model to make predictions evaluator.add(predictions) evaluator.save()
download=True, ) test_loader = DataLoader( test_dataset, batch_size=128, shuffle=False, num_workers=4, pin_memory=True, ) model = model.cuda() model.eval() evaluator = ImageNetEvaluator( model_name='Faster-RCNN-TensorFlow-Python3', paper_arxiv_id='1506.01497') def get_img_id(image_name): return image_name.split('/')[-1].replace('.JPEG', '') with torch.no_grad(): for i, (input, target) in enumerate(test_loader): input = input.to(device='cuda', non_blocking=True) target = target.to(device='cuda', non_blocking=True) output = model(input image_ids = [get_img_id(img[0]) for img in test_loader.dataset.imgs[i*test_loader.batch_size:(i+1)*test_loader.batch_size]] evaluator.add(dict(zip(image_ids, list(output.cpu().numpy())))) evaluator.save()
from sotabencheval.image_classification import ImageNetEvaluator evaluator = ImageNetEvaluator( model_name='ResNeXt-101-32x8d', paper_arxiv_id='1906.06423') evaluator.add(dict(zip(image_ids, batch_output))) evaluator.save()