def benchmark(model: nn.Module,
transform,
batch_size=64,
device=device,
fast: bool = False):
valid_dataset = ImageNet(root="/home/zuppif/Downloads/ImageNet",
split="val",
transform=transform)
valid_loader = torch.utils.data.DataLoader(
valid_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=12,
pin_memory=True,
)
evaluator = ImageNetEvaluator(model_name="test",
paper_arxiv_id="1905.11946")
model.eval().to(device)
num_batches = int(
math.ceil(len(valid_loader.dataset) / float(valid_loader.batch_size)))
start = time.time()
with torch.no_grad():
pbar = tqdm(np.arange(num_batches), leave=False)
for i_val, (images, labels) in enumerate(valid_loader):
images = images.to(device)
labels = torch.squeeze(labels.to(device))
net_out = model(images)
image_ids = [
get_img_id(img[0]) for img in
valid_loader.dataset.imgs[i_val *
valid_loader.batch_size:(i_val + 1) *
valid_loader.batch_size]
]
evaluator.add(dict(zip(image_ids, list(net_out.cpu().numpy()))))
pbar.set_description(f"f1={evaluator.top1.avg:.2f}")
pbar.update(1)
if fast:
break
pbar.close()
stop = time.time()
if fast:
return evaluator.top1.avg, None, None
else:
res = evaluator.get_results()
return res["Top 1 Accuracy"], res["Top 5 Accuracy"], stop - start
def run_eval(model, data, loader, name):
evaluator = ImageNetEvaluator(
model_name=name,
paper_arxiv_id='1912.11370',
paper_results=PAPER_RESULTS.get(name),
)
with torch.no_grad():
for i, (x, _) in enumerate(loader):
print(f'\rEvaluating batch {i}/{len(loader)}', flush=True, end='')
x = x.to(device='cuda', non_blocking=True)
y = model(x).cpu().numpy()
bs = loader.batch_size
evaluator.add({
get_img_id(data, i * bs + j): list(logits)
for j, logits in enumerate(y)
})
if evaluator.cache_exists:
break
for k, v in evaluator.save().to_dict().items():
print(f"{k}: {v}")
batch_size = m['batch_size']
loader = create_loader(
dataset,
input_size=data_config['input_size'],
batch_size=batch_size,
use_prefetcher=True,
interpolation=data_config['interpolation'],
mean=data_config['mean'],
std=data_config['std'],
num_workers=6,
crop_pct=data_config['crop_pct'],
pin_memory=True)
evaluator = ImageNetEvaluator(
root=DATA_ROOT,
model_name=m['paper_model_name'],
paper_arxiv_id=m['paper_arxiv_id'],
model_description=m.get('model_description', None),
)
model.cuda()
model.eval()
with torch.no_grad():
# warmup
input = torch.randn((batch_size,) + data_config['input_size']).cuda()
model(input)
bar = tqdm(desc="Evaluation", mininterval=5, total=50000)
evaluator.reset_time()
sample_count = 0
for input, target in loader:
output = model(input)
num_samples = len(output)
from sotabencheval.image_classification import ImageNetEvaluator
evaluator = ImageNetEvaluator(
# automatically compare to this paper
model_name='ResNeXt-101-32x8d',
paper_arxiv_id='1611.05431'
)
predictions = ... # use your model to make predictions
evaluator.add(predictions)
evaluator.save()
download=True,
)
test_loader = DataLoader(
test_dataset,
batch_size=128,
shuffle=False,
num_workers=4,
pin_memory=True,
)
model = model.cuda()
model.eval()
evaluator = ImageNetEvaluator(
model_name='Faster-RCNN-TensorFlow-Python3',
paper_arxiv_id='1506.01497')
def get_img_id(image_name):
return image_name.split('/')[-1].replace('.JPEG', '')
with torch.no_grad():
for i, (input, target) in enumerate(test_loader):
input = input.to(device='cuda', non_blocking=True)
target = target.to(device='cuda', non_blocking=True)
output = model(input
image_ids = [get_img_id(img[0]) for img in test_loader.dataset.imgs[i*test_loader.batch_size:(i+1)*test_loader.batch_size]]
evaluator.add(dict(zip(image_ids, list(output.cpu().numpy()))))
evaluator.save()
transform=input_transform,
target_transform=None,
)
test_loader = DataLoader(
test_dataset,
batch_size=128,
shuffle=False,
num_workers=4,
pin_memory=True,
)
model = model.cuda()
model.eval()
evaluator = ImageNetEvaluator(model_name=model_name,
paper_arxiv_id='1905.11946')
def get_img_id(image_name):
return image_name.split('/')[-1].replace('.JPEG', '')
with torch.no_grad():
for i, (input, target) in enumerate(test_loader):
input = input.to(device='cuda', non_blocking=True)
target = target.to(device='cuda', non_blocking=True)
output = model(input)
image_ids = [
get_img_id(img[0])
for img in test_loader.dataset.imgs[i *
test_loader.batch_size:(i +
from sotabencheval.image_classification import ImageNetEvaluator
evaluator = ImageNetEvaluator(
model_name='ResNeXt-101-32x8d',
paper_arxiv_id='1906.06423')
evaluator.add(dict(zip(image_ids, batch_output)))
evaluator.save()
download=True,
)
test_loader = DataLoader(
test_dataset,
batch_size=128,
shuffle=False,
num_workers=4,
pin_memory=True,
)
model = model.cuda()
model.eval()
evaluator = ImageNetEvaluator(
model_name='ResNeXt-101-32x8d',
paper_arxiv_id='1611.05431')
def get_img_id(image_name):
return image_name.split('/')[-1].replace('.JPEG', '')
with torch.no_grad():
for i, (input, target) in enumerate(test_loader):
input = input.to(device='cuda', non_blocking=True)
target = target.to(device='cuda', non_blocking=True)
output = model(input
image_ids = [get_img_id(img[0]) for img in test_loader.dataset.imgs[i*test_loader.batch_size:(i+1)*test_loader.batch_size]]
evaluator.add(dict(zip(image_ids, list(output.cpu().numpy()))))
evaluator.save()