class PASCALContext:
dataset = PASCALContext
normalize = Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
transforms = Compose([ToTensor(), normalize])
send_data_to_device = default_data_to_device
collate_fn = default_seg_collate_fn
model_output_transform = default_seg_output_transform
task = "Semantic Segmentation"
@classmethod
def benchmark(cls,
model,
model_description=None,
input_transform=None,
target_transform=None,
transforms=None,
model_output_transform=None,
collate_fn=None,
send_data_to_device=None,
device: str = "cuda",
data_root: str = "./.data/vision/pascalcontext",
num_workers: int = 4,
batch_size: int = 32,
num_gpu: int = 1,
paper_model_name: str = None,
paper_arxiv_id: str = None,
paper_pwc_id: str = None,
paper_results: dict = None,
pytorch_hub_url: str = None,
force: bool = False) -> BenchmarkResult:
config = locals()
model, device = send_model_to_device(model,
device=device,
num_gpu=num_gpu)
model.eval()
if not input_transform or target_transform or transforms:
transforms = cls.transforms
if not model_output_transform:
model_output_transform = cls.model_output_transform
if not send_data_to_device:
send_data_to_device = cls.send_data_to_device
if not collate_fn:
collate_fn = cls.collate_fn
test_dataset = cls.dataset(
root=data_root,
split="val",
transform=input_transform,
target_transform=target_transform,
transforms=transforms,
)
test_loader = DataLoader(
test_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True,
collate_fn=collate_fn,
)
test_loader.no_classes = 59 # Number of classes for PASCALContext
test_results, speed_mem_metrics, run_hash = evaluate_segmentation(
model=model,
test_loader=test_loader,
model_output_transform=model_output_transform,
send_data_to_device=send_data_to_device,
device=device,
force=force)
print(test_results)
return BenchmarkResult(
task=cls.task,
config=config,
dataset=cls.dataset.__name__,
results=test_results,
speed_mem_metrics=speed_mem_metrics,
pytorch_hub_id=pytorch_hub_url,
model=paper_model_name,
model_description=model_description,
arxiv_id=paper_arxiv_id,
pwc_id=paper_pwc_id,
paper_results=paper_results,
)
class PASCALVOC:
"""`PASCALVOC <https://www.sotabench.com/benchmark/pascalvoc2012>`_ benchmark.
Examples:
Evaluate a ResNeXt model from the torchvision repository:
.. code-block:: python
from torchbench.semantic_segmentation import PASCALVOC
from torchbench.semantic_segmentation.transforms import (
Normalize,
Resize,
ToTensor,
Compose,
)
from torchvision.models.segmentation import fcn_resnet101
import torchvision.transforms as transforms
import PIL
def model_output_function(output, labels):
return output['out'].argmax(1).flatten(), target.flatten()
def seg_collate_fn(batch):
images, targets = list(zip(*batch))
batched_imgs = cat_list(images, fill_value=0)
batched_targets = cat_list(targets, fill_value=255)
return batched_imgs, batched_targets
model = fcn_resnet101(num_classes=21, pretrained=True)
normalize = Normalize(
mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
)
my_transforms = Compose([Resize((520, 480)), ToTensor(), normalize])
PASCALVOC.benchmark(batch_size=32,
model=model,
transforms=my_transforms,
model_output_transform=model_output_function,
collate_fn=seg_collate_fn,
paper_model_name='FCN ResNet-101',
paper_arxiv_id='1605.06211')
"""
dataset = datasets.VOCSegmentation
normalize = Normalize(
mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
)
transforms = Compose([Resize((520, 480)), ToTensor(), normalize])
send_data_to_device = default_data_to_device
collate_fn = default_seg_collate_fn
model_output_transform = default_seg_output_transform
task = "Semantic Segmentation"
@classmethod
def benchmark(
cls,
model,
model_description=None,
dataset_year="2012",
input_transform=None,
target_transform=None,
transforms=None,
model_output_transform=None,
collate_fn=None,
send_data_to_device=None,
device: str = "cuda",
data_root: str = "./.data/vision/voc",
num_workers: int = 4,
batch_size: int = 32,
num_gpu: int = 1,
paper_model_name: str = None,
paper_arxiv_id: str = None,
paper_pwc_id: str = None,
paper_results: dict = None,
pytorch_hub_url: str = None,
) -> BenchmarkResult:
config = locals()
model, device = send_model_to_device(
model, device=device, num_gpu=num_gpu
)
model.eval()
if not input_transform or target_transform or transforms:
transforms = cls.transforms
if not model_output_transform:
model_output_transform = cls.model_output_transform
if not send_data_to_device:
send_data_to_device = cls.send_data_to_device
if not collate_fn:
collate_fn = cls.collate_fn
test_dataset = cls.dataset(
root=data_root,
image_set="val",
year=dataset_year,
transform=input_transform,
target_transform=target_transform,
transforms=transforms,
download=True,
)
test_loader = DataLoader(
test_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True,
collate_fn=collate_fn,
)
test_loader.no_classes = 21 # Number of classes for PASCALVOC
test_results, speed_mem_metrics, run_hash = evaluate_segmentation(
model=model,
test_loader=test_loader,
model_output_transform=model_output_transform,
send_data_to_device=send_data_to_device,
device=device,
)
print(test_results)
return BenchmarkResult(
task=cls.task,
config=config,
dataset='PASCAL VOC %s %s' % (dataset_year, "val"),
results=test_results,
speed_mem_metrics=speed_mem_metrics,
pytorch_hub_id=pytorch_hub_url,
model=paper_model_name,
model_description=model_description,
arxiv_id=paper_arxiv_id,
pwc_id=paper_pwc_id,
paper_results=paper_results,
run_hash=run_hash,
)