def list_available_models(
dataset: Union[str, BenchmarkDataset] = BenchmarkDataset.cifar_10,
threat_model: Union[str, ThreatModel] = ThreatModel.Linf,
norm: Optional[str] = None):
dataset_: BenchmarkDataset = BenchmarkDataset(dataset)
if norm is None:
threat_model_: ThreatModel = ThreatModel(threat_model)
else:
threat_model_ = ThreatModel(norm)
warnings.warn(
"`norm` has been deprecated and will be removed in a future version.",
DeprecationWarning)
models = all_models[dataset_][threat_model_].keys()
acc_field = ACC_FIELDS[threat_model_]
json_dicts = []
jsons_dir = Path("./model_info") / dataset_.value / threat_model_.value
for model_name in models:
json_path = jsons_dir / f"{model_name}.json"
# Some models might not yet be in model_info
if not json_path.exists():
continue
with open(json_path, 'r') as model_info:
json_dict = json.load(model_info)
json_dict['model_name'] = model_name
json_dict['venue'] = 'Unpublished' if json_dict[
'venue'] == '' else json_dict['venue']
json_dict[acc_field] = float(json_dict[acc_field]) / 100
json_dict['clean_acc'] = float(json_dict['clean_acc']) / 100
json_dicts.append(json_dict)
json_dicts = sorted(json_dicts, key=lambda d: -d[acc_field])
print('| <sub>#</sub> | <sub>Model ID</sub> | <sub>Paper</sub> | <sub>Clean accuracy</sub> | <sub>Robust accuracy</sub> | <sub>Architecture</sub> | <sub>Venue</sub> |')
print('|:---:|---|---|:---:|:---:|:---:|:---:|')
for i, json_dict in enumerate(json_dicts):
if json_dict['model_name'] == 'Chen2020Adversarial':
json_dict['architecture'] = json_dict[
'architecture'] + ' <br/> (3x ensemble)'
if json_dict['model_name'] != 'Natural':
print(
'| <sub>**{}**</sub> | <sub><sup>**{}**</sup></sub> | <sub>*[{}]({})*</sub> | <sub>{:.2%}</sub> | <sub>{:.2%}</sub> | <sub>{}</sub> | <sub>{}</sub> |'
.format(i + 1, json_dict['model_name'], json_dict['name'],
json_dict['link'], json_dict['clean_acc'],
json_dict[acc_field], json_dict['architecture'],
json_dict['venue']))
else:
print(
'| <sub>**{}**</sub> | <sub><sup>**{}**</sup></sub> | <sub>*{}*</sub> | <sub>{:.2%}</sub> | <sub>{:.2%}</sub> | <sub>{}</sub> | <sub>{}</sub> |'
.format(i + 1, json_dict['model_name'], json_dict['name'],
json_dict['clean_acc'], json_dict[acc_field],
json_dict['architecture'], json_dict['venue']))
def get_leaderboard_latex(dataset: Union[str, BenchmarkDataset], threat_model: Union[str, ThreatModel],
l_keys=['clean_acc', 'autoattack_acc', 'additional_data', 'architecture', 'venue'],
sort_by='autoattack_acc'):
dataset_: BenchmarkDataset = BenchmarkDataset(dataset)
threat_model_: ThreatModel = ThreatModel(threat_model)
jsons_dir = Path("../model_info") / dataset_.value / threat_model_.value
entries = []
for json_path in jsons_dir.glob("*.json"):
model_name = json_path.stem.split("_")[0]
with open(json_path, 'r') as model_info:
model_dict = json.load(model_info)
str_curr = '\\cite{{{}}}'.format(model_name) if not model_name in [
'Standard'] else model_name
for k in l_keys:
if k == 'additional_data':
str_curr += ' & {}'.format('Y' if model_dict[k] else 'N')
else:
str_curr += ' & {}'.format(model_dict[k])
str_curr += '\\\\'
entries.append((str_curr, float(model_dict[sort_by])))
entries = sorted(entries, key=lambda k: k[1], reverse=True)
entries = ['{} &'.format(i + 1) + a for i, (a, b) in enumerate(entries)]
entries = '\n'.join(entries).replace('<br>', ' ')
return entries
def get_leaderboard_bibtex(dataset: Union[str, BenchmarkDataset], threat_model: Union[str, ThreatModel]):
dataset_: BenchmarkDataset = BenchmarkDataset(dataset)
threat_model_: ThreatModel = ThreatModel(threat_model)
jsons_dir = Path("./model_info") / dataset_.value / threat_model_.value
bibtex_entries = set()
for json_path in jsons_dir.glob("*.json"):
model_name = json_path.stem.split("_")[0]
with open(json_path, 'r') as model_info:
model_dict = json.load(model_info)
title = model_dict["name"]
authors = model_dict["authors"]
full_venue = model_dict["venue"]
if full_venue == "N/A":
continue
venue = full_venue.split(" ")[0]
venue = venue.split(",")[0]
year = model_dict["venue"].split(" ")[-1]
bibtex_entry = _get_bibtex_entry(model_name, title, authors, venue, year)
bibtex_entries.add(bibtex_entry)
str_entries = ''
for entry in bibtex_entries:
print(entry)
str_entries += entry
return bibtex_entries, str_entries
def get_dataloaders(opt, mode):
assert (mode != 'train')
assert (mode != 'test')
imagenet_preprocessing_str = all_models_rb[BenchmarkDataset(
opt.dataset_to_use)][ThreatModel(opt.use_robust_bench_threat)][
opt.use_robust_bench_model]['preprocessing']
im_data = load_imagenet(data_dir=opt.imagenet_location,
prepr=imagenet_preprocessing_str)
in_dataset = get_dataset_with_index(
SimpleDataset((im_data[0] - 0.5) / 0.5, im_data[1]), 0)
return return_dataloaders(lambda: in_dataset, opt, split=mode)
def load_model(model_name: str,
model_dir: Union[str, Path] = './models',
dataset: Union[str,
BenchmarkDataset] = BenchmarkDataset.cifar_10,
threat_model: Union[str, ThreatModel] = ThreatModel.Linf,
norm: Optional[str] = None) -> nn.Module:
"""Loads a model from the model_zoo.
The model is trained on the given ``dataset``, for the given ``threat_model``.
:param model_name: The name used in the model zoo.
:param model_dir: The base directory where the models are saved.
:param dataset: The dataset on which the model is trained.
:param threat_model: The threat model for which the model is trained.
:param norm: Deprecated argument that can be used in place of ``threat_model``. If specified, it
overrides ``threat_model``
:return: A ready-to-used trained model.
"""
dataset_: BenchmarkDataset = BenchmarkDataset(dataset)
if norm is None:
threat_model_: ThreatModel = ThreatModel(threat_model)
else:
threat_model_ = ThreatModel(norm)
warnings.warn(
"`norm` has been deprecated and will be removed in a future version.",
DeprecationWarning)
model_dir_ = Path(model_dir) / dataset_.value / threat_model_.value
model_path = model_dir_ / f'{model_name}.pt'
models = all_models[dataset_][threat_model_]
if not isinstance(models[model_name]['gdrive_id'], list):
model = models[model_name]['model']()
if not os.path.exists(model_dir_):
os.makedirs(model_dir_)
if not os.path.isfile(model_path):
download_gdrive(models[model_name]['gdrive_id'], model_path)
checkpoint = torch.load(model_path, map_location=torch.device('cpu'))
if 'Kireev2021Effectiveness' in model_name:
checkpoint = checkpoint['last'] # we take the last model (choices: 'last', 'best')
# needed for the model of `Carmon2019Unlabeled`
try:
state_dict = rm_substr_from_state_dict(checkpoint['state_dict'],
'module.')
except:
state_dict = rm_substr_from_state_dict(checkpoint, 'module.')
model = _safe_load_state_dict(model, model_name, state_dict)
return model.eval()
# If we have an ensemble of models (e.g., Chen2020Adversarial)
else:
model = models[model_name]['model']()
if not os.path.exists(model_dir_):
os.makedirs(model_dir_)
for i, gid in enumerate(models[model_name]['gdrive_id']):
if not os.path.isfile('{}_m{}.pt'.format(model_path, i)):
download_gdrive(gid, '{}_m{}.pt'.format(model_path, i))
checkpoint = torch.load('{}_m{}.pt'.format(model_path, i),
map_location=torch.device('cpu'))
try:
state_dict = rm_substr_from_state_dict(
checkpoint['state_dict'], 'module.')
except KeyError:
state_dict = rm_substr_from_state_dict(checkpoint, 'module.')
model.models[i] = _safe_load_state_dict(model.models[i],
model_name, state_dict)
model.models[i].eval()
return model.eval()
def __init__(self, opt):
super().__init__()
self.model = BigGAN.from_pretrained('biggan-deep-256')
self.imagenet_preprocessing_str = all_models_rb[BenchmarkDataset(
opt.dataset_to_use)][ThreatModel(opt.use_robust_bench_threat)][
opt.use_robust_bench_model]['preprocessing']
corruptions threat model.
:return: A Tuple with the clean accuracy and the accuracy in the given threat model.
"""
if isinstance(model, Sequence) or isinstance(device, Sequence):
# Multiple models evaluation in parallel not yet implemented
raise NotImplementedError
try:
if model.training:
warnings.warn(Warning("The given model is *not* in eval mode."))
except AttributeError:
warnings.warn(Warning("It is not possible to asses if the model is in eval mode"))
dataset_: BenchmarkDataset = BenchmarkDataset(dataset)
threat_model_: ThreatModel = ThreatModel(threat_model)
device = device or torch.device("cpu")
model = model.to(device)
clean_x_test, clean_y_test = load_clean_dataset(dataset_, None, data_dir)
accuracy = clean_accuracy(model,
clean_x_test,
clean_y_test,
batch_size=batch_size,
device=device)
print(f'Clean accuracy: {accuracy:.2%}')
if threat_model_ in {ThreatModel.Linf, ThreatModel.L2}:
if eps is None:
def generate_leaderboard(dataset: Union[str, BenchmarkDataset],
threat_model: Union[str, ThreatModel],
models_folder: str = "model_info") -> str:
"""Prints the HTML leaderboard starting from the .json results.
The result is a <table> that can be put directly into the RobustBench index.html page,
and looks the same as the tables that are already existing.
The .json results must have the same structure as the following:
``
{
"link": "https://arxiv.org/abs/2003.09461",
"name": "Adversarial Robustness on In- and Out-Distribution Improves Explainability",
"authors": "Maximilian Augustin, Alexander Meinke, Matthias Hein",
"additional_data": true,
"number_forward_passes": 1,
"dataset": "cifar10",
"venue": "ECCV 2020",
"architecture": "ResNet-50",
"eps": "0.5",
"clean_acc": "91.08",
"reported": "73.27",
"autoattack_acc": "72.91"
}
``
If the model is robust to common corruptions, then the "autoattack_acc" field should be
"corruptions_acc".
:param dataset: The dataset of the wanted leaderboard.
:param threat_model: The threat model of the wanted leaderboard.
:param models_folder: The base folder of the model jsons (e.g. our "model_info" folder).
:return: The resulting HTML table.
"""
dataset_: BenchmarkDataset = BenchmarkDataset(dataset)
threat_model_: ThreatModel = ThreatModel(threat_model)
folder = Path(models_folder) / dataset_.value / threat_model_.value
acc_field = ACC_FIELDS[threat_model_]
models = []
for model_path in folder.glob("*.json"):
with open(model_path) as fp:
model = json.load(fp)
models.append(model)
models.sort(key=lambda x: x[acc_field], reverse=True)
env = Environment(loader=PackageLoader('robustbench', 'leaderboard'),
autoescape=select_autoescape(['html', 'xml']))
template = env.get_template('leaderboard.html.j2')
result = template.render(threat_model=threat_model,
dataset=dataset,
models=models,
acc_field=acc_field)
print(result)
return result