def get_mobilenet(identifier, init_weights=True, image_size=224):
if 'mobilenet_v1_1.0_224' not in identifier:
model = get_mobilenet_local(False)
if 'random' not in identifier:
model = load_weights(identifier, model)
from model_tools.activations.pytorch import load_preprocess_images
preprocessing = functools.partial(load_preprocess_images,
image_size=image_size)
wrapper = PytorchWrapper(identifier=identifier,
model=model,
preprocessing=preprocessing)
wrapper.image_size = image_size
return wrapper
multiplier = 1.0
version = 1
image_size = 224
identifier = f"mobilenet_v{version}_{multiplier}_{image_size}"
if (version == 1
and multiplier in [.75, .5, .25]) or (version == 2
and multiplier == 1.4):
net_name = f"mobilenet_v{version}_{multiplier * 100:03.0f}"
else:
net_name = f"mobilenet_v{version}"
return TFSlimModel.init(identifier,
preprocessing_type='inception',
image_size=image_size,
net_name=net_name,
model_ctr_kwargs={'depth_multiplier': multiplier})
def robust_model(function, image_size):
from urllib import request
import torch
from model_tools.activations.pytorch import load_preprocess_images
module = import_module(f'torchvision.models')
model_ctr = getattr(module, function)
model = model_ctr()
preprocessing = functools.partial(load_preprocess_images, image_size=image_size)
# load weights
framework_home = os.path.expanduser(os.getenv('CM_HOME', '~/.candidate_models'))
weightsdir_path = os.getenv('CM_TSLIM_WEIGHTS_DIR',
os.path.join(framework_home, 'model-weights', 'resnet-50-robust'))
weights_path = os.path.join(weightsdir_path, 'resnet-50-robust')
if not os.path.isfile(weights_path):
url = 'http://andrewilyas.com/ImageNet.pt'
_logger.debug(f"Downloading weights for resnet-50-robust from {url} to {weights_path}")
os.makedirs(weightsdir_path, exist_ok=True)
request.urlretrieve(url, weights_path)
checkpoint = torch.load(weights_path, map_location=torch.device('cpu'))
# process weights -- remove the attacker and prepocessing weights
weights = checkpoint['model']
weights = {k[len('module.model.'):]: v for k, v in weights.items() if 'attacker' not in k}
weights = {k: weights[k] for k in list(weights.keys())[2:]}
model.load_state_dict(weights)
# wrap model with pytorch wrapper
wrapper = PytorchWrapper(identifier=function+'-robust', model=model, preprocessing=preprocessing)
wrapper.image_size = image_size
return wrapper
def texture_vs_shape(model_identifier, model_name):
from texture_vs_shape.load_pretrained_models import load_model
model = load_model(model_name)
from model_tools.activations.pytorch import load_preprocess_images
preprocessing = functools.partial(load_preprocess_images, image_size=224)
wrapper = PytorchWrapper(identifier=model_identifier, model=model, preprocessing=preprocessing)
wrapper.image_size = 224
return wrapper
def torchvision_model(identifier, image_size):
module = import_module(f'torchvision.models')
model_ctr = getattr(module, identifier)
from model_tools.activations.pytorch import load_preprocess_images
preprocessing = functools.partial(load_preprocess_images, image_size=image_size)
wrapper = PytorchWrapper(identifier=identifier, model=model_ctr(pretrained=True), preprocessing=preprocessing)
wrapper.image_size = image_size
return wrapper
def dcgan(function):
module = import_module(f'cifar10_dcgan.dcgan')
model_ctr = getattr(module, function)
model = model_ctr(pretrained=True)
from model_tools.activations.pytorch import load_preprocess_images
preprocessing = functools.partial(load_preprocess_images, image_size=64)
wrapper = PytorchWrapper(identifier=function, model=model, preprocessing=preprocessing, batch_size=28)
wrapper.image_size = 64
return wrapper
def voneresnet(model_name='resnet50'):
from vonenet import get_model
model = get_model(model_name)
model = model.module
from model_tools.activations.pytorch import load_preprocess_images
preprocessing = functools.partial(load_preprocess_images, image_size=224,
normalize_mean=(0.5, 0.5, 0.5), normalize_std=(0.5, 0.5, 0.5))
wrapper = PytorchWrapper(identifier='vone'+model_name, model=model, preprocessing=preprocessing)
wrapper.image_size = 224
return wrapper
def wsl(c_size):
import torch.hub
model_identifier = f"resnext101_32x{c_size}d_wsl"
model = torch.hub.load('facebookresearch/WSL-Images', model_identifier)
from model_tools.activations.pytorch import load_preprocess_images
preprocessing = functools.partial(load_preprocess_images, image_size=224)
batch_size = {8: 32, 16: 16, 32: 8, 48: 4}
wrapper = PytorchWrapper(identifier=model_identifier, model=model, preprocessing=preprocessing,
batch_size=batch_size[c_size])
wrapper.image_size = 224
return wrapper
def get_hmax(identifier, image_size):
path = os.path.join(os.path.dirname(__file__), 'universal_patch_set.mat')
model = hmax.HMAX(path)
from model_tools.activations.pytorch import load_preprocess_images
preprocessing = functools.partial(load_preprocess_images,
image_size=image_size)
wrapper = PytorchWrapper(identifier=identifier,
model=model,
preprocessing=preprocessing,
batch_size=10)
wrapper.image_size = image_size
return wrapper
def fixres(model_identifier, model_url):
# model
from fixres.hubconf import load_state_dict_from_url
module = import_module('fixres.imnet_evaluate.resnext_wsl')
model_ctr = getattr(module, model_identifier)
model = model_ctr(
pretrained=False
) # the pretrained flag here corresponds to standard resnext weights
pretrained_dict = load_state_dict_from_url(
model_url, map_location=lambda storage, loc: storage)['model']
model_dict = model.state_dict()
for k in model_dict.keys():
assert ('module.' + k) in pretrained_dict.keys()
model_dict[k] = pretrained_dict.get(('module.' + k))
model.load_state_dict(model_dict)
# preprocessing
from fixres.transforms_v2 import get_transforms
# 320 for ResNeXt:
# https://github.com/mschrimpf/FixRes/tree/4ddcf11b29c118dfb8a48686f75f572450f67e5d#example-evaluation-procedure
input_size = 320
# https://github.com/mschrimpf/FixRes/blob/0dc15ab509b9cb9d7002ca47826dab4d66033668/fixres/imnet_evaluate/train.py#L159-L160
transformation = get_transforms(
input_size=input_size,
test_size=input_size,
kind='full',
need=('val', ),
# this is different from standard ImageNet evaluation to show the whole image
crop=False,
# no backbone parameter for ResNeXt following
# https://github.com/mschrimpf/FixRes/blob/0dc15ab509b9cb9d7002ca47826dab4d66033668/fixres/imnet_evaluate/train.py#L154-L156
backbone=None)
transform = transformation['val']
from model_tools.activations.pytorch import load_images
def load_preprocess_images(image_filepaths):
images = load_images(image_filepaths)
images = [transform(image) for image in images]
images = [image.unsqueeze(0) for image in images]
images = np.concatenate(images)
return images
wrapper = PytorchWrapper(
identifier=model_identifier,
model=model,
preprocessing=load_preprocess_images,
batch_size=4) # doesn't fit into 12 GB GPU memory otherwise
wrapper.image_size = input_size
return wrapper
def pytorch_custom():
import torch
from torch import nn
from model_tools.activations.pytorch import load_preprocess_images
class MyModel(nn.Module):
def __init__(self):
super(MyModel, self).__init__()
self.conv1 = torch.nn.Conv2d(in_channels=3,
out_channels=2,
kernel_size=3)
self.relu1 = torch.nn.ReLU()
linear_input_size = np.power((224 - 3 + 2 * 0) / 1 + 1, 2) * 2
self.linear = torch.nn.Linear(int(linear_input_size), 1000)
self.relu2 = torch.nn.ReLU(
) # can't get named ReLU output otherwise
def forward(self, x):
x = self.conv1(x)
x = self.relu1(x)
x = x.view(x.size(0), -1)
x = self.linear(x)
x = self.relu2(x)
return x
preprocessing = functools.partial(load_preprocess_images, image_size=224)
return PytorchWrapper(model=MyModel(), preprocessing=preprocessing)
def pytorch_alexnet():
from torchvision.models.alexnet import alexnet
from model_tools.activations.pytorch import load_preprocess_images
preprocessing = functools.partial(load_preprocess_images, image_size=224)
return PytorchWrapper(model=alexnet(pretrained=True),
preprocessing=preprocessing)
def test_merge_large_layers():
import torch
from torch import nn
from model_tools.activations.pytorch import load_preprocess_images
class LargeModel(nn.Module):
def __init__(self):
super(LargeModel, self).__init__()
self.conv = torch.nn.Conv2d(in_channels=3,
out_channels=4,
kernel_size=3)
self.relu = torch.nn.ReLU()
def forward(self, x):
x = self.conv(x)
x = self.relu(x)
return x
preprocessing = functools.partial(load_preprocess_images, image_size=224)
model = PytorchWrapper(model=LargeModel(), preprocessing=preprocessing)
activations = model(
stimuli=[os.path.join(os.path.dirname(__file__), 'rgb.jpg')] * 64,
layers=['conv', 'relu'])
assert len(activations['neuroid']) == 394272
assert len(set(activations['neuroid_id'].values)) == len(
activations['neuroid'])
assert set(activations['layer'].values) == {'conv', 'relu'}
def pytorch_transformer_substitute():
import torch
from torch import nn
from model_tools.activations.pytorch import load_preprocess_images
class MyTransformer(nn.Module):
def __init__(self):
super(MyTransformer, self).__init__()
self.conv = torch.nn.Conv1d(in_channels=3, out_channels=2, kernel_size=3)
self.relu1 = torch.nn.ReLU()
linear_input_size = (224**2 - 2) * 2
self.linear = torch.nn.Linear(int(linear_input_size), 1000)
self.relu2 = torch.nn.ReLU() # logit out needs to be 1000
def forward(self, x):
x = x.view(*x.shape[:2], -1)
x = self.conv(x)
x = self.relu1(x)
x = x.view(x.shape[0], -1)
x = self.linear(x)
x = self.relu2(x)
return x
preprocessing = functools.partial(load_preprocess_images, image_size=224)
return PytorchWrapper(model=MyTransformer(), preprocessing=preprocessing)
def pytorch_model(function,
identifier,
image_size,
init_weights=True,
transformation=None):
module = importlib.import_module(f'torchvision.models')
model_ctr = getattr(module, function)
if init_weights:
model = model_ctr(pretrained=False)
model = load_weights(identifier, model)
else:
model = model_ctr(pretrained=init_weights)
if transformation:
model = apply_to_net(model, transformation)
from model_tools.activations.pytorch import load_preprocess_images
preprocessing = functools.partial(load_preprocess_images,
image_size=image_size)
wrapper = PytorchWrapper(identifier=identifier,
model=model,
preprocessing=preprocessing)
wrapper.image_size = image_size
return wrapper
def pytorch_alexnet_resize():
from torchvision.models.alexnet import alexnet
from model_tools.activations.pytorch import load_images, torchvision_preprocess
from torchvision import transforms
torchvision_preprocess_input = transforms.Compose([transforms.Resize(224), torchvision_preprocess()])
def preprocessing(paths):
images = load_images(paths)
images = [torchvision_preprocess_input(image) for image in images]
images = np.concatenate(images)
return images
return PytorchWrapper(alexnet(pretrained=True), preprocessing, identifier='alexnet-resize')
def test_newmodel_pytorch(self):
import torch
from torch import nn
from model_tools.activations.pytorch import load_preprocess_images
class MyModel(nn.Module):
def __init__(self):
super(MyModel, self).__init__()
self.conv1 = torch.nn.Conv2d(in_channels=3,
out_channels=2,
kernel_size=3)
self.relu1 = torch.nn.ReLU()
linear_input_size = np.power((224 - 3 + 2 * 0) / 1 + 1, 2) * 2
self.linear = torch.nn.Linear(int(linear_input_size), 1000)
self.relu2 = torch.nn.ReLU(
) # can't get named ReLU output otherwise
# init weights for reproducibility
self.conv1.weight.data.fill_(0.01)
self.conv1.bias.data.fill_(0.01)
self.linear.weight.data.fill_(0.01)
self.linear.bias.data.fill_(0.01)
def forward(self, x):
x = self.conv1(x)
x = self.relu1(x)
x = x.view(x.size(0), -1)
x = self.linear(x)
x = self.relu2(x)
return x
preprocessing = functools.partial(load_preprocess_images,
image_size=224)
model_id = 'new_pytorch'
activations_model = PytorchWrapper(model=MyModel(),
preprocessing=preprocessing,
identifier=model_id)
layer = 'relu2'
candidate = LayerMappedModel(f"{model_id}-{layer}",
activations_model=activations_model,
visual_degrees=8)
candidate.commit('IT', layer)
candidate = TemporalIgnore(candidate)
ceiled_score = score_model(
model_identifier=model_id,
model=candidate,
benchmark_identifier='dicarlo.MajajHong2015.IT-pls')
score = ceiled_score.raw
assert score.sel(aggregation='center') == approx(.0820823, abs=.01)
def pytorch_custom(image_size):
import torch
from torch import nn
from model_tools.activations.pytorch import load_preprocess_images
class MyModel(nn.Module):
def __init__(self):
super(MyModel, self).__init__()
self.conv1 = torch.nn.Conv2d(in_channels=3, out_channels=2, kernel_size=3, bias=False)
self.relu1 = torch.nn.ReLU()
def forward(self, x):
x = self.conv1(x)
x = self.relu1(x)
return x
preprocessing = functools.partial(load_preprocess_images, image_size=image_size)
return PytorchWrapper(model=MyModel(), preprocessing=preprocessing)
