def get_mobilenet(identifier, init_weights=True, image_size=224): if 'mobilenet_v1_1.0_224' not in identifier: model = get_mobilenet_local(False) if 'random' not in identifier: model = load_weights(identifier, model) from model_tools.activations.pytorch import load_preprocess_images preprocessing = functools.partial(load_preprocess_images, image_size=image_size) wrapper = PytorchWrapper(identifier=identifier, model=model, preprocessing=preprocessing) wrapper.image_size = image_size return wrapper multiplier = 1.0 version = 1 image_size = 224 identifier = f"mobilenet_v{version}_{multiplier}_{image_size}" if (version == 1 and multiplier in [.75, .5, .25]) or (version == 2 and multiplier == 1.4): net_name = f"mobilenet_v{version}_{multiplier * 100:03.0f}" else: net_name = f"mobilenet_v{version}" return TFSlimModel.init(identifier, preprocessing_type='inception', image_size=image_size, net_name=net_name, model_ctr_kwargs={'depth_multiplier': multiplier})
def robust_model(function, image_size): from urllib import request import torch from model_tools.activations.pytorch import load_preprocess_images module = import_module(f'torchvision.models') model_ctr = getattr(module, function) model = model_ctr() preprocessing = functools.partial(load_preprocess_images, image_size=image_size) # load weights framework_home = os.path.expanduser(os.getenv('CM_HOME', '~/.candidate_models')) weightsdir_path = os.getenv('CM_TSLIM_WEIGHTS_DIR', os.path.join(framework_home, 'model-weights', 'resnet-50-robust')) weights_path = os.path.join(weightsdir_path, 'resnet-50-robust') if not os.path.isfile(weights_path): url = 'http://andrewilyas.com/ImageNet.pt' _logger.debug(f"Downloading weights for resnet-50-robust from {url} to {weights_path}") os.makedirs(weightsdir_path, exist_ok=True) request.urlretrieve(url, weights_path) checkpoint = torch.load(weights_path, map_location=torch.device('cpu')) # process weights -- remove the attacker and prepocessing weights weights = checkpoint['model'] weights = {k[len('module.model.'):]: v for k, v in weights.items() if 'attacker' not in k} weights = {k: weights[k] for k in list(weights.keys())[2:]} model.load_state_dict(weights) # wrap model with pytorch wrapper wrapper = PytorchWrapper(identifier=function+'-robust', model=model, preprocessing=preprocessing) wrapper.image_size = image_size return wrapper
def texture_vs_shape(model_identifier, model_name): from texture_vs_shape.load_pretrained_models import load_model model = load_model(model_name) from model_tools.activations.pytorch import load_preprocess_images preprocessing = functools.partial(load_preprocess_images, image_size=224) wrapper = PytorchWrapper(identifier=model_identifier, model=model, preprocessing=preprocessing) wrapper.image_size = 224 return wrapper
def torchvision_model(identifier, image_size): module = import_module(f'torchvision.models') model_ctr = getattr(module, identifier) from model_tools.activations.pytorch import load_preprocess_images preprocessing = functools.partial(load_preprocess_images, image_size=image_size) wrapper = PytorchWrapper(identifier=identifier, model=model_ctr(pretrained=True), preprocessing=preprocessing) wrapper.image_size = image_size return wrapper
def dcgan(function): module = import_module(f'cifar10_dcgan.dcgan') model_ctr = getattr(module, function) model = model_ctr(pretrained=True) from model_tools.activations.pytorch import load_preprocess_images preprocessing = functools.partial(load_preprocess_images, image_size=64) wrapper = PytorchWrapper(identifier=function, model=model, preprocessing=preprocessing, batch_size=28) wrapper.image_size = 64 return wrapper
def voneresnet(model_name='resnet50'): from vonenet import get_model model = get_model(model_name) model = model.module from model_tools.activations.pytorch import load_preprocess_images preprocessing = functools.partial(load_preprocess_images, image_size=224, normalize_mean=(0.5, 0.5, 0.5), normalize_std=(0.5, 0.5, 0.5)) wrapper = PytorchWrapper(identifier='vone'+model_name, model=model, preprocessing=preprocessing) wrapper.image_size = 224 return wrapper
def wsl(c_size): import torch.hub model_identifier = f"resnext101_32x{c_size}d_wsl" model = torch.hub.load('facebookresearch/WSL-Images', model_identifier) from model_tools.activations.pytorch import load_preprocess_images preprocessing = functools.partial(load_preprocess_images, image_size=224) batch_size = {8: 32, 16: 16, 32: 8, 48: 4} wrapper = PytorchWrapper(identifier=model_identifier, model=model, preprocessing=preprocessing, batch_size=batch_size[c_size]) wrapper.image_size = 224 return wrapper
def get_hmax(identifier, image_size): path = os.path.join(os.path.dirname(__file__), 'universal_patch_set.mat') model = hmax.HMAX(path) from model_tools.activations.pytorch import load_preprocess_images preprocessing = functools.partial(load_preprocess_images, image_size=image_size) wrapper = PytorchWrapper(identifier=identifier, model=model, preprocessing=preprocessing, batch_size=10) wrapper.image_size = image_size return wrapper
def fixres(model_identifier, model_url): # model from fixres.hubconf import load_state_dict_from_url module = import_module('fixres.imnet_evaluate.resnext_wsl') model_ctr = getattr(module, model_identifier) model = model_ctr( pretrained=False ) # the pretrained flag here corresponds to standard resnext weights pretrained_dict = load_state_dict_from_url( model_url, map_location=lambda storage, loc: storage)['model'] model_dict = model.state_dict() for k in model_dict.keys(): assert ('module.' + k) in pretrained_dict.keys() model_dict[k] = pretrained_dict.get(('module.' + k)) model.load_state_dict(model_dict) # preprocessing from fixres.transforms_v2 import get_transforms # 320 for ResNeXt: # https://github.com/mschrimpf/FixRes/tree/4ddcf11b29c118dfb8a48686f75f572450f67e5d#example-evaluation-procedure input_size = 320 # https://github.com/mschrimpf/FixRes/blob/0dc15ab509b9cb9d7002ca47826dab4d66033668/fixres/imnet_evaluate/train.py#L159-L160 transformation = get_transforms( input_size=input_size, test_size=input_size, kind='full', need=('val', ), # this is different from standard ImageNet evaluation to show the whole image crop=False, # no backbone parameter for ResNeXt following # https://github.com/mschrimpf/FixRes/blob/0dc15ab509b9cb9d7002ca47826dab4d66033668/fixres/imnet_evaluate/train.py#L154-L156 backbone=None) transform = transformation['val'] from model_tools.activations.pytorch import load_images def load_preprocess_images(image_filepaths): images = load_images(image_filepaths) images = [transform(image) for image in images] images = [image.unsqueeze(0) for image in images] images = np.concatenate(images) return images wrapper = PytorchWrapper( identifier=model_identifier, model=model, preprocessing=load_preprocess_images, batch_size=4) # doesn't fit into 12 GB GPU memory otherwise wrapper.image_size = input_size return wrapper
def pytorch_custom(): import torch from torch import nn from model_tools.activations.pytorch import load_preprocess_images class MyModel(nn.Module): def __init__(self): super(MyModel, self).__init__() self.conv1 = torch.nn.Conv2d(in_channels=3, out_channels=2, kernel_size=3) self.relu1 = torch.nn.ReLU() linear_input_size = np.power((224 - 3 + 2 * 0) / 1 + 1, 2) * 2 self.linear = torch.nn.Linear(int(linear_input_size), 1000) self.relu2 = torch.nn.ReLU( ) # can't get named ReLU output otherwise def forward(self, x): x = self.conv1(x) x = self.relu1(x) x = x.view(x.size(0), -1) x = self.linear(x) x = self.relu2(x) return x preprocessing = functools.partial(load_preprocess_images, image_size=224) return PytorchWrapper(model=MyModel(), preprocessing=preprocessing)
def pytorch_alexnet(): from torchvision.models.alexnet import alexnet from model_tools.activations.pytorch import load_preprocess_images preprocessing = functools.partial(load_preprocess_images, image_size=224) return PytorchWrapper(model=alexnet(pretrained=True), preprocessing=preprocessing)
def test_merge_large_layers(): import torch from torch import nn from model_tools.activations.pytorch import load_preprocess_images class LargeModel(nn.Module): def __init__(self): super(LargeModel, self).__init__() self.conv = torch.nn.Conv2d(in_channels=3, out_channels=4, kernel_size=3) self.relu = torch.nn.ReLU() def forward(self, x): x = self.conv(x) x = self.relu(x) return x preprocessing = functools.partial(load_preprocess_images, image_size=224) model = PytorchWrapper(model=LargeModel(), preprocessing=preprocessing) activations = model( stimuli=[os.path.join(os.path.dirname(__file__), 'rgb.jpg')] * 64, layers=['conv', 'relu']) assert len(activations['neuroid']) == 394272 assert len(set(activations['neuroid_id'].values)) == len( activations['neuroid']) assert set(activations['layer'].values) == {'conv', 'relu'}
def pytorch_transformer_substitute(): import torch from torch import nn from model_tools.activations.pytorch import load_preprocess_images class MyTransformer(nn.Module): def __init__(self): super(MyTransformer, self).__init__() self.conv = torch.nn.Conv1d(in_channels=3, out_channels=2, kernel_size=3) self.relu1 = torch.nn.ReLU() linear_input_size = (224**2 - 2) * 2 self.linear = torch.nn.Linear(int(linear_input_size), 1000) self.relu2 = torch.nn.ReLU() # logit out needs to be 1000 def forward(self, x): x = x.view(*x.shape[:2], -1) x = self.conv(x) x = self.relu1(x) x = x.view(x.shape[0], -1) x = self.linear(x) x = self.relu2(x) return x preprocessing = functools.partial(load_preprocess_images, image_size=224) return PytorchWrapper(model=MyTransformer(), preprocessing=preprocessing)
def pytorch_model(function, identifier, image_size, init_weights=True, transformation=None): module = importlib.import_module(f'torchvision.models') model_ctr = getattr(module, function) if init_weights: model = model_ctr(pretrained=False) model = load_weights(identifier, model) else: model = model_ctr(pretrained=init_weights) if transformation: model = apply_to_net(model, transformation) from model_tools.activations.pytorch import load_preprocess_images preprocessing = functools.partial(load_preprocess_images, image_size=image_size) wrapper = PytorchWrapper(identifier=identifier, model=model, preprocessing=preprocessing) wrapper.image_size = image_size return wrapper
def pytorch_alexnet_resize(): from torchvision.models.alexnet import alexnet from model_tools.activations.pytorch import load_images, torchvision_preprocess from torchvision import transforms torchvision_preprocess_input = transforms.Compose([transforms.Resize(224), torchvision_preprocess()]) def preprocessing(paths): images = load_images(paths) images = [torchvision_preprocess_input(image) for image in images] images = np.concatenate(images) return images return PytorchWrapper(alexnet(pretrained=True), preprocessing, identifier='alexnet-resize')
def test_newmodel_pytorch(self): import torch from torch import nn from model_tools.activations.pytorch import load_preprocess_images class MyModel(nn.Module): def __init__(self): super(MyModel, self).__init__() self.conv1 = torch.nn.Conv2d(in_channels=3, out_channels=2, kernel_size=3) self.relu1 = torch.nn.ReLU() linear_input_size = np.power((224 - 3 + 2 * 0) / 1 + 1, 2) * 2 self.linear = torch.nn.Linear(int(linear_input_size), 1000) self.relu2 = torch.nn.ReLU( ) # can't get named ReLU output otherwise # init weights for reproducibility self.conv1.weight.data.fill_(0.01) self.conv1.bias.data.fill_(0.01) self.linear.weight.data.fill_(0.01) self.linear.bias.data.fill_(0.01) def forward(self, x): x = self.conv1(x) x = self.relu1(x) x = x.view(x.size(0), -1) x = self.linear(x) x = self.relu2(x) return x preprocessing = functools.partial(load_preprocess_images, image_size=224) model_id = 'new_pytorch' activations_model = PytorchWrapper(model=MyModel(), preprocessing=preprocessing, identifier=model_id) layer = 'relu2' candidate = LayerMappedModel(f"{model_id}-{layer}", activations_model=activations_model, visual_degrees=8) candidate.commit('IT', layer) candidate = TemporalIgnore(candidate) ceiled_score = score_model( model_identifier=model_id, model=candidate, benchmark_identifier='dicarlo.MajajHong2015.IT-pls') score = ceiled_score.raw assert score.sel(aggregation='center') == approx(.0820823, abs=.01)
def pytorch_custom(image_size): import torch from torch import nn from model_tools.activations.pytorch import load_preprocess_images class MyModel(nn.Module): def __init__(self): super(MyModel, self).__init__() self.conv1 = torch.nn.Conv2d(in_channels=3, out_channels=2, kernel_size=3, bias=False) self.relu1 = torch.nn.ReLU() def forward(self, x): x = self.conv1(x) x = self.relu1(x) return x preprocessing = functools.partial(load_preprocess_images, image_size=image_size) return PytorchWrapper(model=MyModel(), preprocessing=preprocessing)