def __init__(self):
super(Resnet, self).__init__()
self.resnet = resnext.resnet101(shortcut_type='B',
sample_size=64,
sample_duration=16)
self.resnet = nn.DataParallel(self.resnet)
#num_final_in = self.resnet.module.fc.in_features
#self.resnet.fc = nn.Linear(num_final_in, 2)
model_data = torch.load(PRETRAINED_RESNET_PATH)
#new_model_data = OrderedDict()
#for k, v in model_data.items():
# name = k.replace("module.", "")
# new_model_data[name] = v
#self.resnet.load_state_dict(new_model_data)
self.resnet.load_state_dict(model_data['state_dict'])
print(len(list(self.resnet.children())))
#print(list(self.resnet.children()))
print(type(self.resnet.children()))
#self.resnet = nn.Sequential(*[*list(self.resnet.children()), Flatten()])
for param in self.resnet.parameters():
param.requires_grad = False
for layer in self.resnet.children():
layer.fc = nn.Linear(4096, 2)
def load_temporal_model(model_name, model_depth):
verbose = False
#model_name = 'i3d' # resnext resnet i3d
if model_name == 'i3d':
model_path = '/home/vsharma/Documents/Audio_Visual_Text/models/i3d/rgb_imagenet.pt'
model = InceptionI3d(400, in_channels=3)
model.load_state_dict(torch.load(model_path))
arch = model_name
model.train(False) # Set model to evaluate mode
elif (model_name == 'resnet') or (model_name == 'resnext'):
#model_depth = 50 # 101 50
arch = '{}-{}'.format(model_name, model_depth)
model_path = '/home/vsharma/Documents/Audio_Visual_Text/models/resnet3d'
model_path = '{}/{}-kinetics.pth'.format(model_path, arch)
if arch == 'resnet-50':
model = resnet.resnet50(num_classes=400,
shortcut_type='B',
sample_size=112,
sample_duration=16,
last_fc=True)
elif arch == 'resnext-101':
model = resnext.resnet101(num_classes=400,
shortcut_type='B',
cardinality=32,
sample_size=112,
sample_duration=16,
last_fc=True)
model_data = torch.load(model_path)
assert arch == model_data['arch']
#model.load_state_dict(model_data['state_dict'])
state_dict = model_data['state_dict']
new_state_dict = OrderedDict()
for k, v in state_dict.items():
name = k[7:] # remove `module.`
new_state_dict[name] = v
model.load_state_dict(new_state_dict)
# Removing the last 2 layers: fc and softmax
model = nn.Sequential(*list(model.children())[:-2])
model.eval()
if verbose:
print(model)
return model
def initialize_model(pretrained_path):
torch_module = resnext.resnet101(sample_size=112,
sample_duration=16,
num_classes=27)
state_dict = torch.load(pretrained_path)["state_dict"]
from collections import OrderedDict
new_state_dict = OrderedDict()
for k, v in state_dict.items():
name = k[7:] # remove `module.`
new_state_dict[name] = v
torch_module.load_state_dict(new_state_dict)
# freeze feature-extraction
for param in torch_module.parameters():
param.requires_grad = False
# unfreeze fc-layer
torch_module.fc.weight.requires_grad = True
return torch_module
def generate_resnext_model(mode, model_depth = 101, n_classes = 400, sample_size = 112, sample_duration = 16, resnet_shortcut = 'B', resnext_cardinality = 32):
assert mode in ['score', 'feature']
if mode == 'score':
last_fc = True
elif mode == 'feature':
last_fc = False
assert model_depth in [50, 101, 152]
if model_depth == 50:
model = resnext.resnet50(num_classes=n_classes, shortcut_type=resnet_shortcut, cardinality=resnext_cardinality,
sample_size=sample_size, sample_duration=sample_duration,
last_fc=last_fc)
elif model_depth == 101:
model = resnext.resnet101(num_classes=n_classes, shortcut_type=resnet_shortcut, cardinality=resnext_cardinality,
sample_size=sample_size, sample_duration=sample_duration,
last_fc=last_fc)
elif model_depth == 152:
model = resnext.resnet152(num_classes=n_classes, shortcut_type=resnet_shortcut, cardinality=resnext_cardinality,
sample_size=sample_size, sample_duration=sample_duration,
last_fc=last_fc)
return model
# values don't matter as we care about network structure.
# But they can also be real inputs.
def get_mean():
return [114.7748, 107.7354, 99.4750]
mean = get_mean()
arch = 'resnext-101'
sample_size = 112
sample_duration = 16
n_classes = 400
model = resnext.resnet101(num_classes=n_classes,
sample_size=sample_size,
sample_duration=sample_duration,
shortcut_type='B')
dummy_input = torch.randn(64, 3, 7, 7, 7)
model_data = torch.load(
'C:\\Users\\ribeirfi\\git\\videoextractor\\weights\\resnext-101-kinetics.pth',
map_location=torch.device('cpu'))
print(model_data['arch'])
assert arch == model_data['arch']
model.load_state_dict(model_data['state_dict'], strict=False)
model.eval()
# Invoke export
torch.onnx.export(model, dummy_input, "resnet-101-kinetics.onnx")
import torch
import torch.nn as nn
from dataset import make_dataset
from sklearn.metrics import f1_score
import pickle
from mobilenetv2 import get_model
from resnext import resnet101
import numpy as np
from collections import OrderedDict
from tqdm import tqdm
big_model = resnet101(sample_size=112,
sample_duration=32,
num_classes=27,
shortcut_type='A')
checkpoint = torch.load('./pretrain/models/jester_resnext_101_RGB_32.pth')
weights = OrderedDict()
for w_name in checkpoint['state_dict']:
_w_name = '.'.join(w_name.split('.')[1:])
weights[_w_name] = checkpoint['state_dict'][w_name]
big_model.load_state_dict(weights)
big_model.cuda()
model = get_model(num_classes=27, sample_size=224, width_mult=1.0)
try:
checkpoint = torch.load('./best_classifier_checkpoint.pth.tar',
map_location=torch.device('cpu'))
model.load_state_dict(checkpoint)
except:
checkpoint = torch.load(
'./pretrain/models/imagenet_mobilenetv2_autoencoder.pth')
def generate_model(opt):
assert opt.model in [
'resnet', 'preresnet', 'wideresnet', 'resnext', 'densenet'
]
if opt.model == 'resnet':
assert opt.model_depth in [10, 18, 34, 50, 101, 152, 200]
from resnet import get_fine_tuning_parameters
if opt.model_depth == 10:
model = resnet.resnet10(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 18:
model = resnet.resnet18(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 34:
model = resnet.resnet34(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 50:
model = resnet.resnet50(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 101:
model = resnet.resnet101(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 152:
model = resnet.resnet152(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 200:
model = resnet.resnet200(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model == 'wideresnet':
assert opt.model_depth in [50]
from models.wide_resnet import get_fine_tuning_parameters
if opt.model_depth == 50:
model = wide_resnet.resnet50(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
k=opt.wide_resnet_k,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model == 'resnext':
assert opt.model_depth in [50, 101, 152]
from models.resnext import get_fine_tuning_parameters
if opt.model_depth == 50:
model = resnext.resnet50(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
cardinality=opt.resnext_cardinality,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 101:
model = resnext.resnet101(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
cardinality=opt.resnext_cardinality,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 152:
model = resnext.resnet152(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
cardinality=opt.resnext_cardinality,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model == 'preresnet':
assert opt.model_depth in [18, 34, 50, 101, 152, 200]
from models.pre_act_resnet import get_fine_tuning_parameters
if opt.model_depth == 18:
model = pre_act_resnet.resnet18(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 34:
model = pre_act_resnet.resnet34(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 50:
model = pre_act_resnet.resnet50(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 101:
model = pre_act_resnet.resnet101(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 152:
model = pre_act_resnet.resnet152(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 200:
model = pre_act_resnet.resnet200(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model == 'densenet':
assert opt.model_depth in [121, 169, 201, 264]
from models.densenet import get_fine_tuning_parameters
if opt.model_depth == 121:
model = densenet.densenet121(num_classes=opt.n_classes,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 169:
model = densenet.densenet169(num_classes=opt.n_classes,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 201:
model = densenet.densenet201(num_classes=opt.n_classes,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 264:
model = densenet.densenet264(num_classes=opt.n_classes,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
if not opt.no_cuda:
model = model.cuda()
model = nn.DataParallel(model, device_ids=None)
if opt.pretrain_path:
print('loading pretrained model {}'.format(opt.pretrain_path))
pretrain = torch.load(opt.pretrain_path)
assert opt.arch == pretrain['arch']
model.load_state_dict(pretrain['state_dict'])
if opt.model == 'densenet':
model.module.classifier = nn.Linear(
model.module.classifier.in_features,
opt.n_finetune_classes)
model.module.classifier = model.module.classifier.cuda()
else:
model.module.fc = nn.Linear(model.module.fc.in_features,
opt.n_finetune_classes)
model.module.fc = model.module.fc.cuda()
parameters = get_fine_tuning_parameters(model, opt.ft_begin_index)
return model, parameters
else:
if opt.pretrain_path:
print('loading pretrained model {}'.format(opt.pretrain_path))
pretrain = torch.load(opt.pretrain_path)
assert opt.arch == pretrain['arch']
model.load_state_dict(pretrain['state_dict'])
if opt.model == 'densenet':
model.classifier = nn.Linear(model.classifier.in_features,
opt.n_finetune_classes)
else:
model.fc = nn.Linear(model.fc.in_features,
opt.n_finetune_classes)
parameters = get_fine_tuning_parameters(model, opt.ft_begin_index)
return model, parameters
return model, model.parameters()
trainloader = torch.utils.data.DataLoader(trainset, batch_size=args.train_batch_size, shuffle=args.shuffle, num_workers=args.num_workers)
valset = DatasetType(args.tensor_dir, args.video_dir, args.raw_path, 'val',
transform=test_transform, slice_transform=test_slice_transform,
num_frames=args.num_frames, eval_mode=eval_mode, num_slices=args.test_slices, preload=args.preload, storage=args.storage)
valloader = torch.utils.data.DataLoader(valset, batch_size=args.test_batch_size, shuffle=args.shuffle, num_workers=args.num_workers)
print('==> Building model..')
if args.mode == 'train':
#net = resnext.resnet101(sample_size=args.resolution, sample_duration=args.num_frames, num_classes=2)
#net = densenet.densenet121(sample_size=args.resolution, sample_duration=args.num_frames)#, num_classes=2)
if args.load_epoch is None:
#net = wide_resnet.resnet50(sample_size=args.resolution, sample_duration=args.num_frames, num_classes=2)
#net = densenet.densenet264(sample_size=args.resolution, sample_duration=args.num_frames, num_classes=2)
net = resnext.resnet101(sample_size=args.resolution, sample_duration=args.num_frames, num_classes=400)
net.cuda()
net = torch.nn.DataParallel(net, device_ids=range(torch.cuda.device_count()))
#state_dict = torch.load('./pretrained/resnext-101-64f-kinetics.pth')['state_dict']
state_dict = torch.load('./pretrained/resnext-101-kinetics.pth')['state_dict']
net.load_state_dict(state_dict)
net.module.fc = torch.nn.Linear(2048, 2).cuda()
#net = net.module
#net.cuda()
#net = torch.nn.DataParallel(net, device_ids=range(torch.cuda.device_count()))
#cudnn.benchmark = True
#torch.backends.cudnn.enabled = False
print('==> Loaded model')
else:
model_path = os.path.join(args.model_dir, '{}_{}_{}.t7'.format(args.load_modality, args.method, args.load_epoch))
checkpoint = torch.load(model_path)
3,
config.sample_duration,
config.sample_size,
config.sample_size,
),
)
if not config.transfer:
inputs = torch.from_numpy(data).cuda(
config.device).float().div(255.0).add(-0.5)
if config.half_mode == 1:
inputs = inputs.half()
labels = torch.from_numpy(np.random.choice(
10, size=config.batch_size)).cuda(config.device)
model = resnext.resnet101(
num_classes=10,
sample_size=config.sample_size,
sample_duration=config.sample_duration,
).cuda(config.device)
if config.bench_train:
model.train()
else:
model.eval()
if config.all_gpus:
model = nn.DataParallel(model,
device_ids=range(torch.cuda.device_count()))
if config.half_mode == 1:
model = model.half()
optimizer = optim.Adam(model.parameters(), lr=0.001)
criterion = nn.CrossEntropyLoss().cuda(config.device)
if config.half_mode == 2:
def generate_model(opt):
assert opt.mode in ['score', 'feature']
if opt.mode == 'score':
last_fc = True
elif opt.mode == 'feature':
last_fc = False
assert opt.model_name in [
'resnet', 'preresnet', 'wideresnet', 'resnext', 'densenet'
]
if opt.model_name == 'resnet':
assert opt.model_depth in [10, 18, 34, 50, 101, 152, 200]
if opt.model_depth == 10:
model = resnet.resnet10(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 18:
model = resnet.resnet18(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 34:
model = resnet.resnet34(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 50:
model = resnet.resnet50(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 101:
model = resnet.resnet101(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 152:
model = resnet.resnet152(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 200:
model = resnet.resnet200(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_name == 'wideresnet':
assert opt.model_depth in [50]
if opt.model_depth == 50:
model = wide_resnet.resnet50(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
k=opt.wide_resnet_k,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_name == 'resnext':
assert opt.model_depth in [50, 101, 152]
if opt.model_depth == 50:
model = resnext.resnet50(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
cardinality=opt.resnext_cardinality,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 101:
model = resnext.resnet101(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
cardinality=opt.resnext_cardinality,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 152:
model = resnext.resnet152(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
cardinality=opt.resnext_cardinality,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_name == 'preresnet':
assert opt.model_depth in [18, 34, 50, 101, 152, 200]
if opt.model_depth == 18:
model = pre_act_resnet.resnet18(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 34:
model = pre_act_resnet.resnet34(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 50:
model = pre_act_resnet.resnet50(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 101:
model = pre_act_resnet.resnet101(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 152:
model = pre_act_resnet.resnet152(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 200:
model = pre_act_resnet.resnet200(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_name == 'densenet':
assert opt.model_depth in [121, 169, 201, 264]
if opt.model_depth == 121:
model = densenet.densenet121(num_classes=opt.n_classes,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 169:
model = densenet.densenet169(num_classes=opt.n_classes,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 201:
model = densenet.densenet201(num_classes=opt.n_classes,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 264:
model = densenet.densenet264(num_classes=opt.n_classes,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
if not opt.no_cuda:
model = model.cuda()
model = nn.DataParallel(model, device_ids=None)
return model