def generate_model(opt): assert opt.model in ['resnet', 'resnetl', 'resnext', 'c3d'] if opt.model == 'resnet': assert opt.model_depth in [10] from models.resnet import get_fine_tuning_parameters if opt.model_depth == 10: model = resnet.resnet10(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model == 'resnetl': assert opt.model_depth in [10] from models.resnetl import get_fine_tuning_parameters if opt.model_depth == 10: model = resnetl.resnetl10(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model == 'resnext': assert opt.model_depth in [101] from models.resnext import get_fine_tuning_parameters if opt.model_depth == 101: model = resnext.resnet101(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, cardinality=opt.resnext_cardinality, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model == 'c3d': assert opt.model_depth in [10] from models.c3d import get_fine_tuning_parameters if opt.model_depth == 10: model = c3d.c3d_v1(sample_size=opt.sample_size, sample_duration=opt.sample_duration, num_classes=opt.n_classes) if not opt.no_cuda: model = model.cuda() model = nn.DataParallel(model, device_ids=None) if opt.pretrain_path: print('loading pretrained model {}'.format(opt.pretrain_path)) pretrain = torch.load(opt.pretrain_path) assert opt.arch == pretrain['arch'] model.load_state_dict(pretrain['state_dict']) if opt.model == 'c3d': # CHECK HERE model.module.fc = nn.Linear(model.module.fc[0].in_features, opt.n_finetune_classes) else: model.module.fc = nn.Linear(model.module.fc.in_features, opt.n_finetune_classes) model.module.fc = model.module.fc.cuda() if opt.modality == 'RGB' and opt.model != 'c3d': print("[INFO]: RGB model is used for init model") model = _modify_first_conv_layer( model, 3, 3) ##### Check models trained (3,7,7) or (7,7,7) elif opt.modality == 'Depth': print( "[INFO]: Converting the pretrained model to Depth init model") model = _construct_depth_model(model) print("[INFO]: Done. Flow model ready.") elif opt.modality == 'RGB-D': print( "[INFO]: Converting the pretrained model to RGB+D init model") model = _construct_rgbdepth_model(model) print("[INFO]: Done. RGB-D model ready.") modules = list(model.modules()) first_conv_idx = list( filter(lambda x: isinstance(modules[x], nn.Conv3d), list(range(len(modules)))))[0] conv_layer = modules[first_conv_idx] if conv_layer.kernel_size[0] > opt.sample_duration: model = _modify_first_conv_layer(model, int(opt.sample_duration / 2), 1) parameters = get_fine_tuning_parameters(model, opt.ft_begin_index) return model, parameters else: if opt.pretrain_path: print('loading pretrained model {}'.format(opt.pretrain_path)) pretrain = torch.load(opt.pretrain_path) assert opt.arch == pretrain['arch'] model.load_state_dict(pretrain['state_dict']) if opt.modality == 'RGB' and opt.model != 'c3d': print("[INFO]: RGB model is used for init model") model = _modify_first_conv_layer(model, 3, 3) elif opt.modality == 'Depth': print( "[INFO]: Converting the pretrained model to Depth init model") model = _construct_depth_model(model) print("[INFO]: Deoth model ready.") elif opt.modality == 'RGB-D': print( "[INFO]: Converting the pretrained model to RGB-D init model") model = _construct_rgbdepth_model(model) print("[INFO]: Done. RGB-D model ready.") modules = list(model.modules()) first_conv_idx = list( filter(lambda x: isinstance(modules[x], nn.Conv3d), list(range(len(modules)))))[0] conv_layer = modules[first_conv_idx] if conv_layer.kernel_size[0] > opt.sample_duration: print("[INFO]: RGB model is used for init model") model = _modify_first_conv_layer(model, int(opt.sample_duration / 2), 1) if opt.model == 'c3d': # CHECK HERE model.fc = nn.Linear(model.fc[0].in_features, model.fc[0].out_features) else: model.fc = nn.Linear(model.fc.in_features, opt.n_finetune_classes) parameters = get_fine_tuning_parameters(model, opt.ft_begin_index) return model, parameters
def generate_model(opt): assert opt.model in [ 'c3d', 'squeezenet', 'mobilenet', 'resnext', 'resnet', 'resnetl', 'shufflenet', 'mobilenetv2', 'shufflenetv2' ] if opt.model == 'c3d': from models.c3d import get_fine_tuning_parameters model = c3d.get_model(num_classes=opt.n_classes, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model == 'squeezenet': from models.squeezenet import get_fine_tuning_parameters model = squeezenet.get_model(version=opt.version, num_classes=opt.n_classes, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model == 'shufflenet': from models.shufflenet import get_fine_tuning_parameters model = shufflenet.get_model(groups=opt.groups, width_mult=opt.width_mult, num_classes=opt.n_classes) elif opt.model == 'shufflenetv2': from models.shufflenetv2 import get_fine_tuning_parameters model = shufflenetv2.get_model(num_classes=opt.n_classes, sample_size=opt.sample_size, width_mult=opt.width_mult) elif opt.model == 'mobilenet': from models.mobilenet import get_fine_tuning_parameters model = mobilenet.get_model(num_classes=opt.n_classes, sample_size=opt.sample_size, width_mult=opt.width_mult) elif opt.model == 'mobilenetv2': from models.mobilenetv2 import get_fine_tuning_parameters model = mobilenetv2.get_model(num_classes=opt.n_classes, sample_size=opt.sample_size, width_mult=opt.width_mult) elif opt.model == 'resnext': assert opt.model_depth in [50, 101, 152] from models.resnext import get_fine_tuning_parameters if opt.model_depth == 50: model = resnext.resnet50(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, cardinality=opt.resnext_cardinality, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model_depth == 101: model = resnext.resnet101(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, cardinality=opt.resnext_cardinality, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model_depth == 152: model = resnext.resnet152(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, cardinality=opt.resnext_cardinality, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model == 'resnetl': assert opt.model_depth in [10] from models.resnetl import get_fine_tuning_parameters if opt.model_depth == 10: model = resnetl.resnetl10(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model == 'resnet': assert opt.model_depth in [10, 18, 34, 50, 101, 152, 200] from models.resnet import get_fine_tuning_parameters if opt.model_depth == 10: model = resnet.resnet10(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model_depth == 18: model = resnet.resnet18(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model_depth == 34: model = resnet.resnet34(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model_depth == 50: model = resnet.resnet50(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model_depth == 101: model = resnet.resnet101(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model_depth == 152: model = resnet.resnet152(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model_depth == 200: model = resnet.resnet200(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) if not opt.no_cuda: model = model.cuda() model = nn.DataParallel(model, device_ids=None) pytorch_total_params = sum(p.numel() for p in model.parameters() if p.requires_grad) print("Total number of trainable parameters: ", pytorch_total_params) if opt.pretrain_path: print('loading pretrained model {}'.format(opt.pretrain_path)) pretrain = torch.load(opt.pretrain_path, map_location=torch.device('cpu')) # print(opt.arch) # print(pretrain['arch']) # assert opt.arch == pretrain['arch'] model.load_state_dict(pretrain['state_dict']) if opt.model in [ 'mobilenet', 'mobilenetv2', 'shufflenet', 'shufflenetv2' ]: model.module.classifier = nn.Sequential( nn.Dropout(0.5), nn.Linear(model.module.classifier[1].in_features, opt.n_finetune_classes)) model.module.classifier = model.module.classifier.cuda() elif opt.model == 'squeezenet': model.module.classifier = nn.Sequential( nn.Dropout(p=0.5), nn.Conv3d(model.module.classifier[1].in_channels, opt.n_finetune_classes, kernel_size=1), nn.ReLU(inplace=True), nn.AvgPool3d((1, 4, 4), stride=1)) model.module.classifier = model.module.classifier.cuda() else: model.module.fc = nn.Linear(model.module.fc.in_features, opt.n_finetune_classes) model.module.fc = model.module.fc.cuda() # model = _modify_first_conv_layer(model) # model = model.cuda() parameters = get_fine_tuning_parameters(model, opt.ft_portion) return model, parameters else: if opt.pretrain_path: print('loading pretrained model {}'.format(opt.pretrain_path)) pretrain = torch.load(opt.pretrain_path) assert opt.arch == pretrain['arch'] model.load_state_dict(pretrain['state_dict']) if opt.model in [ 'mobilenet', 'mobilenetv2', 'shufflenet', 'shufflenetv2' ]: model.module.classifier = nn.Sequential( nn.Dropout(0.9), nn.Linear(model.module.classifier[1].in_features, opt.n_finetune_classes)) elif opt.model == 'squeezenet': model.module.classifier = nn.Sequential( nn.Dropout(p=0.5), nn.Conv3d(model.module.classifier[1].in_channels, opt.n_finetune_classes, kernel_size=1), nn.ReLU(inplace=True), nn.AvgPool3d((1, 4, 4), stride=1)) else: model.module.fc = nn.Linear(model.module.fc.in_features, opt.n_finetune_classes) parameters = get_fine_tuning_parameters(model, opt.ft_begin_index) return model, parameters return model, model.parameters()
def generate_model(opt): assert opt.model in [ 'resnet', 'resnetl', 'resnext', 'c3d', 'mobilenetv2', 'shufflenetv2', "mstcn" ] if opt.model == 'resnet': assert opt.model_depth in [10, 50] from models.resnet import get_fine_tuning_parameters if opt.model_depth == 10: model = resnet.resnet10(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model_depth == 50: model = resnet.resnet50(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model == 'resnetl': assert opt.model_depth in [10, 18] from models.resnetl import get_fine_tuning_parameters if opt.model_depth == 10: model = resnetl.resnetl10(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model_depth == 18: model = resnetl.resnetl10(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model == 'resnext': assert opt.model_depth in [101] from models.resnext import get_fine_tuning_parameters if opt.model_depth == 101: model = resnext.resnet101(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, cardinality=opt.resnext_cardinality, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model == 'c3d': assert opt.model_depth in [10] from models.c3d import get_fine_tuning_parameters if opt.model_depth == 10: model = c3d.c3d_v1(sample_size=opt.sample_size, sample_duration=opt.sample_duration, num_classes=opt.n_classes) elif opt.model == 'mobilenetv2': from models.mobilenetv2 import get_fine_tuning_parameters model = mobilenetv2.mob_v2(num_classes=opt.n_classes, sample_size=opt.sample_size, width_mult=opt.width_mult) elif opt.model == 'shufflenetv2': from models.shufflenetv2 import get_fine_tuning_parameters model = shufflenetv2.shf_v2(num_classes=opt.n_classes, sample_size=opt.sample_size, width_mult=opt.width_mult) elif opt.model == "mstcn": model = MultiStageTemporalConvNet( embed_size=opt.embedding_dim, encoder=opt.tcn_encoder, n_classes=opt.n_classes, input_size=(opt.sample_size, opt.sample_size), input_channels=4 if opt.modality != "RGB" else 3, num_stages=opt.tcn_stages, causal_config=opt.tcn_causality, CTHW_layout=True, use_preprocessing=opt.use_preprocessing) if not opt.no_cuda: if opt.pretrain_path: print('loading pretrained model {}'.format(opt.pretrain_path)) pretrain = torch.load(opt.pretrain_path) assert opt.arch == pretrain['arch'] if opt.pretrain_dataset == 'jester': if opt.sample_duration < 32 and opt.model != 'c3d': model = _modify_first_conv_layer(model, 3, 3) if opt.model in ['mobilenetv2', 'shufflenetv2']: del pretrain['state_dict']['module.classifier.1.weight'] del pretrain['state_dict']['module.classifier.1.bias'] else: del pretrain['state_dict']['module.fc.weight'] del pretrain['state_dict']['module.fc.bias'] model.load_state_dict(pretrain['state_dict'], strict=False) if opt.modality in ['RGB', 'flo'] and opt.model != 'c3d': print("[INFO]: RGB model is used for init model") if opt.dataset != 'jester' and not opt.no_first_lay: model = _modify_first_conv_layer( model, 3, 3) ##### Check models trained (3,7,7) or (7,7,7) elif opt.modality in ['Depth', 'seg']: print( "[INFO]: Converting the pretrained model to Depth init model") model = _construct_depth_model(model) print("[INFO]: Done. Flow model ready.") elif opt.modality in ['RGB-D', 'RGB-flo', 'RGB-seg']: if opt.model != "mstcn": print( "[INFO]: Converting the pretrained model to RGB+D init model" ) model = _construct_rgbdepth_model(model) if opt.no_first_lay: model = _modify_first_conv_layer( model, 3, 4) ##### Check models trained (3,7,7) or (7,7,7) print("[INFO]: Done. RGB-D model ready.") if opt.pretrain_dataset == opt.dataset: model.load_state_dict(pretrain['state_dict']) elif opt.pretrain_dataset in ['egogesture', 'nv', 'denso']: del pretrain['state_dict']['module.fc.weight'] del pretrain['state_dict']['module.fc.bias'] model.load_state_dict(pretrain['state_dict'], strict=False) # Check first kernel size if opt.model != "mstcn": modules = list(model.modules()) first_conv_idx = list( filter(lambda x: isinstance(modules[x], nn.Conv3d), list(range(len(modules)))))[0] conv_layer = modules[first_conv_idx] if conv_layer.kernel_size[0] > opt.sample_duration: print("[INFO]: RGB model is used for init model") model = _modify_first_conv_layer(model, int(opt.sample_duration / 2), 1) if opt.model == 'c3d': # CHECK HERE model.module.fc = nn.Linear(model.module.fc[0].in_features, model.module.fc[0].out_features) model.module.fc = model.module.fc.cuda() elif opt.model in ['mobilenetv2', 'shufflenetv2']: model.module.classifier = nn.Sequential( nn.Dropout(0.9), nn.Linear(model.module.classifier[1].in_features, opt.n_finetune_classes)) model.module.classifier = model.module.classifier.cuda() elif opt.model != "mstcn": model.module.fc = nn.Linear(model.module.fc.in_features, opt.n_finetune_classes) model.module.fc = model.module.fc.cuda() if opt.model != "mstcn": parameters = get_fine_tuning_parameters(model, opt.ft_begin_index) else: parameters = model.trainable_parameters() model = nn.DataParallel(model, device_ids=None) model = model.cuda() return model, parameters else: print('ERROR no cuda') return model, model.parameters()
def generate_model(opt, modality): assert opt.model in ['resnet', 'resnetl', 'resnext', 'c3d'] if opt.model == 'resnet': # assert opt.model_depth in [10] from models.resnet import get_fine_tuning_parameters if opt.model_depth == 10: model = resnet.resnet10(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.n_frames_per_clip) elif opt.model_depth == 18: model = resnet.resnet18(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.n_frames_per_clip) elif opt.model_depth == 50: model = resnet.resnet50(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.n_frames_per_clip) elif opt.model == 'resnetl': assert opt.model_depth in [10] from models.resnetl import get_fine_tuning_parameters if opt.model_depth == 10: model = resnetl.resnetl10(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.n_frames_per_clip) elif opt.model == 'resnext': assert opt.model_depth in [101] from models.resnext import get_fine_tuning_parameters if opt.model_depth == 101: model = resnext.resnet101(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, cardinality=opt.resnext_cardinality, sample_size=opt.sample_size, sample_duration=opt.n_frames_per_clip) elif opt.model == 'c3d': assert opt.model_depth in [10] from models.c3d import get_fine_tuning_parameters if opt.model_depth == 10: model = c3d.c3d_v1(sample_size=opt.sample_size, sample_duration=opt.n_frames_per_clip, num_classes=opt.n_classes) if not opt.no_cuda: model = model.cuda() model = nn.DataParallel(model, device_ids=None) if opt.pretrain_path: print('loading pretrained model {}'.format(opt.pretrain_path)) pretrain = torch.load(opt.pretrain_path, map_location='cpu') # assert opt.arch == pretrain['arch'] feature_state_dict = { key: value for key, value in pretrain['state_dict'].items() if key not in ['module.fc.weight', 'module.fc.bias'] } state_dict = deepcopy(model.state_dict()) state_dict.update(feature_state_dict) model.load_state_dict(state_dict) # model.load_state_dict(pretrain['state_dict']) if modality == 'RGB' and opt.model != 'c3d': print("[INFO]: RGB model is used for init model") # model = _modify_first_conv_layer(model,3,3) ##### Check models trained (3,7,7) or (7,7,7) elif modality == 'Depth': print( "[INFO]: Converting the pretrained model to Depth init model") model = _construct_depth_model(model) print("[INFO]: Done. Flow model ready.") elif modality == 'RGB-D': print( "[INFO]: Converting the pretrained model to RGB+D init model") model = _construct_rgbdepth_model(model) print("[INFO]: Done. RGB-D model ready.") # Check first kernel size modules = list(model.modules()) first_conv_idx = list( filter(lambda x: isinstance(modules[x], nn.Conv3d), list(range(len(modules)))))[0] conv_layer = modules[first_conv_idx] if conv_layer.kernel_size[0] > opt.n_frames_per_clip: print("[INFO]: RGB model is used for init model") model = _modify_first_conv_layer(model, int(opt.n_frames_per_clip / 2), 1) if opt.model == 'c3d': # CHECK HERE model.module.fc = nn.Linear(model.module.fc[0].in_features, model.module.fc[0].out_features) model.module.fc = model.module.fc.cuda() else: model.module.fc = nn.Linear(model.module.fc.in_features, opt.n_finetune_classes) model.module.fc = model.module.fc.cuda() parameters = get_fine_tuning_parameters(model, opt.ft_begin_index) return model, parameters else: if opt.pretrain_path: print('loading pretrained model {}'.format(opt.pretrain_path)) pretrain = torch.load(opt.pretrain_path, map_location='cpu') # assert opt.arch == pretrain['arch'] feature_state_dict = { key.replace('module.', ''): value for key, value in pretrain['state_dict'].items() if key not in ['module.fc.weight', 'module.fc.bias'] } state_dict = deepcopy(model.state_dict()) state_dict.update(feature_state_dict) model.load_state_dict(state_dict) # model.load_state_dict(pretrain['state_dict']) if modality == 'RGB' and opt.model != 'c3d': print("[INFO]: RGB model is used for init model") model = _modify_first_conv_layer(model, 3, 3) elif modality == 'Depth': print( "[INFO]: Converting the pretrained model to Depth init model") model = _construct_depth_model(model) print("[INFO]: Deoth model ready.") elif modality == 'RGB-D': print( "[INFO]: Converting the pretrained model to RGB-D init model") model = _construct_rgbdepth_model(model) print("[INFO]: Done. RGB-D model ready.") # Check first kernel size modules = list(model.modules()) first_conv_idx = list( filter(lambda x: isinstance(modules[x], nn.Conv3d), list(range(len(modules)))))[0] conv_layer = modules[first_conv_idx] if conv_layer.kernel_size[0] > opt.n_frames_per_clip: print("[INFO]: RGB model is used for init model") model = _modify_first_conv_layer(model, int(opt.n_frames_per_clip / 2), 1) if opt.model == 'c3d': # CHECK HERE model.fc = nn.Linear(model.fc[0].in_features, model.fc[0].out_features) else: model.fc = nn.Linear(model.fc.in_features, opt.n_finetune_classes) parameters = get_fine_tuning_parameters(model, opt.ft_begin_index) return model, parameters return model, model.parameters()
def generate_model(opt): assert opt.model in [ 'c3d', 'squeezenet', 'mobilenet', 'resnext', 'resnet', 'resnetl', 'shufflenet', 'mobilenetv2', 'shufflenetv2' ] if opt.model == 'c3d': from models.c3d import get_fine_tuning_parameters model = c3d.get_model(num_classes=opt.n_classes, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model == 'squeezenet': from models.squeezenet import get_fine_tuning_parameters model = squeezenet.get_model(version=opt.version, num_classes=opt.n_classes, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model == 'shufflenet': from models.shufflenet import get_fine_tuning_parameters model = shufflenet.get_model(groups=opt.groups, width_mult=opt.width_mult, num_classes=opt.n_classes) elif opt.model == 'shufflenetv2': from models.shufflenetv2 import get_fine_tuning_parameters model = shufflenetv2.get_model(num_classes=opt.n_classes, sample_size=opt.sample_size, width_mult=opt.width_mult) elif opt.model == 'mobilenet': from models.mobilenet import get_fine_tuning_parameters model = mobilenet.get_model(num_classes=opt.n_classes, sample_size=opt.sample_size, width_mult=opt.width_mult) elif opt.model == 'mobilenetv2': from models.mobilenetv2 import get_fine_tuning_parameters model = mobilenetv2.get_model(num_classes=opt.n_classes, sample_size=opt.sample_size, width_mult=opt.width_mult) elif opt.model == 'resnext': assert opt.model_depth in [50, 101, 152] from models.resnext import get_fine_tuning_parameters if opt.model_depth == 50: model = resnext.resnet50(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, cardinality=opt.resnext_cardinality, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model_depth == 101: model = resnext.resnet101(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, cardinality=opt.resnext_cardinality, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model_depth == 152: model = resnext.resnet152(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, cardinality=opt.resnext_cardinality, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model == 'resnetl': assert opt.model_depth in [10] from models.resnetl import get_fine_tuning_parameters if opt.model_depth == 10: model = resnetl.resnetl10(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model == 'resnet': assert opt.model_depth in [10, 18, 34, 50, 101, 152, 200] from models.resnet import get_fine_tuning_parameters if opt.model_depth == 10: model = resnet.resnet10(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model_depth == 18: model = resnet.resnet18(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model_depth == 34: model = resnet.resnet34(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model_depth == 50: model = resnet.resnet50(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model_depth == 101: model = resnet.resnet101(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model_depth == 152: model = resnet.resnet152(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model_depth == 200: model = resnet.resnet200(num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, sample_size=opt.sample_size, sample_duration=opt.sample_duration) if not opt.no_cuda: model = model.cuda() model = nn.DataParallel(model, device_ids=None) pytorch_total_params = sum(p.numel() for p in model.parameters() if p.requires_grad) print("Total number of trainable parameters: ", pytorch_total_params) if opt.pretrain_path: print('loading pretrained model {}'.format(opt.pretrain_path)) pretrain = torch.load(opt.pretrain_path, map_location=torch.device('cpu')) # print(opt.arch) # print(pretrain['arch']) # assert opt.arch == pretrain['arch'] model.load_state_dict(pretrain['state_dict']) if opt.model in [ 'mobilenet', 'mobilenetv2', 'shufflenet', 'shufflenetv2' ]: model.module.classifier = nn.Sequential( nn.Dropout(0.5), nn.Linear(model.module.classifier[1].in_features, opt.n_finetune_classes)) model.module.classifier = model.module.classifier.cuda() elif opt.model == 'squeezenet': model.module.classifier = nn.Sequential( nn.Dropout(p=0.5), nn.Conv3d(model.module.classifier[1].in_channels, opt.n_finetune_classes, kernel_size=1), nn.ReLU(inplace=True), nn.AvgPool3d((1, 4, 4), stride=1)) model.module.classifier = model.module.classifier.cuda() else: model.module.fc = nn.Linear(model.module.fc.in_features, opt.n_finetune_classes) model.module.fc = model.module.fc.cuda() if opt.modality == 'RGB' and opt.model != 'c3d': print("[INFO]: RGB model is used for init model") model = _modify_first_conv_layer( model, 3, 3) ##### Check models trained (3,7,7) or (7,7,7) elif opt.modality == 'Depth': print( "[INFO]: Converting the pretrained model to Depth init model") model = _construct_depth_model(model) print("[INFO]: Done. Flow model ready.") elif opt.modality == 'RGB-D': print( "[INFO]: Converting the pretrained model to RGB+D init model") model = _construct_rgbdepth_model(model) print("[INFO]: Done. RGB-D model ready.") modules = list(model.modules()) first_conv_idx = list( filter(lambda x: isinstance(modules[x], nn.Conv3d), list(range(len(modules)))))[0] conv_layer = modules[first_conv_idx] if conv_layer.kernel_size[0] > opt.sample_duration: model = _modify_first_conv_layer(model, int(opt.sample_duration / 2), 1) parameters = get_fine_tuning_parameters(model, opt.ft_portion) return model, parameters else: if opt.pretrain_path: print('loading pretrained model {}'.format(opt.pretrain_path)) pretrain = torch.load(opt.pretrain_path) assert opt.arch == pretrain['arch'] model.load_state_dict(pretrain['state_dict']) if opt.model in [ 'mobilenet', 'mobilenetv2', 'shufflenet', 'shufflenetv2' ]: model.module.classifier = nn.Sequential( nn.Dropout(0.9), nn.Linear(model.module.classifier[1].in_features, opt.n_finetune_classes)) elif opt.model == 'squeezenet': model.module.classifier = nn.Sequential( nn.Dropout(p=0.5), nn.Conv3d(model.module.classifier[1].in_channels, opt.n_finetune_classes, kernel_size=1), nn.ReLU(inplace=True), nn.AvgPool3d((1, 4, 4), stride=1)) else: model.module.fc = nn.Linear(model.module.fc.in_features, opt.n_finetune_classes) if opt.modality == 'RGB' and opt.model != 'c3d': print("[INFO]: RGB model is used for init model") model = _modify_first_conv_layer(model, 3, 3) elif opt.modality == 'Depth': print( "[INFO]: Converting the pretrained model to Depth init model") model = _construct_depth_model(model) print("[INFO]: Deoth model ready.") elif opt.modality == 'RGB-D': print( "[INFO]: Converting the pretrained model to RGB-D init model") model = _construct_rgbdepth_model(model) print("[INFO]: Done. RGB-D model ready.") modules = list(model.modules()) first_conv_idx = list( filter(lambda x: isinstance(modules[x], nn.Conv3d), list(range(len(modules)))))[0] conv_layer = modules[first_conv_idx] if conv_layer.kernel_size[0] > opt.sample_duration: print("[INFO]: RGB model is used for init model") model = _modify_first_conv_layer(model, int(opt.sample_duration / 2), 1) if opt.model == 'c3d': # CHECK HERE model.fc = nn.Linear(model.fc[0].in_features, model.fc[0].out_features) else: model.fc = nn.Linear(model.fc.in_features, opt.n_finetune_classes) parameters = get_fine_tuning_parameters(model, opt.ft_begin_index) return model, parameters return model, model.parameters()
def generate_model(opt): assert opt.model in ['c3d', 'squeezenet', 'mobilenet', 'resnext', 'resnet', 'resnetl', 'shufflenet', 'mobilenetv2', 'shufflenetv2'] if opt.model == 'resnetl': assert opt.model_depth in [10] # 깊이는 10만 된다! from models.resnetl import get_fine_tuning_parameters # 전이학습을 위함. if opt.model_depth == 10: model = resnetl.resnetl10( num_classes=opt.n_classes, # 클래스 개수. shortcut_type=opt.resnet_shortcut, # 디폴트 값 : 'B' sample_size=opt.sample_size, # 디폴트 값 : 112 sample_duration=opt.sample_duration) # 디폴트 값 : 16 , 입력 프레임 elif opt.model == 'resnext': assert opt.model_depth in [50, 101, 152] from models.resnext import get_fine_tuning_parameters if opt.model_depth == 50: model = resnext.resnext50( num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, cardinality=opt.resnext_cardinality, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model_depth == 101: model = resnext.resnext101( num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, cardinality=opt.resnext_cardinality, sample_size=opt.sample_size, sample_duration=opt.sample_duration) elif opt.model_depth == 152: model = resnext.resnext152( num_classes=opt.n_classes, shortcut_type=opt.resnet_shortcut, cardinality=opt.resnext_cardinality, sample_size=opt.sample_size, sample_duration=opt.sample_duration) if not opt.no_cuda: if opt.gpus == '0': model = model.cuda() else: opt.gpus = opt.local_rank torch.cuda.set_device(opt.gpus) model = model.cuda() #model = nn.DataParallel(model, device_ids=None) # 병렬처리를 위함인데, 안쓸 것 같음. pytorch_total_params = sum(p.numel() for p in model.parameters() if p.requires_grad) # grad를 하는 파라미터들을 모두 더한다. print("Total number of trainable parameters: ", pytorch_total_params) # 파라미터 값 출력 if opt.pretrain_path: # 전이학습. print('loading pretrained model {}'.format(opt.pretrain_path)) pretrain = torch.load(opt.pretrain_path, map_location=torch.device('cpu')) # print(opt.arch) # print(pretrain['arch']) # assert opt.arch == pretrain['arch'] model = modify_kernels(opt, model, opt.pretrain_modality) model.load_state_dict(pretrain['state_dict']) if opt.model in ['mobilenet', 'mobilenetv2', 'shufflenet', 'shufflenetv2']: model.module.classifier = nn.Sequential( nn.Dropout(0.5), nn.Linear(model.module.classifier[1].in_features, opt.n_finetune_classes)) model.module.classifier = model.module.classifier.cuda() elif opt.model == 'squeezenet': model.module.classifier = nn.Sequential( nn.Dropout(p=0.5), nn.Conv3d(model.module.classifier[1].in_channels, opt.n_finetune_classes, kernel_size=1), nn.ReLU(inplace=True), nn.AvgPool3d((1,4,4), stride=1)) model.module.classifier = model.module.classifier.cuda() else: model.module.fc = nn.Linear(model.module.fc.in_features, opt.n_finetune_classes) model.module.fc = model.module.fc.cuda() model = modify_kernels(opt, model, opt.modality) else: # 전이학습이 아닐때 pass model = modify_kernels(opt, model, opt.modality) parameters = get_fine_tuning_parameters(model, opt.ft_portion) # 전이학습할때만 적용 지금은 그냥 파라미터 그대로 반환됨. return model, parameters