def __init__(self):
super(DeepCPH, self).__init__()
self.ct_model = backbone.generate_model(18, n_input_channels=1)
self.pet_model = backbone.generate_model(18, n_input_channels=1)
self.linear1 = nn.Linear(1024, 256, bias=True)
self.linear2 = nn.Linear(256, 32, bias=True)
self.linear3 = nn.Linear(32, 1, bias=True)
self.relu = nn.ReLU()
self.dropout = nn.Dropout(p=0.2, inplace=True)
self.reset_parameters()
def generate_model(opt): # 调用各个模型中的generate_model函数，定义相应的模型
assert opt.model in [
'resnet', 'resnet2p1d', 'preresnet', 'wideresnet', 'resnext', 'densenet'
]
if opt.model == 'resnet':
model = resnet.generate_model(model_depth=opt.model_depth,
n_classes=opt.n_classes,
n_input_channels=opt.n_input_channels,
shortcut_type=opt.resnet_shortcut,
conv1_t_size=opt.conv1_t_size,
conv1_t_stride=opt.conv1_t_stride,
no_max_pool=opt.no_max_pool,
widen_factor=opt.resnet_widen_factor)
elif opt.model == 'resnet2p1d':
model = resnet2p1d.generate_model(model_depth=opt.model_depth,
n_classes=opt.n_classes,
n_input_channels=opt.n_input_channels,
shortcut_type=opt.resnet_shortcut,
conv1_t_size=opt.conv1_t_size,
conv1_t_stride=opt.conv1_t_stride,
no_max_pool=opt.no_max_pool,
widen_factor=opt.resnet_widen_factor)
elif opt.model == 'wideresnet':
model = wide_resnet.generate_model(
model_depth=opt.model_depth,
k=opt.wide_resnet_k,
n_classes=opt.n_classes,
n_input_channels=opt.n_input_channels,
shortcut_type=opt.resnet_shortcut,
conv1_t_size=opt.conv1_t_size,
conv1_t_stride=opt.conv1_t_stride,
no_max_pool=opt.no_max_pool)
elif opt.model == 'resnext':
model = resnext.generate_model(model_depth=opt.model_depth,
cardinality=opt.resnext_cardinality,
n_classes=opt.n_classes,
n_input_channels=opt.n_input_channels,
shortcut_type=opt.resnet_shortcut,
conv1_t_size=opt.conv1_t_size,
conv1_t_stride=opt.conv1_t_stride,
no_max_pool=opt.no_max_pool)
elif opt.model == 'preresnet':
model = pre_act_resnet.generate_model(
model_depth=opt.model_depth,
n_classes=opt.n_classes,
n_input_channels=opt.n_input_channels,
shortcut_type=opt.resnet_shortcut,
conv1_t_size=opt.conv1_t_size,
conv1_t_stride=opt.conv1_t_stride,
no_max_pool=opt.no_max_pool)
elif opt.model == 'densenet':
model = densenet.generate_model(model_depth=opt.model_depth,
n_classes=opt.n_classes,
n_input_channels=opt.n_input_channels,
conv1_t_size=opt.conv1_t_size,
conv1_t_stride=opt.conv1_t_stride,
no_max_pool=opt.no_max_pool)
return model
def nn_model(config):
data_transforms = transforms.Compose([transforms.ToTensor()])
train_set = BalletDancer(data_transforms)
validation_set = GoalKeeper(data_transforms)
#Loading train and validation set
train_set_loader = DataLoader(
train_set,
batch_size=config.batch_size,
shuffle=False,
num_workers=configuration.training_configuration.number_workers)
validation_set_loader = DataLoader(
validation_set,
batch_size=config.batch_size,
shuffle=False,
num_workers=configuration.training_configuration.number_workers)
#Build the model
intermediate_net = resnet.generate_model(config.resnet_depth)
siamese_net = siamese.Siamese(intermediate_net)
if configuration.training_configuration.device.type == 'cuda':
siamese_net.cuda()
loss_function = loss.ContrastiveLoss(config.loss_margin)
optimizer = torch.optim.Adam(siamese_net.parameters(), lr=config.lr)
return siamese_net, train_set_loader, validation_set_loader, loss_function, optimizer
def get_model(model_name, encoder, num_classes):
if model_name == 'lstm':
return LSTM(encoder, num_classes)
elif model_name == 'cnn':
return CNN(encoder, num_classes)
elif model_name == 'transformer':
return Transformer(encoder, num_classes)
elif model_name == 'cnn3d':
if 'efficientnet' in encoder:
return EfficientNet3D.from_name(
encoder,
override_params={'num_classes': num_classes},
in_channels=1)
elif 'resnet' in encoder:
if encoder == 'resnet':
encoder = 'resnet34'
return resnet.generate_model(model_depth=int(
encoder.split('resnet')[-1]),
n_classes=num_classes,
n_input_channels=1,
shortcut_type='B',
conv1_t_size=7,
conv1_t_stride=1,
no_max_pool=False,
widen_factor=1.)
elif 'resnext' in encoder:
return resnext.generate_model(model_depth=int(
encoder.split('resnext')[-1]),
n_classes=num_classes,
n_input_channels=1,
cardinality=32,
shortcut_type='B',
conv1_t_size=7,
conv1_t_stride=1,
no_max_pool=False)
elif 'resnet2p1d' in encoder:
return resnet2p1d.generate_model(model_depth=int(
encoder.split('resnet2p1d')[-1]),
n_classes=num_classes,
n_input_channels=1,
shortcut_type='B',
conv1_t_size=7,
conv1_t_stride=1,
no_max_pool=False,
widen_factor=1.)
elif 'densenet' in encoder:
return densenet.generate_model(model_depth=int(
encoder.split('densenet')[-1]),
num_classes=num_classes,
n_input_channels=1,
conv1_t_size=7,
conv1_t_stride=1,
no_max_pool=False)
else:
print(encoder)
raise
else:
print(model_name)
raise
def generate_model(opt):
if opt.model == "ECO":
model = ECO.generate_model(opt)
elif opt.model == "I3D":
model = I3D.generate_model(opt)
elif opt.model == "resnet" or opt.model == "resnext":
model = ResNet.generate_model(opt)
return model
def generate_model():
model = resnet.generate_model(model_depth = 50,
n_classes = 1139,
n_input_channels = 3,
shortcut_type = 'B',
conv1_t_size = 7,
conv1_t_stride = 1,
no_max_pool = False,
widen_factor = 1.0)
return model
def generate_model(opt):
model = resnet.generate_model(model_depth=opt.model_depth,
n_classes=opt.n_classes,
n_input_channels=opt.n_input_channels,
shortcut_type=opt.resnet_shortcut,
conv1_t_size=opt.conv1_t_size,
conv1_t_stride=opt.conv1_t_stride,
no_max_pool=opt.no_max_pool,
widen_factor=opt.resnet_widen_factor)
return model
def __init__(self):
super(DeepCPH, self).__init__()
self.ct_model = backbone.generate_model(34, n_input_channels=1)
self.pet_model = backbone.generate_model(34, n_input_channels=1)
# self.histology = nn.Linear(4, 1, bias=True)
# self.gender = nn.Linear(2, 1, bias=True)
self.linear1 = nn.Linear(1024, 128, bias=True)
self.bn1 = nn.BatchNorm1d(num_features=128)
self.linear2 = nn.Linear(128, 32, bias=True)
self.bn2 = nn.BatchNorm1d(num_features=32)
self.linear3 = nn.Linear(39, 1, bias=False)
self.relu = nn.ReLU()
self.dropout = nn.Dropout(p=0.2, inplace=True)
self.reset_parameters()
def __init__(self):
super(DeepCPH_image, self).__init__()
self.img_model = backbone.generate_model(34, n_input_channels=1)
self.linear1 = nn.Linear(512, 128, bias=True)
self.linear2 = nn.Linear(128, 32, bias=True)
self.linear3 = nn.Linear(32, 1, bias=False)
self.relu = nn.ReLU()
self.dropout = nn.Dropout(p=0.2, inplace=True)
self.reset_parameters()
def main():
parser = argparse.ArgumentParser(description="Run model against images")
parser.add_argument(
'--input-glob',
default=
'data/kinetics_videos/jpg/yoga/0wHOYxjRmlw_000041_000051/image_000{41,42,43,44,45,46,47,48,49,50,41,42,43,44,45,46}.jpg',
help="inputs")
parser.add_argument("--depth", default="50", help="which model depth")
args = parser.parse_args()
model_file = model_files[args.depth]
model_depth = int(args.depth)
model = resnet.generate_model(model_depth=model_depth,
n_classes=700,
n_input_channels=3,
shortcut_type="B",
conv1_t_size=7,
conv1_t_stride=1,
no_max_pool=False,
widen_factor=1.0)
# model = load_pretrained_model(model, args.model, "resnet", 700)
checkpoint = torch.load(model_file, map_location='cpu')
arch = '{}-{}'.format("resnet", model_depth)
print(arch, checkpoint['arch'])
assert arch == checkpoint['arch']
if hasattr(model, 'module'):
# I think this only for legacy models
model.module.load_state_dict(checkpoint['state_dict'])
else:
model.load_state_dict(checkpoint['state_dict'])
model.eval()
image_clips = []
files = real_glob(args.input_glob)
files = extend_to_length(files, 16)
print(files)
for f in files:
img = Image.open(f).convert("RGB")
image_clips.append(img)
# print("EARLY", image_clips[0][0:4,0:4,0])
mean = [0.4345, 0.4051, 0.3775]
std = [0.2768, 0.2713, 0.2737]
normalize = Normalize(mean, std)
sample_size = 112
spatial_transform = [Resize(sample_size)]
spatial_transform.append(CenterCrop(sample_size))
spatial_transform.append(ToTensor())
spatial_transform.extend([ScaleValue(1), normalize])
spatial_transform = Compose(spatial_transform)
# c = spatial_transform(image_clips[0])
# c.save("raw.png")
model_clips = []
clip = [spatial_transform(img) for img in image_clips]
model_clips.append(torch.stack(clip, 0).permute(1, 0, 2, 3))
model_clips = torch.stack(model_clips, 0)
print("Final", model_clips.shape)
print("PEEK", model_clips[0, 0, 0, 0:4, 0:4])
with torch.no_grad():
outputs = model(model_clips)
print(outputs[0][0:10])
outputs = F.softmax(outputs, dim=1).cpu()
sorted_scores, locs = torch.topk(outputs[0], k=3)
print(locs[0])
video_results = []
for i in range(sorted_scores.size(0)):
video_results.append({
'label': magic_labels_700[locs[i].item()],
'score': sorted_scores[i].item()
})
print(video_results)
torch.cuda.manual_seed_all(args.seed)
########### model ##############
if args.dataset == 'ucf101':
class_num = 101
elif args.dataset == 'hmdb51':
class_num = 51
elif args.dataset == 'faceforensics':
class_num = 5
if args.model == 'c3d':
print(class_num)
model = generate_model(model_depth=18,
n_classes=700,
n_input_channels=3,
shortcut_type='B',
conv1_t_size=7,
conv1_t_stride=1,
no_max_pool=True,
widen_factor=1.0).to(device)
if args.mode == 'train': ########### Train #############
if args.ckpt: # resume training
model = nn.DataParallel(model)
model.load_state_dict(torch.load(args.ckpt))
log_dir = os.path.dirname(args.ckpt)
parameters = model.parameters()
else:
if args.pretrain:
model = load_pretrained_resnet(model, args.pretrain, class_num)
model = nn.DataParallel(model).to(device)
parameters = get_fine_tuning_parameters(model, 'layer4')
type='model')
artifact_dir = artifact.download()
# Set device
device = torch.device("cuda")
config = wandb.config
config.sequence_length = 10
config.skip_frames = 1
config.model_depth = 18
config.max_target = 30
config.grayscale = True
if config.grayscale:
model = generate_model(model_depth=config.model_depth,
n_classes=1,
n_input_channels=1)
tfms = transforms.Compose([transforms.Grayscale()])
else:
model = generate_model(model_depth=config.model_depth,
n_classes=1,
n_input_channels=3)
tfms = None
validset = VideoFrameDataset(os.path.join("data", "route"),
int(config.sequence_length),
1,
skip_frames=int(config.skip_frames),
transform=tfms)