def i3d_feature(frame_dir, save_dir):
for file in os.listdir(frame_dir):
if os.path.isdir(frame_dir + '/' + file):
sub_dir = frame_dir + '/' + file
else:
sub_dir = frame_dir
input_shape = (224, 224)
i3d = InceptionI3d(400, in_channels=3)
#i3d.replace_logits(157)
i3d.load_state_dict(torch.load('../checkpoints/rgb_imagenet.pt'))
i3d = torch.nn.DataParallel(i3d, device_ids=[0, 1, 2, 4])
#i3d.cuda()
features = []
# b,c,t,h,w = inputs.shape
# if t > 1600:
# features = []
# for start in range(1, t - 56, 1600):
# end = min(t - 1, start + 1600 + 56)
# start = max(1, start - 48)
# ip = Variable(torch.from_numpy(inputs.numpy()[:, :, start:end]).cuda(), volatile=True)
# features.append(i3d.module.extract_features(ip).squeeze(0).permute(1, 2, 3, 0).data.cpu().numpy())
# np.save(os.path.join(save_dir, name[0]), np.concatenate(features, axis=0))
# else:
# # wrap them in Variable
# inputs = Variable(inputs.cuda(), volatile=True)
# features = i3d.extract_features(inputs)
# np.save(os.path.join(save_dir, name[0]), features.squeeze(0).permute(1, 2, 3, 0).data.cpu().numpy())
# for x in os.listdir(sub_dir):
# frame = mmcv.imread(sub_dir + '/' + x)
# frame = scipy.misc.imresize(crop_center(frame).astype(np.float32), input_shape)
frames = load_rgb_frames(frame_dir, file)
ip = Variable(torch.from_numpy(frames), volatile=True)
features.append(
i3d.module.extract_features(ip).squeeze(0).permute(
1, 2, 3, 0).data.cpu().numpy())
feat_filepath = os.path.join(save_dir, file + '.npy')
with open(feat_filepath, 'wb') as f:
np.save(f, np.concatenate(features, axis=0))
def __init__(self, i3d_pretrained=None, mlp_pretrained=None):
super(i3d_mlp, self).__init__()
if i3d_pretrained:
self.i3d = i3d_pretrained
else:
self.i3d = InceptionI3d(
num_classes=1064,
spatiotemporal_squeeze=True,
final_endpoint="Logits",
name="inception_i3d",
in_channels=3,
dropout_keep_prob=0.5,
num_in_frames=16,
include_embds=True,
)
if mlp_pretrained:
self.mlp = mlp_pretrained
else:
self.mlp = Mlp()
def get_models(args):
"""
Get the i3d backbone and the evaluator with parameters moved to GPU.
"""
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
i3d = InceptionI3d().cuda()
i3d.load_state_dict(torch.load(i3d_pretrained_path))
if args.type == 'USDL':
evaluator = Evaluator(output_dim=output_dim['USDL'],
model_type='USDL').cuda()
else:
evaluator = Evaluator(output_dim=output_dim['MUSDL'],
model_type='MUSDL',
num_judges=num_judges).cuda()
if len(args.gpu.split(',')) > 1:
i3d = nn.DataParallel(i3d)
evaluator = nn.DataParallel(evaluator)
return i3d, evaluator
def get_models(args):
"""
Get the i3d i3d and the evaluator with parameters moved to GPU.
Use ModuleList to perform 4-fold training.
"""
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
i3ds = nn.ModuleList([InceptionI3d() for _ in range(4)]).cuda()
for i3d in i3ds:
i3d.load_state_dict(torch.load(i3d_pretrained_path))
if args.type == 'USDL':
evaluators = nn.ModuleList([Evaluator(output_dim=output_dim['USDL'], model_type='USDL')
for _ in range(4)]).cuda()
else:
evaluators = nn.ModuleList([Evaluator(output_dim=output_dim['MUSDL'], model_type='MUSDL', num_judges=num_judges)
for _ in range(4)]).cuda()
if len(args.gpu.split(',')) > 1:
i3ds = nn.ModuleList([nn.DataParallel(i3d) for i3d in i3ds])
evaluators = nn.ModuleList([nn.DataParallel(evaluator) for evaluator in evaluators])
return i3ds, evaluators
def factory(*args, **kwargs):
if kwargs["model_name"] == "c3d":
model = C3dNet(*args, **kwargs)
elif kwargs["model_name"] == "c3d_top":
model = C3dTOPNet(*args, **kwargs)
elif kwargs["model_name"] == "c3d_lstm":
model = C3dLstmNet(*args, **kwargs)
elif kwargs["model_name"] == "c3d_block_lstm":
model = C3dBlockLstmNet(*args, **kwargs)
elif kwargs["model_name"] == "cnn3d":
model = CNN3dNet(*args, **kwargs)
elif kwargs["model_name"] == "cnn3d_lstm":
model = CNN3dLSTMNet(*args, **kwargs)
elif kwargs["model_name"] == "cnn3d_block_lstm":
model = Cnn3dBlockLstmNet(*args, **kwargs)
# resnets
elif kwargs["model_name"] == "resnet3d_101":
model = resnet101(*args, **kwargs)
elif kwargs["model_name"] == "resnet3d_18":
model = resnet18(*args, **kwargs)
# i3d
elif kwargs["model_name"] == "i3d":
model = InceptionI3d(*args, **kwargs)
elif kwargs["model_name"] == "crnn":
model = CRNNNet(*args, **kwargs)
elif kwargs["model_name"] == "crnn_nf":
model = CRNNNFNet(*args, **kwargs)
elif kwargs["model_name"] == "crnn_simple_sum":
model = CRNNSimpleSum(*args, **kwargs)
elif kwargs["model_name"] == "crnn_skip":
model = CRNNSkip(*args, **kwargs)
elif kwargs["model_name"] == "crnn_attention":
model = CRNNAttention(*args, **kwargs)
elif kwargs["model_name"] == "crnn_attention_nn":
model = CRNNAttentionNN(*args, **kwargs)
elif kwargs["model_name"] == "inceptionv3_lstm":
model = InceptionV3LstmNet(*args, **kwargs)
elif kwargs["model_name"] == "vgg16_lstm":
model = VGG16LstmNet(*args, **kwargs)
elif kwargs["model_name"] == "resnet101_lstm":
model = ResNet101LstmNet(*args, **kwargs)
elif kwargs["model_name"] == "resnet18_lstm":
model = ResNet18LstmNet(*args, **kwargs)
else:
assert 0, "Bad model_name of model creation: " + kwargs[
"model_name"]
if torch.cuda.device_count() > 1:
print("Let's use", torch.cuda.device_count(), "GPUs!")
model = torch.nn.DataParallel(model)
return model
if args.seed:
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if args.gpu:
torch.cuda.manual_seed_all(args.seed)
########### model ##############
if args.model == 'c3d':
base = C3D(with_classifier=False)
elif args.model == 'r3d':
base = R3DNet(layer_sizes=(1, 1, 1, 1), with_classifier=False)
elif args.model == 'r21d':
base = R2Plus1DNet(layer_sizes=(1, 1, 1, 1), with_classifier=False)
elif args.model == 'i3d':
base = InceptionI3d(400, in_channels=3)
base.load_state_dict(
torch.load('../pytorch-i3d/models/rgb_imagenet.pt'))
# for name,param in base.named_parameters():
# if('Mixed_5c' not in name):
# param.requires_grad = False
vcopn = VCOPN(base_network=base, feature_size=512,
tuple_len=args.tl).to(device)
# vcopn = VCOPN(base_network=base, feature_size=1024, tuple_len=args.tl).to(device) # for i3d
if args.mode == 'train': ########### Train #############
if args.ckpt: # resume training
vcopn.load_state_dict(torch.load(args.ckpt))
log_dir = os.path.dirname(args.ckpt)
else:
if args.desp: