def __init__(self, args):
super().__init__()
# Initialize Stem to adjust number of channels
self.stem = nn.Conv3d(1, 3, (1, 3, 3), stride=1, padding=(0, 1, 1))
# Initialize 3D Resnet Model
self.resnet3d = None
if args.resnet3d_model == "r3d_18":
# 18 layer Resnet3D
self.resnet3d = r3d_18(pretrained=True)
elif args.resnet3d_model == "mc3_18":
# 18 layer Mixed Convolution network
self.resnet3d = mc3_18(pretrained=True)
else:
# 18 layer deep R(2+1)D network
self.resnet3d = r2plus1d_18(pretrained=True)
self.resnet3d_out_features = self.resnet3d.fc.out_features
self.features = args.features
# # FC layers between resnet3d and the heads
self.x1 = nn.Linear(self.resnet3d_out_features,
self.resnet3d_out_features)
nn.init.kaiming_normal_(self.x1.weight)
self.dropout1 = nn.Dropout(p=0.2)
self.x2 = nn.Linear(self.resnet3d_out_features,
self.resnet3d_out_features // 2)
nn.init.kaiming_normal_(self.x2.weight)
self.dropout2 = nn.Dropout(p=0.2)
for feature in self.features:
setattr(self, f"{feature}_head",
ClassifierHead(self.resnet3d_out_features // 2, 1))
def __init__(self,
num_classes=2,
sequence_length=8,
contains_dropout=False):
super().__init__(num_classes, sequence_length, contains_dropout)
self.mc3 = mc3_18(pretrained=True)
self.mc3.layer4 = nn.Identity()
self.mc3.fc = nn.Linear(256, self.num_classes)
def __init__(self):
"""Generic resnet video generator.
Args:
block (nn.Module): resnet building block
conv_makers (list(functions)): generator function for each layer
layers (List[int]): number of blocks per layer
stem (nn.Module, optional): Resnet stem, if None, defaults to conv-bn-relu. Defaults to None.
num_classes (int, optional): Dimension of the final FC layer. Defaults to 400.
zero_init_residual (bool, optional): Zero init bottleneck residual BN. Defaults to False.
"""
super(ResNetM3D, self).__init__()
self.backbone = mc3_18(pretrained=True)
def __init__(self, sequence_length):
super().__init__()
self.sequence_length = sequence_length
self.mc3 = mc3_18(pretrained=True)
self.mc3.reduce = nn.Sequential(
Conv3DNoTemporal(256, 64, stride=2),
nn.BatchNorm3d(64),
nn.ELU(),
Conv3DNoTemporal(64, 32, stride=2),
nn.BatchNorm3d(32),
nn.ELU(),
Conv3DNoTemporal(32, 16, stride=2),
nn.BatchNorm3d(16),
nn.ELU(),
)
# remove unnecessary stuff
self.mc3.layer4 = nn.Identity()
self.mc3.fc = nn.Identity()
def test_mc3_18_video(self):
x = Variable(torch.randn(1, 3, 4, 112, 112).fill_(1.0))
self.exportTest(toC(mc3_18()), toC(x), rtol=1e-3, atol=1e-5)
def main(model_name,
mode,
root,
val_split,
ckpt,
batch_per_gpu):
num_gpus = MPI.COMM_WORLD.Get_size()
distributed = False
if num_gpus > 1:
distributed = True
local_rank = MPI.COMM_WORLD.Get_rank() % torch.cuda.device_count()
if distributed:
torch.cuda.set_device(local_rank)
host = os.environ["MASTER_ADDR"] if "MASTER_ADDR" in os.environ else "127.0.0.1"
torch.distributed.init_process_group(
backend="nccl",
init_method='tcp://{}:12345'.format(host),
rank=MPI.COMM_WORLD.Get_rank(),
world_size=MPI.COMM_WORLD.Get_size()
)
synchronize()
val_dataloader = make_dataloader(root,
val_split,
mode,
model_name,
seq_len=16, #64,
overlap=8, #32,
phase='val',
max_iters=None,
batch_per_gpu=batch_per_gpu,
num_workers=16,
shuffle=False,
distributed=distributed,
with_normal=False)
if model_name == 'i3d':
if mode == 'flow':
model = InceptionI3d(val_dataloader.dataset.num_classes, in_channels=2, dropout_keep_prob=0.5)
else:
model = InceptionI3d(val_dataloader.dataset.num_classes, in_channels=3, dropout_keep_prob=0.5)
model.replace_logits(val_dataloader.dataset.num_classes)
elif model_name == 'r3d_18':
model = r3d_18(pretrained=False, num_classes=val_dataloader.dataset.num_classes)
elif model_name == 'mc3_18':
model = mc3_18(pretrained=False, num_classes=val_dataloader.dataset.num_classes)
elif model_name == 'r2plus1d_18':
model = r2plus1d_18(pretrained=False, num_classes=val_dataloader.dataset.num_classes)
elif model_name == 'c3d':
model = C3D(pretrained=False, num_classes=val_dataloader.dataset.num_classes)
else:
raise NameError('unknown model name:{}'.format(model_name))
# pdb.set_trace()
for param in model.parameters():
pass
device = torch.device('cuda')
model.to(device)
if distributed:
model = apex.parallel.convert_syncbn_model(model)
model = DDP(model.cuda(), delay_allreduce=True)
framewise_transforms=True,
i3d_norm=False,
max_frames=32)
dataloader_train = DataLoader(dataset_train,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=N_WORKERS,
pin_memory=True)
dataloader_test = DataLoader(dataset_test,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=N_WORKERS,
pin_memory=True)
model = mc3_18(pretrained=True)
model.fc = nn.Linear(in_features=512, out_features=2)
model.load_state_dict(
torch.load(
"../../data/saved_models/mc3_18_112_1cy_lilaug_nonorm_e9_l0.1905.model"
))
model = model.to(DEVICE)
for p in model.parameters():
p.requires_grad = True
criterion = torch.nn.CrossEntropyLoss()
lr = 5e-5
optimizer = torch.optim.Adam(
def __init__(self, experiment, device):
config_file = os.path.join(CONFIG_DIR, experiment + '.json')
assert os.path.exists(
config_file), 'config file {} does not exist'.format(config_file)
self.experiment = experiment
with open(config_file, 'r') as f:
configs = json.load(f)
self.device = int(device)
self.lr = configs['lr']
self.max_epochs = configs['max-epochs']
self.train_batch_size = configs['train-batch-size']
self.test_batch_size = configs['test-batch-size']
self.n_epochs = 0
self.n_test_segments = configs['n-test-segments']
self.log_dir = os.path.join(LOG_DIR, experiment)
if not os.path.exists(self.log_dir):
os.makedirs(self.log_dir)
self.tboard_writer = tensorboardX.SummaryWriter(log_dir=self.log_dir)
self.checkpoint_dir = os.path.join(CHECKPOINT_DIR, experiment)
if not os.path.exists(self.checkpoint_dir):
os.makedirs(self.checkpoint_dir)
model_id = configs['model-id']
if model_id == 'r3d':
self.model = models.r3d_18(pretrained=True)
elif model_id == 'mc3':
self.model = models.mc3_18(pretrained=True)
elif model_id == 'r2plus1d':
self.model = models.r2plus1d_18(pretrained=True)
else:
raise ValueError('no such model')
# replace the last layer.
self.model.fc = nn.Linear(self.model.fc.in_features,
out_features=breakfast.N_CLASSES,
bias=self.model.fc.bias is not None)
self.model = self.model.cuda(self.device)
self.loss_fn = nn.CrossEntropyLoss().cuda(self.device)
if configs['optim'] == 'adam':
self.optimizer = optim.Adam(self.model.parameters(), lr=self.lr)
elif configs['optim'] == 'sgd':
self.optimizer = optim.SGD(self.model.parameters(),
lr=self.lr,
momentum=configs['momentum'],
nesterov=configs['nesterov'])
else:
raise ValueError('no such optimizer')
if configs['scheduler'] == 'step':
self.scheduler = optim.lr_scheduler.StepLR(
self.optimizer,
step_size=configs['lr-step'],
gamma=configs['lr-decay'])
elif configs['scheduler'] == 'plateau':
self.scheduler = optim.lr_scheduler.ReduceLROnPlateau(
self.optimizer, mode='min', patience=configs['lr-step'])
else:
raise ValueError('no such scheduler')
self._load_checkpoint()
self.frame_stride = configs['frame-stride']
def main(ckpt,
model_name,
mode='rgb',
root='/home/data/vision7/A3D_2.0/frames/',
split_file='A3D_2.0_val.json',
split='val',
with_normal=True,
batch_per_gpu=16,
save_dir=''):
device = torch.device('cuda')
num_gpus = MPI.COMM_WORLD.Get_size()
distributed = False
if num_gpus > 1:
distributed = True
local_rank = MPI.COMM_WORLD.Get_rank() % torch.cuda.device_count()
# logger must be initialized after distributed!
if args.use_wandb:
cfg = {'PROJECT': 'i3d_a3d'}
logger = Logger(
"I3D",
cfg, #convert_to_dict(cfg, []),
project='i3d_a3d',
viz_backend="wandb")
save_dir = os.path.join(save_dir, logger.run_id)
else:
logger = logging.Logger('test_VAR_final')
logger.info("Using {} GPUs".format(num_gpus))
if distributed:
torch.cuda.set_device(local_rank)
host = os.environ[
"MASTER_ADDR"] if "MASTER_ADDR" in os.environ else "127.0.0.1"
torch.distributed.init_process_group(
backend="nccl",
init_method='tcp://{}:12345'.format(host),
rank=MPI.COMM_WORLD.Get_rank(),
world_size=MPI.COMM_WORLD.Get_size())
synchronize()
dataloader = make_dataloader(
root,
split_file,
mode,
model_name=model_name,
seq_len=16, #64,
overlap=15, #32,
phase='val',
max_iters=None,
batch_per_gpu=batch_per_gpu,
num_workers=16,
shuffle=False,
distributed=distributed,
with_normal=with_normal)
# evaluator = ActionClassificationEvaluator(cfg=None,
# dataset=dataloader.dataset,
# split='val',
# mode='accuracy',#'mAP',
# output_dir=save_dir,
# with_normal=with_normal)
# setup the model
# set dropout_keep_prob=0.0 for overfit
if model_name == 'i3d':
if mode == 'flow':
model = InceptionI3d(dataloader.dataset.num_classes,
in_channels=2,
dropout_keep_prob=0.5)
else:
model = InceptionI3d(dataloader.dataset.num_classes,
in_channels=3,
dropout_keep_prob=0.5)
model.replace_logits(dataloader.dataset.num_classes)
elif model_name == 'r3d_18':
model = r3d_18(pretrained=False,
num_classes=dataloader.dataset.num_classes)
elif model_name == 'mc3_18':
model = mc3_18(pretrained=False,
num_classes=dataloader.dataset.num_classes)
elif model_name == 'r2plus1d_18':
model = r2plus1d_18(pretrained=False,
num_classes=dataloader.dataset.num_classes)
elif model_name == 'c3d':
model = C3D(pretrained=False,
num_classes=dataloader.dataset.num_classes)
else:
raise NameError('unknown model name:{}'.format(model_name))
model.load_state_dict(torch.load(ckpt))
# do_test(i3d, dataloader, device, distributed=distributed,logger=logger, output_dir=save_dir, train_iters=0, evaluator=evaluator)
model.to(device)
model.eval()
do_val(model_name,
model,
dataloader,
device,
distributed,
logger,
output_dir=os.path.join('test_output'),
train_iters=0)