def forward(self, input):
# resnet
input_i=input[:,:,0:3,:,:]#[64,4,3,224,224]
input_m=input[:,:,3:5,:,:]#mv
input_r=input[:,:,5:8,:,:]
# print(input_i.shape)
# print(input_m.shape)
# print(input_r.shape)
input_i = input_i.view((-1, ) + input_i.size()[-3:])
input_m = input_m.view((-1, ) + input_m.size()[-3:])
input_r = input_r.view((-1, ) + input_r.size()[-3:])
# a=input_i.view((-1, ) + input_i.size()[-4:])
# print(a.shape)
input_m = self.data_bn_m(input_m)
input_r = self.data_bn_r(input_r)
base_out_i = self.base_model_i(input_i)
base_out_m = self.base_model_m(input_m)
base_out_r = self.base_model_r(input_r)
# print(base_out_i.shape)
# print(base_out_m.shape)
# print(base_out_r.shape)
base_out = torch.cat((base_out_i, base_out_m, base_out_r), 1)#[batchsize*?num_gop, 3072, 1, 1]
print(base_out.shape)
# lstm
args = parser.parse_args()
input_l = base_out.view((args.batch_size,-1) + base_out.size()[1:])#>[batchsize,num_gop, 3072, 1, 1]
print(input_l.shape)
input_l = torch.squeeze(input_l, 3)#[batchsize,num_gop, 3072, 1]
print(input_l.shape)
input_l = torch.squeeze(input_l, 3)#[batchsize,num_gop, 3072]
print(input_l.shape)
_,(input_l,_)=self.lstm(input_l)
lstm_out=input_l[-1,:,:]
print(lstm_out.shape)
lstm_out=lstm_out.view(-1,lstm_out.size()[-1])
print(lstm_out.shape)
lstm_out=self.linear_1(lstm_out)
print(lstm_out.shape)
return lstm_out
def main():
global args
global best_prec1
args = parser.parse_args()
print('Training arguments:')
for k, v in vars(args).items():
print('\t{}: {}'.format(k, v))
if args.data_name == 'ucf101':
num_class = 101
elif args.data_name == 'hmdb51':
num_class = 51
else:
raise ValueError('Unknown dataset ' + args.data_name)
model = Model(num_class,
args.num_segments,
args.representation,
base_model=args.arch)
print(model)
train_loader = torch.utils.data.DataLoader(CoviarDataSet(
args.data_root,
args.data_name,
video_list=args.train_list,
num_segments=args.num_segments,
representation=args.representation,
transform=model.get_augmentation(),
is_train=True,
accumulate=(not args.no_accumulation),
),
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
val_loader = torch.utils.data.DataLoader(CoviarDataSet(
args.data_root,
args.data_name,
video_list=args.test_list,
num_segments=args.num_segments,
representation=args.representation,
transform=torchvision.transforms.Compose([
GroupScale(int(model.scale_size)),
GroupCenterCrop(model.crop_size),
]),
is_train=False,
accumulate=(not args.no_accumulation),
),
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
model = torch.nn.DataParallel(model, device_ids=args.gpus).cuda()
cudnn.benchmark = True
params_dict = dict(model.named_parameters())
params = []
for key, value in params_dict.items():
decay_mult = 0.0 if 'bias' in key else 1.0
if ('module.base_model.conv1' in key or 'module.base_model.bn1' in key
or 'data_bn'
in key) and args.representation in ['mv', 'residual']:
lr_mult = 0.1
elif '.fc.' in key:
lr_mult = 1.0
else:
lr_mult = 0.01
params += [{
'params': value,
'lr': args.lr,
'lr_mult': lr_mult,
'decay_mult': decay_mult
}]
optimizer = torch.optim.Adam(params,
weight_decay=args.weight_decay,
eps=0.001)
criterion = torch.nn.CrossEntropyLoss().cuda()
for epoch in range(args.epochs):
cur_lr = adjust_learning_rate(optimizer, epoch, args.lr_steps,
args.lr_decay)
train(train_loader, model, criterion, optimizer, epoch, cur_lr)
if epoch % args.eval_freq == 0 or epoch == args.epochs - 1:
prec1 = validate(val_loader, model, criterion)
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
if is_best or epoch % SAVE_FREQ == 0:
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
},
is_best,
filename='checkpoint.pth.tar')
import copy
import torch
import torch.nn as nn
import torch.optim as optim
import torch.optim.lr_scheduler as LS
from torch.autograd import Variable
import network
from dataset import get_loader, default_loader
from evaluate import run_eval
from train_options import parser
from util import get_models, init_lstm, set_train, set_eval
from util import prepare_inputs, forward_ctx
args = parser.parse_args()
print(args)
############### Data ###############
train_loader = get_loader(is_train=True,
root=args.train,
mv_dir=args.train_mv,
args=args)
def get_eval_loaders():
# We can extend this dict to evaluate on multiple datasets.
eval_loaders = {
'TVL':
get_loader(is_train=False,
root=args.eval,
def main():
print(torch.cuda.device_count())
global args
global devices
global WRITER
args = parser.parse_args()
global description
description = '%s_bt_%d_seg_%d_%s' % (args.arch, args.batch_size,
args.num_segments, "iframe+mv*0.5")
log_name = './log/%s' % description
WRITER = SummaryWriter(log_name)
print('Training arguments:')
for k, v in vars(args).items():
print('\t{}: {}'.format(k, v))
model = Model(2,
args.num_segments,
args.representation,
base_model=args.arch)
# add continue train from before
if CONTINUE_FROM_LAST:
checkpoint = torch.load(LAST_SAVE_PATH)
# print("model epoch {} best prec@1: {}".format(checkpoint['epoch'], checkpoint['best_prec1']))
print("model epoch {} lowest loss {}".format(checkpoint['epoch'],
checkpoint['loss_min']))
base_dict = {
'.'.join(k.split('.')[1:]): v
for k, v in list(checkpoint['state_dict'].items())
}
loss_min = checkpoint['loss_min']
model.load_state_dict(base_dict)
start_epochs = checkpoint['epoch']
else:
loss_min = 10000
start_epochs = 0
# print(model)
# WRITER.add_graph(model, (torch.randn(10,5, 2, 224, 224),))
devices = [torch.device("cuda:%d" % device) for device in args.gpus]
global DEVICES
DEVICES = devices
train_loader = torch.utils.data.DataLoader(CoviarDataSet(
args.data_root,
video_list=args.train_list,
num_segments=args.num_segments,
representation=args.representation,
is_train=True,
accumulate=(not args.no_accumulation),
),
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=False)
val_loader = torch.utils.data.DataLoader(CoviarDataSet(
args.data_root,
video_list=args.test_list,
num_segments=args.num_segments,
representation=args.representation,
is_train=False,
accumulate=(not args.no_accumulation),
),
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=False)
model = torch.nn.DataParallel(model, device_ids=args.gpus)
model = model.to(devices[0])
cudnn.benchmark = True
params_dict = dict(model.named_parameters())
params = []
for key, value in params_dict.items():
decay_mult = 0.0 if 'bias' in key else 1.0
if ('module.base_model.conv1' in key or 'module.base_model.bn1' in key
or 'data_bn'
in key) and args.representation in ['mv', 'residual']:
lr_mult = 0.1
elif '.fc.' in key:
lr_mult = 1.0
else:
lr_mult = 0.01
params += [{
'params': value,
'lr': args.lr,
'lr_mult': lr_mult,
'decay_mult': decay_mult
}]
if FINETUNE:
optimizer = torch.optim.SGD(params, lr=1e-5, momentum=0.9)
else:
optimizer = torch.optim.Adam(params,
weight_decay=args.weight_decay,
eps=0.001)
criterions = []
siamese_loss = ContrastiveLoss(margin=2.0).to(devices[0])
classifiy_loss = nn.CrossEntropyLoss().to(devices[0])
# classifiy_loss = LabelSmoothingLoss(2,0.1,-1)
criterions.append(siamese_loss)
criterions.append(classifiy_loss)
# try to use ReduceOnPlatue to adjust lr
scheduler = ReduceLROnPlateau(optimizer,
mode='min',
factor=0.2,
patience=20 // args.eval_freq,
verbose=True)
for epoch in range(start_epochs, args.epochs):
# about optimizer
WRITER.add_scalar('Lr/epoch', get_lr(optimizer), epoch)
loss_train_s, loss_train_c = train(train_loader, model, criterions,
optimizer, epoch)
loss_train = WEI_S * loss_train_s + WEI_C * loss_train_c
if epoch % args.eval_freq == 0 or epoch == args.epochs - 1:
loss_val_s, loss_val_c, acc = validate(val_loader, model,
criterions, epoch)
loss_val = WEI_S * loss_val_s + WEI_C * loss_val_c
scheduler.step(loss_val_c)
is_best = (loss_val_c < loss_min)
loss_min = min(loss_val_c, loss_min)
# visualization
WRITER.add_scalar('Accuracy/epoch', acc, epoch)
WRITER.add_scalars('Siamese Loss/epoch', {
'Train': loss_train_s,
'Val': loss_val_s
}, epoch)
WRITER.add_scalars('Classification Loss/epoch', {
'Train': loss_train_c,
'Val': loss_val_c
}, epoch)
WRITER.add_scalars('Combine Loss/epoch', {
'Train': loss_train,
'Val': loss_val
}, epoch)
if is_best or epoch % SAVE_FREQ == 0:
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'loss_min': loss_min,
},
is_best,
filename='checkpoint.pth.tar')
WRITER.close()
def main():
global args
global best_prec1
args = parser.parse_args()
print('Training arguments:')
for k, v in vars(args).items():
print('\t{}: {}'.format(k, v))
if args.data_name == 'ucf101':
num_class = 101
elif args.data_name == 'hmdb51':
num_class = 51
elif args.data_name == 'mine':
num_class = 2
else:
raise ValueError('Unknown dataset ' + args.data_name)
model = Model(num_class,
args.num_segments,
args.representation,
base_model=args.arch)
print(model)
if 'resnet3D' in args.arch:
train_crop_min_ratio = 0.75
train_crop_min_scale = 0.25
mean = [0.4345, 0.4051, 0.3775]
std = [0.2768, 0.2713, 0.2737]
value_scale = 1
train_transform = Compose([
RandomResizedCrop(
model.crop_size, (train_crop_min_scale, 1.0),
(train_crop_min_ratio, 1.0 / train_crop_min_ratio)),
RandomHorizontalFlip(),
ToTensor(),
ScaleValue(value_scale),
Normalize(mean, std)
])
test_trainsform = Compose([
Resize(model.crop_size),
CenterCrop(model.crop_size),
ToTensor(), # range [0, 255] -> [0.0,1.0]
ScaleValue(1),
Normalize(mean, std)
])
train_loader = torch.utils.data.DataLoader(
CoviarDataSet(
args.data_root,
args.data_name,
video_list=args.train_list,
num_segments=args.num_segments,
representation=args.representation,
transform=model.get_augmentation(), #train_transform,
is_train=True,
accumulate=(not args.no_accumulation),
model_name=args.arch),
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True,
worker_init_fn=worker_init_fn)
val_loader = torch.utils.data.DataLoader(
CoviarDataSet(
args.data_root,
args.data_name,
video_list=args.test_list,
num_segments=args.num_segments,
representation=args.representation,
transform=torchvision.transforms.Compose([
GroupScale(int(model.scale_size)),
GroupCenterCrop(model.crop_size)
]), #test_trainsform,
is_train=True,
accumulate=(not args.no_accumulation),
model_name=args.arch),
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True,
worker_init_fn=worker_init_fn)
model = torch.nn.DataParallel(model, device_ids=args.gpus).cuda()
cudnn.benchmark = True
params_dict = dict(model.named_parameters())
params = []
for key, value in params_dict.items():
decay_mult = 0.0 if 'bias' in key else 1.0
if ('module.base_model.conv1' in key or 'module.base_model.bn1' in key
or 'data_bn'
in key) and args.representation in ['mv', 'residual']:
lr_mult = 0.1
elif '.fc.' in key:
lr_mult = 1.0
else:
lr_mult = 0.01
params += [{
'params': value,
'lr': args.lr,
'lr_mult': lr_mult,
'decay_mult': decay_mult
}]
#optimizer = torch.optim.SGD(params, weight_decay=0.001, momentum=0.9, nesterov=False)
#scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=10)
optimizer = torch.optim.Adam(params,
weight_decay=args.weight_decay,
eps=0.001)
criterion = torch.nn.CrossEntropyLoss().cuda()
for epoch in range(args.epochs):
cur_lr = adjust_learning_rate(optimizer, epoch, args.lr_steps,
args.lr_decay)
#cur_lr = get_lr(optimizer)
train(train_loader, model, criterion, optimizer, epoch, cur_lr)
#prec1, prev_val_loss = validate(val_loader, model, criterion)
#scheduler.step(prev_val_loss)
if epoch % args.eval_freq == 0 or epoch == args.epochs - 1:
prec1, _ = validate(val_loader, model, criterion)
# 紀錄訓練歷程
np.savez("train_history/train_history.npz",
loss=np.array(train_loss),
top1=np.array(train_prec),
lr=np.array(train_lr))
np.savez("train_history/valid_history.npz",
loss=np.array(valid_loss),
top1=np.array(valid_prec))
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
if is_best or epoch % SAVE_FREQ == 0:
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
},
is_best,
filename='checkpoint.pth.tar')
def main():
global args
global best_prec1
args = parser.parse_args()
print('Training arguments:')
for k, v in vars(args).items():
print('\t{}: {}'.format(k, v))
if args.data_name == 'ucf101':
num_class = 101
elif args.data_name == 'hmdb51':
num_class = 51
else:
raise ValueError('Unknown dataset ' + args.data_name)
# num_class: total number of classes
# num_segments: number of TSN segments, default=3
# representation: iframe, mv, residual
# base_model: base architecture
model = Model(num_class,
args.num_segments,
args.representation,
base_model=args.arch,
mv_stack_size=args.mv_stack_size)
print(model)
# dataset (Dataset) – dataset from which to load the data.
# batch_size – how many samples per batch to load (default: 1).
# shuffle – set to True to have the data reshuffled at every epoch.
# num_workers – how many subprocesses to use for data loading. 0 means that the data will be loaded in the main process. (default: 0)
# pin_memory – If True, the data loader will copy tensors into CUDA pinned memory before returning them.
train_loader = torch.utils.data.DataLoader(
CoviarDataSet(
args.data_root,
args.data_name,
video_list=args.train_list,
num_segments=args.num_segments,
representation=args.representation,
transform=model.get_augmentation(),
# get_augmentation() =
# GroupMultiScaleCrop + GroupRandomHorizontalFlip
# GroupMultiScaleCrop contains stack mv
# seems np.stack in resize_mv() called in GroupMultiScaleCrop
# has the same effects as Stack() in TSN
# -----------------------
# TSN:
# transform=torchvision.transforms.Compose([
# train_augmentation, # train_augmentation = model.get_augmentation(), same
# Stack(roll=args.arch == 'BNInception'), # this line seems important
# ToTorchFormatTensor(div=args.arch != 'BNInception'),
# normalize, # used for RGBDiff
# ])),
# ----------------------
is_train=True,
accumulate=(not args.no_accumulation),
mv_stack_size=args.mv_stack_size),
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
val_loader = torch.utils.data.DataLoader(
CoviarDataSet(
args.data_root,
args.data_name,
video_list=args.test_list,
num_segments=args.num_segments,
representation=args.representation,
transform=torchvision.transforms.
Compose([ # seems important to stacking
GroupScale(int(model.scale_size)),
GroupCenterCrop(
model.crop_size
), # here they both use model.crop_size (instead of TSN's net.input_size in test_model.py)
]), # this function contains stack
# seems np.stack in resize_mv() called in GroupCenterCrop
# has the same effects as Stack() in TSN
# -----------------------
# TSN:
# transform=torchvision.transforms.Compose([
# GroupScale(int(scale_size)),
# GroupCenterCrop(crop_size),
# Stack(roll=args.arch == 'BNInception'), # this line seems important
# ToTorchFormatTensor(div=args.arch != 'BNInception'),
# normalize,
# ])),
# -----------------------
is_train=False,
accumulate=(not args.no_accumulation),
mv_stack_size=args.mv_stack_size),
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
# parallel gpu setting
model = torch.nn.DataParallel(model, device_ids=args.gpus).cuda()
cudnn.benchmark = True
params_dict = dict(model.named_parameters())
params = []
for key, value in params_dict.items():
decay_mult = 0.0 if 'bias' in key else 1.0
if ('module.base_model.conv1' in key or 'module.base_model.bn1' in key
or 'data_bn'
in key) and args.representation in ['mv', 'residual']:
lr_mult = 0.1
elif '.fc.' in key:
lr_mult = 1.0
else:
lr_mult = 0.01
params += [{
'params': value,
'lr': args.lr,
'lr_mult': lr_mult,
'decay_mult': decay_mult
}]
optimizer = torch.optim.Adam(params,
weight_decay=args.weight_decay,
eps=0.001)
criterion = torch.nn.CrossEntropyLoss().cuda()
for epoch in range(args.epochs):
cur_lr = adjust_learning_rate(optimizer, epoch, args.lr_steps,
args.lr_decay)
train(train_loader, model, criterion, optimizer, epoch, cur_lr)
if epoch % args.eval_freq == 0 or epoch == args.epochs - 1:
prec1 = validate(val_loader, model, criterion)
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
if is_best or epoch % SAVE_FREQ == 0:
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
},
is_best,
filename='checkpoint.pth.tar')
def main():
global args
global best_prec1
args = parser.parse_args()
print('Training arguments:')
for k, v in vars(args).items():
print('\t{}: {}'.format(k, v))
if args.data_name == 'ucf101':
num_class = 101
elif args.data_name == 'hmdb51':
num_class = 51
else:
raise ValueError('Unknown dataset '+ args.data_name)
model = Model(num_class, args.num_segments, args.representation,
base_model=args.arch)
print(model)
train_loader = torch.utils.data.DataLoader(
CoviarDataSet(
args.data_root,
args.data_name,
video_list=args.train_list,
num_segments=args.num_segments,
representation=args.representation,
transform=model.get_augmentation(),
is_train=True,
accumulate=(not args.no_accumulation),
),
batch_size=args.batch_size, shuffle=True,
num_workers=args.workers, pin_memory=True)
val_loader = torch.utils.data.DataLoader(
CoviarDataSet(
args.data_root,
args.data_name,
video_list=args.test_list,
num_segments=args.num_segments,
representation=args.representation,
transform=torchvision.transforms.Compose([
GroupScale(int(model.scale_size)),
GroupCenterCrop(model.crop_size),
]),
is_train=False,
accumulate=(not args.no_accumulation),
),
batch_size=args.batch_size, shuffle=False,
num_workers=args.workers, pin_memory=True)
model = torch.nn.DataParallel(model, device_ids=args.gpus).cuda()
cudnn.benchmark = True
params_dict = dict(model.named_parameters())
params = []
for key, value in params_dict.items():
decay_mult = 0.0 if 'bias' in key else 1.0
if ('module.base_model.conv1' in key
or 'module.base_model.bn1' in key
or 'data_bn' in key) and args.representation in ['mv', 'residual']:
lr_mult = 0.1
elif '.fc.' in key:
lr_mult = 1.0
else:
lr_mult = 0.01
params += [{'params': value, 'lr': args.lr, 'lr_mult': lr_mult, 'decay_mult': decay_mult}]
optimizer = torch.optim.Adam(
params,
weight_decay=args.weight_decay,
eps=0.001)
criterion = torch.nn.CrossEntropyLoss().cuda()
for epoch in range(args.epochs):
cur_lr = adjust_learning_rate(optimizer, epoch, args.lr_steps, args.lr_decay)
train(train_loader, model, criterion, optimizer, epoch, cur_lr)
if epoch % args.eval_freq == 0 or epoch == args.epochs - 1:
prec1 = validate(val_loader, model, criterion)
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
if is_best or epoch % SAVE_FREQ == 0:
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
},
is_best,
filename='checkpoint.pth.tar')
def main():
# loading input arguments for training
global args
global best_prec1
global start_epoch
start_epoch = 0
args = parser.parse_args()
print('Training arguments:')
for k, v in vars(args).items():
print('\t{}: {}'.format(k, v))
if args.data_name == 'ucf101':
num_class = 101
elif args.data_name == 'hmdb51':
num_class = 51
elif args.data_name == 'kinetics400':
num_class = 400
else:
raise ValueError('Unknown dataset ' + args.data_name)
# define the model architecture
model = Model(num_class, args.num_segments, args.representation,
base_model=args.arch,
new_length=args.new_length,
use_databn=args.use_databn,
gen_flow_or_delta=args.gen_flow_or_delta,
gen_flow_ds_factor=args.gen_flow_ds_factor,
arch_estimator=args.arch_estimator,
arch_d=args.arch_d,
att=args.att)
print(model)
# load the pre-trained model
if args.weights is not None:
checkpoint = torch.load(args.weights, map_location=lambda storage, loc: storage)
print("model epoch {} best prec@1: {}".format(checkpoint['epoch'], checkpoint['best_prec1']))
base_dict = {'.'.join(k.split('.')[1:]): v for k,v in list(checkpoint['state_dict'].items())}
model.load_state_dict(base_dict, strict=False)
# define the data loader for reading training data
train_loader = torch.utils.data.DataLoader(
CoviarDataSet(
args.data_root,
args.flow_root,
args.data_name,
video_list=args.train_list,
num_segments=args.num_segments,
representation=args.representation,
new_length=args.new_length,
flow_ds_factor=args.flow_ds_factor,
upsample_interp=args.upsample_interp,
transform=model.get_augmentation(),
is_train=True,
accumulate=(not args.no_accumulation),
gop=args.gop,
flow_folder=args.data_flow,
mv_minmaxnorm=args.mv_minmaxnorm,
),
batch_size=args.batch_size, shuffle=True,
num_workers=args.workers, pin_memory=True)
# define the data loader for reading val data
val_loader = torch.utils.data.DataLoader(
CoviarDataSet(
args.data_root,
args.flow_root,
args.data_name,
video_list=args.test_list,
num_segments=args.num_segments,
representation=args.representation,
new_length=args.new_length,
flow_ds_factor=args.flow_ds_factor,
upsample_interp=args.upsample_interp,
transform=torchvision.transforms.Compose([
GroupScale(int(model.scale_size)),
GroupCenterCrop(model.crop_size),
]),
is_train=False,
accumulate=(not args.no_accumulation),
gop=args.gop,
flow_folder=args.data_flow,
mv_minmaxnorm=args.mv_minmaxnorm,
),
batch_size=args.batch_size, shuffle=False,
num_workers=args.workers, pin_memory=True)
model = torch.nn.DataParallel(model, device_ids=args.gpus).cuda(args.gpus[0])
cudnn.benchmark = True
# define optimizer and specify the corresponding parameters
params_dict = dict(model.named_parameters())
params_cls = []
params_gf = []
params_d = []
for key, value in params_dict.items():
if 'base_model' in key:
decay_mult = 0.0 if 'bias' in key else 1.0
lr_mult = args.lr_cls_mult # for cls, just finetune. if '.fc.' in key: lr_mult = 1.0
params_cls += [{'params': value, 'lr': args.lr, 'lr_mult': lr_mult, 'decay_mult': decay_mult}]
if 'gen_flow_model' in key:
decay_mult = 0.0 if 'bias' in key else 1.0
lr_mult = args.lr_mse_mult
params_gf += [{'params': value, 'lr': args.lr, 'lr_mult': lr_mult, 'decay_mult': decay_mult}]
if 'discriminator' in key:
decay_mult = 0.0 if 'bias' in key else 1.0
lr_mult = args.lr_d_mult
params_d += [{'params': value, 'lr': args.lr, 'lr_mult': lr_mult, 'decay_mult': decay_mult}]
optimizer_cls = torch.optim.Adam(
params_cls,
weight_decay=args.weight_decay,
eps=0.001)
optimizer_gf = torch.optim.Adam(
params_gf,
weight_decay=args.weight_decay,
eps=0.001)
optimizer_d = torch.optim.Adam(
params_d,
weight_decay=args.weight_decay,
eps=0.001)
# resume training from previous checkpoint
if args.resume is not None:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume, map_location=lambda storage, loc: storage)
start_epoch = checkpoint['epoch']
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
if 'optimizer_cls' in checkpoint.keys():
optimizer_cls.load_state_dict(checkpoint['optimizer_cls'])
optimizer_gf.load_state_dict(checkpoint['optimizer_gf'])
optimizer_d.load_state_dict(checkpoint['optimizer_d'])
def load_opt_update_cuda(optimizer, cuda_id):
for state in optimizer.state.values():
for k, v in state.items():
if torch.is_tensor(v):
state[k] = v.cuda(cuda_id)
load_opt_update_cuda(optimizer_cls, args.gpus[0])
load_opt_update_cuda(optimizer_gf, args.gpus[0])
load_opt_update_cuda(optimizer_d, args.gpus[0])
print("=> loaded checkpoint '{}' (epoch {})"
.format(args.resume, checkpoint['epoch']))
# define several loss functions
criterion = torch.nn.CrossEntropyLoss().cuda(args.gpus[0])
if args.loss_mse == 'MSELoss':
criterion_mse = torch.nn.MSELoss().cuda(args.gpus[0])
elif args.loss_mse == 'SmoothL1Loss':
criterion_mse = torch.nn.SmoothL1Loss().cuda(args.gpus[0])
elif args.loss_mse == 'L1':
criterion_mse = torch.nn.L1Loss().cuda(args.gpus[0])
# finally done with setup and start to train model
for epoch in range(start_epoch, args.epochs):
# determine the learning rate for the current epoch
cur_lr_cls = adjust_learning_rate(optimizer_cls, epoch, args.lr_steps, args.lr_decay) #, freeze=True, epoch_thre=args.epoch_thre)
cur_lr_gf = adjust_learning_rate(optimizer_gf, epoch, args.lr_steps, args.lr_decay)
cur_lr_d = adjust_learning_rate(optimizer_d, epoch, args.lr_steps, args.lr_decay)
# perform training
train(train_loader, model, criterion, criterion_mse, optimizer_cls,
optimizer_gf, optimizer_d, epoch, cur_lr_cls, cur_lr_gf, cur_lr_d, args.lr_cls, args.lr_adv_g, args.lr_adv_d, args.lr_mse, args.att)
# perform validation if needed
if epoch % args.eval_freq == 0 or epoch == args.epochs - 1:
prec1 = validate(val_loader, model, criterion, criterion_mse, args.lr_cls, args.lr_adv_g, args.lr_adv_d, args.lr_mse, args.att)
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
if is_best or epoch % SAVE_FREQ == 0:
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer_cls': optimizer_cls.state_dict(),
'optimizer_gf': optimizer_gf.state_dict(),
'optimizer_d': optimizer_d.state_dict(),
},
is_best,
filename='checkpoint.pth.tar')
def main():
print(torch.cuda.device_count())
global args
global devices
global WRITER
args = parser.parse_args()
global description
description = 'bt_%d_seg_%d_%s' % (args.batch_size * ACCUMU_STEPS,
args.num_segments, "finetune_from_vcdb")
log_name = r'/home/sjhu/projects/compressed_video_compare/imqfusion/log/%s' % description
WRITER = SummaryWriter(log_name)
print('Training arguments:')
for k, v in vars(args).items():
print('\t{}: {}'.format(k, v))
model = Model(2,
args.num_segments,
args.representation,
base_model=args.arch)
# add continue train from before
if CONTINUE_FROM_LAST:
checkpoint = torch.load(LAST_SAVE_PATH)
# print("model epoch {} best prec@1: {}".format(checkpoint['epoch'], checkpoint['best_prec1']))
print("model epoch {} lowest loss {}".format(checkpoint['epoch'],
checkpoint['loss_min']))
base_dict = {
'.'.join(k.split('.')[1:]): v
for k, v in list(checkpoint['state_dict'].items())
}
loss_min = checkpoint['loss_min']
model.load_state_dict(base_dict)
start_epochs = checkpoint['epoch']
else:
loss_min = 10000
start_epochs = 0
devices = [torch.device("cuda:%d" % device) for device in args.gpus]
global DEVICES
DEVICES = devices
# deal the unbalance between pos and neg samples
train_dataset = CoviarDataSet(
args.data_root,
video_list=args.train_list,
num_segments=args.num_segments,
is_train=True,
)
target = train_dataset._labels_list
class_sample_count = torch.tensor([(target == t).sum()
for t in np.unique(target)])
weight = 1. / class_sample_count.float()
samples_weights = weight[target]
train_sampler = WeightedRandomSampler(samples_weights, len(train_dataset),
True)
train_loader = torch.utils.data.DataLoader(CoviarDataSet(
args.data_root,
video_list=args.train_list,
num_segments=args.num_segments,
is_train=True,
),
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True,
sampler=train_sampler)
val_loader = torch.utils.data.DataLoader(CoviarDataSet(
args.data_root,
video_list=args.test_list,
num_segments=args.num_segments,
is_train=False,
),
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
model = torch.nn.DataParallel(model, device_ids=args.gpus)
model = model.to(devices[0])
cudnn.benchmark = True
params_dict = dict(model.named_parameters())
params = []
for key, value in params_dict.items():
decay_mult = 0.0 if 'bias' in key else 1.0
if 'module.fc' in key:
params += [{
'params': [value],
'lr': args.lr * 10,
'decay_mult': decay_mult
}]
elif 'module.fusion' in key:
params += [{
'params': [value],
'lr': args.lr * 10,
'decay_mult': decay_mult
}]
elif 'module.mvnet' in key:
params += [{
'params': [value],
'lr': args.lr * 10,
'decay_mult': decay_mult
}]
else:
params += [{
'params': [value],
'lr': args.lr * 1,
'decay_mult': decay_mult
}]
# loss_weights = torch.FloatTensor([1.01,1])
optimizer = torch.optim.SGD(params, lr=args.lr, momentum=0.9)
criterions = []
siamese_loss = ContrastiveLoss(margin=2.0).to(devices[0])
classifiy_loss = nn.CrossEntropyLoss().to(devices[0])
# classifiy_loss = LabelSmoothingLoss(2,0.1,-1)
criterions.append(siamese_loss)
criterions.append(classifiy_loss)
# try to use ReduceOnPlatue to adjust lr
# scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.2, patience=20 // args.eval_freq, verbose=True)
scheduler = WarmStartCosineAnnealingLR(optimizer,
T_max=args.epochs,
T_warm=10)
for epoch in range(start_epochs, args.epochs):
# about optimizer:
WRITER.add_scalar('Lr/epoch', get_lr(optimizer), epoch)
loss_train_s, loss_train_c = train(train_loader, model, criterions,
optimizer, epoch)
loss_train = WEI_S * loss_train_s + WEI_C * loss_train_c
scheduler.step(epoch)
if epoch % EVAL_FREQ == 0 or epoch == args.epochs - 1:
loss_val_s, loss_val_c, acc, report = validate(
val_loader, model, criterions, epoch)
loss_val = WEI_S * loss_val_s + WEI_C * loss_val_c
is_best = (loss_val_c < loss_min)
loss_min = min(loss_val_c, loss_min)
# visualization
WRITER.add_text(tag='Classification Report',
text_string=report,
global_step=epoch)
WRITER.add_scalar('Accuracy/epoch', acc, epoch)
WRITER.add_scalars('Siamese Loss/epoch', {
'Train': loss_train_s,
'Val': loss_val_s
}, epoch)
WRITER.add_scalars('Classification Loss/epoch', {
'Train': loss_train_c,
'Val': loss_val_c
}, epoch)
WRITER.add_scalars('Combine Loss/epoch', {
'Train': loss_train,
'Val': loss_val
}, epoch)
if is_best or epoch % SAVE_FREQ == 0:
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'loss_min': loss_min,
},
is_best,
filename='checkpoint.pth.tar')
WRITER.close()
def main():
global args
best_prec = 0
args = parser.parse_args()
# Data Transform and data loading
traindir = os.path.join(args.data, 'train_data')
valdir = os.path.join(args.data, 'valid_data')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.339, 0.224, 0.225])
transform = (transforms.Compose([
transforms.Resize(224),
transforms.CenterCrop(224),
transforms.ToTensor(), normalize
]),
transforms.Compose([
transforms.Resize(224),
transforms.CenterCrop(224),
transforms.ToTensor()
]))
train_dataset = dataset.loadedDataset(traindir, transform)
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
val_loader = torch.utils.data.DataLoader(dataset.loadedDataset(
valdir, transform),
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
if os.path.exists(args.model):
# load existing model
model_info = torch.load(args.model)
print("==> loading existing model '{}' ".format(model_info['arch']))
original_model = models.__dict__[model_info['arch']](pretrained=False)
model = LSTMModel(original_model, model_info['arch'],
model_info['num_classes'], model_info['lstm_layers'],
model_info['hidden_size'], model_info['fc_size'])
print(model)
model.cuda()
model.load_state_dict(model_info['state_dict'])
best_prec = model_info['best_prec']
cur_epoch = model_info['epoch']
else:
# load and create model
print("==> creating model '{}' ".format(args.arch))
original_model = models.__dict__[args.arch](pretrained=True)
model = LSTMModel(original_model, args.arch,
len(train_dataset.classes), args.lstm_layers,
args.hidden_size, args.fc_size)
print(model)
model.cuda()
cur_epoch = 0
# loss criterion and optimizer
criterion = nn.CrossEntropyLoss(reduction='none')
criterion = criterion.cuda()
if args.optim == 'sgd':
optimizer = torch.optim.SGD([{
'params': model.fc_pre.parameters()
}, {
'params': model.rnn.parameters()
}, {
'params': model.fc.parameters()
}],
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
elif args.optim == 'rmsprop':
optimizer = torch.optim.RMSprop([{
'params': model.fc_pre.parameters()
}, {
'params': model.rnn.parameters()
}, {
'params': model.fc.parameters()
}],
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
elif args.optim == 'adam':
optimizer = torch.optim.Adam([{
'params': model.fc_pre.parameters()
}, {
'params': model.rnn.parameters()
}, {
'params': model.fc.parameters()
}],
lr=args.lr,
weight_decay=args.weight_decay)
# Training on epochs
for epoch in range(cur_epoch, args.epochs):
optimizer = adjust_learning_rate(optimizer, epoch)
print(
"---------------------------------------------------Training---------------------------------------------------"
)
# train on one epoch
train(train_loader, model, criterion, optimizer, epoch)
print(
"--------------------------------------------------Validation--------------------------------------------------"
)
# evaluate on validation set
prec1, prec5 = validate(val_loader, model, criterion)
print("------Validation Result------")
print(" Top1 accuracy: {prec: .2f} %".format(prec=prec1.item()))
print(" Top5 accuracy: {prec: .2f} %".format(prec=prec5.item()))
print("-----------------------------")
# remember best top1 accuracy and save checkpoint
is_best = prec1 > best_prec
best_prec = max(prec1, best_prec)
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'num_classes': len(train_dataset.classes),
'lstm_layers': args.lstm_layers,
'hidden_size': args.hidden_size,
'fc_size': args.fc_size,
'state_dict': model.state_dict(),
'best_prec': best_prec,
'optimizer': optimizer.state_dict(),
}, is_best)
