def get_pretrained_model(args):
"""
Load a pretrained model according to the arguments
:param args: object, a config parser object containing the model-related information
:return: DataParallel, a model with the trained weights and bias
"""
print("Loading pretrained model...")
cuda = True if torch.cuda.is_available() else False
arch = '{}-{}'.format(args.model, args.model_depth)
model, _ = generate_model(args)
cur_dir = os.path.dirname(os.path.realpath(__file__))
split_path = os.path.split(args.resume_path)
resume_path = os.path.join(cur_dir, *split_path)
if resume_path:
print('Loading checkpoint {}'.format(resume_path))
if cuda:
checkpoint = torch.load(resume_path)
else:
checkpoint = torch.load(resume_path, map_location=torch.device('cpu'))
assert arch == checkpoint['arch']
model.load_state_dict(checkpoint['state_dict'])
model.eval()
if cuda:
model = model.cuda()
return model
def main():
opts = parse_opts()
print(opts)
opts.arch = '{}-{}'.format(opts.model, opts.model_depth)
torch.manual_seed(opts.manual_seed)
opts.input_channels = 3
# initialize model
print("Loading model {}.".format(opts.arch))
model, _ = generate_model(opts)
resume_path = opts.resume_path1
# resuming model {}
print("Resuming model {}.".format(resume_path))
checkpoint = torch.load(resume_path)
assert opts.arch == checkpoint['arch']
model.load_state_dict(checkpoint['state_dict'])
model.eval()
model = model.cuda()
# initial detection model
device = select_device()
cur_dir = os.path.dirname(os.path.realpath(__file__))
weights = os.path.join(cur_dir, 'detection/best.pt')
detect_model = attempt_load(weights, map_location=device)
if device.type != 'cpu':
detect_model.half()
print("Processing case validation data.")
val_data = globals()['{}'.format(opts.dataset)](train=2,
opt=opts,
detect_model=detect_model)
val_dataloader = DataLoader(val_data,
batch_size=1,
shuffle=True,
num_workers=opts.n_workers,
pin_memory=True,
drop_last=False)
print("Length of case validation dataloder = {}.".format(
len(val_dataloader)))
validation(opts, model, val_dataloader)
print("Processing control validation data.")
val_data_2 = globals()['{}'.format(opts.dataset)](
train=3, opt=opts, detect_model=detect_model)
val_dataloader_2 = DataLoader(val_data_2,
batch_size=1,
shuffle=False,
num_workers=opts.n_workers,
pin_memory=True,
drop_last=False)
print("Length of control validation_2 data = ", len(val_data_2))
validation(opts, model, val_dataloader_2)
def test():
print("Using GPU {}".format(device_id))
print("Device: {}".format(device))
correct = 0
opt = parse_opts()
print(opt)
opt.arch = "{}-{}".format(opt.model, opt.model_depth)
wandb.init(project="MARS", name="MARS test", notes=str(opt))
print("Preprocessing validation data ...")
data = globals()["{}_test".format(opt.dataset)](split=opt.split, train=0, opt=opt)
print("Length of validation data = ", len(data))
print("Preparing datatloaders ...")
val_dataloader = DataLoader(
data,
batch_size=1,
shuffle=False,
num_workers=opt.n_workers,
pin_memory=False,
drop_last=False,
)
print("Length of validation datatloader = ", len(val_dataloader))
result_path = "{}/{}/".format(opt.result_path, opt.dataset)
if not os.path.exists(result_path):
os.makedirs(result_path)
# define the model
print("Loading models... ", opt.model, opt.model_depth)
model1, parameters1 = generate_model(opt)
model1 = model1.to(device)
# if testing RGB+Flow streams change input channels
if not opt.only_RGB:
opt.input_channels = 2
model2, parameters2 = generate_model(opt)
model2 = model2.to(device)
if opt.resume_path1:
print("loading checkpoint {}".format(opt.resume_path1))
checkpoint = torch.load(opt.resume_path1)
assert opt.arch == checkpoint["arch"]
model1.load_state_dict(checkpoint["state_dict"])
if opt.resume_path2:
print("loading checkpoint {}".format(opt.resume_path2))
checkpoint = torch.load(opt.resume_path2)
assert opt.arch == checkpoint["arch"]
model2.load_state_dict(checkpoint["state_dict"])
model1.eval()
model2.eval()
wandb.watch(model1)
wandb.watch(model2)
accuracies = AverageMeter()
if opt.log:
if opt.only_RGB:
f = open(
"test_RGB_MARS_{}{}_{}_{}_{}.txt".format(
opt.model,
opt.model_depth,
opt.dataset,
opt.split,
opt.sample_duration,
),
"w+",
)
else:
f = open(
"test_RGB_Flow_{}{}_{}_{}_{}.txt".format(
opt.model,
opt.model_depth,
opt.dataset,
opt.split,
opt.sample_duration,
),
"w+",
)
f.write(str(opt))
f.write("\n")
f.flush()
with torch.no_grad():
for i, (clip, label) in enumerate(val_dataloader):
clip = clip.to(device)
label = label.to(device)
clip = torch.squeeze(clip)
if opt.only_RGB:
inputs = torch.Tensor(
int(clip.shape[1] / opt.sample_duration),
3,
opt.sample_duration,
opt.sample_size,
opt.sample_size,
)
for k in range(inputs.shape[0]):
inputs[k, :, :, :, :] = clip[
:, k * opt.sample_duration : (k + 1) * opt.sample_duration, :, :
]
inputs_var1 = Variable(inputs)
inputs_var2 = Variable(inputs)
else:
RGB_clip = clip[0:3, :, :, :]
Flow_clip = clip[3:, :, :, :]
inputs1 = torch.Tensor(
int(RGB_clip.shape[1] / opt.sample_duration),
3,
opt.sample_duration,
opt.sample_size,
opt.sample_size,
).to(device)
inputs2 = torch.Tensor(
int(Flow_clip.shape[1] / opt.sample_duration),
2,
opt.sample_duration,
opt.sample_size,
opt.sample_size,
).to(device)
for k in range(inputs1.shape[0]):
inputs1[k, :, :, :, :] = RGB_clip[
:, k * opt.sample_duration : (k + 1) * opt.sample_duration, :, :
]
inputs2[k, :, :, :, :] = Flow_clip[
:, k * opt.sample_duration : (k + 1) * opt.sample_duration, :, :
]
inputs_var1 = Variable(inputs1).to(device)
inputs_var2 = Variable(inputs2).to(device)
outputs_var1 = model1(inputs_var1).to(device)
outputs_var2 = model2(inputs_var2).to(device)
outputs_var = torch.mean(
torch.cat((outputs_var1, outputs_var2), dim=0), dim=0
).unsqueeze(0)
pred5 = np.array(outputs_var.topk(5, 1, True)[1].cpu().data[0])
if str(pred5[0]) in total_predictions:
total_predictions[str(pred5[0])] += 1
else:
total_predictions[str(pred5[0])] = 1
acc = float(pred5[0] == label[0])
if str(pred5[0]) in class_map:
pred_list.append(class_map[str(pred5[0])])
else:
pred_list.append("other")
true_list.append(class_map[str(int(label[0]))])
accuracies.update(acc, 1)
float_acc_log = {"Average Accuracy": accuracies.avg}
wandb.log(float_acc_log)
if int(pred5[0]) == int(label[0]):
prediction = "correct"
correct += 1
else:
prediction = "wrong"
line = (
"Video["
+ str(i)
+ "] : \t top5 "
+ str(pred5)
+ "\t top1 = "
+ str(pred5[0])
+ "\t true = "
+ str(label[0])
+ "\t video = "
+ str(accuracies.avg)
)
if prediction == "correct":
line2 = "Prediction correct!"
else:
line2 = "Prediction wrong"
line2 += " - Correct predictions: {}/{}\n".format(correct, i + 1)
print(line)
print(line2)
if opt.log:
f.write(line + "\n")
f.flush()
print(total_predictions)
cm = confusion_matrix(true_list, pred_list)
df_cm = pd.DataFrame(
cm, index=["clapping", "jogging", "other", "punching", "push up"], columns=["clapping", "jogging", "other", "punching", "push up"]
)
plt.figure(figsize = (15,10))
sn.set(font_scale=1.4) # for label size
sn.heatmap(df_cm, annot=True, annot_kws={"size": 16}) # font size
plt.savefig('cm.png')
wandb.save('cm.png')
print("Video accuracy = ", accuracies.avg)
line = "Video accuracy = " + str(accuracies.avg) + "\n"
if opt.log:
f.write(line)
opt.model, opt.model_depth, opt.ft_begin_index,
opt.output_layers[0], opt.MARS_alpha)),
['epoch', 'loss', 'acc'], opt.MARS_resume_path,
opt.begin_epoch)
if opt.pretrain_path != '' and opt.dataset != 'Kinetics':
opt.weight_decay = 1e-5
opt.learning_rate = 0.001
if opt.nesterov: dampening = 0
else: dampening = opt.dampening
# define the model
print("Loading MARS model... ", opt.model, opt.model_depth)
opt.input_channels = 3
model_MARS, parameters_MARS = generate_model(opt)
print("Loading Flow model... ", opt.model, opt.model_depth)
opt.input_channels = 2
if opt.pretrain_path != '':
opt.pretrain_path = ''
if opt.dataset == 'HMDB51':
opt.n_classes = 51
elif opt.dataset == 'Kinetics':
opt.n_classes = 400
elif opt.dataset == 'UCF101':
opt.n_classes = 101
model_Flow, parameters_Flow = generate_model(opt)
criterion_MARS = nn.CrossEntropyLoss().cuda()
def test():
opt = parse_opts()
opt.modality = 'RGB_Flow'
opt.dataset = 'UCF101'
opt.model = 'tc3d'
opt.model_depth = 101
opt.log = 1
opt.only_RGB = False
opt.n_classes = 101
opt.batch_size = 32
opt.sample_duration = 12
#opt.resume_path2 = "/home/haoyu/Documents/6_ECCV_competition/vipriors-challenges-toolkit-master/action-recognition/baselines/MARS-master/results/server_results/UCF101/with validation/Flow_train_batch64_sample112_clip12_modeltc3d101_epoch152_acc0.7428977272727273.pth"
opt.resume_path1 = "/home/haoyu/Documents/6_ECCV_competition/vipriors-challenges-toolkit-master/action-recognition/baselines/MARS-threestream_hybrid/results/RGB_Flow_train_batch64_sample112_clip12_modeltc3d50_epoch102_acc0.6988636363636364_withvalid_1.pth"
opt.resume_path2 = "/home/haoyu/Documents/6_ECCV_competition/vipriors-challenges-toolkit-master/action-recognition/baselines/MARS-threestream_hybrid/results/Flow_train_batch64_sample112_clip12_modeltc3d101_epoch152_acc0.7428977272727273.pth"
#opt.resume_path1 = "/home/haoyu/Documents/6_ECCV_competition/vipriors-challenges-toolkit-master/action-recognition/baselines/MARS-master/results/server_results/UCF101/with validation/RGB_train_batch64_sample112_clip12_modeltc3d101_epoch118_acc0.5553977272727273.pth"
opt.frame_dir = '/home/haoyu/Documents/6_ECCV_competition/vipriors-challenges-toolkit-master/action-recognition/data/mod-ucf101/rp/'
opt.offrame_dir = '/home/haoyu/Documents/6_ECCV_competition/vipriors-challenges-toolkit-master/action-recognition/data/mod-ucf101/opticalflows/'
opt.annotation_path = "/home/haoyu/Documents/6_ECCV_competition/vipriors-challenges-toolkit-master/action-recognition/data/mod-ucf101/annotations/"
opt.result_path = "results/"
opt.rpframe_dir = ''
print(opt)
opt.arch = '{}-{}'.format(opt.model, opt.model_depth)
print("Preprocessing validation data ...")
data = globals()['{}_test'.format(opt.dataset)](split=opt.split,
train=0,
opt=opt)
print("Length of validation data = ", len(data))
print("Preparing datatloaders ...")
val_dataloader = DataLoader(data,
batch_size=1,
shuffle=False,
num_workers=opt.n_workers,
pin_memory=True,
drop_last=False)
print("Length of validation datatloader = ", len(val_dataloader))
result_path = "{}/{}/".format(opt.result_path, opt.dataset)
if not os.path.exists(result_path):
os.makedirs(result_path)
# define the model
print("Loading models... ", opt.model, opt.model_depth)
opt.model_depth = 50
opt.input_channels = 5
model1, parameters1 = generate_model(opt)
# if testing RGB+Flow streams change input channels
if not opt.only_RGB:
opt.input_channels = 2
opt.model_depth = 101
opt.input_channels = 2
model2, parameters2 = generate_model(opt)
if opt.resume_path1:
print('loading checkpoint {}'.format(opt.resume_path1))
checkpoint = torch.load(opt.resume_path1)
# assert opt.arch == checkpoint['arch']
model1.load_state_dict(checkpoint['state_dict'])
if opt.resume_path2:
print('loading checkpoint {}'.format(opt.resume_path2))
checkpoint = torch.load(opt.resume_path2)
# assert opt.arch == checkpoint['arch']
model2.load_state_dict(checkpoint['state_dict'])
model1.eval()
model2.eval()
accuracies1 = AverageMeter()
accuracies2 = AverageMeter()
accuracies3 = AverageMeter()
accuracies4 = AverageMeter()
accuracies5 = AverageMeter()
accuracies6 = AverageMeter()
accuracies7 = AverageMeter()
accuracies8 = AverageMeter()
accuracies9 = AverageMeter()
if opt.log:
f = open(
os.path.join(
result_path,
"test_{}{}_{}_{}_{}_{}.txt".format(opt.model, opt.model_depth,
opt.dataset, opt.split,
opt.modality,
opt.sample_duration)), 'w+')
f.write(str(opt))
f.write('\n')
f.flush()
submission = open(
os.path.join(
result_path, "submit_hybrid_{}{}_{}_{}_{}_{}.txt".format(
opt.model, opt.model_depth, opt.dataset, opt.split,
opt.modality, opt.sample_duration)), 'w+')
RGB_candidate_top5 = []
Flow_candidate_top5 = []
submission1 = []
submission2 = []
submission3 = []
submission4 = []
submission5 = []
submission6 = []
submission6 = []
submission7 = []
submission8 = []
submission9 = []
with torch.no_grad():
for i, (clip, videoID, label) in tqdm(enumerate(val_dataloader)):
clip = torch.squeeze(clip)
if opt.only_RGB:
inputs = torch.Tensor(int(clip.shape[1] / opt.sample_duration),
3, opt.sample_duration, opt.sample_size,
opt.sample_size)
for k in range(inputs.shape[0]):
inputs[k, :, :, :, :] = clip[:, k *
opt.sample_duration:(k + 1) *
opt.sample_duration, :, :]
inputs_var1 = Variable(inputs)
inputs_var2 = Variable(inputs)
else:
RGB_clip = clip
Flow_clip = clip[3:, :, :, :]
inputs1 = torch.Tensor(
int(RGB_clip.shape[1] / opt.sample_duration), 5,
opt.sample_duration, opt.sample_size, opt.sample_size)
inputs2 = torch.Tensor(
int(Flow_clip.shape[1] / opt.sample_duration), 2,
opt.sample_duration, opt.sample_size, opt.sample_size)
for k in range(inputs1.shape[0]):
inputs1[
k, :, :, :, :] = RGB_clip[:, k *
opt.sample_duration:(k + 1) *
opt.sample_duration, :, :]
inputs2[
k, :, :, :, :] = Flow_clip[:, k *
opt.sample_duration:(k +
1) *
opt.sample_duration, :, :]
inputs_var1 = Variable(inputs1)
inputs_var2 = Variable(inputs2)
outputs_var1 = model1(inputs_var1)
outputs_var2 = model2(inputs_var2)
# print(outputs_var1.shape)
#outputs_var_0_4 = outputs_var1 + outputs_var2*3/2
outputs_var1 = model1(inputs_var1)
outputs_var2 = model2(inputs_var2)
# print(outputs_var1.shape)
outputs_var_0_1 = outputs_var1 + outputs_var2 * 7 / 3
outputs_var_0_2 = outputs_var1 + outputs_var2 * 6.75 / 3.75
outputs_var_0_3 = outputs_var1 + outputs_var2 * 6.5 / 3.5
outputs_var_0_4 = outputs_var1 + outputs_var2 * 6.25 / 3.75
outputs_var_0_5 = outputs_var1 + outputs_var2 * 6 / 4
outputs_var_0_6 = outputs_var1 + outputs_var2 * 5.75 / 4.25
outputs_var_0_7 = outputs_var1 + outputs_var2 * 5.5 / 4.5
outputs_var_0_8 = outputs_var1 + outputs_var2 * 5.25 / 4.75
outputs_var_0_9 = outputs_var1 + outputs_var2 * 5 / 5
pred5_1 = np.array(
torch.mean(outputs_var_0_1, dim=0,
keepdim=True).topk(5, 1, True)[1].cpu().data[0])
pred5_2 = np.array(
torch.mean(outputs_var_0_2, dim=0,
keepdim=True).topk(5, 1, True)[1].cpu().data[0])
pred5_3 = np.array(
torch.mean(outputs_var_0_3, dim=0,
keepdim=True).topk(5, 1, True)[1].cpu().data[0])
pred5_4 = np.array(
torch.mean(outputs_var_0_4, dim=0,
keepdim=True).topk(5, 1, True)[1].cpu().data[0])
pred5_5 = np.array(
torch.mean(outputs_var_0_5, dim=0,
keepdim=True).topk(5, 1, True)[1].cpu().data[0])
pred5_6 = np.array(
torch.mean(outputs_var_0_6, dim=0,
keepdim=True).topk(5, 1, True)[1].cpu().data[0])
pred5_7 = np.array(
torch.mean(outputs_var_0_7, dim=0,
keepdim=True).topk(5, 1, True)[1].cpu().data[0])
pred5_8 = np.array(
torch.mean(outputs_var_0_8, dim=0,
keepdim=True).topk(5, 1, True)[1].cpu().data[0])
pred5_9 = np.array(
torch.mean(outputs_var_0_9, dim=0,
keepdim=True).topk(5, 1, True)[1].cpu().data[0])
# print(outputs_var)
acc1 = float(pred5_1[0] == label[0])
acc2 = float(pred5_2[0] == label[0])
acc3 = float(pred5_3[0] == label[0])
acc4 = float(pred5_4[0] == label[0])
acc5 = float(pred5_5[0] == label[0])
acc6 = float(pred5_6[0] == label[0])
acc7 = float(pred5_7[0] == label[0])
acc8 = float(pred5_8[0] == label[0])
acc9 = float(pred5_9[0] == label[0])
accuracies1.update(acc1, 1)
accuracies2.update(acc2, 1)
accuracies3.update(acc3, 1)
accuracies4.update(acc4, 1)
accuracies5.update(acc5, 1)
accuracies6.update(acc6, 1)
accuracies7.update(acc7, 1)
accuracies8.update(acc8, 1)
accuracies9.update(acc9, 1)
# line = "Video[" + str(i) + "\t video = " + str(accuracies5.avg)
# print(line)
# line = "Video[" + str(i) + "] : \t top5 " + str(pred5) + "\t top1 = " + str(pred5[0]) + "\t true = " +str(int(label[0])) + "\t video = " + str(accuracies.avg)
# print(line)
resulttop5_1 = videoID[0] + ' ' + str(pred5_1[0] + 1) + ' ' + str(
pred5_1[1] + 1) + ' ' + str(pred5_1[2] + 1) + ' ' + str(
pred5_1[3] + 1) + ' ' + str(pred5_1[4] + 1)
resulttop5_2 = videoID[0] + ' ' + str(pred5_2[0] + 1) + ' ' + str(
pred5_2[1] + 1) + ' ' + str(pred5_2[2] + 1) + ' ' + str(
pred5_2[3] + 1) + ' ' + str(pred5_2[4] + 1)
resulttop5_3 = videoID[0] + ' ' + str(pred5_3[0] + 1) + ' ' + str(
pred5_3[1] + 1) + ' ' + str(pred5_3[2] + 1) + ' ' + str(
pred5_3[3] + 1) + ' ' + str(pred5_3[4] + 1)
resulttop5_4 = videoID[0] + ' ' + str(pred5_4[0] + 1) + ' ' + str(
pred5_4[1] + 1) + ' ' + str(pred5_4[2] + 1) + ' ' + str(
pred5_4[3] + 1) + ' ' + str(pred5_4[4] + 1)
resulttop5_5 = videoID[0] + ' ' + str(pred5_5[0] + 1) + ' ' + str(
pred5_5[1] + 1) + ' ' + str(pred5_5[2] + 1) + ' ' + str(
pred5_5[3] + 1) + ' ' + str(pred5_5[4] + 1)
resulttop5_6 = videoID[0] + ' ' + str(pred5_6[0] + 1) + ' ' + str(
pred5_6[1] + 1) + ' ' + str(pred5_6[2] + 1) + ' ' + str(
pred5_6[3] + 1) + ' ' + str(pred5_6[4] + 1)
resulttop5_7 = videoID[0] + ' ' + str(pred5_7[0] + 1) + ' ' + str(
pred5_7[1] + 1) + ' ' + str(pred5_7[2] + 1) + ' ' + str(
pred5_7[3] + 1) + ' ' + str(pred5_7[4] + 1)
resulttop5_8 = videoID[0] + ' ' + str(pred5_8[0] + 1) + ' ' + str(
pred5_8[1] + 1) + ' ' + str(pred5_8[2] + 1) + ' ' + str(
pred5_8[3] + 1) + ' ' + str(pred5_8[4] + 1)
resulttop5_9 = videoID[0] + ' ' + str(pred5_9[0] + 1) + ' ' + str(
pred5_9[1] + 1) + ' ' + str(pred5_9[2] + 1) + ' ' + str(
pred5_9[3] + 1) + ' ' + str(pred5_9[4] + 1)
submission1.append(resulttop5_1)
submission2.append(resulttop5_2)
submission3.append(resulttop5_3)
submission4.append(resulttop5_4)
submission5.append(resulttop5_5)
submission6.append(resulttop5_6)
submission7.append(resulttop5_7)
submission8.append(resulttop5_8)
submission9.append(resulttop5_9)
# print(resulttop5)
print("Video accuracy = ", accuracies1.avg)
print("Video accuracy = ", accuracies2.avg)
print("Video accuracy = ", accuracies3.avg)
print("Video accuracy = ", accuracies4.avg)
print("Video accuracy = ", accuracies5.avg)
print("Video accuracy = ", accuracies6.avg)
print("Video accuracy = ", accuracies7.avg)
print("Video accuracy = ", accuracies8.avg)
print("Video accuracy = ", accuracies9.avg)
acc_list = []
acc_list.append(accuracies1.avg)
acc_list.append(accuracies2.avg)
acc_list.append(accuracies3.avg)
acc_list.append(accuracies4.avg)
acc_list.append(accuracies5.avg)
acc_list.append(accuracies6.avg)
acc_list.append(accuracies7.avg)
acc_list.append(accuracies8.avg)
acc_list.append(accuracies9.avg)
best_index = acc_list.index(max(acc_list))
if best_index == 0:
submission = submission1
elif best_index == 1:
submission = submission2
elif best_index == 2:
submission = submission3
elif best_index == 3:
submission = submission4
elif best_index == 4:
submission = submission5
elif best_index == 5:
submission = submission6
elif best_index == 6:
submission = submission7
elif best_index == 7:
submission = submission8
elif best_index == 8:
submission = submission9
else:
print("Invalid month")
# submission.write(resulttop5 + '\n')
# submission.flush()
# if opt.log:
#
# f.write(line + '\n')
# f.flush()
# print("Video accuracy = ", accuracies.avg)
# line = "Video accuracy = " + str(accuracies.avg) + '\n'
# if opt.log:
# f.write(line)
submission_file = open(
os.path.join(
result_path, "best_submit_{}{}_{}_{}_{}_{}.txt".format(
opt.model, opt.model_depth, opt.dataset, opt.split,
opt.modality, opt.sample_duration)), 'w+')
for lines in submission:
submission_file.write(lines + '\n')
submission_file.flush()
def test(opts):
if torch.cuda.is_available():
opts.cuda = True
opts.arch = '{}-{}'.format(opts.model, opts.model_depth)
print("Preprocessing testing data ...")
test_data = globals()['{}'.format(opts.dataset)](split=opts.split,
train=0,
opt=opts)
print("Length of testing data = ", len(test_data))
if opts.modality == 'RGB': opts.input_channels = 3
elif opts.modality == 'Flow': opts.input_channels = 2
print("Preparing datatloaders ...")
test_dataloader = DataLoader(test_data,
batch_size=opts.batch_size,
shuffle=False,
num_workers=opts.n_workers,
pin_memory=True,
drop_last=False)
print("Length of validation datatloader = ", len(test_dataloader))
# Loading model and checkpoint
model, parameters = generate_model(opts)
accuracies = AverageMeter()
#Path to store results
result_path = "{}/{}/".format(opts.result_path, opts.dataset)
if not os.path.exists(result_path):
os.makedirs(result_path)
if opts.log:
f = open(
os.path.join(
result_path, "test_{}{}_{}_{}_{}_{}_online_{}_{}".format(
opts.model, opts.model_depth, opts.dataset, opts.split,
opts.modality, opts.sample_duration, opts.test_file,
opts.resume_path1.split('/')[1])), 'w+')
f.write(str(opts))
f.write('\n')
f.flush()
prob_list = open(
os.path.join(
result_path, "prob_FP_{}{}_{}_{}_{}_{}_online_{}_{}".format(
opts.model, opts.model_depth, opts.dataset, opts.split,
opts.modality, opts.sample_duration, opts.test_file,
opts.resume_path1.split('/')[1])), 'w+')
softmax = nn.Softmax(dim=1)
model.eval()
with torch.no_grad():
for i, (clip, targets, video_name) in enumerate(test_dataloader):
clip = torch.squeeze(clip)
if opts.modality == 'RGB':
inputs = torch.Tensor(
int(clip.shape[1] / opts.sample_duration) + 1, 3,
opts.sample_duration, opts.sample_size, opts.sample_size)
for k in range(inputs.shape[0] - 1):
inputs[k, :, :, :, :] = clip[:,
k * opts.sample_duration:(k + 1) *
opts.sample_duration, :, :]
inputs[-1, :, :, :, :] = clip[:, -opts.sample_duration:, :, :]
if opts.cuda:
inputs = inputs.cuda()
outputs = model(inputs)
pre_label = torch.sum(outputs.topk(1)[1]).item()
prob_outputs = softmax(outputs)
if targets.item() == 0:
if pre_label > 0:
acc = 0
line = 'False Positive: name={}, prob={}'.format(
video_name, prob_outputs)
prob_list.write(line + '\n')
prob_list.flush()
else:
acc = 1
# print(prob_outputs)
# if pre_label > 1:
# for h in range(10):
# if 0 < prob_outputs[h][1] - prob_outputs[h][0] < 0.3:
# # if prob of case bigger than control and their gap smaller than 0.3, it should be control
# pre_label = pre_label - 1
# if pre_label > 1:
# # acc = 0 means it's a case
# acc = 0
# else:
# acc = 1
# else:
# acc = 1
else:
if pre_label > 0:
acc = 1
else:
acc = 0
accuracies.update(acc, inputs.size(0))
line = "Video[" + str(i) + "] : " + "\t predict = " + str(pre_label) + \
"\t true = " +str(int(targets[0])) + "\t acc = " + str(accuracies.avg)
print(line)
# print(outputs.topk(1)[1])
# print(prob_outputs)
if opts.log:
f.write(line + '\n')
f.flush()
print("Video accuracy = ", accuracies.avg)
line = "Video accuracy = " + str(accuracies.avg) + '\n'
if opts.log:
f.write(line)
def test():
opt = parse_opts()
print(opt)
opt.arch = '{}-{}'.format(opt.model, opt.model_depth)
print("Preprocessing validation data ...")
data = globals()['{}_test'.format(opt.dataset)](split=opt.split,
train=0,
opt=opt)
print("Length of validation data = ", len(data))
print("Preparing datatloaders ...")
val_dataloader = DataLoader(data,
batch_size=1,
shuffle=False,
num_workers=opt.n_workers,
pin_memory=True,
drop_last=False)
print("Length of validation datatloader = ", len(val_dataloader))
result_path = "{}/{}/".format(opt.result_path, opt.dataset)
if not os.path.exists(result_path):
os.makedirs(result_path)
# define the model
print("Loading models... ", opt.model, opt.model_depth)
model1, parameters1 = generate_model(opt)
# if testing RGB+Flow streams change input channels
if not opt.only_RGB:
opt.input_channels = 2
model2, parameters2 = generate_model(opt)
if opt.resume_path1:
print('loading checkpoint {}'.format(opt.resume_path1))
checkpoint = torch.load(opt.resume_path1)
assert opt.arch == checkpoint['arch']
model1.load_state_dict(checkpoint['state_dict'])
if opt.resume_path2:
print('loading checkpoint {}'.format(opt.resume_path2))
checkpoint = torch.load(opt.resume_path2)
assert opt.arch == checkpoint['arch']
model2.load_state_dict(checkpoint['state_dict'])
model1.eval()
model2.eval()
accuracies = AverageMeter()
if opt.log:
if opt.only_RGB:
f = open(
os.path.join(
root_dir, "test_RGB_MARS_{}{}_{}_{}_{}.txt".format(
opt.model, opt.model_depth, opt.dataset, opt.split,
opt.sample_duration)), 'w+')
else:
f = open(
os.path.join(
root_dir, "test_RGB_Flow_{}{}_{}_{}_{}.txt".format(
opt.model, opt.model_depth, opt.dataset, opt.split,
opt.sample_duration)), 'w+')
f.write(str(opt))
f.write('\n')
f.flush()
with torch.no_grad():
for i, (clip, label) in enumerate(val_dataloader):
clip = torch.squeeze(clip)
if opt.only_RGB:
inputs = torch.Tensor(int(clip.shape[1] / opt.sample_duration),
3, opt.sample_duration, opt.sample_size,
opt.sample_size)
for k in range(inputs.shape[0]):
inputs[k, :, :, :, :] = clip[:, k *
opt.sample_duration:(k + 1) *
opt.sample_duration, :, :]
inputs_var1 = Variable(inputs)
inputs_var2 = Variable(inputs)
else:
RGB_clip = clip[0:3, :, :, :]
Flow_clip = clip[3:, :, :, :]
inputs1 = torch.Tensor(
int(RGB_clip.shape[1] / opt.sample_duration), 3,
opt.sample_duration, opt.sample_size, opt.sample_size)
inputs2 = torch.Tensor(
int(Flow_clip.shape[1] / opt.sample_duration), 2,
opt.sample_duration, opt.sample_size, opt.sample_size)
for k in range(inputs1.shape[0]):
inputs1[
k, :, :, :, :] = RGB_clip[:, k *
opt.sample_duration:(k + 1) *
opt.sample_duration, :, :]
inputs2[
k, :, :, :, :] = Flow_clip[:, k *
opt.sample_duration:(k +
1) *
opt.sample_duration, :, :]
inputs_var1 = Variable(inputs1)
inputs_var2 = Variable(inputs2)
outputs_var1 = model1(inputs_var1)
outputs_var2 = model2(inputs_var2)
outputs_var = torch.mean(torch.cat((outputs_var1, outputs_var2),
dim=0),
dim=0).unsqueeze(0)
pred5 = np.array(outputs_var.topk(5, 1, True)[1].cpu().data[0])
acc = float(pred5[0] == label[0])
accuracies.update(acc, 1)
line = "Video[" + str(i) + "] : \t top5 " + str(
pred5) + "\t top1 = " + str(pred5[0]) + "\t true = " + str(
label[0]) + "\t video = " + str(accuracies.avg)
print(line)
if opt.log:
f.write(line + '\n')
f.flush()
print("Video accuracy = ", accuracies.avg)
line = "Video accuracy = " + str(accuracies.avg) + '\n'
if opt.log:
f.write(line)
print("Length of testing data = ", len(test_data))
if opts.modality == 'RGB': opts.input_channels = 3
elif opts.modality == 'Flow': opts.input_channels = 2
print("Preparing datatloaders ...")
test_dataloader = DataLoader(test_data,
batch_size=opts.batch_size,
shuffle=False,
num_workers=opts.n_workers,
pin_memory=True,
drop_last=False)
print("Length of validation datatloader = ", len(test_dataloader))
# Loading model and checkpoint
model, parameters = generate_model(opts)
criterion_rl = nn.CrossEntropyLoss(reduction='none').cuda()
accuracies = AverageMeter()
clip_accuracies = AverageMeter()
#Path to store results
result_path = "{}/{}/".format(opts.result_path, opts.dataset)
if not os.path.exists(result_path):
os.makedirs(result_path)
if opts.log:
f = open(
os.path.join(
result_path, "test_{}{}_{}_{}_{}_{}_plusone.txt".format(
opts.model_name, opts.model_depth, opts.dataset,
opts.split, opts.modality, opts.sample_duration)), 'w+')
def train():
#读取配置文件
opt = parse_opts()
print(opt)
opt.arch = '{}-{}'.format(opt.model, opt.model_depth)
#
with fluid.dygraph.guard(place = fluid.CUDAPlace(0)):
#训练数据加载器
print("Preprocessing train data ...")
train_data = globals()['{}_test'.format(opt.dataset)](split = opt.split, train = 1, opt = opt)
train_dataloader = paddle.batch(train_data, batch_size=opt.batch_size, drop_last=True)
#训练数据加载器
print("Preprocessing validation data ...")
val_data = globals()['{}_test'.format(opt.dataset)](split = opt.split, train = 2, opt = opt)
val_dataloader = paddle.batch(val_data, batch_size=opt.batch_size, drop_last=True)
#如果使用光流图像进行训练，输入通道数为2
opt.input_channels = 2
#构建网络模型结构
print("Loading Flow model... ", opt.model, opt.model_depth)
model,parameters = generate_model(opt)
print("Initializing the optimizer ...")
if opt.Flow_premodel_path:
opt.weight_decay = 1e-5
opt.learning_rate = 0.001
print("lr = {} \t momentum = {}, \t nesterov = {} \t LR patience = {} "
.format(opt.learning_rate, opt.momentum, opt.nesterov, opt.lr_patience))
#构建优化器
optimizer = fluid.optimizer.MomentumOptimizer(learning_rate=opt.learning_rate,
momentum=opt.momentum, parameter_list=parameters,
use_nesterov=opt.nesterov)
scheduler = ReduceLROnPlateau(opt.learning_rate, mode='min', patience=opt.lr_patience)
if opt.continue_train and opt.Flow_resume_path != '':
resume_params(model, optimizer, opt)
print('run')
losses_avg=np.zeros((1,),dtype=np.float)
for epoch in range(opt.begin_epoch, opt.n_epochs+1):
#设置模型为训练模式，模型中的参数可以被训练优化
model.train()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
end_time = time.time()
for i, data in enumerate(train_dataloader()):
#输入视频图像或者光流
inputs = np.array([x[0] for x in data]).astype('float32')
# 输入视频图像或者光流的标签
targets = np.array([x[1] for x in data]).astype('int')
inputs = fluid.dygraph.base.to_variable(inputs)
targets = fluid.dygraph.base.to_variable(targets)
targets.stop_gradient = True
data_time.update(time.time() - end_time)
#计算网络输出结果
outputs = model(inputs)
#计算网络输出和标签的交叉熵损失
loss = fluid.layers.cross_entropy(outputs, targets)
avg_loss = fluid.layers.mean(loss)
#计算网络预测精度
acc = calculate_accuracy(outputs, targets)
losses.update(avg_loss.numpy()[0], inputs.shape[0])
accuracies.update(acc[0], inputs.shape[0])
#反向传播梯度
optimizer.clear_gradients()
avg_loss.backward()
#最小化损失来优化网络中的权重
#print(avg_loss)
#pdb.set_trace()
optimizer.minimize(avg_loss)
batch_time.update(time.time() - end_time)
end_time = time.time()
print('Epoch: [{0}][{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss val:{loss.val:.4f} (avg:{loss.avg:.4f})\t'
'Acc val:{acc.val:.3f} (avg:{acc.avg:.3f})'.format(
epoch,
i + 1,
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
losses_avg[0]=losses.avg
scheduler.step(losses_avg)
if epoch % opt.checkpoint == 0 and epoch != 0:
fluid.dygraph.save_dygraph(model.state_dict(),os.path.join(opt.Flow_resume_path,'model_Flow_'+str(epoch)+'_saved'))
fluid.dygraph.save_dygraph(optimizer.state_dict(), os.path.join(opt.Flow_resume_path,'model_Flow_'+str(epoch)+'_saved'))
#设置模型为验证模式，对验证数据集进行验证
model.eval()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
end_time = time.time()
for i, data in enumerate(val_dataloader()):
data_time.update(time.time() - end_time)
inputs = np.array([x[0] for x in data]).astype('float32')
targets = np.array([x[1] for x in data]).astype('int')
inputs = fluid.dygraph.base.to_variable(inputs)
targets = fluid.dygraph.base.to_variable(targets)
targets.stop_gradient = True
outputs = model(inputs)
loss = fluid.layers.cross_entropy(outputs, targets)
avg_loss = fluid.layers.mean(loss)
acc = calculate_accuracy(outputs, targets)
losses.update(avg_loss.numpy()[0], inputs.shape[0])
accuracies.update(acc[0], inputs.shape[0])
batch_time.update(time.time() - end_time)
end_time = time.time()
print('Val_Epoch: [{0}][{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(
epoch,
i + 1,
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
A list of parsed arguments.
"""
parser = argparse.ArgumentParser(description="Action Recognition Demo")
parser.add_argument("--video-path",
type=str,
default='./video_list/sword.avi',
help="Video path selection")
parser.add_argument("--resume-path",
type=str,
default='./RGB_HMDB51_64f.pth')
return parser.parse_args()
if __name__ == '__main__':
args = get_arguments()
model = generate_model()
checkpoint = torch.load(args.resume_path)
model.load_state_dict(checkpoint['state_dict'], strict=False)
model.eval()
stack_frame, end_frame = get_video_frame(args)
stack_frame = stack_frame.cuda()
pred5 = []
grad_cam = GradCam(model=model,
feature_module=model.module.layer4,
target_layer_names=["2"])
if end_frame < 128:
a = 1
else:
a = 2
def main(args):
# Setup manual seed
reset_seed(args.manual_seed)
# Architecture
args.arch = 'i3d_{}'.format(args.modality)
print(args, '\n')
# Augumentation setting
args.scales = [args.initial_scale]
for i in range(1, args.n_scales):
args.scales.append(args.scales[-1] * args.scale_step)
assert args.train_crop in ['random', 'corner', 'center']
if args.train_crop == 'random':
crop_method = MultiScaleRandomCrop(args.scales, args.sample_size)
elif args.train_crop == 'corner':
crop_method = MultiScaleCornerCrop(args.scales, args.sample_size)
elif args.train_crop == 'center':
crop_method = MultiScaleCornerCrop(
args.scales, args.sample_size, crop_positions=['c'])
# Spatial transform
train_spatial_transform = Compose([
crop_method,
RandomHorizontalFlip(),
ToTensor(args.norm_value)])
val_spatial_transform = Compose([
Scale(args.sample_size),
CenterCrop(args.sample_size),
ToTensor(args.norm_value)])
# Load training dataset
ann_list = get_ann_list(args.ann_dir, args.ann_type)
class_label_map = get_action_labels(ann_list, args.action_coarse, args.SIL)
print('{} classes'.format(len(class_label_map)))
print(class_label_map, '\n')
args.num_classes = len(class_label_map)
train, val = split_breakfast_train_val(ann_list)
# BreakFast dataset class
print('Initialize train dataset')
train_dataset = BreakFast(train,
class_label_map, args,
spatial_transform=train_spatial_transform)
print('Initialize valid dataset')
val_dataset = BreakFast(val,
class_label_map, args,
spatial_transform=val_spatial_transform)
assert (len(train_dataset) != 0)
assert (len(val_dataset) != 0)
print('train: {} samples'.format(len(train_dataset)))
print('val: {} samples\n'.format(len(val_dataset)))
# Training iterotor
train_loader = torch.utils.data.DataLoader(
train_dataset, batch_size=args.batch_size,
shuffle=True, num_workers=args.n_threads,
pin_memory=True, drop_last=False)
val_loader = torch.utils.data.DataLoader(
val_dataset, batch_size=args.batch_size,
shuffle=False, num_workers=args.n_threads,
pin_memory=True, drop_last=True)
# Build model
model, parameters = generate_model(args)
# Loss function
criterion = nn.CrossEntropyLoss()
if not args.no_cuda:
criterion = criterion.cuda()
# Setup optimizer
if args.nesterov:
dampening = 0
else:
dampening = args.dampening
optimizer = optim.SGD(parameters, lr=args.learning_rate,
momentum=args.momentum, dampening=dampening,
weight_decay=args.weight_decay, nesterov=args.nesterov)
# Learning rate scheduler
scheduler = lr_scheduler.ReduceLROnPlateau(
optimizer, 'min', patience=args.lr_patience)
# Prepare result directory
if not os.path.exists(args.result_path):
os.makedirs(args.result_path)
if args.resume_path:
args.result_path = os.path.join(args.result_path,
args.resume.split('/')[-2])
else:
out_counter = len([None for out in os.listdir(args.result_path) \
if args.arch in out])
args.result_path = os.path.join(args.result_path,
args.arch + '-' + str(out_counter+1))
if not os.path.exists(args.result_path):
os.makedirs(args.result_path)
# Save configs
with open(os.path.join(args.result_path, 'config.json'), 'w') as f:
f.write(json.dumps(vars(args), indent=4))
# Logger
log_dir = os.path.join(args.result_path, 'log')
if not os.path.exists(log_dir):
os.makedirs(log_dir)
train_logger = Logger(os.path.join(log_dir, 'train.log'),
['epoch', 'loss', 'acc', 'lr'])
train_batch_logger = Logger(os.path.join(log_dir, 'train_batch.log'),
['epoch', 'batch', 'iter', 'loss', 'acc', 'lr'])
val_logger = Logger(os.path.join(log_dir, 'val.log'),
['epoch', 'loss', 'acc'])
# Resume pretrained weights
if args.resume_path:
print('loading checkpoint {}'.format(args.resume_path))
checkpoint = torch.load(args.resume_path)
assert args.arch == checkpoint['arch']
args.start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
# Training Loop
for i in range(args.start_epoch, args.max_epochs + 1):
# Train
train_epoch(i, train_loader, model, criterion,
optimizer, args, train_logger, train_batch_logger)
# Validate
validation_loss = val_epoch(i, val_loader, model, criterion,
args, val_logger)
# Learning rate scheduler
scheduler.step(validation_loss)
def main():
opts = parse_opts()
if torch.cuda.is_available():
opts.cuda = True
opts.device = torch.device(opts.device if opts.cuda else 'cpu')
print(opts)
opts.arch = '{}-{}'.format(opts.model_name, opts.model_depth)
torch.manual_seed(opts.manual_seed)
print("Preprocessing train data ...")
train_data = globals()['{}'.format(opts.dataset)](data_type='train',
opts=opts,
split=opts.split)
print("Length of train data = ", len(train_data))
print("Preprocessing validation data ...")
val_data = globals()['{}'.format(opts.dataset)](data_type='val',
opts=opts,
split=opts.split)
print("Length of validation data = ", len(val_data))
if opts.modality == 'RGB': opts.input_channels = 3
elif opts.modality == 'Flow': opts.input_channels = 2
print("Preparing dataloaders ...")
train_dataloader = DataLoader(train_data,
batch_size=opts.batch_size,
shuffle=True,
num_workers=opts.n_workers,
pin_memory=True,
drop_last=True)
val_dataloader = DataLoader(val_data,
batch_size=opts.batch_size,
shuffle=True,
num_workers=opts.n_workers,
pin_memory=True,
drop_last=True)
print("Length of train dataloader = ", len(train_dataloader))
print("Length of validation dataloader = ", len(val_dataloader))
# define the model
print("Loading model... ", opts.model_name, opts.model_depth)
model, parameters = generate_model(opts)
if not opts.pretrained_path:
opts.learning_rate = 0.1
opts.weight_decay = 5e-4
criterion = nn.CrossEntropyLoss().cuda()
if opts.resume_md_path:
print('loading checkpoint {}'.format(opts.resume_path1))
checkpoint = torch.load(opts.resume_path1)
assert opts.arch == checkpoint['arch']
opts.begin_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
log_path = os.path.join(opts.result_path, opts.dataset)
if not os.path.exists(log_path):
os.makedirs(log_path)
if opts.log == 1:
if opts.resume_md_path:
begin_epoch = int(opts.resume_path1.split('/')[-1].split('_')[1])
train_logger = Logger(os.path.join(
log_path,
'{}_train_clip{}model{}{}.log'.format(opts.dataset,
opts.sample_duration,
opts.model_name,
opts.model_depth)),
['epoch', 'loss', 'acc', 'lr'],
overlay=False)
val_logger = Logger(os.path.join(
log_path,
'{}_val_clip{}model{}{}.log'.format(opts.dataset,
opts.sample_duration,
opts.model_name,
opts.model_depth)),
['epoch', 'loss', 'acc'],
overlay=False)
else:
begin_epoch = 0
train_logger = Logger(os.path.join(
log_path,
'{}_train_clip{}model{}{}.log'.format(opts.dataset,
opts.sample_duration,
opts.model_name,
opts.model_depth)),
['epoch', 'loss', 'acc', 'lr'],
overlay=True)
val_logger = Logger(os.path.join(
log_path,
'{}_val_clip{}model{}{}.log'.format(opts.dataset,
opts.sample_duration,
opts.model_name,
opts.model_depth)),
['epoch', 'loss', 'acc'],
overlay=True)
print("Initializing the optimizer ...")
if opts.nesterov: dampening = 0
else: dampening = opts.dampening
print(
"lr = {} \t momentum = {} \t dampening = {} \t weight_decay = {}, \t nesterov = {}"
.format(opts.learning_rate, opts.momentum, dampening,
opts.weight_decay, opts.nesterov))
print("LR patience = ", opts.lr_patience)
optimizer = optim.SGD(parameters,
lr=opts.learning_rate,
momentum=opts.momentum,
dampening=dampening,
weight_decay=opts.weight_decay,
nesterov=opts.nesterov)
if opts.resume_md_path:
optimizer.load_state_dict(
torch.load(opts.pretrained_path)['optimizer'])
scheduler = lr_scheduler.ReduceLROnPlateau(optimizer,
'min',
patience=opts.lr_patience)
for epoch in range(begin_epoch, opts.n_epochs + 1):
print('Start training epoch {}'.format(epoch))
train_epoch(epoch, train_dataloader, model, criterion, optimizer, opts,
train_logger)
print('Start validating epoch {}'.format(epoch))
with torch.no_grad():
val_epoch(epoch, val_dataloader, model, criterion, optimizer, opts,
val_logger, scheduler)
def train():
#读取配置文件
opt = parse_opts()
print(opt)
opt.arch = '{}-{}'.format(opt.model, opt.model_depth)
with fluid.dygraph.guard(place=fluid.CUDAPlace(0)):
#训练数据加载函数
print("Preprocessing train data ...")
train_data = globals()['{}_test'.format(opt.dataset)](split=opt.split,
train=1,
opt=opt)
train_dataloader = paddle.batch(train_data,
batch_size=opt.batch_size,
drop_last=True)
#验证数据加载函数
print("Preprocessing validation data ...")
val_data = globals()['{}_test'.format(opt.dataset)](split=opt.split,
train=2,
opt=opt)
val_dataloader = paddle.batch(val_data,
batch_size=opt.batch_size,
drop_last=False)
#构建光流网络模型
print("Loading Flow model... ", opt.model, opt.model_depth)
opt.input_channels = 2
opt.n_classes = 51
model_Flow, _ = generate_model(opt)
#如果光流部分网络有预定义模型，则加载预定义模型
if opt.Flow_resume_path:
print('loading Flow checkpoint {}'.format(opt.Flow_resume_path))
para_dict, _ = fluid.dygraph.load_dygraph(opt.Flow_resume_path)
model_Flow.set_dict(para_dict)
#构建图像网络模型
print("Loading MARS model... ", opt.model, opt.model_depth)
opt.input_channels = 3
opt.n_classes = 400
model_MARS, parameters = generate_model(opt)
print("Initializing the optimizer ...")
if opt.RGB_premodel_path:
opt.weight_decay = 1e-5
opt.learning_rate = 0.001
print(
"lr = {} \t momentum = {} \t weight_decay = {}, \t nesterov = {}".
format(opt.learning_rate, opt.momentum, opt.weight_decay,
opt.nesterov))
print("LR patience = ", opt.lr_patience)
#定义优化器
optimizer = fluid.optimizer.MomentumOptimizer(
learning_rate=opt.learning_rate,
momentum=opt.momentum,
parameter_list=parameters,
use_nesterov=opt.nesterov)
scheduler = ReduceLROnPlateau(opt.learning_rate,
mode='min',
patience=opt.lr_patience)
if opt.MARS_resume_path != '' and opt.continue_train:
resume_params(model_MARS, optimizer, opt)
print('run')
#在网络训练过程中，光流网络参数保持固定
# model_Flow.eval()
losses_avg = np.zeros((1, ), dtype=np.float)
for epoch in range(opt.begin_epoch, opt.n_epochs + 1):
#设置模型为训练模式，模型中的参数可以被训练优化
model_MARS.train()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
losses_MARS = AverageMeter()
losses_MSE = AverageMeter()
accuracies = AverageMeter()
end_time = time.time()
for i, data in enumerate(train_dataloader()):
#输入视频图像、光流
inputs = np.array([x[0] for x in data]).astype('float32')
#输入视频图像、光流的标签
targets = np.array([x[1] for x in data]).astype('int')
#将视频图像和光流分离开
inputs_MARS = inputs[:, 0:3, :, :, :]
inputs_FLOW = inputs[:, 3:, :, :, :]
inputs_MARS = fluid.dygraph.base.to_variable(inputs_MARS)
inputs_FLOW = fluid.dygraph.base.to_variable(inputs_FLOW)
targets = fluid.dygraph.base.to_variable(targets)
targets.stop_gradient = True
data_time.update(time.time() - end_time)
#计算图像的网络输出结果
outputs_MARS = model_MARS(inputs_MARS)
#计算光流的网络输出结果
#pdb.set_trace()
outputs_Flow = model_Flow(inputs_FLOW)[1].detach()
#outputs_Flow.stop_gradient = True
#计算图像网络输出和标签的交叉熵损失
loss_MARS = fluid.layers.cross_entropy(outputs_MARS[0],
targets)
loss_MARS = fluid.layers.mean(loss_MARS)
#计算图像网络和光流网络提取特征的mse损失
loss_MSE = opt.MARS_alpha * fluid.layers.mse_loss(
outputs_MARS[1], outputs_Flow)
#计算总的损失
loss = loss_MARS + loss_MSE
optimizer.clear_gradients()
#反向传播梯度
loss.backward()
#最小化损失来优化网络中的权重
optimizer.minimize(loss)
#计算网络预测精度
acc = calculate_accuracy(outputs_MARS[0], targets)
losses.update(loss.numpy()[0], inputs.shape[0])
losses_MARS.update(loss_MARS.numpy()[0], inputs.shape[0])
losses_MSE.update(loss_MSE.numpy()[0], inputs.shape[0])
accuracies.update(acc[0], inputs.shape[0])
batch_time.update(time.time() - end_time)
end_time = time.time()
print('Epoch: [{0}][{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'lr {lr:.5f}\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Loss_MARS {loss_MARS.val:.4f} ({loss_MARS.avg:.4f})\t'
'Loss_MSE {loss_MSE.val:.4f} ({loss_MSE.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(
epoch,
i + 1,
batch_time=batch_time,
data_time=data_time,
lr=optimizer.current_step_lr(),
loss=losses,
loss_MARS=losses_MARS,
loss_MSE=losses_MSE,
acc=accuracies))
losses_avg[0] = losses.avg
scheduler.step(losses_avg)
if epoch % opt.checkpoint == 0:
fluid.dygraph.save_dygraph(
model_MARS.state_dict(),
os.path.join(opt.MARS_resume_path,
'model_MARS_' + str(epoch) + '_saved'))
fluid.dygraph.save_dygraph(
optimizer.state_dict(),
os.path.join(opt.MARS_resume_path,
'model_MARS_' + str(epoch) + '_saved'))
#设置图像网络模型为验证模式，对验证数据集进行验证
model_MARS.eval()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
end_time = time.time()
for i, data in enumerate(val_dataloader()):
data_time.update(time.time() - end_time)
inputs = np.array([x[0] for x in data]).astype('float32')
targets = np.array([x[1] for x in data]).astype('int')
inputs_MARS = inputs[:, 0:3, :, :, :]
inputs_MARS = fluid.dygraph.base.to_variable(inputs_MARS)
targets = fluid.dygraph.base.to_variable(targets)
targets.stop_gradient = True
outputs_MARS = model_MARS(inputs_MARS)
loss = fluid.layers.cross_entropy(outputs_MARS[0], targets)
acc = calculate_accuracy(outputs_MARS[0], targets)
losses.update(loss.numpy()[0], inputs.shape[0])
accuracies.update(acc[0], inputs.shape[0])
batch_time.update(time.time() - end_time)
end_time = time.time()
print('Val_Epoch: [{0}][{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(
epoch,
i + 1,
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
opt.input_channels = 3
model_STS, parameters_STS = generate_model_t(opt)
model_STS_dict = model_STS.state_dict()
print("Loading Flow model... ", opt.model, opt.model_depth)
opt.input_channels = 2
if opt.pretrain_path != '':
if opt.dataset == 'HMDB51':
opt.n_classes = 51
elif opt.dataset == 'J-Kinetics':
opt.n_classes = 200
elif opt.dataset == 'UCF101':
opt.n_classes = 101
model_Flow, parameters_Flow = generate_model(opt)
criterion_STS = nn.CrossEntropyLoss().cuda()
criterion_MSE = nn.MSELoss().cuda()
if opt.resume_path1:
print('loading checkpoint {}'.format(opt.resume_path1))
checkpoint = torch.load(opt.resume_path1)
model_Flow.load_state_dict(checkpoint['state_dict'])
if opt.resume_path2:
print('loading checkpoint {}'.format(opt.resume_path2))
checkpoint = torch.load(opt.resume_path2)
model_STS.load_state_dict(checkpoint['state_dict'])
print("Initializing the optimizer ...")
def main():
opts = parse_opts()
if torch.cuda.is_available():
opts.cuda = True
opts.arch = '{}-{}'.format(opts.model, opts.model_depth)
# load the human activity recognition model
print("[INFO] loading human activity recognition model...")
# net = cv2.dnn.readNet(args["model"])
model, parameters = generate_model(opts)
if opts.resume_path1:
print('loading checkpoint {}'.format(opts.resume_path1))
checkpoint = torch.load(opts.resume_path1)
assert opts.arch == checkpoint['arch']
model.load_state_dict(checkpoint['state_dict'])
model.eval()
# grab a pointer to the input video stream
print("[INFO] accessing video stream...")
vs = cv2.VideoCapture(opts.inputs)
fourcc = cv2.VideoWriter_fourcc(*'XVID')
v_writer = cv2.VideoWriter('mice.avi', fourcc, 10, (1280, 720))
# loop until we explicitly break from it
while True:
# initialize the batch of frames that will be passed through the
# model
frames = []
ori_frames = []
# loop over the number of required sample frames
for i in range(0, opts.sample_duration):
# read a frame from the video stream
(grabbed, frame) = vs.read()
# if the frame was not grabbed then we've reached the end of
# the video stream so exit the script
if not grabbed:
print("[INFO] no frame read from stream - exiting")
sys.exit(0)
ori_frame = frame.copy()
ori_frames.append(ori_frame)
# otherwise, the frame was read so resize it and add it to
# our frames list
frame = imutils.resize(frame, height=112)
frames.append(frame)
# now that our frames array is filled we can construct our blob
blob = cv2.dnn.blobFromImages(frames,
1.0,
(opts.sample_size, opts.sample_size),
(114.7748, 107.7354, 99.4750),
swapRB=True,
crop=True)
# blob = cv2.dnn.blobFromImages(frames, 1.0,
# (opts.sample_size, opts.sample_size), (0, 0, 0),
# swapRB=True, crop=True)
# img = blob[0, :, :, :]
# img = np.array(img, dtype=np.uint8)
# img = np.transpose(img, (1, 2, 0))
# cv2.imshow('img', img)
# cv2.waitKey(0)
# import pdb; pdb.set_trace()
blob = np.transpose(blob, (1, 0, 2, 3))
blob = np.expand_dims(blob, axis=0)
# pass the blob through the network to obtain our human activity
# recognition predictions
inputs = torch.from_numpy(blob)
outputs = model(inputs)
label = np.array(
torch.mean(outputs, dim=0, keepdim=True).topk(1)[1].cpu().data[0])
print(label)
if label[0] == 0:
text = 'Normal'
elif label[0] == 1:
text = 'Ictal'
else:
text = 'NoneType'
# loop over our frames
for frame in ori_frames:
# draw the predicted activity on the frame
# cv2.rectangle(frame, (0, 0), (300, 40), (0, 0, 0), -1)
cv2.putText(frame, text, (40, 40), cv2.FONT_HERSHEY_SIMPLEX, 1,
(255, 255, 255), 2)
v_writer.write(frame)
# display the frame to our screen
cv2.imshow("Activity Recognition", frame)
key = cv2.waitKey(1) & 0xFF
# if the `q` key was pressed, break from the loop
if key == ord("q"):
break
v_writer.release()
def main_baseline_model(opt):
torch.manual_seed(opt.manual_seed)
np.random.seed(opt.manual_seed)
model, parameters = generate_model(opt)
print(model)
criterion = nn.CrossEntropyLoss()
if not opt.no_cuda:
criterion = criterion.cuda()
if opt.no_mean_norm and not opt.std_norm:
norm_method = Normalize([0, 0, 0], [1, 1, 1])
elif not opt.std_norm:
norm_method = Normalize(opt.mean, [1, 1, 1])
else:
norm_method = Normalize(opt.mean, opt.std)
if not opt.no_train:
assert opt.train_crop in ['random', 'corner', 'center']
if opt.train_crop == 'random':
crop_method = MultiScaleRandomCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'corner':
crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'center':
crop_method = MultiScaleCornerCrop(opt.scales,
opt.sample_size,
crop_positions=['c'])
spatial_transform = Compose([
crop_method,
RandomHorizontalFlip(),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = TemporalRandomCrop(opt.sample_duration)
target_transform = ClassLabel()
training_data = get_training_set(opt, spatial_transform,
temporal_transform, target_transform)
train_loader = torch.utils.data.DataLoader(training_data,
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.n_threads,
pin_memory=True)
train_logger = Logger(os.path.join(opt.result_path, 'train.log'),
['epoch', 'loss', 'acc', 'lr'])
train_batch_logger = Logger(
os.path.join(opt.result_path, 'train_batch.log'),
['epoch', 'batch', 'iter', 'loss', 'acc', 'lr'])
if opt.nesterov:
dampening = 0
else:
dampening = opt.dampening
if opt.dataset in [
'kinetics_adv', 'kinetics_bkgmsk', 'kinetics_adv_msk'
]:
first_optimizer = optim.SGD(
parameters[1], # new parameters only
lr=opt.new_layer_lr,
momentum=opt.momentum,
dampening=dampening,
weight_decay=opt.weight_decay,
nesterov=opt.nesterov)
first_scheduler = lr_scheduler.ReduceLROnPlateau(
first_optimizer, 'min', patience=opt.lr_patience)
second_optimizer = optim.SGD(
[
{
'params': parameters[0]
}, # pretrained parameters
{
'params': parameters[1]
} # new parameters
],
lr=opt.learning_rate,
momentum=opt.momentum,
dampening=dampening,
weight_decay=opt.weight_decay,
nesterov=opt.nesterov)
second_scheduler = lr_scheduler.ReduceLROnPlateau(
second_optimizer, 'min', patience=opt.lr_patience)
else:
optimizer = optim.SGD(parameters,
lr=opt.learning_rate,
momentum=opt.momentum,
dampening=dampening,
weight_decay=opt.weight_decay,
nesterov=opt.nesterov)
scheduler = lr_scheduler.ReduceLROnPlateau(
optimizer, 'min', patience=opt.lr_patience)
if not opt.no_val:
spatial_transform = Compose([
Scale(opt.sample_size),
CenterCrop(opt.sample_size),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = LoopPadding(opt.sample_duration)
target_transform = ClassLabel()
validation_data = get_validation_set(opt, spatial_transform,
temporal_transform,
target_transform)
val_loader = torch.utils.data.DataLoader(validation_data,
batch_size=opt.val_batch_size,
shuffle=False,
num_workers=opt.n_threads,
pin_memory=True)
val_logger = Logger(os.path.join(opt.result_path, 'val.log'),
['epoch', 'loss', 'acc'])
if opt.resume_path:
print('loading checkpoint {}'.format(opt.resume_path))
checkpoint = torch.load(opt.resume_path)
assert opt.arch == checkpoint['arch']
opt.begin_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
if not opt.no_train:
optimizer.load_state_dict(checkpoint['optimizer'])
print('run')
tr_btime_avg = AverageMeter()
tr_dtime_avg = AverageMeter()
val_btime_avg = AverageMeter()
val_dtime_avg = AverageMeter()
for i in range(opt.begin_epoch, opt.n_epochs + 1):
if opt.dataset in [
'kinetics_adv', 'kinetics_bkgmsk', 'kinetics_adv_msk'
]:
if i < opt.warm_up_epochs:
optimizer = first_optimizer
scheduler = first_scheduler
else:
optimizer = second_optimizer
scheduler = second_scheduler
if not opt.no_train:
tr_btime, tr_dtime = train_epoch(i, train_loader, model, criterion,
optimizer, opt, train_logger,
train_batch_logger)
if not opt.no_val:
validation_loss, val_btime, val_dtime = val_epoch(
i, val_loader, model, criterion, opt, val_logger)
if not opt.no_train and not opt.no_val:
scheduler.step(validation_loss)
tr_btime_avg.update(tr_btime)
tr_dtime_avg.update(tr_dtime)
val_btime_avg.update(val_btime)
val_dtime_avg.update(val_dtime)
print('One epoch tr btime = {:.2f}sec, tr dtime = {:.2f}'.format(
tr_btime_avg.avg, tr_dtime_avg.avg))
print('One epoch val btime = {:.2f}sec, val dtime = {:.2f}'.format(
val_btime_avg.avg, val_dtime_avg.avg))
sys.stdout.flush()
if opt.test:
spatial_transform = Compose([
Scale(int(opt.sample_size / opt.scale_in_test)),
CornerCrop(opt.sample_size, opt.crop_position_in_test),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = LoopPadding(opt.sample_duration)
target_transform = VideoID()
test_data = get_test_set(opt, spatial_transform, temporal_transform,
target_transform)
test_loader = torch.utils.data.DataLoader(
test_data,
batch_size=opt.val_batch_size,
shuffle=False,
num_workers=opt.n_threads,
pin_memory=True)
test(test_loader, model, opt, test_data.class_names)
def main_sdn_full_model(opt):
torch.manual_seed(opt.manual_seed)
np.random.seed(opt.manual_seed)
model, parameters = generate_model(opt)
print(model)
criterion = nn.CrossEntropyLoss()
if opt.is_place_soft:
places_criterion = SoftCrossEntropy()
print('using soft cross entropy for places')
else:
places_criterion = nn.CrossEntropyLoss()
if opt.is_place_entropy:
places_entropy_criterion = HLoss(is_maximization=opt.is_entropy_max)
if opt.is_mask_cross_entropy:
mask_criterion = nn.CrossEntropyLoss()
elif opt.is_mask_entropy:
mask_criterion = HLoss(is_maximization=True)
if not opt.no_cuda:
criterion = criterion.cuda()
places_criterion = places_criterion.cuda()
if opt.is_place_entropy:
places_entropy_criterion = places_entropy_criterion.cuda()
mask_criterion = mask_criterion.cuda()
criterions = dict()
criterions['action_cross_entropy'] = criterion
criterions['places_cross_entropy'] = places_criterion
criterions['mask_criterion'] = mask_criterion
if opt.is_place_entropy:
criterions['places_entropy'] = places_entropy_criterion
if opt.no_mean_norm and not opt.std_norm:
norm_method = Normalize([0, 0, 0], [1, 1, 1])
elif not opt.std_norm:
norm_method = Normalize(opt.mean, [1, 1, 1])
else:
norm_method = Normalize(opt.mean, opt.std)
if not opt.no_train:
assert opt.train_crop in ['random', 'corner', 'center']
if opt.train_crop == 'random':
crop_method = MultiScaleRandomCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'corner':
crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'center':
crop_method = MultiScaleCornerCrop(opt.scales,
opt.sample_size,
crop_positions=['c'])
spatial_transform = Compose([
crop_method,
RandomHorizontalFlip(),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = TemporalRandomCrop(opt.sample_duration)
target_transform = ClassLabel()
training_data = get_training_set(opt, spatial_transform,
temporal_transform, target_transform)
train_loader_unmasked = torch.utils.data.DataLoader(
training_data[0],
batch_size=opt.batch_size,
shuffle=True,
num_workers=int(opt.n_threads / 2),
pin_memory=True,
drop_last=True)
train_loader_masked = torch.utils.data.DataLoader(
training_data[1],
batch_size=opt.batch_size,
shuffle=True,
num_workers=int(opt.n_threads / 2),
pin_memory=True,
drop_last=True)
train_loaders = [train_loader_unmasked, train_loader_masked]
train_logger = Logger(os.path.join(opt.result_path, 'train.log'), [
'epoch', 'loss total', 'loss act', 'loss place', 'acc act',
'acc place', 'lr'
])
train_batch_logger = Logger(
os.path.join(opt.result_path, 'train_batch.log'), [
'epoch', 'batch', 'iter', 'loss total', 'loss act',
'loss place', 'acc act', 'acc place', 'lr'
])
if opt.nesterov:
dampening = 0
else:
dampening = opt.dampening
if len(parameters[1]) > 0:
first_optimizer = optim.SGD(
parameters[1], # new parameters only
lr=opt.new_layer_lr,
momentum=opt.momentum,
dampening=dampening,
weight_decay=opt.weight_decay,
nesterov=opt.nesterov)
first_scheduler = lr_scheduler.ReduceLROnPlateau(
first_optimizer, 'min', patience=opt.lr_patience)
second_optimizer = optim.SGD(
[
{
'params': parameters[0]
}, # pretrained parameters
{
'params': parameters[1]
} # new parameters
],
lr=opt.learning_rate,
momentum=opt.momentum,
dampening=dampening,
weight_decay=opt.weight_decay,
nesterov=opt.nesterov)
second_scheduler = lr_scheduler.ReduceLROnPlateau(
second_optimizer, 'min', patience=opt.lr_patience)
if not opt.no_val:
spatial_transform = Compose([
Scale(opt.sample_size),
CenterCrop(opt.sample_size),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = LoopPadding(opt.sample_duration)
target_transform = ClassLabel()
validation_data = get_validation_set(opt, spatial_transform,
temporal_transform,
target_transform)
val_loader_unmasked = torch.utils.data.DataLoader(
validation_data[0],
batch_size=opt.val_batch_size,
shuffle=False,
num_workers=int(opt.n_threads / 2),
pin_memory=True,
drop_last=True)
val_loader_masked = torch.utils.data.DataLoader(
validation_data[1],
batch_size=opt.val_batch_size,
shuffle=False,
num_workers=int(opt.n_threads / 2),
pin_memory=True,
drop_last=True)
val_loaders = [val_loader_unmasked, val_loader_masked]
val_logger = Logger(os.path.join(opt.result_path, 'val.log'), [
'epoch', 'loss total', 'loss act', 'loss place', 'acc act',
'acc place'
])
if opt.resume_path:
print('loading checkpoint {}'.format(opt.resume_path))
checkpoint = torch.load(opt.resume_path)
assert opt.arch == checkpoint['arch']
opt.begin_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
if not opt.no_train:
optimizer.load_state_dict(checkpoint['optimizer'])
print('run')
for i in range(opt.begin_epoch, opt.n_epochs + 1):
if i < opt.warm_up_epochs:
optimizer = first_optimizer
scheduler = first_scheduler
else:
optimizer = second_optimizer
scheduler = second_scheduler
if not opt.no_train:
if not opt.is_mask_adv:
if i < opt.warm_up_epochs:
train_adv_epoch(i, train_loaders[0], model, criterions,
optimizer, opt, train_logger,
train_batch_logger, opt.warm_up_epochs)
else:
train_adv_msk_epoch(i, train_loaders, model, criterions,
optimizer, opt, train_logger,
train_batch_logger, opt.warm_up_epochs)
else:
train_adv_msk_epoch(i, train_loaders, model, criterions,
optimizer, opt, train_logger,
train_batch_logger, opt.warm_up_epochs)
if not opt.no_val:
if not opt.is_mask_adv:
if i < opt.warm_up_epochs:
validation_loss = val_adv_epoch(i, val_loaders[0], model,
criterions, opt,
val_logger)
else:
validation_loss = val_adv_msk_epoch(
i, val_loaders, model, criterions, opt, val_logger)
else:
validation_loss = val_adv_msk_epoch(i, val_loaders, model,
criterions, opt,
val_logger)
if not opt.no_train and not opt.no_val:
scheduler.step(validation_loss)
sys.stdout.flush()
if opt.test:
spatial_transform = Compose([
Scale(int(opt.sample_size / opt.scale_in_test)),
CornerCrop(opt.sample_size, opt.crop_position_in_test),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = LoopPadding(opt.sample_duration)
target_transform = VideoID()
test_data = get_test_set(opt, spatial_transform, temporal_transform,
target_transform)
test_loader = torch.utils.data.DataLoader(
test_data,
batch_size=opt.val_batch_size,
shuffle=False,
num_workers=opt.n_threads,
pin_memory=True)
test(test_loader, model, opt, test_data.class_names)
if __name__=="__main__":
opt = parse_opts()
print(opt)
opt.arch = '{}-{}'.format(opt.model, opt.model_depth)
with fluid.dygraph.guard(place = fluid.CUDAPlace(0)):
print("Preprocessing validation data ...")
test_data = globals()['{}_test'.format(opt.dataset)](split = opt.split, train = 0, opt = opt)
test_dataloader = paddle.batch(test_data, batch_size=opt.batch_size, drop_last=False)
if opt.modality=='Flow': opt.input_channels = 2
else: opt.input_channels = 3
# Loading model and checkpoint
model,_ = generate_model(opt)
if opt.modality=='RGB' and opt.RGB_resume_path!='':
para_dict, _ = fluid.dygraph.load_dygraph(opt.RGB_resume_path)
model.set_dict(para_dict)
if opt.modality=='Flow' and opt.Flow_resume_path!='':
para_dict, _ = fluid.dygraph.load_dygraph(opt.Flow_resume_path)
model.set_dict(para_dict)
model.eval()
accuracies = AverageMeter()
clip_accuracies = AverageMeter()
#Path to store results
result_path = "{}/{}/".format(opt.result_path, opt.dataset)
if not os.path.exists(result_path):
os.makedirs(result_path)
def train():
opt = parse_opts()
print(opt)
opt.arch = '{}-{}'.format(opt.model, opt.model_depth)
torch.manual_seed(opt.manual_seed)
print("Preprocessing train data ...")
train_data = globals()['{}_test'.format(opt.dataset)](split=opt.split, train=1, opt=opt)
print("Length of train data = ", len(train_data))
print("Preprocessing validation data ...")
val_data = globals()['{}_test'.format(opt.dataset)](split=opt.split, train=2, opt=opt)
print("Length of validation data = ", len(val_data))
if opt.modality=='RGB': opt.input_channels = 3
elif opt.modality=='Flow': opt.input_channels = 2
print("Preparing datatloaders ...")
train_dataloader = DataLoader(train_data, batch_size = opt.batch_size, shuffle=True, num_workers = opt.n_workers, pin_memory = True, drop_last=True)
val_dataloader = DataLoader(val_data, batch_size = opt.batch_size, shuffle=True, num_workers = opt.n_workers, pin_memory = True, drop_last=True)
print("Length of train datatloader = ",len(train_dataloader))
print("Length of validation datatloader = ",len(val_dataloader))
log_path = os.path.join(opt.result_path, opt.dataset)
if not os.path.exists(log_path):
os.makedirs(log_path)
result_path = "{}/{}/".format(opt.result_path, opt.dataset)
if not os.path.exists(result_path):
os.makedirs(result_path)
if opt.log == 1:
epoch_logger = Logger_MARS(os.path.join(log_path, 'Fusion_{}_{}_train_batch{}_sample{}_clip{}_lr{}_nesterov{}_manualseed{}_model{}{}_ftbeginidx{}_alpha{}.log'
.format(opt.dataset, opt.split, opt.batch_size, opt.sample_size, opt.sample_duration, opt.learning_rate, opt.nesterov, opt.manual_seed, opt.model, opt.model_depth, opt.ft_begin_index
, opt.MARS_alpha))
,['epoch', 'loss', 'acc', 'lr'], opt.MARS_resume_path, opt.begin_epoch)
val_logger = Logger_MARS(os.path.join(log_path, 'Fusion_{}_{}_val_batch{}_sample{}_clip{}_lr{}_nesterov{}_manualseed{}_model{}{}_ftbeginidx{}_alpha{}.log'
.format(opt.dataset,opt.split, opt.batch_size, opt.sample_size, opt.sample_duration, opt.learning_rate, opt.nesterov, opt.manual_seed, opt.model, opt.model_depth, opt.ft_begin_index,
opt.MARS_alpha))
,['epoch', 'loss', 'acc'], opt.MARS_resume_path, opt.begin_epoch)
if opt.nesterov: dampening = 0
else: dampening = opt.dampening
# define the model
print("Loading models... ", opt.model, opt.model_depth)
model1, parameters1 = generate_model(opt)
# if testing RGB+Flow streams change input channels
opt.input_channels = 2
model2, parameters2 = generate_model(opt)
model_fusion = new_fusion_model(opt.n_finetune_classes)
model_fusion = model_fusion.cuda()
model_fusion = nn.DataParallel(model_fusion)
if opt.resume_path1:
print('Loading MARS model {}'.format(opt.resume_path1))
checkpoint = torch.load(opt.resume_path1)
assert opt.arch == checkpoint['arch']
model1.load_state_dict(checkpoint['state_dict'])
if opt.resume_path2:
print('Loading Flow model {}'.format(opt.resume_path2))
checkpoint = torch.load(opt.resume_path2)
assert opt.arch == checkpoint['arch']
model2.load_state_dict(checkpoint['state_dict'])
if opt.resume_path3:
print('Loading Fusion model {}'.format(opt.resume_path3))
checkpoint = torch.load(opt.resume_path3)
assert opt.arch == checkpoint['arch']
model2.load_state_dict(checkpoint['state_dict'])
model1.eval()
model2.eval()
model_fusion.train()
for p in model1.parameters():
# if p.requires_grad:
# print("Need to freeze the parameters")
p.requires_grad = False
for p in model2.parameters():
# if p.requires_grad:
# print("Need to freeze the parameters..")
p.requires_grad = False
print("Initializing the optimizer ...")
print("lr = {} \t momentum = {} \t dampening = {} \t weight_decay = {}, \t nesterov = {}"
.format(opt.learning_rate, opt.momentum, dampening, opt.weight_decay, opt.nesterov))
print("LR patience = ", opt.lr_patience)
optimizer = optim.SGD(
model_fusion.parameters(),
lr=opt.learning_rate,
momentum=opt.momentum,
dampening=dampening,
weight_decay=opt.weight_decay,
nesterov=opt.nesterov)
scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=opt.lr_patience)
criterion = nn.CrossEntropyLoss().cuda()
print('run')
for epoch in range(opt.begin_epoch, opt.n_epochs + 1):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
weights=AverageMeter()
end_time = time.time()
for i, (inputs, targets) in enumerate(train_dataloader):
data_time.update(time.time() - end_time)
inputs_MARS = inputs[:, 0:3, :, :, :]
inputs_Flow = inputs[:, 3:, :, :, :]
targets = targets.cuda(non_blocking=True)
inputs_MARS = Variable(inputs_MARS)
inputs_Flow = Variable(inputs_Flow)
targets = Variable(targets)
outputs_MARS = model1(inputs_MARS)
outputs_Flow = model2(inputs_Flow)
weight,outputs_var =model_fusion(outputs_MARS.detach(),outputs_Flow.detach())
loss=criterion(outputs_var,targets)
acc = calculate_accuracy(outputs_var, targets)
losses.update(loss.data, inputs.size(0))
accuracies.update(acc, inputs.size(0))
weights.update(weight[0][0].data, inputs.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end_time)
end_time = time.time()
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})\t'
'Weight {weight.val:.3f} ({weight.avg:.3f})'.format(
epoch,
i + 1,
len(train_dataloader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies,
weight=weights
))
if opt.log == 1:
epoch_logger.log({
'epoch': epoch,
'loss': losses.avg,
'acc': accuracies.avg,
'lr': optimizer.param_groups[0]['lr']
})
if epoch % opt.checkpoint == 0:
if opt.pretrain_path != '':
save_file_path = os.path.join(log_path,
'Fusion_{}_{}_train_batch{}_sample{}_clip{}_lr{}_nesterov{}_manualseed{}_model{}{}_ftbeginidx{}_alpha{}_{}.pth'
.format(opt.dataset, opt.split, opt.batch_size, opt.sample_size,
opt.sample_duration, opt.learning_rate, opt.nesterov,
opt.manual_seed, opt.model, opt.model_depth,
opt.ft_begin_index,
opt.MARS_alpha, epoch))
else:
save_file_path = os.path.join(log_path,
'Fusion_{}_{}_train_batch{}_sample{}_clip{}_lr{}_nesterov{}_manualseed{}_model{}{}_ftbeginidx{}_alpha{}_{}.pth'
.format(opt.dataset, opt.split, opt.batch_size, opt.sample_size,
opt.sample_duration, opt.learning_rate, opt.nesterov,
opt.manual_seed, opt.model, opt.model_depth,
opt.ft_begin_index,
opt.MARS_alpha, epoch))
states = {
'epoch': epoch + 1,
'arch': opt.arch,
'state_dict': model_fusion.state_dict(),
'optimizer': optimizer.state_dict(),
}
torch.save(states, save_file_path)
model_fusion.eval()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
end_time = time.time()
with torch.no_grad():
for i, (inputs, targets) in enumerate(val_dataloader):
data_time.update(time.time() - end_time)
inputs_MARS = inputs[:, 0:3, :, :, :]
inputs_Flow = inputs[:, 3:, :, :, :]
targets = targets.cuda(non_blocking=True)
inputs_MARS = Variable(inputs_MARS)
inputs_Flow=Variable(inputs_Flow)
targets = Variable(targets)
outputs_MARS = model1(inputs_MARS)
outputs_Flow = model2(inputs_Flow)
_,outputs_var=model_fusion(outputs_MARS,outputs_Flow)
loss = criterion(outputs_var, targets)
acc = calculate_accuracy(outputs_var, targets)
losses.update(loss.data, inputs.size(0))
accuracies.update(acc, inputs.size(0))
batch_time.update(time.time() - end_time)
end_time = time.time()
print('Val_Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(
epoch,
i + 1,
len(val_dataloader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
if opt.log == 1:
val_logger.log({'epoch': epoch, 'loss': losses.avg, 'acc': accuracies.avg})
scheduler.step(losses.avg)
