train_frame_basedir = '/data/SaliencyDataset/Video/ActionInTheEye/Hollywood2/frames' train_density_basedir = '/data/SaliencyDataset/Video/ActionInTheEye/Hollywood2/density' elif dataset == 'dhf1k': train_frame_basedir = '/data/SaliencyDataset/Video/DHF1K/frames' train_density_basedir = '/data/SaliencyDataset/Video/DHF1K/density' elif dataset == 'ucf': train_frame_basedir = '' train_density_basedir = '' elif dataset == 'voc': train_frame_basedir = '' train_density_basedir = '' else: raise NotImplementedError tranining_dataset = VideoDataset(train_frame_basedir, train_density_basedir, img_size=(args.width, args.height), bgr_mean_list=[98, 102, 90], sort='rgb') tranining_dataset.setup_video_dataset_c3d( overlap=args.overlap, training_example_props=args.training_example_props, debug=args.debug) plot_figure_dir = os.path.join(snapshot_dir, 'figure') if not os.path.isdir(plot_figure_dir): os.makedirs(plot_figure_dir) print "Loss and metric figure will be save to", plot_figure_dir start_time = time.time() max_iter = 5000000
solver.net.copy_from(pretrained_model_path) # untrained.caffemodel else: solver.restore(snapshot_path) """End of A2""" # ╔═╗┌─┐┌┬┐┬ ┬┌─┐ ┌┬┐┌─┐┌┬┐┌─┐┌─┐┌─┐┌┬┐ # ╚═╗├┤ │ │ │├─┘ ││├─┤ │ ├─┤└─┐├┤ │ # ╚═╝└─┘ ┴ └─┘┴ ─┴┘┴ ┴ ┴ ┴ ┴└─┘└─┘ ┴ print "Loading data..." if training_base == 'msu': train_frame_basedir = '/data/sunnycia/SaliencyDataset/Video/MSU/frames' train_density_basedir = '/data/sunnycia/SaliencyDataset/Video/MSU/density/sigma32' tranining_dataset = VideoDataset(train_frame_basedir, train_density_basedir, img_size=(112, 112), bgr_mean_list=[98, 102, 90], sort='rgb') tranining_dataset.setup_video_dataset_c3d() # ╔╦╗╦╔═╗╔═╗ # ║║║║╚═╗║ # ╩ ╩╩╚═╝╚═╝ plot_figure_dir = '../figure' ## Figure dir plot_figure_dir = os.path.join(plot_figure_dir, postfix_str) if not os.path.isdir(plot_figure_dir): os.makedirs(plot_figure_dir) print "Loss figure will be save to", plot_figure_dir ####################################
solver.net.copy_from(pretrained_model_path) # untrained.caffemodel else: solver.restore(snapshot_path) """End of A2""" # ╔═╗┌─┐┌┬┐┬ ┬┌─┐ ┌┬┐┌─┐┌┬┐┌─┐┌─┐┌─┐┌┬┐ # ╚═╗├┤ │ │ │├─┘ ││├─┤ │ ├─┤└─┐├┤ │ # ╚═╝└─┘ ┴ └─┘┴ ─┴┘┴ ┴ ┴ ┴ ┴└─┘└─┘ ┴ print "Loading data..." train_frame_basedir = '/data/sunnycia/SaliencyDataset/Video/MSU/frames' train_density_basedir = '/data/sunnycia/SaliencyDataset/Video/MSU/density/sigma32' validation_frame_basedir = '/data/sunnycia/SaliencyDataset/Image/SALICON/DATA/train_val/val2014/images' validation_density_basedir = '/data/sunnycia/SaliencyDataset/Image/SALICON/DATA/train_val/val2014/density' tranining_dataset = VideoDataset(train_frame_basedir, train_density_basedir, img_size=img_size, video_length=key_frame_interval) tranining_dataset.setup_video_dataset_stack(overlap=args.overlap) # validation_dataset = StaticDataset(train_frame_basedir, train_density_basedir, debug=debug_mode) # ╔╦╗╦╔═╗╔═╗ # ║║║║╚═╗║ # ╩ ╩╩╚═╝╚═╝ plot_figure_dir = '../figure' ## Figure dir plot_figure_dir = os.path.join(plot_figure_dir, postfix_str) if not os.path.isdir(plot_figure_dir): os.makedirs(plot_figure_dir) print "Loss figure will be save to", plot_figure_dir ####################################
print >> solver_af, '\nnet:"' + network_path + '"' solver_af.close() log_path = os.path.join(savemodel_dir, output_log) result_path = os.path.join(savemodel_dir, output_results) ### finetune frame_basedir, density_basedir = get_frame_and_density_dir(dataset) if not frame_basedir: print "None preset dataset." exit() dataset = VideoDataset(frame_basedir, density_basedir, img_size=(112, 112), bgr_mean_list=[98, 102, 90], sort='rgb') dataset.setup_video_dataset_c3d_with_fold(overlap=args.overlap, fold=fold) output_log = time_stamp + '_' + output_log #save some critical information log_f = open(os.path.join(savemodel_dir, output_log), 'w') print >> log_f, 'dataset' print >> log_f, 'fold', fold print >> log_f, 'epoch', epoch solver_rf = open(solver_path, 'r') solver_lines = solver_rf.readlines() print >> log_f, 'solver info'
def main(): global args, best_prec1 cudnn.benchmark = True args = parser.parse_args() if not os.path.exists(args.log_dir): os.mkdir(args.log_dir) if not os.path.exists(args.model_dir): os.mkdir(args.model_dir) strat_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') log = open(os.path.join(args.log_dir, strat_time + '.txt'), 'w') print(args.description) log.write(args.description + '\n') log.flush() print( '=======================Experimental Settings=======================\n' ) log.write( '=======================Experimental Settings=======================\n' ) log.flush() print('Using_Dataset:{0} Batch_Size:{1} Epoch:{2} '.format( args.dataset, args.batch_size, args.epoch)) log.write('Using_Dataset:{0} Batch_Size:{1} Epoch:{2}'.format( args.dataset, args.batch_size, args.epoch) + '\n') log.flush() print('Num_segments:{0} Learning_rate:{1} Attention_type:{2}\n'.format( args.segments, args.learning_rate, args.attention_type)) log.write('Num_segments:{0} Learning_rate:{1} Attention_type:{2}\n'. format(args.segments, args.learning_rate, args.attention_type) + '\n') log.flush() print( '===================================================================\n' ) log.write( '===================================================================\n' ) log.flush() train_loader = torch.utils.data.DataLoader(VideoDataset( root=args.root, list=args.train_video_list, num_segments=args.segments, num_frames=args.frames, transform=transforms.Compose([ transforms.Resize(256), transforms.RandomCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ])), batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True, drop_last=True) test_loader = torch.utils.data.DataLoader(VideoDataset( root=args.root, list=args.test_video_list, num_segments=args.segments, num_frames=args.frames, test_mode=True, transform=transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ])), batch_size=1, shuffle=False, num_workers=args.workers, pin_memory=True, drop_last=True) net = Spatial_TemporalNet(basemodel=args.base_model, dataset=args.dataset, segment=args.segments, attention_type=args.attention_type, hidden_size=args.hidden_size, img_dim=args.feature_size, kernel_size=args.kernel_size) net = torch.nn.DataParallel(net).cuda() for param in net.parameters(): param.requires_grad = True def attentionloss(baseline, attention, target): target_temp = target.unsqueeze(1) baseline_temp = torch.gather(baseline, 1, target_temp).squeeze(0) attention_temp = torch.gather(attention, 1, target_temp).squeeze(0) selfloss = torch.max( torch.zeros(1).cuda(), baseline_temp - attention_temp + 0.1) return selfloss.mean() if args.pretrained: dicts_net = net.state_dict() weights = torch.load('./model/Kinetics_cnnlstm.pkl') count = 0 for key, value in weights.items(): for key_net, value_net in dicts_net.items(): if key.startswith('module.spatial.net') and key == key_net: dicts_net[key_net] = value count += 1 print('loading {0} pretrained weights from Kinetics-600'.format(count)) net.load_state_dict(dicts_net) if args.cross: net.load_state_dict(torch.load('./model/2019-03-04 23:12:5914.pkl')) if args.target_dataset == 'hmdb': num_class = 51 if args.target_dataset == 'ucf': num_class = 101 setattr(net.module.temporal.reason_learned, 'fc', nn.Linear(1024, num_class).cuda()) setattr(net.module.temporal.reason_auto, 'fc', nn.Linear(1024, num_class).cuda()) setattr(net.module.temporal.reason_average, 'fc', nn.Linear(1024, num_class).cuda()) print('load pre-trained weights on Kinetics successfully ') if args.get_scores: net.load_state_dict(torch.load('./model/2019-03-11 08:45:40.pkl')) print('begin to get class scores 2019-03-11 08:45:40') criterion = torch.nn.CrossEntropyLoss() optimizer = torch.optim.SGD(net.parameters(), lr=args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) print('doing experiments on ' + args.test_video_list) best_prec1 = 0 for epoch in range(args.epoch): if not args.get_scores: adjust_learning_rate(optimizer, epoch, args) train(train_loader, net, criterion, optimizer, epoch, log, attentionloss) # torch.save(net.state_dict(), os.path.join(args.model_dir, strat_time + str(epoch) + '.pkl')) if (epoch + 1) % args.eval_freq == 0: prec1 = test(test_loader, net, epoch, log) if prec1 > best_prec1: best_prec1 = prec1 torch.save( net.state_dict(), os.path.join(args.model_dir, strat_time + '.pkl')) else: print('Begin Get Scores') gets(test_loader, net, epoch, args) print('Done') break
def main(args): global best_prec1 # Benchmark模式会提升计算速度,但是由于计算中有随机性,每次网络前馈结果略有差异。 cudnn.benchmark = True # 如果想要避免这种结果波动,设置: cudnn.deterministic = True # 创建模型文件夹 if not os.path.exists(args.model_dir): os.mkdir(args.model_dir) strat_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # 训练数据加载 train_loader = torch.utils.data.DataLoader(VideoDataset( root=args.root, list=args.train_video_list, num_segments=args.segments, num_frames=args.frames, transform=transforms.Compose([ transforms.Resize(256), transforms.RandomCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ])), batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True, drop_last=True) # 加载测试数据 test_loader = torch.utils.data.DataLoader(VideoDataset( root=args.root, list=args.test_video_list, num_segments=args.segments, num_frames=args.frames, test_mode=True, transform=transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ])), batch_size=1, shuffle=False, num_workers=args.workers, pin_memory=True, drop_last=True) # 网络模型创建 net = Spatial_TemporalNet(basemodel=args.base_model, dataset=args.dataset, segment=args.segments, attention_type=args.attention_type, hidden_size=args.hidden_size, img_dim=args.feature_size, kernel_size=args.kernel_size) # 多GPU训练的代码 net = torch.nn.DataParallel(net).cuda() # 获得梯度为True for param in net.parameters(): param.requires_grad = True # 注意力损失函数 def attentionloss(baseline, attention, target): target_temp = target.unsqueeze(1) baseline_temp = torch.gather(baseline, 1, target_temp).squeeze(0) attention_temp = torch.gather(attention, 1, target_temp).squeeze(0) selfloss = torch.max( torch.zeros(1).cuda(), baseline_temp - attention_temp + 0.1) return selfloss.mean() # # if args.cross: # net.load_state_dict(torch.load('./model/model.pkl')) # if args.target_dataset == 'hmdb': # num_class = 51 # if args.target_dataset == 'ucf': # num_class = 101 # setattr(net.module.temporal.reason_learned, 'fc', nn.Linear(1024, num_class).cuda()) # setattr(net.module.temporal.reason_auto, 'fc', nn.Linear(1024, num_class).cuda()) # setattr(net.module.temporal.reason_average, 'fc', nn.Linear(1024, num_class).cuda()) # print ('load pre-trained weights on Kinetics successfully ') # # if args.get_scores: # net.load_state_dict(torch.load('./model/model.pkl')) # print ('begin to get class scores model') # 分类损失为交叉熵损失 criterion = torch.nn.CrossEntropyLoss() # 优化器梯度下降 optimizer = torch.optim.SGD(net.parameters(), lr=args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) print('doing experiments on ' + args.test_video_list) best_prec1 = 0 # 迭代训练并优化 for epoch in range(args.epoch): if not args.get_scores: adjust_learning_rate(optimizer, epoch, args) # 训练 train_model(train_loader, net, criterion, optimizer, epoch, attentionloss) if (epoch + 1) % args.eval_freq == 0: # 测试 prec1 = test_model(test_loader, net, epoch) if prec1 > best_prec1: best_prec1 = prec1 torch.save( net.state_dict(), os.path.join(args.model_dir, strat_time + '.pkl')) print('保存在', os.path.join(args.model_dir, strat_time + '.pkl')) else: print('开始获得分数') gets(test_loader, net, epoch, args) print('完成') break
# ┌─┐┌─┐┌┬┐┬ ┬┌─┐ ┌─┐┌─┐┬ ┬┌─┐┌┐┌┌─┐┬ ┬ ┌┐┌┌─┐┌┬┐ # └─┐├┤ │ │ │├─┘ └─┐├─┤│ │├┤ ││││ └┬┘ │││├┤ │ # └─┘└─┘ ┴ └─┘┴ └─┘┴ ┴┴─┘┴└─┘┘└┘└─┘ ┴ ┘└┘└─┘ ┴ saliencynet = ImageSaliencyNet(saliency_deploy_prototxt_path, saliency_caffemodel_path) # ╔═╗┌─┐┌┬┐┬ ┬┌─┐ ┌┬┐┌─┐┌┬┐┌─┐┌─┐┌─┐┌┬┐ # ╚═╗├┤ │ │ │├─┘ ││├─┤ │ ├─┤└─┐├┤ │ # ╚═╝└─┘ ┴ └─┘┴ ─┴┘┴ ┴ ┴ ┴ ┴└─┘└─┘ ┴ print "Loading data..." train_frame_basedir = '/data/sunnycia/SaliencyDataset/Video/MSU/frames' train_density_basedir = '/data/sunnycia/SaliencyDataset/Video/MSU/density/sigma32' validation_frame_basedir = '/data/sunnycia/SaliencyDataset/Image/SALICON/DATA/train_val/val2014/images' validation_density_basedir = '/data/sunnycia/SaliencyDataset/Image/SALICON/DATA/train_val/val2014/density' tranining_dataset = VideoDataset(train_frame_basedir, train_density_basedir) tranining_dataset.setup_video_dataset_flow() # validation_dataset = StaticDataset(train_frame_basedir, train_density_basedir, debug=debug_mode) # ╔╦╗╦╔═╗╔═╗ # ║║║║╚═╗║ # ╩ ╩╩╚═╝╚═╝ plot_figure_dir = '../figure' ## Figure dir plot_figure_dir = os.path.join(plot_figure_dir, postfix_str) if not os.path.isdir(plot_figure_dir): os.makedirs(plot_figure_dir) print "Loss figure will be save to", plot_figure_dir #################################### # /| / / #
def main(): global args, best_prec1 cudnn.benchmark = True args = parser.parse_args() if not os.path.exists(args.log_dir): os.mkdir(args.log_dir) if not os.path.exists(args.model_dir): os.mkdir(args.model_dir) strat_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') log = open(os.path.join(args.log_dir, strat_time + '.txt'), 'w') print(args.description) log.write(args.description + '\n') log.flush() print( '=======================Experimental Settings=======================\n' ) log.write( '=======================Experimental Settings=======================\n' ) log.flush() print('Using_Dataset:{0} Batch_Size:{1} Epochs:{2} '.format( args.dataset, args.batch_size, args.epoch)) log.write('Using_Dataset:{0} Batch_Size:{1} Epochs:{2}'.format( args.dataset, args.batch_size, args.epoch) + '\n') log.flush() print('Num_segments:{0} Num_frames:{1} Base_model:{2}\n'.format( args.segments, args.frames, args.base_model)) log.write('Num_segments:{0} Num_frames:{1} Base_model:{2}\n'.format( args.segments, args.frames, args.base_model) + '\n') log.flush() print( '===================================================================\n' ) log.write( '===================================================================\n' ) log.flush() test_loader = torch.utils.data.DataLoader(VideoDataset( root=args.root, list=args.test_video_list, num_segments=args.segments, num_frames=args.frames, test_mode=True, transform=transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ])), batch_size=1, shuffle=False, num_workers=args.workers * 2, pin_memory=True, drop_last=True) net = Net(basemodel=args.base_model, num_segments=args.segments, num_frames=args.frames, dataset=args.dataset, d_model=args.d_model, start=args.start) net = torch.nn.DataParallel(net).cuda() net.load_state_dict(torch.load('./model/2018-08-18 08_16_57.pkl')) prec1 = test(test_loader, net) print('The testing accuracy is: {0}'.format(prec1))
clip_length = args.clip_length if dataset == 'videoset': frame_dir = '/data/SaliencyDataset/Video/VideoSet/ImageSet/Seperate/frame' density_dir = '/data/SaliencyDataset/Video/VideoSet/ImageSet/Seperate/density/sigma32' fixation_dir = '/data/SaliencyDataset/Video/VideoSet/ImageSet/Seperate/fixation' if not os.path.isdir(result_path): os.makedirs(result_path) log_name = 'result.txt' log_path = os.path.join(result_path, log_name) log_f = open(log_path, 'w') dataset = VideoDataset(frame_dir, density_dir, img_size=(112, 112), bgr_mean_list=[98, 102, 90], sort='rgb') video_frame_list = os.listdir(frame_dir) video_density_list = os.listdir(density_dir) video_fixation_list = os.listdir(fixation_dir) ### Load test clip list testset_path_list = glob.glob(os.path.join(working_directory, 'testset*pkl')) testset_path_list.sort() solver_path = glob.glob(os.path.join(working_directory, '*solver*prototxt'))[0] timestamp = working_directory.split('_')[-1] cc_list = [] sim_list = []