def main(argv=None):
#Getting arguments from config file and command line
#Building the arg reader
argreader = ArgReader(argv)
########### PLOT TSNE ################
argreader.parser.add_argument('--tsne',action='store_true',help='To plot t-sne representation of feature extracted. Also plots the representation of a video side by side to make an image. \
The --exp_id, --model_id, --seed and --dataset_test arguments should be set.')
########### PLOT SCORE EVOLUTION ALONG VIDEO ##################
argreader.parser.add_argument('--plot_score',action="store_true",help='To plot the scene change probability of produced by a model for all the videos processed by this model during validation for all epochs.\
The --model_id argument must be set, along with the --exp_id, --dataset_test and --epoch_to_plot arguments.')
argreader.parser.add_argument('--epoch_to_plot',type=int,metavar="N",help='The epoch at which to plot the predictions when using the --plot_score argument')
argreader.parser.add_argument('--untrained_exp_and_model_id',default=[None,None],type=str,nargs=2,help='To plot the distance between features computed by the network before training when using the --plot_score arg.\
The values are the exp_id and the model_id of the model not trained on scene change detection (i.e. the model with the ImageNet weights)')
argreader.parser.add_argument('--plot_dist',action="store_true",help='To plot the distance when using the --plot_score argument')
########## COMPUTE METRICS AND PUT THEM IN AN LATEX TABLE #############
argreader.parser.add_argument('--eval_model_leave_one_out',type=float,nargs=3,help='To evaluate a model by tuning its decision threshold on the video on which it is not\
evaluated. The --model_id argument must be set, along with the --model_name, --exp_id and --dataset_test arguments. \
The values of this args are the epoch at which to evaluate , followed by the minimum and maximum decision threshold \
to evaluate. Use the --len_pond to ponderate the overflow and coverage by scene lengths.')
argreader.parser.add_argument('--model_name',type=str,metavar="NAME",help='The name of the model as will appear in the latex table produced by the --eval_model_leave_one_out argument.')
argreader.parser.add_argument('--len_pond',action="store_true",help='Use this argument to ponderate the coverage and overflow by the GT scene length when using --eval_model_leave_one_out.')
argreader.parser.add_argument('--fine_tuned_thres',action="store_true",help='To automatically fine tune the decision threshold of the model. Only useful for the --bbc_annot_dist arg. \
Check the help of this arg.')
######### Make a gif out of predictions ################
argreader.parser.add_argument('--gif',action="store_true",help='Make a gif out of the predictions of a model on one dataset.')
argreader.parser.add_argument('--model_id1',type=str,metavar="ID",help='The id of the first model to plot with --gif')
argreader.parser.add_argument('--model_id2',type=str,metavar="ID",help='The id of the second model to plot with --gif')
argreader.parser.add_argument('--model_name1',type=str,metavar="NAME",help='The name of the first model to plot with --gif')
argreader.parser.add_argument('--model_name2',type=str,metavar="NAME",help='The name of the second model to plot with --gif')
argreader = load_data.addArgs(argreader)
#Reading the comand line arg
argreader.getRemainingArgs()
#Getting the args from command line and config file
args = argreader.args
if args.tsne:
tsne(args.dataset_test,args.exp_id,args.model_id,args.seed)
if args.plot_score:
plotScore(args.exp_id,args.model_id,args.untrained_exp_and_model_id[0],args.untrained_exp_and_model_id[1],args.dataset_test,args.plot_dist,args.epoch_to_plot)
if args.eval_model_leave_one_out:
epoch = int(args.eval_model_leave_one_out[0])
thresMin = args.eval_model_leave_one_out[1]
thresMax = args.eval_model_leave_one_out[2]
evalModel_leaveOneOut(args.exp_id,args.model_id,args.model_name,args.dataset_test,epoch,thresMin,thresMax,args.len_pond)
if args.gif:
makeGif(args.exp_id,args.model_id1,args.model_id2,args.model_name1,args.model_name2,args.dataset_test)
def main(argv=None):
#Getting arguments from config file and command line
#Building the arg reader
argreader = ArgReader(argv)
argreader.parser.add_argument('--comp_feat', action='store_true',help='To compute and write in a file the features of all images in the test set. All the arguments used to \
build the model and the test data loader should be set.')
argreader.parser.add_argument('--no_train', type=str,nargs=2,help='To use to re-evaluate a model at each epoch after training. At each epoch, the model is not trained but \
the weights of the corresponding epoch are loaded and then the model is evaluated.\
The values of this argument are the exp_id and the model_id of the model to get the weights from.')
argreader = addInitArgs(argreader)
argreader = addLossArgs(argreader)
argreader = addOptimArgs(argreader)
argreader = addValArgs(argreader)
argreader = modelBuilder.addArgs(argreader)
argreader = load_data.addArgs(argreader)
#Reading the comand line arg
argreader.getRemainingArgs()
args = argreader.args
if args.redirect_out:
sys.stdout = open("python.out", 'w')
#The folders where the experience file will be written
if not (os.path.exists("../vis/{}".format(args.exp_id))):
os.makedirs("../vis/{}".format(args.exp_id))
if not (os.path.exists("../results/{}".format(args.exp_id))):
os.makedirs("../results/{}".format(args.exp_id))
if not (os.path.exists("../models/{}".format(args.exp_id))):
os.makedirs("../models/{}".format(args.exp_id))
#Write the arguments in a config file so the experiment can be re-run
argreader.writeConfigFile("../models/{}/{}.ini".format(args.exp_id,args.model_id))
writer = SummaryWriter("../results/{}".format(args.exp_id))
print("Model :",args.model_id,"Experience :",args.exp_id)
if args.comp_feat:
testLoader = load_data.TestLoader(args.val_l,args.dataset_test,args.test_part_beg,args.test_part_end,args.img_size,\
args.resize_image,args.exp_id,args.random_frame_val)
if args.feat != "None":
featModel = modelBuilder.buildFeatModel(args.feat,args.pretrain_dataset,args.lay_feat_cut)
if args.cuda:
featModel = featModel.cuda()
if args.init_path_visual != "None":
featModel.load_state_dict(torch.load(args.init_path_visual))
elif args.init_path != "None":
model = modelBuilder.netBuilder(args)
params = torch.load(args.init_path)
state_dict = {k.replace("module.cnn.","cnn.module."): v for k,v in params.items()}
model.load_state_dict(state_dict)
featModel = model.featModel
featModel.eval()
else:
featModel = None
with torch.no_grad():
evalAllImages(args.exp_id,args.model_id,featModel,testLoader,args.cuda,args.log_interval)
else:
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
paramToOpti = []
trainLoader,trainDataset = load_data.buildSeqTrainLoader(args)
valLoader = load_data.TestLoader(args.val_l,args.dataset_val,args.val_part_beg,args.val_part_end,\
args.img_size,args.resize_image,\
args.exp_id,args.random_frame_val)
#Building the net
net = modelBuilder.netBuilder(args)
if args.cuda:
net = net.cuda()
trainFunc = epochSeqTr
valFunc = epochSeqVal
kwargsTr = {'log_interval':args.log_interval,'loader':trainLoader,'args':args,'writer':writer}
kwargsVal = kwargsTr.copy()
kwargsVal['loader'] = valLoader
kwargsVal.update({"metricEarlyStop":args.metric_early_stop,"maximiseMetric":args.maximise_metric})
if args.adv_weight > 0:
kwargsTr["discrModel"] = modelBuilder.Discriminator(net.nbFeat,args.discr_dropout)
kwargsTr["discrModel"] = kwargsTr["discrModel"].cuda() if args.cuda else kwargsTr["discrModel"].cpu()
kwargsTr["discrLoader"] = load_data.buildFrameTrainLoader(args)
kwargsTr["discrOptim"] = torch.optim.SGD(kwargsTr["discrModel"].parameters(), lr=args.lr,momentum=args.momentum)
else:
kwargsTr["discrModel"],kwargsTr["discrLoader"],kwargsTr["discrOptim"] = None,None,None
for p in net.parameters():
paramToOpti.append(p)
paramToOpti = (p for p in paramToOpti)
#Getting the contructor and the kwargs for the choosen optimizer
optimConst,kwargsOpti = get_OptimConstructor_And_Kwargs(args.optim,args.momentum)
startEpoch = initialize_Net_And_EpochNumber(net,args.exp_id,args.model_id,args.cuda,args.start_mode,args.init_path,args.init_path_visual_temp)
#If no learning rate is schedule is indicated (i.e. there's only one learning rate),
#the args.lr argument will be a float and not a float list.
#Converting it to a list with one element makes the rest of processing easier
if type(args.lr) is float:
args.lr = [args.lr]
lrCounter = 0
metricLastVal = None
outDictEpochs = {}
targDictEpochs = {}
for epoch in range(startEpoch, args.epochs + 1):
kwargsOpti,kwargsTr,lrCounter = update.updateLR(epoch,args.epochs,args.lr,startEpoch,kwargsOpti,kwargsTr,lrCounter,net,optimConst)
kwargsTr["epoch"],kwargsVal["epoch"] = epoch,epoch
kwargsTr["model"],kwargsVal["model"] = net,net
kwargsTr = resetAdvIter(kwargsTr)
if not args.no_train:
trainFunc(**kwargsTr)
else:
net.load_state_dict(torch.load("../models/{}/model{}_epoch{}".format(args.no_train[0],args.no_train[1],epoch)))
kwargsVal["metricLastVal"] = metricLastVal
#Checking if validation has already been done
if len(glob.glob("../results/{}/{}_epoch{}_*".format(args.exp_id,args.model_id,epoch))) < len(kwargsVal["loader"].videoPaths):
with torch.no_grad():
metricLastVal,outDict,targDict = valFunc(**kwargsVal)
outDictEpochs[epoch] = outDict
targDictEpochs[epoch] = targDict
update.updateHist(writer,args.model_id,outDictEpochs,targDictEpochs)
else:
print("Validation epoch {} already done !".format(epoch))
def main(argv=None):
# Getting arguments from config file and command line
# Building the arg reader
argreader = ArgReader(argv)
argreader.parser.add_argument('--no_train', type=str2bool, help='To use to re-evaluate a model at each epoch after training. At each epoch, the model is not trained but \
the weights of the corresponding epoch are loaded and then the model is evaluated.\
The arguments --exp_id_no_train and the --model_id_no_train must be set')
argreader.parser.add_argument('--exp_id_no_train', type=str,
help="To use when --no_train is set to True. This is the exp_id of the model to get the weights from.")
argreader.parser.add_argument('--model_id_no_train', type=str,
help="To use when --no_train is set to True. This is the model_id of the model to get the weights from.")
argreader.parser.add_argument('--no_val', type=str2bool, help='To not compute the validation')
argreader.parser.add_argument('--only_test', type=str2bool, help='To only compute the test')
argreader.parser.add_argument('--do_test_again', type=str2bool, help='Does the test evaluation even if it has already been done')
argreader.parser.add_argument('--compute_latency', type=str2bool, help='To write in a file the latency at each forward pass.')
argreader.parser.add_argument('--grad_cam', type=str2bool, help='To compute grad cam instead of training or testing.')
argreader.parser.add_argument('--optuna', type=str2bool, help='To run a hyper-parameter study')
argreader.parser.add_argument('--optuna_trial_nb', type=int, help='The number of hyper-parameter trial to run.')
argreader.parser.add_argument('--opt_data_aug', type=str2bool, help='To optimise data augmentation hyper-parameter.')
argreader.parser.add_argument('--max_batch_size', type=int, help='To maximum batch size to test.')
argreader = addInitArgs(argreader)
argreader = addOptimArgs(argreader)
argreader = addValArgs(argreader)
argreader = addLossTermArgs(argreader)
argreader = modelBuilder.addArgs(argreader)
argreader = load_data.addArgs(argreader)
# Reading the comand line arg
argreader.getRemainingArgs()
args = argreader.args
if args.redirect_out:
sys.stdout = open("python.out", 'w')
# The folders where the experience file will be written
if not (os.path.exists("../vis/{}".format(args.exp_id))):
os.makedirs("../vis/{}".format(args.exp_id))
if not (os.path.exists("../results/{}".format(args.exp_id))):
os.makedirs("../results/{}".format(args.exp_id))
if not (os.path.exists("../models/{}".format(args.exp_id))):
os.makedirs("../models/{}".format(args.exp_id))
args = updateSeedAndNote(args)
# Update the config args
argreader.args = args
# Write the arguments in a config file so the experiment can be re-run
argreader.writeConfigFile("../models/{}/{}.ini".format(args.exp_id, args.model_id))
print("Model :", args.model_id, "Experience :", args.exp_id)
if args.distributed:
size = args.distrib_size
processes = []
for rank in range(size):
p = Process(target=init_process, args=(args, rank, size, run))
p.start()
processes.append(p)
for p in processes:
p.join()
else:
if args.optuna:
def objective(trial):
return run(args,trial=trial)
study = optuna.create_study(direction="maximize" if args.maximise_val_metric else "minimize",\
storage="sqlite:///../results/{}/{}_hypSearch.db".format(args.exp_id,args.model_id), \
study_name=args.model_id,load_if_exists=True)
con = sqlite3.connect("../results/{}/{}_hypSearch.db".format(args.exp_id,args.model_id))
curr = con.cursor()
failedTrials = 0
for elem in curr.execute('SELECT trial_id,value FROM trials WHERE study_id == 1').fetchall():
if elem[1] is None:
failedTrials += 1
trialsAlreadyDone = len(curr.execute('SELECT trial_id,value FROM trials WHERE study_id == 1').fetchall())
if trialsAlreadyDone-failedTrials < args.optuna_trial_nb:
studyDone = False
while not studyDone:
try:
print("N trials",args.optuna_trial_nb-trialsAlreadyDone+failedTrials)
study.optimize(objective,n_trials=args.optuna_trial_nb-trialsAlreadyDone+failedTrials)
studyDone = True
except RuntimeError as e:
if str(e).find("CUDA out of memory.") != -1:
gc.collect()
torch.cuda.empty_cache()
args.max_batch_size -= 5
else:
raise RuntimeError(e)
curr.execute('SELECT trial_id,value FROM trials WHERE study_id == 1')
query_res = curr.fetchall()
query_res = list(filter(lambda x:not x[1] is None,query_res))
trialIds = [id_value[0] for id_value in query_res]
values = [id_value[1] for id_value in query_res]
trialIds = trialIds[:args.optuna_trial_nb]
values = values[:args.optuna_trial_nb]
bestTrialId = trialIds[np.array(values).argmax()]
curr.execute('SELECT param_name,param_value from trial_params WHERE trial_id == {}'.format(bestTrialId))
query_res = curr.fetchall()
bestParamDict = {key:value for key,value in query_res}
args.lr,args.batch_size = bestParamDict["lr"],int(bestParamDict["batch_size"])
args.optim = OPTIM_LIST[int(bestParamDict["optim"])]
args.only_test = True
bestPath = glob.glob("../models/{}/model{}_trial{}_best_epoch*".format(args.exp_id,args.model_id,bestTrialId-1))[0]
copyfile(bestPath, bestPath.replace("_trial{}".format(bestTrialId-1),""))
run(args)
else:
run(args)