cuda = None # the gpu id, e.g., 0 or 1, otherwise, set it as None indicating to use cpu debug = True # in a debug mode, we just check whether the model can operate config_with_json = False # specify configuration with json files or not models_to_run = [ #'IRGAN_Point', #'IRGAN_Pair', #'IRGAN_List' 'IRFGAN_Point', #'IRFGAN_Pair', #'IRFGAN_List' ] evaluator = AdLTREvaluator(cuda=cuda) if config_with_json: # specify configuration with json files # the directory of json files #dir_json = '/home/dl-box/WorkBench/ExperimentBench/ALTR/ecir2021/irgan/mq2008_json/' dir_json = '/home/dl-box/WorkBench/ExperimentBench/ALTR/ecir2021/irgan/ms30k_json/' #dir_json = '/home/dl-box/WorkBench/ExperimentBench/ALTR/ecir2021/irfgan/mq2008_json/' dir_json = '/home/dl-box/WorkBench/ExperimentBench/ALTR/ecir2021/irfgan/ms30k_json/' #dir_json = '/Users/dryuhaitao/WorkBench/Dropbox/CodeBench/GitPool/irgan_ptranking/testing/ltr_adversarial/json/' for model_id in models_to_run: evaluator.run(debug=debug, model_id=model_id, config_with_json=config_with_json, dir_json=dir_json) else: # specify configuration manually
| IRGAN_MQ2008_Semi | ----------------------------------------------------------------------------------------- """ ''' selected dataset ''' data_id = 'MQ2007_Super' ''' location of the adopted data ''' dir_data = os.path.join(DATASET_DIR, 'MQ2007/') #dir_data = '/home/dl-box/WorkBench/Datasets/L2R/LETOR4.0/MQ2008/' #dir_data = '/Users/solar/WorkBench/Datasets/L2R/LETOR4.0/MQ2008/' ''' output directory ''' dir_output = os.path.join(PROJECT_OUTPUT_DIR, 'NeuralLTR/Listwise/') #dir_output = '/home/dl-box/WorkBench/CodeBench/PyCharmProject/Project_output/Out_L2R/Listwise/' #dir_output = '/Users/solar/WorkBench/CodeBench/PyCharmProject/Project_output/Out_L2R/' debug = True # with a debug mode, we can make a quick test, e.g., check whether the model can operate or not grid_search = False # with grid_search, we can explore the effects of different hyper-parameters of a model evaluator = AdLTREvaluator() to_run_models = ['IRGAN_Pair'] for model_id in to_run_models: evaluator.run(debug=debug, model_id=model_id, data_id=data_id, dir_data=dir_data, dir_output=dir_output, grid_search=grid_search)
----------------------------------------------------------------------------------------- | Yahoo_LTR | Set1 % Set2 | ----------------------------------------------------------------------------------------- | ISTELLA_LTR | Istella_S | Istella | Istella_X | ----------------------------------------------------------------------------------------- """ args_obj = ArgsUtil(given_root='./') l2r_args = args_obj.get_l2r_args() if l2r_args.model in LTR_ADHOC_MODEL: evaluator = LTREvaluator(cuda=l2r_args.cuda) elif l2r_args.model in LTR_ADVERSARIAL_MODEL: evaluator = AdLTREvaluator(cuda=l2r_args.cuda) elif l2r_args.model in LTR_TREE_MODEL: evaluator = TreeLTREvaluator() else: args_obj.args_parser.print_help() sys.exit() print('Started evaluation with pt_ranking !') evaluator.run(model_id=l2r_args.model, dir_json=l2r_args.dir_json, debug=l2r_args.debug, config_with_json=True) print('Finished evaluation with pt_ranking !')
----------------------------------------------------------------------------------------- | LETTOR | MQ2007_Super % MQ2008_Super % MQ2007_Semi % MQ2008_Semi | ----------------------------------------------------------------------------------------- | MSLRWEB | MSLRWEB10K % MSLRWEB30K | ----------------------------------------------------------------------------------------- | Yahoo_LTR | Set1 % Set2 | ----------------------------------------------------------------------------------------- | ISTELLA_LTR | Istella_S | Istella | Istella_X | ----------------------------------------------------------------------------------------- """ print('Started PT_Ranking ...') args_obj = ArgsUtil(given_root='./') l2r_args = args_obj.get_l2r_args() if l2r_args.model in LTR_ADHOC_MODEL: evaluator = LTREvaluator() elif l2r_args.model in LTR_ADVERSARIAL_MODEL: evaluator = AdLTREvaluator() elif l2r_args.model in LTR_TREE_MODEL: evaluator = TreeLTREvaluator() else: raise NotImplementedError evaluator.run(debug=True, model_id=l2r_args.model, data_id=l2r_args.data_id, dir_data=l2r_args.dir_data, dir_output=l2r_args.dir_output, grid_search=False)