def run_svm_on_validation_set_and_pick_the_best(self, models_dir,
validation_file,
score_directory,
scores_in_trec_format_path,
qrel_path):
models_dirs = os.walk(models_dir)
evaluation = evaluator.evaluator()
evaluation.prepare_index_of_test_file(validation_file)
for models_dir in models_dirs:
if not models_dir[1]:
for model in models_dir[2]:
model_file = models_dir[0] + "/" + model
self.run_model_svm(model_file, validation_file,
score_directory)
for scores_dir_data in os.walk(score_directory):
if not scores_dir_data[1]: # no subdirectories
for scores_file_name in scores_dir_data[2]:
scores_file = scores_dir_data[0] + "/" + scores_file_name
evaluation.create_file_in_trec_eval_format(
scores_file, scores_in_trec_format_path, '')
for final_score_dir in os.walk(scores_in_trec_format_path):
if not final_score_dir[1]:
evaluation.run_trec_eval_on_evaluation_set(
final_score_dir[0], qrel_path)
fold = os.path.basename(final_score_dir[0])
self.chosen_models[fold] = evaluation.chosen_model
def sort_and_evaluate_final_file(self, final_file_name, qrel_path):
final_file_as_txt = final_file_name.replace("tmp", "txt")
command = "sort -k1,1 -k5nr " + final_file_name + " > " + final_file_as_txt
for output_line in self.run_command(command):
print(output_line)
evaluation = evaluator.evaluator()
evaluation.run_trec_eval_on_test_file(qrel_path, final_file_as_txt)
def run_svm_on_test_set(self, fold, models_path, test_file, score_dir,
final_score_directory, qrel_path):
key = fold
print(self.chosen_models.keys())
sys.stdout.flush()
model_file_name = models_path + "/" + self.chosen_models[key]
score_file = self.run_model_svm(model_file_name, test_file, score_dir)
evaluation = evaluator.evaluator()
evaluation.prepare_index_of_test_file(test_file)
final_score_trec_file = evaluation.create_file_in_trec_eval_format(
score_file, final_score_directory, '')
return final_score_trec_file
def run_chosen_model_on_test_lambda_mart(self, fold, models_path,
test_file, score_dir,
final_score_directory, qrel_path):
key = fold
model_file_name = models_path + "/" + self.chosen_models[key]
score_file = self.run_model_lmbda_mart(model_file_name, test_file,
score_dir)
evaluation = evaluator.evaluator()
evaluation.prepare_index_of_test_file(test_file)
final_score_trec_file = evaluation.create_file_in_trec_eval_format(
score_file, final_score_directory, 'RANKLIB')
return final_score_trec_file