def test_msmarco_doc_qrels(self): qrels = search.get_qrels('msmarco-doc-dev') self.assertEqual(len(qrels), 5193) self.assertTrue(isinstance(next(iter(qrels.keys())), int))
def test_msmarco_passage_qrels(self): qrels = search.get_qrels('msmarco-passage-dev-subset') self.assertEqual(len(qrels), 6980) self.assertTrue(isinstance(next(iter(qrels.keys())), int))
def test_trec1_adhoc_qrels(self): qrels = search.get_qrels('trec1-adhoc') self.assertEqual(len(qrels), 50) self.assertTrue(isinstance(next(iter(qrels.keys())), int))
def test_dl20_passage_qrels(self): qrels = search.get_qrels('dl20-passage') self.assertEqual(len(qrels), 54) self.assertFalse(isinstance(next(iter(qrels.keys())), str))
def test_trec2019_bl_qrels(self): qrels = search.get_qrels('trec2019-bl') self.assertEqual(len(qrels), 57) self.assertTrue(isinstance(next(iter(qrels.keys())), int))
def test_dl20_doc_qrels(self): qrels = search.get_qrels('dl20-doc') self.assertEqual(len(qrels), 45) self.assertFalse(isinstance(next(iter(qrels.keys())), str))
def test_trec2004_terabyte_qrels(self): qrels = search.get_qrels('trec2004-terabyte') self.assertEqual(len(qrels), 49) self.assertTrue(isinstance(next(iter(qrels.keys())), int))
def test_trec2002_ar_qrels(self): qrels = search.get_qrels('trec2002-ar') self.assertEqual(len(qrels), 50) self.assertTrue(isinstance(next(iter(qrels.keys())), int))
import os import subprocess import sys from pyserini.search import get_qrels from pyserini.util import download_evaluation_script script_path = download_evaluation_script('msmarco_passage_eval') cmd_prefix = ['python3', script_path] args = sys.argv if len(args) > 1: cmd = cmd_prefix + args[1:] if not os.path.exists(cmd[-2]): cmd[-2] = get_qrels(cmd[-2]) else: cmd = cmd_prefix print(f'Running command: {cmd}') process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() if stderr: print('Results:') print(stderr.decode("utf-8")) else: print(stdout.decode("utf-8"))
import os import subprocess import sys from pyserini.search import get_qrels from pyserini.util import download_evaluation_script script_path = download_evaluation_script('msmarco_doc_eval') cmd_prefix = ['python3', script_path] args = sys.argv if len(args) > 1: cmd = cmd_prefix + args[1:] for i in range(len(cmd) - 1): if cmd[i] == '--judgments': if not os.path.exists(cmd[i + 1]): cmd[i + 1] = get_qrels(cmd[i + 1]) else: cmd = cmd_prefix print(f'Running command: {cmd}') process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() if stderr: print('Results:') print(stderr.decode("utf-8")) else: print(stdout.decode("utf-8"))
def test_wt10g_qrels(self): qrels = search.get_qrels('wt10g') self.assertEqual(len(qrels), 100) self.assertTrue(isinstance(next(iter(qrels.keys())), int))
def test_core18_qrels(self): qrels = search.get_qrels('core18') self.assertEqual(len(qrels), 50) self.assertTrue(isinstance(next(iter(qrels.keys())), int))
def test_robust05_qrels(self): qrels = search.get_qrels('robust05') self.assertEqual(len(qrels), 50) self.assertTrue(isinstance(next(iter(qrels.keys())), int))
def test_ntcir8_zh_qrels(self): qrels = search.get_qrels('ntcir8-zh') self.assertEqual(len(qrels), 100) self.assertTrue(isinstance(next(iter(qrels.keys())), str))
def test_mb14_qrels(self): qrels = search.get_qrels('mb14') self.assertEqual(len(qrels), 55) self.assertTrue(isinstance(next(iter(qrels.keys())), int))
def test_clef2006_fr_qrels(self): qrels = search.get_qrels('clef2006-fr') self.assertEqual(len(qrels), 49) self.assertTrue(isinstance(next(iter(qrels.keys())), str))
def test_car20_qrels(self): qrels = search.get_qrels('car17v2.0-benchmarkY1test') self.assertEqual(len(qrels), 2254) self.assertFalse(isinstance(next(iter(qrels.keys())), int))
def test_fire2012_hi_qrels(self): qrels = search.get_qrels('fire2012-hi') self.assertEqual(len(qrels), 50) self.assertTrue(isinstance(next(iter(qrels.keys())), int))
def test_covid_round4_qrels(self): qrels = search.get_qrels('covid-round4') self.assertEqual(len(qrels), 45) self.assertTrue(isinstance(next(iter(qrels.keys())), int))