def test_msmarco_doc_qrels(self):
qrels = search.get_qrels('msmarco-doc-dev')
self.assertEqual(len(qrels), 5193)
self.assertTrue(isinstance(next(iter(qrels.keys())), int))
def test_msmarco_passage_qrels(self):
qrels = search.get_qrels('msmarco-passage-dev-subset')
self.assertEqual(len(qrels), 6980)
self.assertTrue(isinstance(next(iter(qrels.keys())), int))
def test_trec1_adhoc_qrels(self):
qrels = search.get_qrels('trec1-adhoc')
self.assertEqual(len(qrels), 50)
self.assertTrue(isinstance(next(iter(qrels.keys())), int))
def test_dl20_passage_qrels(self):
qrels = search.get_qrels('dl20-passage')
self.assertEqual(len(qrels), 54)
self.assertFalse(isinstance(next(iter(qrels.keys())), str))
def test_trec2019_bl_qrels(self):
qrels = search.get_qrels('trec2019-bl')
self.assertEqual(len(qrels), 57)
self.assertTrue(isinstance(next(iter(qrels.keys())), int))
def test_dl20_doc_qrels(self):
qrels = search.get_qrels('dl20-doc')
self.assertEqual(len(qrels), 45)
self.assertFalse(isinstance(next(iter(qrels.keys())), str))
def test_trec2004_terabyte_qrels(self):
qrels = search.get_qrels('trec2004-terabyte')
self.assertEqual(len(qrels), 49)
self.assertTrue(isinstance(next(iter(qrels.keys())), int))
def test_trec2002_ar_qrels(self):
qrels = search.get_qrels('trec2002-ar')
self.assertEqual(len(qrels), 50)
self.assertTrue(isinstance(next(iter(qrels.keys())), int))
import os
import subprocess
import sys
from pyserini.search import get_qrels
from pyserini.util import download_evaluation_script
script_path = download_evaluation_script('msmarco_passage_eval')
cmd_prefix = ['python3', script_path]
args = sys.argv
if len(args) > 1:
cmd = cmd_prefix + args[1:]
if not os.path.exists(cmd[-2]):
cmd[-2] = get_qrels(cmd[-2])
else:
cmd = cmd_prefix
print(f'Running command: {cmd}')
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
if stderr:
print('Results:')
print(stderr.decode("utf-8"))
else:
print(stdout.decode("utf-8"))
import os
import subprocess
import sys
from pyserini.search import get_qrels
from pyserini.util import download_evaluation_script
script_path = download_evaluation_script('msmarco_doc_eval')
cmd_prefix = ['python3', script_path]
args = sys.argv
if len(args) > 1:
cmd = cmd_prefix + args[1:]
for i in range(len(cmd) - 1):
if cmd[i] == '--judgments':
if not os.path.exists(cmd[i + 1]):
cmd[i + 1] = get_qrels(cmd[i + 1])
else:
cmd = cmd_prefix
print(f'Running command: {cmd}')
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
if stderr:
print('Results:')
print(stderr.decode("utf-8"))
else:
print(stdout.decode("utf-8"))
def test_wt10g_qrels(self):
qrels = search.get_qrels('wt10g')
self.assertEqual(len(qrels), 100)
self.assertTrue(isinstance(next(iter(qrels.keys())), int))
def test_core18_qrels(self):
qrels = search.get_qrels('core18')
self.assertEqual(len(qrels), 50)
self.assertTrue(isinstance(next(iter(qrels.keys())), int))
def test_robust05_qrels(self):
qrels = search.get_qrels('robust05')
self.assertEqual(len(qrels), 50)
self.assertTrue(isinstance(next(iter(qrels.keys())), int))
def test_ntcir8_zh_qrels(self):
qrels = search.get_qrels('ntcir8-zh')
self.assertEqual(len(qrels), 100)
self.assertTrue(isinstance(next(iter(qrels.keys())), str))
def test_mb14_qrels(self):
qrels = search.get_qrels('mb14')
self.assertEqual(len(qrels), 55)
self.assertTrue(isinstance(next(iter(qrels.keys())), int))
def test_clef2006_fr_qrels(self):
qrels = search.get_qrels('clef2006-fr')
self.assertEqual(len(qrels), 49)
self.assertTrue(isinstance(next(iter(qrels.keys())), str))
def test_car20_qrels(self):
qrels = search.get_qrels('car17v2.0-benchmarkY1test')
self.assertEqual(len(qrels), 2254)
self.assertFalse(isinstance(next(iter(qrels.keys())), int))
def test_fire2012_hi_qrels(self):
qrels = search.get_qrels('fire2012-hi')
self.assertEqual(len(qrels), 50)
self.assertTrue(isinstance(next(iter(qrels.keys())), int))
def test_covid_round4_qrels(self):
qrels = search.get_qrels('covid-round4')
self.assertEqual(len(qrels), 45)
self.assertTrue(isinstance(next(iter(qrels.keys())), int))
