def test_core18(self):
qrels_path = search.get_qrels_file('core18')
lines = read_file_lines(qrels_path)
length = len(lines)
first_line = lines[0].rstrip()
mid_line = lines[length // 2].rstrip()
last_line = lines[-1].rstrip()
self.assertEqual(length, 26233)
self.assertEqual(first_line, "321 0 004c6120d0aa69da29cc045da0562168 0")
self.assertEqual(mid_line, "646 0 260365e8-eb18-11e2-a301-ea5a8116d211 0")
self.assertEqual(last_line, "825 0 ff3a25b0-0ba4-11e4-8341-b8072b1e7348 0")
def test_covid_round3(self):
qrels_path = search.get_qrels_file('covid-round3')
lines = read_file_lines(qrels_path)
length = len(lines)
first_line = lines[0].rstrip()
mid_line = lines[length // 2].rstrip()
last_line = lines[-1].rstrip()
self.assertEqual(length, 12713)
self.assertEqual(first_line, "1 2.5 0194oljo 1")
self.assertEqual(mid_line, "19 2.5 s0o0egw8 2")
self.assertEqual(last_line, "40 3 zsx7wfyj 1")
def test_trec2018_bl(self):
qrels_path = search.get_qrels_file('trec2018-bl')
lines = read_file_lines(qrels_path)
length = len(lines)
first_line = lines[0].rstrip()
mid_line = lines[length // 2].rstrip()
last_line = lines[-1].rstrip()
self.assertEqual(length, 8508)
self.assertEqual(first_line, "321 0 00f57310e5c8ec7833d6756ba637332e 16")
self.assertEqual(mid_line, "809 0 921073ca-c0a3-11e1-9ce8-ff26651238d0 0")
self.assertEqual(last_line, "825 0 f66b624ba8689d704872fa776fb52860 0")
def test_robust04(self):
qrels_path = search.get_qrels_file('robust04')
lines = read_file_lines(qrels_path)
length = len(lines)
first_line = lines[0].rstrip()
mid_line = lines[length // 2].rstrip()
last_line = lines[-1].rstrip()
self.assertEqual(length, 311410)
self.assertEqual(first_line, "301 0 FBIS3-10082 1")
self.assertEqual(mid_line, "409 0 LA010189-0112 0")
self.assertEqual(last_line, "700 0 LA123090-0137 0")
def test_covid_round4_cumulative(self):
qrels_path = search.get_qrels_file('covid-round4-cumulative')
lines = read_file_lines(qrels_path)
length = len(lines)
first_line = lines[0].rstrip()
mid_line = lines[length // 2].rstrip()
last_line = lines[-1].rstrip()
self.assertEqual(length, 46203)
self.assertEqual(first_line, "1 4 00fmeepz 1")
self.assertEqual(mid_line, "19 1 bt5857p3 0")
self.assertEqual(last_line, "45 4 zzrsk1ls 2")
def test_covid_complete(self):
qrels_path = search.get_qrels_file('covid-complete')
lines = read_file_lines(qrels_path)
length = len(lines)
first_line = lines[0].rstrip()
mid_line = lines[length // 2].rstrip()
last_line = lines[-1].rstrip()
self.assertEqual(length, 69318)
self.assertEqual(first_line, "1 4.5 005b2j4b 2")
self.assertEqual(mid_line, "23 5 71jjbyds 0")
self.assertEqual(last_line, "50 5 zz8wvos9 1")
def test_covid_round5(self):
qrels_path = search.get_qrels_file('covid-round5')
lines = read_file_lines(qrels_path)
length = len(lines)
first_line = lines[0].rstrip()
mid_line = lines[length // 2].rstrip()
last_line = lines[-1].rstrip()
self.assertEqual(length, 23151)
self.assertEqual(first_line, "1 4.5 005b2j4b 2")
self.assertEqual(mid_line, "36 4.5 ylgmn69k 0")
self.assertEqual(last_line, "50 5 zz8wvos9 1")
def test_covid_round3_cumulative(self):
qrels_path = search.get_qrels_file('covid-round3-cumulative')
lines = read_file_lines(qrels_path)
length = len(lines)
first_line = lines[0].rstrip()
mid_line = lines[length // 2].rstrip()
last_line = lines[-1].rstrip()
self.assertEqual(length, 33068)
self.assertEqual(first_line, "1 0.5 010vptx3 2")
self.assertEqual(mid_line, "17 1.5 4txctk7k 0")
self.assertEqual(last_line, "40 3 zsx7wfyj 1")
def test_trec2019_bl(self):
qrels_path = search.get_qrels_file('trec2019-bl')
lines = read_file_lines(qrels_path)
length = len(lines)
first_line = lines[0].rstrip()
mid_line = lines[length // 2].rstrip()
last_line = lines[-1].rstrip()
self.assertEqual(length, 15655)
self.assertEqual(first_line, "826 0 0154349511cd8c49ab862d6cb0d8f6a8 2")
self.assertEqual(mid_line, "853 0 2444d88d62539b0b88dc919909cb9701 2")
self.assertEqual(last_line, "885 0 fde80cb0-b4f0-11e2-bbf2-a6f9e9d79e19 0")
def test_covid_round1(self):
qrels_path = search.get_qrels_file('covid-round1')
lines = read_file_lines(qrels_path)
length = len(lines)
first_line = lines[0].rstrip()
mid_line = lines[length // 2].rstrip()
last_line = lines[-1].rstrip()
self.assertEqual(length, 8691)
self.assertEqual(first_line, "1 0.5 010vptx3 2")
self.assertEqual(mid_line, "15 0.5 01rdlf8l 0")
self.assertEqual(last_line, "30 0.5 zn87f1lk 1")
def test_covid_round2(self):
qrels_path = search.get_qrels_file('covid-round2')
lines = read_file_lines(qrels_path)
length = len(lines)
first_line = lines[0].rstrip()
mid_line = lines[length // 2].rstrip()
last_line = lines[-1].rstrip()
self.assertEqual(length, 12037)
self.assertEqual(first_line, "1 1.5 08efpohc 0")
self.assertEqual(mid_line, "16 2 uj0i2anr 0")
self.assertEqual(last_line, "35 2 zzmfhr2s 0")
def test_msmarco_passage(self):
qrels_path = search.get_qrels_file('msmarco-passage-dev-subset')
lines = read_file_lines(qrels_path)
length = len(lines)
first_line = lines[0].rstrip()
mid_line = lines[length // 2].rstrip()
last_line = lines[-1].rstrip()
self.assertEqual(length, 7437)
self.assertEqual(first_line, "300674 0 7067032 1")
self.assertEqual(mid_line, "573452 0 3182069 1")
self.assertEqual(last_line, "195199 0 8009377 1")
def test_msmarco_doc(self):
qrels_path = search.get_qrels_file('msmarco-doc-dev')
lines = read_file_lines(qrels_path)
length = len(lines)
first_line = lines[0].rstrip()
mid_line = lines[length // 2].rstrip()
last_line = lines[-1].rstrip()
self.assertEqual(length, 5193)
self.assertEqual(first_line, "2 0 D1650436 1")
self.assertEqual(mid_line, "855050 0 D2851565 1")
self.assertEqual(last_line, "1102400 0 D677570 1")
def test_core17(self):
qrels_path = search.get_qrels_file('core17')
lines = read_file_lines(qrels_path)
length = len(lines)
first_line = lines[0].rstrip()
mid_line = lines[length // 2].rstrip()
last_line = lines[-1].rstrip()
self.assertEqual(length, 30030)
self.assertEqual(first_line, "307 0 1001536 1")
self.assertEqual(mid_line, "393 0 1586039 2")
self.assertEqual(last_line, "690 0 996059 0")
def test_robust05(self):
qrels_path = search.get_qrels_file('robust05')
lines = read_file_lines(qrels_path)
length = len(lines)
first_line = lines[0].rstrip()
mid_line = lines[length // 2].rstrip()
last_line = lines[-1].rstrip()
self.assertEqual(length, 37798)
self.assertEqual(first_line, "303 0 APW19980609.1531 2")
self.assertEqual(mid_line, "397 0 XIE19960920.0297 0")
self.assertEqual(last_line, "689 0 XIE20000925.0055 0")
def test_covid_round4(self):
qrels_path = search.get_qrels_file('covid-round4')
lines = read_file_lines(qrels_path)
length = len(lines)
first_line = lines[0].rstrip()
mid_line = lines[length // 2].rstrip()
last_line = lines[-1].rstrip()
self.assertEqual(length, 13262)
self.assertEqual(first_line, "1 4 00fmeepz 1")
self.assertEqual(mid_line, "27 4 hmh4s3w4 0")
self.assertEqual(last_line, "45 4 zzrsk1ls 2")
def test_car20(self):
qrels_path = search.get_qrels_file('car17v2.0-benchmarkY1test')
lines = read_file_lines(qrels_path)
length = len(lines)
first_line = lines[0].rstrip()
mid_line = lines[length // 2].rstrip()
last_line = lines[-1].rstrip()
self.assertEqual(length, 6192)
self.assertEqual(first_line, "enwiki:Aftertaste 0 327cca6c4d38953196fa6789f615546f03287b25 1")
self.assertEqual(mid_line, "enwiki:Insular%20cortex/Function/Interoceptive%20awareness"
" 0 f037f925acd4c59e802a58aa74430fc6aa163606 1")
self.assertEqual(last_line, "enwiki:Yellowstone%20National%20Park/Recreation"
" 0 b812fca195f74f8c563db4262260554fe3ff3731 1")
def test_car15(self):
qrels_path = search.get_qrels_file('car17v1.5-benchmarkY1test')
lines = read_file_lines(qrels_path)
length = len(lines)
first_line = lines[0].rstrip()
mid_line = lines[length // 2].rstrip()
last_line = lines[-1].rstrip()
self.assertEqual(length, 5820)
self.assertEqual(first_line, "Aftertaste/Aftertaste%20processing%20in%20the%20cerebral%20cortex "
"0 38c1bd25ddca2705164677a3f598c46df85afba7 1")
self.assertEqual(mid_line, "Insular%20cortex/Function/Interoceptive%20awareness "
"0 f037f925acd4c59e802a58aa74430fc6aa163606 1")
self.assertEqual(last_line, "Yellowstone%20National%20Park/Recreation"
" 0 e80b5185da1493edde41bea19a389a3f62167369 1")
#
import os
import subprocess
import sys
import platform
from pyserini.search import get_qrels_file
from pyserini.util import download_evaluation_script
script_path = download_evaluation_script('trec_eval')
cmd_prefix = ['java', '-jar', script_path]
args = sys.argv
if len(args) > 1:
cmd = cmd_prefix + args[1:]
if not os.path.exists(cmd[-2]):
cmd[-2] = get_qrels_file(cmd[-2])
else:
cmd = cmd_prefix
print(f'Running command: {cmd}')
shell = platform.system() == "Windows"
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=shell)
stdout, stderr = process.communicate()
if stderr:
print(stderr.decode("utf-8"))
print('Results:')
print(stdout.decode("utf-8"))
import os
import subprocess
import sys
from pyserini.search import get_qrels_file
from pyserini.util import download_evaluation_script
script_path = download_evaluation_script('msmarco_doc_eval')
cmd_prefix = ['python3', script_path]
args = sys.argv
if len(args) > 1:
cmd = cmd_prefix + args[1:]
for i in range(len(cmd) - 1):
if cmd[i] == '--judgments':
if not os.path.exists(cmd[i + 1]):
cmd[i + 1] = get_qrels_file(cmd[i + 1])
else:
cmd = cmd_prefix
print(f'Running command: {cmd}')
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
if stderr:
print(stderr.decode("utf-8"))
else:
print('Results:')
print(stdout.decode("utf-8"))
