def main():
apb = ArgumentParserBuilder()
apb.add_opts(
opt('--task', type=str, default='msmarco'),
opt('--dataset', type=Path, required=True),
opt('--index-dir', type=Path, required=True),
opt('--method', required=True, type=str, choices=METHOD_CHOICES),
opt('--model',
required=True,
type=str,
help='Path to pre-trained model or huggingface model name'),
opt('--duo_model',
type=str,
help='Path to pre-trained model or huggingface model name'),
opt('--mono_hits',
type=int,
default=50,
help='Top k candidates from mono for duo reranking'),
opt('--output-file', type=Path, default='.'),
opt('--overwrite-output', action='store_true'),
opt('--split', type=str, default='dev', choices=('dev', 'eval')),
opt('--batch-size', '-bsz', type=int, default=96),
opt('--device', type=str, default='cuda:0'),
opt('--is-duo', action='store_true'),
opt('--from-tf', action='store_true'),
opt('--metrics',
type=str,
nargs='+',
default=metric_names(),
choices=metric_names()), opt('--model-type', type=str),
opt('--tokenizer-name', type=str))
args = apb.parser.parse_args()
options = PassageRankingEvaluationOptions(**vars(args))
logging.info("Preprocessing Queries & Passages:")
ds = MsMarcoDataset.from_folder(str(options.dataset),
split=options.split,
is_duo=options.is_duo)
examples = ds.to_relevance_examples(str(options.index_dir),
is_duo=options.is_duo)
logging.info("Loading Ranker & Tokenizer:")
construct_map = dict(transformer=construct_transformer,
bm25=construct_bm25,
t5=construct_t5,
duo_t5=construct_duo_t5,
seq_class_transformer=construct_seq_class_transformer,
random=lambda _: RandomReranker())
reranker = construct_map[options.method](options)
writer = MsMarcoWriter(args.output_file, args.overwrite_output)
if options.method == 'duo_t5':
evaluator = DuoRerankerEvaluator(mono_reranker=reranker[0],
duo_reranker=reranker[1],
metric_names=options.metrics,
mono_hits=options.mono_hits,
writer=writer)
else:
evaluator = RerankerEvaluator(reranker, options.metrics, writer=writer)
width = max(map(len, args.metrics)) + 1
logging.info("Reranking:")
for metric in evaluator.evaluate(examples):
logging.info(f'{metric.name:<{width}}{metric.value:.5}')
def main():
apb = ArgumentParserBuilder()
apb.add_opts(
opt('--dataset', type=Path, default='data/kaggle-lit-review-0.1.json'),
opt('--method', required=True, type=str, choices=METHOD_CHOICES),
opt('--model-name', type=str),
opt('--split', type=str, default='nq', choices=('nq', 'kq')),
opt('--batch-size', '-bsz', type=int, default=96),
opt('--device', type=str, default='cuda:0'),
opt('--tokenizer-name', type=str),
opt('--do-lower-case', action='store_true'),
opt('--metrics',
type=str,
nargs='+',
default=metric_names(),
choices=metric_names()))
args = apb.parser.parse_args()
options = KaggleEvaluationOptions(**vars(args))
ds = LitReviewDataset.from_file(str(options.dataset))
examples = ds.to_senticized_dataset(SETTINGS.cord19_index_path,
split=options.split)
construct_map = dict(transformer=construct_transformer,
bm25=construct_bm25,
t5=construct_t5,
seq_class_transformer=construct_seq_class_transformer,
qa_transformer=construct_qa_transformer,
random=lambda _: RandomReranker())
reranker = construct_map[options.method](options)
evaluator = RerankerEvaluator(reranker, options.metrics)
width = max(map(len, args.metrics)) + 1
stdout = []
for metric in evaluator.evaluate(examples):
logging.info(f'{metric.name:<{width}}{metric.value:.5}')
stdout.append(f'{metric.name}\t{metric.value}')
print('\n'.join(stdout))
def main():
apb = ArgumentParserBuilder()
apb.add_opts(opt('--dataset',
type=str,
default='msmarco'),
opt('--data-dir', type=Path, default='/content/data/msmarco'),
opt('--method',
required=True,
type=str,
choices=METHOD_CHOICES),
opt('--model-name-or-path', type=str),
opt('--output-file', type=Path, default='.'),
opt('--overwrite-output', action='store_true'),
opt('--split',
type=str,
default='dev',
choices=('dev', 'eval')),
opt('--batch-size', '-bsz', type=int, default=96),
opt('--device', type=str, default='cuda:0'),
opt('--is-duo', action='store_true'),
opt('--metrics',
type=str,
nargs='+',
default=metric_names(),
choices=metric_names()),
opt('--model-type', type=str, default='bert-base'),
opt('--tokenizer-name', type=str),
opt('--index-dir', type=Path))
args = apb.parser.parse_args()
options = PassageRankingEvaluationOptions(**vars(args))
ds = MsMarcoDataset.from_folder(str(options.data_dir), split=options.split,
is_duo=options.is_duo)
examples = ds.to_relevance_examples(SETTINGS.msmarco_index_path,
is_duo=options.is_duo)
construct_map = dict(transformer=construct_transformer,
bm25=construct_bm25,
t5=construct_t5,
seq_class_transformer=construct_seq_class_transformer,
random=lambda _: RandomReranker())
reranker = construct_map[options.method](options)
writer = MsMarcoWriter(args.output_file, args.overwrite_output)
evaluator = RerankerEvaluator(reranker, options.metrics, writer=writer)
width = max(map(len, args.metrics)) + 1
stdout = []
for metric in evaluator.evaluate(examples):
logging.info(f'{metric.name:<{width}}{metric.value:.5}')
stdout.append(f'{metric.name}\t{metric.value}')
print('\n'.join(stdout))
def main():
apb = ArgumentParserBuilder()
apb.add_opts(opt('--dataset', type=Path, required=True),
opt('--index-dir', type=Path, required=True),
opt('--method',
required=True,
type=str,
choices=METHOD_CHOICES),
opt('--model-name', type=str),
opt('--split', type=str, default='nq', choices=('nq', 'kq')),
opt('--batch-size', '-bsz', type=int, default=96),
opt('--device', type=str, default='cuda:0'),
opt('--tokenizer-name', type=str),
opt('--do-lower-case', action='store_true'),
opt('--metrics',
type=str,
nargs='+',
default=metric_names(),
choices=metric_names()))
args = apb.parser.parse_args()
options = KaggleEvaluationOptions(**vars(args))
ds = LitReviewDataset.from_file(str(options.dataset))
examples = ds.to_senticized_dataset(str(options.index_dir),
split=options.split)
construct_map = dict(transformer=construct_transformer,
bm25=construct_bm25,
t5=construct_t5,
seq_class_transformer=construct_seq_class_transformer,
qa_transformer=construct_qa_transformer,
random=lambda _: RandomReranker())
reranker = construct_map[options.method](options)
evaluator = RerankerEvaluator(reranker, options.metrics)
width = max(map(len, args.metrics)) + 1
stdout = []
import time
start = time.time()
with open(f'{options.model_name.replace("/","_")}.csv', 'w') as fd:
logging.info('writing %s.csv', options.model_name)
for metric in evaluator.evaluate(examples):
logging.info(f'{metric.name:<{width}}{metric.value:.5}')
stdout.append(f'{metric.name}\t{metric.value:.3}')
fd.write(f"{metric.name}\t{metric.value:.3}\n")
end = time.time()
fd.write(f"time\t{end-start:.3}\n")
print('\n'.join(stdout))
def main():
apb = ArgumentParserBuilder()
apb.add_opts(opt('--task',
type=str,
default='trec-covid',
help='A string correspondding to the task to execute. By default, this is "trec-covid".'),
opt('--dataset',
type=Path,
required=True,
help='Path to the directory containing the topics file, qrels file, and run file.'),
opt('--index-dir',
type=Path,
required=True,
help='Path to the input Anserini index.'),
opt('--method',
required=True,
type=str,
choices=METHOD_CHOICES,
help='Specifies the type of reranker to use.'),
opt('--model',
required=True,
type=str,
help='Path to pre-trained model or huggingface model name.'),
opt('--output-file',
type=Path,
default='.',
help='A path to the output file.'),
opt('--overwrite-output',
action='store_true',
help='If set to true, output will be overwritten if the output file already exists. Otherwise, '
'output will be appended to the existing file.'),
opt('--batch-size',
'-bsz',
type=int,
default=96,
help='The batch size for tokenization.'),
opt('--device',
type=str,
default='cuda:0',
help='The CUDA device to use for reranking.'),
opt('--from-tf',
action='store_true',
help='A boolean of whether the pretrained model is being loaded from a Tensorflow checkpoint. '
'If flag is unused, assumed to be false.'),
opt('--metrics',
type=str,
nargs='+',
default=metric_names(),
choices=metric_names(),
help='The list of metrics to collect while evaluating the reranker.'),
opt('--model-type',
type=str,
help='The T5 tokenizer name.'),
opt('--tokenizer-name',
type=str,
help='The name of the tokenizer to pull from huggingface using the AutoTokenizer class. If '
'left empty, this will be set to the model name.'),
opt('--seg-size',
type=int,
default=10,
help='The number of sentences in each segment. For example, given a document with sentences'
'[1,2,3,4,5], a seg_size of 3, and a stride of 2, the document will be broken into segments'
'[[1, 2, 3], [3, 4, 5], and [5]].'),
opt('--seg-stride',
type=int,
default=5,
help='The number of sentences to increment between each segment. For example, given a document'
'with sentences [1,2,3,4,5], a seg_size of 3, and a stride of 2, the document will be broken into'
'segments [[1, 2, 3], [3, 4, 5], and [5]].'),
opt('--aggregate-method',
type=str,
default="max",
help='Aggregation method for combining scores across sentence segments of the same document.'))
args = apb.parser.parse_args()
options = DocumentRankingEvaluationOptions(**vars(args))
logging.info("Preprocessing Queries & Docs:")
ds = TRECCovidDataset.from_folder(str(options.dataset))
examples = ds.to_relevance_examples(str(options.index_dir))
logging.info("Loading Ranker & Tokenizer:")
construct_map = dict(transformer=construct_transformer,
bm25=construct_bm25,
t5=construct_t5,
seq_class_transformer=construct_seq_class_transformer,
random=lambda _: RandomReranker())
# Retrieve the correct reranker from the options map based on the input flag.
reranker = construct_map[options.method](options)
writer = TrecWriter(args.output_file, args.overwrite_output)
evaluator = RerankerEvaluator(reranker, options.metrics, writer=writer)
width = max(map(len, args.metrics)) + 1
logging.info("Reranking:")
for metric in evaluator.evaluate_by_segments(examples,
options.seg_size,
options.seg_stride,
options.aggregate_method):
logging.info(f'{metric.name:<{width}}{metric.value:.5}')