コード例 #1
0
def main():
    apb = ArgumentParserBuilder()
    apb.add_opts(
        opt('--task', type=str, default='msmarco'),
        opt('--dataset', type=Path, required=True),
        opt('--index-dir', type=Path, required=True),
        opt('--method', required=True, type=str, choices=METHOD_CHOICES),
        opt('--model',
            required=True,
            type=str,
            help='Path to pre-trained model or huggingface model name'),
        opt('--duo_model',
            type=str,
            help='Path to pre-trained model or huggingface model name'),
        opt('--mono_hits',
            type=int,
            default=50,
            help='Top k candidates from mono for duo reranking'),
        opt('--output-file', type=Path, default='.'),
        opt('--overwrite-output', action='store_true'),
        opt('--split', type=str, default='dev', choices=('dev', 'eval')),
        opt('--batch-size', '-bsz', type=int, default=96),
        opt('--device', type=str, default='cuda:0'),
        opt('--is-duo', action='store_true'),
        opt('--from-tf', action='store_true'),
        opt('--metrics',
            type=str,
            nargs='+',
            default=metric_names(),
            choices=metric_names()), opt('--model-type', type=str),
        opt('--tokenizer-name', type=str))
    args = apb.parser.parse_args()
    options = PassageRankingEvaluationOptions(**vars(args))
    logging.info("Preprocessing Queries & Passages:")
    ds = MsMarcoDataset.from_folder(str(options.dataset),
                                    split=options.split,
                                    is_duo=options.is_duo)
    examples = ds.to_relevance_examples(str(options.index_dir),
                                        is_duo=options.is_duo)
    logging.info("Loading Ranker & Tokenizer:")
    construct_map = dict(transformer=construct_transformer,
                         bm25=construct_bm25,
                         t5=construct_t5,
                         duo_t5=construct_duo_t5,
                         seq_class_transformer=construct_seq_class_transformer,
                         random=lambda _: RandomReranker())
    reranker = construct_map[options.method](options)
    writer = MsMarcoWriter(args.output_file, args.overwrite_output)
    if options.method == 'duo_t5':
        evaluator = DuoRerankerEvaluator(mono_reranker=reranker[0],
                                         duo_reranker=reranker[1],
                                         metric_names=options.metrics,
                                         mono_hits=options.mono_hits,
                                         writer=writer)
    else:
        evaluator = RerankerEvaluator(reranker, options.metrics, writer=writer)
    width = max(map(len, args.metrics)) + 1
    logging.info("Reranking:")
    for metric in evaluator.evaluate(examples):
        logging.info(f'{metric.name:<{width}}{metric.value:.5}')
コード例 #2
0
def main():
    apb = ArgumentParserBuilder()
    apb.add_opts(
        opt('--dataset', type=Path, default='data/kaggle-lit-review-0.1.json'),
        opt('--method', required=True, type=str, choices=METHOD_CHOICES),
        opt('--model-name', type=str),
        opt('--split', type=str, default='nq', choices=('nq', 'kq')),
        opt('--batch-size', '-bsz', type=int, default=96),
        opt('--device', type=str, default='cuda:0'),
        opt('--tokenizer-name', type=str),
        opt('--do-lower-case', action='store_true'),
        opt('--metrics',
            type=str,
            nargs='+',
            default=metric_names(),
            choices=metric_names()))
    args = apb.parser.parse_args()
    options = KaggleEvaluationOptions(**vars(args))
    ds = LitReviewDataset.from_file(str(options.dataset))
    examples = ds.to_senticized_dataset(SETTINGS.cord19_index_path,
                                        split=options.split)
    construct_map = dict(transformer=construct_transformer,
                         bm25=construct_bm25,
                         t5=construct_t5,
                         seq_class_transformer=construct_seq_class_transformer,
                         qa_transformer=construct_qa_transformer,
                         random=lambda _: RandomReranker())
    reranker = construct_map[options.method](options)
    evaluator = RerankerEvaluator(reranker, options.metrics)
    width = max(map(len, args.metrics)) + 1
    stdout = []
    for metric in evaluator.evaluate(examples):
        logging.info(f'{metric.name:<{width}}{metric.value:.5}')
        stdout.append(f'{metric.name}\t{metric.value}')
    print('\n'.join(stdout))
コード例 #3
0
def main():
    apb = ArgumentParserBuilder()
    apb.add_opts(opt('--dataset',
                     type=str,
                     default='msmarco'),
                 opt('--data-dir', type=Path, default='/content/data/msmarco'),
                 opt('--method',
                     required=True,
                     type=str,
                     choices=METHOD_CHOICES),
                 opt('--model-name-or-path', type=str),
                 opt('--output-file', type=Path, default='.'),
                 opt('--overwrite-output', action='store_true'),
                 opt('--split',
                     type=str,
                     default='dev',
                     choices=('dev', 'eval')),
                 opt('--batch-size', '-bsz', type=int, default=96),
                 opt('--device', type=str, default='cuda:0'),
                 opt('--is-duo', action='store_true'),
                 opt('--metrics',
                     type=str,
                     nargs='+',
                     default=metric_names(),
                     choices=metric_names()),
                 opt('--model-type', type=str, default='bert-base'),
                 opt('--tokenizer-name', type=str),
                 opt('--index-dir', type=Path))
    args = apb.parser.parse_args()
    options = PassageRankingEvaluationOptions(**vars(args))
    ds = MsMarcoDataset.from_folder(str(options.data_dir), split=options.split,
                                    is_duo=options.is_duo)
    examples = ds.to_relevance_examples(SETTINGS.msmarco_index_path,
                                        is_duo=options.is_duo)
    construct_map = dict(transformer=construct_transformer,
                         bm25=construct_bm25,
                         t5=construct_t5,
                         seq_class_transformer=construct_seq_class_transformer,
                         random=lambda _: RandomReranker())
    reranker = construct_map[options.method](options)
    writer = MsMarcoWriter(args.output_file, args.overwrite_output)
    evaluator = RerankerEvaluator(reranker, options.metrics, writer=writer)
    width = max(map(len, args.metrics)) + 1
    stdout = []
    for metric in evaluator.evaluate(examples):
        logging.info(f'{metric.name:<{width}}{metric.value:.5}')
        stdout.append(f'{metric.name}\t{metric.value}')
    print('\n'.join(stdout))
コード例 #4
0
def main():
    apb = ArgumentParserBuilder()
    apb.add_opts(opt('--dataset', type=Path, required=True),
                 opt('--index-dir', type=Path, required=True),
                 opt('--method',
                     required=True,
                     type=str,
                     choices=METHOD_CHOICES),
                 opt('--model-name', type=str),
                 opt('--split', type=str, default='nq', choices=('nq', 'kq')),
                 opt('--batch-size', '-bsz', type=int, default=96),
                 opt('--device', type=str, default='cuda:0'),
                 opt('--tokenizer-name', type=str),
                 opt('--do-lower-case', action='store_true'),
                 opt('--metrics',
                     type=str,
                     nargs='+',
                     default=metric_names(),
                     choices=metric_names()))
    args = apb.parser.parse_args()
    options = KaggleEvaluationOptions(**vars(args))
    ds = LitReviewDataset.from_file(str(options.dataset))
    examples = ds.to_senticized_dataset(str(options.index_dir),
                                        split=options.split)
    construct_map = dict(transformer=construct_transformer,
                         bm25=construct_bm25,
                         t5=construct_t5,
                         seq_class_transformer=construct_seq_class_transformer,
                         qa_transformer=construct_qa_transformer,
                         random=lambda _: RandomReranker())
    reranker = construct_map[options.method](options)
    evaluator = RerankerEvaluator(reranker, options.metrics)
    width = max(map(len, args.metrics)) + 1
    stdout = []
    import time
    start = time.time()
    with open(f'{options.model_name.replace("/","_")}.csv', 'w') as fd:
        logging.info('writing %s.csv', options.model_name)
        for metric in evaluator.evaluate(examples):
            logging.info(f'{metric.name:<{width}}{metric.value:.5}')
            stdout.append(f'{metric.name}\t{metric.value:.3}')
            fd.write(f"{metric.name}\t{metric.value:.3}\n")
        end = time.time()
        fd.write(f"time\t{end-start:.3}\n")

    print('\n'.join(stdout))
コード例 #5
0
def main():
    apb = ArgumentParserBuilder()
    apb.add_opts(opt('--task',
                     type=str,
                     default='trec-covid',
                     help='A string correspondding to the task to execute. By default, this is "trec-covid".'),
                 opt('--dataset',
                     type=Path,
                     required=True,
                     help='Path to the directory containing the topics file, qrels file, and run file.'),
                 opt('--index-dir',
                     type=Path,
                     required=True,
                     help='Path to the input Anserini index.'),
                 opt('--method',
                     required=True,
                     type=str,
                     choices=METHOD_CHOICES,
                     help='Specifies the type of reranker to use.'),
                 opt('--model',
                     required=True,
                     type=str,
                     help='Path to pre-trained model or huggingface model name.'),
                 opt('--output-file',
                     type=Path,
                     default='.',
                     help='A path to the output file.'),
                 opt('--overwrite-output',
                     action='store_true',
                     help='If set to true, output will be overwritten if the output file already exists. Otherwise, '
                          'output will be appended to the existing file.'),
                 opt('--batch-size',
                     '-bsz',
                     type=int,
                     default=96,
                     help='The batch size for tokenization.'),
                 opt('--device',
                     type=str,
                     default='cuda:0',
                     help='The CUDA device to use for reranking.'),
                 opt('--from-tf',
                     action='store_true',
                     help='A boolean of whether the pretrained model is being loaded from a Tensorflow checkpoint. '
                          'If flag is unused, assumed to be false.'),
                 opt('--metrics',
                     type=str,
                     nargs='+',
                     default=metric_names(),
                     choices=metric_names(),
                     help='The list of metrics to collect while evaluating the reranker.'),
                 opt('--model-type',
                     type=str,
                     help='The T5 tokenizer name.'),
                 opt('--tokenizer-name',
                     type=str,
                     help='The name of the tokenizer to pull from huggingface using the AutoTokenizer class. If '
                     'left empty, this will be set to the model name.'),
                 opt('--seg-size',
                     type=int,
                     default=10,
                     help='The number of sentences in each segment. For example, given a document with sentences'
                     '[1,2,3,4,5], a seg_size of 3, and a stride of 2, the document will be broken into segments'
                     '[[1, 2, 3], [3, 4, 5], and [5]].'),
                 opt('--seg-stride',
                     type=int,
                     default=5,
                     help='The number of sentences to increment between each segment. For example, given a document'
                     'with sentences [1,2,3,4,5], a seg_size of 3, and a stride of 2, the document will be broken into'
                     'segments [[1, 2, 3], [3, 4, 5], and [5]].'),
                 opt('--aggregate-method',
                     type=str,
                     default="max",
                     help='Aggregation method for combining scores across sentence segments of the same document.'))
    args = apb.parser.parse_args()
    options = DocumentRankingEvaluationOptions(**vars(args))
    logging.info("Preprocessing Queries & Docs:")
    ds = TRECCovidDataset.from_folder(str(options.dataset))
    examples = ds.to_relevance_examples(str(options.index_dir))
    logging.info("Loading Ranker & Tokenizer:")
    construct_map = dict(transformer=construct_transformer,
                         bm25=construct_bm25,
                         t5=construct_t5,
                         seq_class_transformer=construct_seq_class_transformer,
                         random=lambda _: RandomReranker())
    # Retrieve the correct reranker from the options map based on the input flag.
    reranker = construct_map[options.method](options)
    writer = TrecWriter(args.output_file, args.overwrite_output)
    evaluator = RerankerEvaluator(reranker, options.metrics, writer=writer)
    width = max(map(len, args.metrics)) + 1
    logging.info("Reranking:")
    for metric in evaluator.evaluate_by_segments(examples,
                                                 options.seg_size,
                                                 options.seg_stride,
                                                 options.aggregate_method):
        logging.info(f'{metric.name:<{width}}{metric.value:.5}')