def construct_transformer( options: PassageRankingEvaluationOptions) -> Reranker: device = torch.device(options.device) model = AutoModel.from_pretrained( options.model, from_tf=options.from_tf).to(device).eval() tokenizer = SimpleBatchTokenizer( AutoTokenizer.from_pretrained(options.tokenizer_name), options.batch_size) provider = CosineSimilarityMatrixProvider() return UnsupervisedTransformerReranker(model, tokenizer, provider)
def construct_transformer(options: KaggleEvaluationOptions) -> Reranker: device = torch.device(options.device) try: model = AutoModel.from_pretrained(options.model_name).to(device).eval() except OSError: model = AutoModel.from_pretrained(options.model_name, from_tf=True).to(device).eval() tokenizer = SimpleBatchTokenizer(AutoTokenizer.from_pretrained(options.tokenizer_name, do_lower_case=options.do_lower_case), options.batch_size) provider = CosineSimilarityMatrixProvider() return UnsupervisedTransformerReranker(model, tokenizer, provider)