Exemplos de UnsupervisedTransformerReranker em Python

Exemplo n.º 1

def construct_transformer(
        options: PassageRankingEvaluationOptions) -> Reranker:
    device = torch.device(options.device)
    model = AutoModel.from_pretrained(
        options.model, from_tf=options.from_tf).to(device).eval()
    tokenizer = SimpleBatchTokenizer(
        AutoTokenizer.from_pretrained(options.tokenizer_name),
        options.batch_size)
    provider = CosineSimilarityMatrixProvider()
    return UnsupervisedTransformerReranker(model, tokenizer, provider)

Exemplo n.º 2

Arquivo: evaluate_kaggle_highlighter.py Projeto: dragomirradev/pygaggle

def construct_transformer(options: KaggleEvaluationOptions) -> Reranker:
    device = torch.device(options.device)
    try:
        model = AutoModel.from_pretrained(options.model_name).to(device).eval()
    except OSError:
        model = AutoModel.from_pretrained(options.model_name, from_tf=True).to(device).eval()
    tokenizer = SimpleBatchTokenizer(AutoTokenizer.from_pretrained(options.tokenizer_name,
                                                                   do_lower_case=options.do_lower_case),
                                     options.batch_size)
    provider = CosineSimilarityMatrixProvider()
    return UnsupervisedTransformerReranker(model, tokenizer, provider)