def __init__(self, model_path, cuda_device=-1, predicted_pages=False, merge_google=False, score_format=False, verbose=False): logger.info("Load model from {1} on device".format( model_path, cuda_device)) archive = load_archive(model_path, cuda_device=cuda_device) logger.info("Loading FEVER Reader") ds_params = archive.config["dataset_reader"] ds_params["cuda_device"] = cuda_device self.reader = FEVERReader.from_params(ds_params) self.open_ie_predictor = AllenNLPPredictor.from_path( "https://s3-us-west-2.amazonaws.com/allennlp/models/openie-model.2018-08-20.tar.gz" ) self.model = archive.model self.model.eval() self.reverse_labels = { j: i for i, j in self.reader.label_lookup.items() } self.predicted_pages = predicted_pages self.merge_google = merge_google self.score_format = score_format self.verbose = verbose
def __init__(self, database_path, add_claim=False, k_wiki_results=None): self.db = FeverDocDB(database_path) self.add_claim = add_claim self.k_wiki_results = k_wiki_results self.proter_stemm = nltk.PorterStemmer() self.tokenizer = nltk.word_tokenize self.predictor = Predictor.from_path( "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo-constituency-parser-2018.03.14.tar.gz" )
def load_predictor(): """Load model from AllenNLP, which we've downloaded""" return Predictor.from_path("elmo-constituency-parser-2018.03.14.tar.gz")
from allennlp.service.predictors import Predictor from allennlp.models import load_archive from drqa import retriever from doc.getDocuments import getDocsBatch, GoogleConfig if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('in_file', type=str) parser.add_argument('out_file', type=str) parser.add_argument('--config') parser.add_argument('--cuda_device', type=int, default=-1) args = parser.parse_args() with open(args.config) as f: config = json.load(f) ner_predictor = Predictor.from_path( "https://s3-us-west-2.amazonaws.com/allennlp/models/fine-grained-ner-model-elmo-2018.12.21.tar.gz" ) google_config = GoogleConfig(**config['retrieval']['google']) ranker = retriever.get_class('tfidf')( tfidf_path=config['retrieval']['tfidf']['index']) with open(args.out_file, 'w') as outfile: for docs in getDocsBatch(args.in_file, google_config, ner_predictor, ranker): print(json.dumps(docs), file=outfile)
import json #Takes all sentences from the blind test jsonl and writes to a file blind_test.txt #and then calls pretrained model to get the Named entities and stores them in file in order f = open('NER_dev_txt','w') from allennlp.service.predictors import Predictor predictor = Predictor.from_path("https://s3-us-west-2.amazonaws.com/allennlp/models/ner-model-2018.04.26.tar.gz") for line in open('NER_shared_dev','r'): line = line.strip() results = predictor.predict(sentence=line[0]) for word, tag in zip(results["words"], results["tags"]): f.write(word+'\t'+tag+'\n') f.write('\n')