def __init__(self,
model_path,
cuda_device=-1,
predicted_pages=False,
merge_google=False,
score_format=False,
verbose=False):
logger.info("Load model from {1} on device".format(
model_path, cuda_device))
archive = load_archive(model_path, cuda_device=cuda_device)
logger.info("Loading FEVER Reader")
ds_params = archive.config["dataset_reader"]
ds_params["cuda_device"] = cuda_device
self.reader = FEVERReader.from_params(ds_params)
self.open_ie_predictor = AllenNLPPredictor.from_path(
"https://s3-us-west-2.amazonaws.com/allennlp/models/openie-model.2018-08-20.tar.gz"
)
self.model = archive.model
self.model.eval()
self.reverse_labels = {
j: i
for i, j in self.reader.label_lookup.items()
}
self.predicted_pages = predicted_pages
self.merge_google = merge_google
self.score_format = score_format
self.verbose = verbose
def __init__(self, database_path, add_claim=False, k_wiki_results=None):
self.db = FeverDocDB(database_path)
self.add_claim = add_claim
self.k_wiki_results = k_wiki_results
self.proter_stemm = nltk.PorterStemmer()
self.tokenizer = nltk.word_tokenize
self.predictor = Predictor.from_path(
"https://s3-us-west-2.amazonaws.com/allennlp/models/elmo-constituency-parser-2018.03.14.tar.gz"
)
def load_predictor():
"""Load model from AllenNLP, which we've downloaded"""
return Predictor.from_path("elmo-constituency-parser-2018.03.14.tar.gz")
from allennlp.service.predictors import Predictor
from allennlp.models import load_archive
from drqa import retriever
from doc.getDocuments import getDocsBatch, GoogleConfig
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('in_file', type=str)
parser.add_argument('out_file', type=str)
parser.add_argument('--config')
parser.add_argument('--cuda_device', type=int, default=-1)
args = parser.parse_args()
with open(args.config) as f:
config = json.load(f)
ner_predictor = Predictor.from_path(
"https://s3-us-west-2.amazonaws.com/allennlp/models/fine-grained-ner-model-elmo-2018.12.21.tar.gz"
)
google_config = GoogleConfig(**config['retrieval']['google'])
ranker = retriever.get_class('tfidf')(
tfidf_path=config['retrieval']['tfidf']['index'])
with open(args.out_file, 'w') as outfile:
for docs in getDocsBatch(args.in_file, google_config, ner_predictor,
ranker):
print(json.dumps(docs), file=outfile)
import json
#Takes all sentences from the blind test jsonl and writes to a file blind_test.txt
#and then calls pretrained model to get the Named entities and stores them in file in order
f = open('NER_dev_txt','w')
from allennlp.service.predictors import Predictor
predictor = Predictor.from_path("https://s3-us-west-2.amazonaws.com/allennlp/models/ner-model-2018.04.26.tar.gz")
for line in open('NER_shared_dev','r'):
line = line.strip()
results = predictor.predict(sentence=line[0])
for word, tag in zip(results["words"], results["tags"]):
f.write(word+'\t'+tag+'\n')
f.write('\n')
