def eval_model(db: FeverDocDB, args) -> Model:
archive = load_archive(args.archive_file, cuda_device=args.cuda_device)
config = archive.config
ds_params = config["dataset_reader"]
model = archive.model
model.eval()
reader = FEVERReader(db,
sentence_level=ds_params.pop("sentence_level", False),
wiki_tokenizer=Tokenizer.from_params(
ds_params.pop('wiki_tokenizer', {})),
claim_tokenizer=Tokenizer.from_params(
ds_params.pop('claim_tokenizer', {})),
token_indexers=TokenIndexer.dict_from_params(
ds_params.pop('token_indexers', {})))
logger.info("Reading training data from %s", args.in_file)
data = reader.read(args.in_file).instances
actual = []
predicted = []
if args.log is not None:
f = open(args.log, "w+")
for item in tqdm(data):
if item.fields["premise"] is None or item.fields[
"premise"].sequence_length() == 0:
cls = "NOT ENOUGH INFO"
else:
prediction = model.forward_on_instance(item, args.cuda_device)
cls = model.vocab._index_to_token["labels"][np.argmax(
prediction["label_probs"])]
if "label" in item.fields:
actual.append(item.fields["label"].label)
predicted.append(cls)
if args.log is not None:
if "label" in item.fields:
f.write(
json.dumps({
"actual": item.fields["label"].label,
"predicted": cls
}) + "\n")
else:
f.write(json.dumps({"predicted": cls}) + "\n")
if args.log is not None:
f.close()
if len(actual) > 0:
print(accuracy_score(actual, predicted))
print(classification_report(actual, predicted))
print(confusion_matrix(actual, predicted))
return model
def train_model(db: FeverDocDB, params: Union[Params, Dict[str, Any]],
cuda_device: int, serialization_dir: str,
filtering: str) -> Model:
"""
This function can be used as an entry point to running models in AllenNLP
directly from a JSON specification using a :class:`Driver`. Note that if
you care about reproducibility, you should avoid running code using Pytorch
or numpy which affect the reproducibility of your experiment before you
import and use this function, these libraries rely on random seeds which
can be set in this function via a JSON specification file. Note that this
function performs training and will also evaluate the trained model on
development and test sets if provided in the parameter json.
Parameters
----------
params: Params, required.
A parameter object specifying an AllenNLP Experiment.
serialization_dir: str, required
The directory in which to save results and logs.
"""
SimpleRandom.set_seeds()
os.makedirs(serialization_dir, exist_ok=True)
sys.stdout = TeeLogger(os.path.join(serialization_dir, "stdout.log"),
sys.stdout) # type: ignore
sys.stderr = TeeLogger(os.path.join(serialization_dir, "stderr.log"),
sys.stderr) # type: ignore
handler = logging.FileHandler(
os.path.join(serialization_dir, "python_logging.log"))
handler.setLevel(logging.INFO)
handler.setFormatter(
logging.Formatter(
'%(asctime)s - %(levelname)s - %(name)s - %(message)s'))
logging.getLogger().addHandler(handler)
serialization_params = deepcopy(params).as_dict(quiet=True)
with open(os.path.join(serialization_dir, "model_params.json"),
"w") as param_file:
json.dump(serialization_params, param_file, indent=4)
# Now we begin assembling the required parts for the Trainer.
ds_params = params.pop('dataset_reader', {})
dataset_reader = FEVERReader(db,
sentence_level=ds_params.pop(
"sentence_level", False),
wiki_tokenizer=Tokenizer.from_params(
ds_params.pop('wiki_tokenizer', {})),
claim_tokenizer=Tokenizer.from_params(
ds_params.pop('claim_tokenizer', {})),
token_indexers=TokenIndexer.dict_from_params(
ds_params.pop('token_indexers', {})),
filtering=filtering)
train_data_path = params.pop('train_data_path')
logger.info("Reading training data from %s", train_data_path)
train_data = dataset_reader.read(train_data_path)
all_datasets = [train_data]
datasets_in_vocab = ["train"]
validation_data_path = params.pop('validation_data_path', None)
if validation_data_path is not None:
logger.info("Reading validation data from %s", validation_data_path)
validation_data = dataset_reader.read(validation_data_path)
all_datasets.append(validation_data)
datasets_in_vocab.append("validation")
else:
validation_data = None
logger.info("Creating a vocabulary using %s data.",
", ".join(datasets_in_vocab))
vocab = Vocabulary.from_params(
params.pop("vocabulary", {}),
Dataset([
instance for dataset in all_datasets
for instance in dataset.instances
]))
vocab.save_to_files(os.path.join(serialization_dir, "vocabulary"))
model = Model.from_params(vocab, params.pop('model'))
iterator = DataIterator.from_params(params.pop("iterator"))
train_data.index_instances(vocab)
if validation_data:
validation_data.index_instances(vocab)
trainer_params = params.pop("trainer")
if cuda_device is not None:
trainer_params["cuda_device"] = cuda_device
trainer = Trainer.from_params(model, serialization_dir, iterator,
train_data, validation_data, trainer_params)
trainer.train()
# Now tar up results
archive_model(serialization_dir)
return model
def eval_model_fnc_data(db: FeverDocDB, args, mithun_logger,
name_of_trained_model_to_use,
path_to_trained_models_folder, cuda_device, operation,
path_to_fnc_annotated_data) -> Model:
print("got inside eval_model_fnc_data")
archive = load_archive(
path_to_trained_models_folder + name_of_trained_model_to_use,
cuda_device)
config = archive.config
ds_params = config["dataset_reader"]
model = archive.model
model.eval()
reader = FEVERReader(db,
sentence_level=ds_params.pop("sentence_level", False),
wiki_tokenizer=Tokenizer.from_params(
ds_params.pop('wiki_tokenizer', {})),
claim_tokenizer=Tokenizer.from_params(
ds_params.pop('claim_tokenizer', {})),
token_indexers=TokenIndexer.dict_from_params(
ds_params.pop('token_indexers', {})))
# do annotation on the fly using pyprocessors. i.e creating NER tags, POS Tags etcThis takes along time.
# so almost always we do it only once, and load it from disk . Hence do_annotation_live = False
do_annotation_live = False
data = reader.read_annotated_fnc_and_do_ner_replacement(
args, operation, do_annotation_live, mithun_logger,
path_to_fnc_annotated_data).instances
joblib.dump(data, "fever_dev_dataset_format.pkl")
#
###################end of running model and saving
path = os.getcwd()
#data=joblib.load(path+"fever_dev_dataset_format")
actual = []
predicted = []
if args.log is not None:
f = open(args.log, "w+")
if_ctr, else_ctr = 0, 0
pred_dict = defaultdict(int)
for item in tqdm(data):
if item.fields["premise"] is None or item.fields[
"premise"].sequence_length() == 0:
cls = "NOT ENOUGH INFO"
if_ctr += 1
else:
else_ctr += 1
prediction = model.forward_on_instance(item, args.cuda_device)
cls = model.vocab._index_to_token["labels"][np.argmax(
prediction["label_probs"])]
if "label" in item.fields:
actual.append(item.fields["label"].label)
predicted.append(cls)
pred_dict[cls] += 1
if args.log is not None:
if "label" in item.fields:
f.write(
json.dumps({
"actual": item.fields["label"].label,
"predicted": cls
}) + "\n")
else:
f.write(json.dumps({"predicted": cls}) + "\n")
print(f'if_ctr = {if_ctr}')
print(f'else_ctr = {else_ctr}')
print(f'pred_dict = {pred_dict}')
if args.log is not None:
f.close()
if len(actual) > 0:
print(accuracy_score(actual, predicted))
print(classification_report(actual, predicted))
print(confusion_matrix(actual, predicted))
return model
def eval_model(db: FeverDocDB, args) -> Model:
archive = load_archive(args.archive_file,
cuda_device=args.cuda_device,
overrides=args.overrides)
config = archive.config
ds_params = config["dataset_reader"]
model = archive.model
model.eval()
reader = FEVERReader(db,
sentence_level=ds_params.pop("sentence_level", False),
wiki_tokenizer=Tokenizer.from_params(
ds_params.pop('wiki_tokenizer', {})),
claim_tokenizer=Tokenizer.from_params(
ds_params.pop('claim_tokenizer', {})),
token_indexers=TokenIndexer.dict_from_params(
ds_params.pop('token_indexers', {})))
while True:
claim = input("enter claim (or q to quit) >>")
if claim.lower() == "q":
break
ranker = retriever.get_class('tfidf')(tfidf_path=args.model)
p_lines = []
pages, _ = ranker.closest_docs(claim, 5)
for page in pages:
lines = db.get_doc_lines(page)
lines = [
line.split("\t")[1] if len(line.split("\t")[1]) > 1 else ""
for line in lines.split("\n")
]
p_lines.extend(zip(lines, [page] * len(lines), range(len(lines))))
scores = tf_idf_sim(claim, [pl[0] for pl in p_lines])
scores = list(
zip(scores, [pl[1] for pl in p_lines], [pl[2] for pl in p_lines],
[pl[0] for pl in p_lines]))
scores = list(filter(lambda score: len(score[3].strip()), scores))
sentences_l = list(
sorted(scores, reverse=True, key=lambda elem: elem[0]))
sentences = [s[3] for s in sentences_l[:5]]
evidence = " ".join(sentences)
print("Best pages: {0}".format(repr(pages)))
print("Evidence:")
for idx, sentence in enumerate(sentences_l[:5]):
print("{0}\t{1}\t\t{2}\t{3}".format(idx + 1, sentence[0],
sentence[1], sentence[3]))
item = reader.text_to_instance(evidence, claim)
prediction = model.forward_on_instance(item, args.cuda_device)
cls = model.vocab._index_to_token["labels"][np.argmax(
prediction["label_probs"])]
print("PREDICTED: {0}".format(cls))
print()
reader = FEVERReader(
db,
sentence_level=ds_params.pop("sentence_level", False),
wiki_tokenizer=Tokenizer.from_params(
ds_params.pop('wiki_tokenizer',
{"word_splitter": {
"type": "indexed_spaces"
}})),
claim_tokenizer=Tokenizer.from_params(
ds_params.pop('claim_tokenizer',
{"word_splitter": {
"type": "indexed_spaces"
}})),
token_indexers=TokenIndexer.dict_from_params(
ds_params.pop('token_indexers',
{'tokens': SingleIdTokenIndexer()})))
print("")
print("")
print("")
while True:
claim = input("enter claim (or q to quit) >>\t")
if claim.lower() == "q":
break
if len(claim.strip()) < 2:
continue
print("Pages:")
pages = evidence_retriever.get_docs_for_claim(claim)
archive = load_archive(
path_to_trained_models_folder + name_of_trained_model_to_use,
cuda_device)
config = archive.config
ds_params = config["dataset_reader"]
model = archive.model
model.eval()
mithun_logger.info(f"going to initiate FEVERReaderUofa.")
fever_reader = FEVERReaderUofa(
db,
sentence_level=ds_params.pop("sentence_level", False),
wiki_tokenizer=Tokenizer.from_params(
ds_params.pop('wiki_tokenizer', {})),
claim_tokenizer=Tokenizer.from_params(
ds_params.pop('claim_tokenizer', {})),
token_indexers=TokenIndexer.dict_from_params(
ds_params.pop('token_indexers', {})))
cwd = os.getcwd()
mithun_logger.info(f"going to start reading data.")
zipped_annotated_data, length_data = fever_reader.read(
mithun_logger, cwd + path_to_pyproc_annotated_data_folder)
mithun_logger.info(
f"done with reading data. going to generate features.")
data = None
for feature in features:
# todo: right now there is only one feature, NER ONE, so you will get away with data inside this for loop. However, need to dynamically add features
fdl = feature + "_details"
mithun_logger.info(f"value of fdl is:{fdl}")
mithun_logger.info(f"value of feature is:{feature}")