def eval_da(dataset_to_work_on, args, operation, mithun_logger):
LogHelper.setup()
LogHelper.get_logger("allennlp.training.trainer")
LogHelper.get_logger(__name__)
params = Params.from_file(args.param_path, args.overrides)
uofa_params = params.pop('uofa_params', {})
path_to_saved_db = uofa_params.pop("path_to_saved_db")
db = FeverDocDB(path_to_saved_db)
mithun_logger.info("inside main function going to call eval on " +
str(dataset_to_work_on))
mithun_logger.info("path_to_pyproc_annotated_data_folder " +
str(path_to_pyproc_annotated_data_folder))
mithun_logger.info("value of name_of_trained_model_to_use: " +
str(name_of_trained_model_to_use))
mithun_logger.info("value of dataset_to_work_on: " +
str(dataset_to_work_on))
if (dataset_to_work_on == "fnc"):
fever_dataset_details = uofa_params.pop('fever_dataset_details', {})
dev_partition_details = fever_dataset_details.pop(
'dev_partition_details', {})
name_of_trained_model_to_use = dev_partition_details.pop(
'name_of_trained_model_to_use', {})
path_to_pyproc_annotated_data_folder = dev_partition_details.pop(
'path_to_pyproc_annotated_data_folder', {})
debug_mode = uofa_params.pop('debug_mode', {})
path_to_trained_models_folder = uofa_params.pop(
'path_to_trained_models_folder', {})
path_to_fnc_annotated_data = dev_partition_details.pop(
'path_to_pyproc_annotated_data_folder', {})
eval_model_fnc_data(db, args, mithun_logger,
name_of_trained_model_to_use,
path_to_trained_models_folder, cuda_device,
operation, path_to_fnc_annotated_data)
elif (dataset_to_work_on == "fever"):
fever_dataset_details = uofa_params.pop('fever_dataset_details', {})
dev_partition_details = fever_dataset_details.pop(
'dev_partition_details', {})
name_of_trained_model_to_use = dev_partition_details.pop(
'name_of_trained_model_to_use', {})
path_to_pyproc_annotated_data_folder = dev_partition_details.pop(
'path_to_pyproc_annotated_data_folder', {})
debug_mode = uofa_params.pop('debug_mode', {})
path_to_trained_models_folder = uofa_params.pop(
'path_to_trained_models_folder', {})
eval_model(db, args, mithun_logger, path_to_trained_models_folder,
name_of_trained_model_to_use)
def main(args=NullArgs()):
LogHelper.setup()
logger = LogHelper.get_logger(
os.path.splitext(os.path.basename(__file__))[0])
args.mode = Mode.PREDICT
if args.config is not None:
Config.load_config(args.config)
if args.out_file is not None:
Config.relative_path_submission = args.out_file
if args.in_file is not None:
Config.relative_path_test_file = args.in_file
if args.database is not None:
Config.relative_path_db = args.database
print("relative_path_db " + Config.relative_path_db)
print("raw_test_set " + Config.raw_test_set())
if os.path.exists(Config.test_doc_file):
os.remove(Config.test_doc_file)
if os.path.exists(Config.test_set_file):
os.remove(Config.test_set_file)
if args.mode in {Mode.PIPELINE, Mode.PREDICT, Mode.PREDICT_ALL_DATASETS}:
logger.info(
"=========================== Sub-task 1. Document Retrieval =========================================="
)
document_retrieval(logger, args.mode)
if args.mode in {
Mode.PIPELINE_NO_DOC_RETR, Mode.PIPELINE, Mode.PREDICT,
Mode.PREDICT_NO_DOC_RETR, Mode.PREDICT_ALL_DATASETS,
Mode.PREDICT_NO_DOC_RETR_ALL_DATASETS
}:
logger.info(
"=========================== Sub-task 2. Sentence Retrieval =========================================="
)
sentence_retrieval_ensemble(logger, args.mode)
logger.info(
"=========================== Sub-task 3. Claim Validation ============================================"
)
rte(logger, args, args.mode)
def main(mode: RTERunPhase, config=None, estimator=None):
LogHelper.setup()
logger = LogHelper.get_logger(
os.path.splitext(os.path.basename(__file__))[0] + "_" + str(mode))
if config is not None and isinstance(config, str):
logger.info("model: " + str(mode) + ", config: " + str(config))
Config.load_config(config)
if hasattr(Config, 'is_snopes'):
is_snopes = Config.is_snopes
else:
is_snopes = False
logger.debug("is_snopes: " + str(is_snopes))
if mode == RTERunPhase.train:
# training mode
if hasattr(Config, 'training_dump') and os.path.exists(
Config.training_dump):
with open(Config.training_dump, 'rb') as f:
(X_train, Y_labels_train, X_valid,
Y_labels_valid) = pickle.load(f)
else:
# process training JSONL file
X_train, Y_labels_train = read_data_set_from_jsonl(
Config.training_set_file,
Config.db_path,
num_sentences=Config.max_sentences,
is_snopes=is_snopes)
X_valid, Y_labels_valid = read_data_set_from_jsonl(
Config.dev_set_file,
Config.db_path,
num_sentences=Config.max_sentences,
is_snopes=is_snopes)
b_train = X_train['b']
X_train['b_sizes'] = get_num_sents_of_bodies(b_train)
for i, sample in enumerate(b_train):
if len(sample) < Config.max_sentences:
for _ in range(Config.max_sentences - len(sample)):
sample.append(" ")
b_train[i] = np.asarray(sample)
b_train = np.asarray(b_train)
X_train['b'] = b_train
logger.debug("b_train.shape: " + str(b_train.shape))
b_valid = X_valid['b']
X_valid['b_sizes'] = get_num_sents_of_bodies(b_valid)
for i, sample in enumerate(b_valid):
if len(sample) < Config.max_sentences:
for _ in range(Config.max_sentences - len(sample)):
sample.append(" ")
b_valid[i] = np.asarray(sample)
b_valid = np.asarray(b_valid)
X_valid['b'] = b_valid
logger.debug("b_valid.shape: " + str(b_valid.shape))
if hasattr(Config, 'training_dump'):
with open(Config.training_dump, 'wb') as f:
pickle.dump(
(X_train, Y_labels_train, X_valid, Y_labels_valid),
f,
protocol=pickle.HIGHEST_PROTOCOL)
if estimator is None:
estimator = get_estimator(Config.estimator_name,
Config.ckpt_folder)
if 'CUDA_VISIBLE_DEVICES' not in os.environ or not str(
os.environ['CUDA_VISIBLE_DEVICES']).strip():
os.environ['CUDA_VISIBLE_DEVICES'] = str(
GPUtil.getFirstAvailable(maxLoad=1.0,
maxMemory=1.0 -
Config.max_gpu_memory)[0])
estimator.fit(X_train, Y_labels_train, X_valid, Y_labels_valid)
save_model(estimator, Config.model_folder, Config.pickle_name, logger)
else:
# testing mode
restore_param_required = estimator is None
if estimator is None:
estimator = load_model(Config.model_folder, Config.pickle_name)
if estimator is None:
estimator = get_estimator(Config.estimator_name,
Config.ckpt_folder)
X_test, Y_labels_test = read_data_set_from_jsonl(
Config.test_set_file,
Config.db_path,
num_sentences=Config.max_sentences,
is_snopes=is_snopes)
b_test = X_test['b']
X_test['b_sizes'] = get_num_sents_of_bodies(b_test)
for i, sample in enumerate(b_test):
if len(sample) < Config.max_sentences:
for _ in range(Config.max_sentences - len(sample)):
sample.append(" ")
b_test[i] = np.asarray(sample)
b_test = np.asarray(b_test)
X_test['b'] = b_test
logger.debug("b_test.shape: " + str(b_test.shape))
if 'CUDA_VISIBLE_DEVICES' not in os.environ or not str(
os.environ['CUDA_VISIBLE_DEVICES']).strip():
os.environ['CUDA_VISIBLE_DEVICES'] = str(
GPUtil.getFirstAvailable(maxLoad=1.0,
maxMemory=1.0 -
Config.max_gpu_memory)[0])
predictions = estimator.predict(X_test, restore_param_required)
generate_submission(predictions, X_test['id'], Config.test_set_file,
Config.submission_file)
if Y_labels_test:
print_metrics(Y_labels_test, predictions, logger)
return estimator
# LICENSE file in the root directory of this source tree.
"""A script to read in and store documents in a sqlite database."""
import argparse
import importlib.util
import json
import os
import sqlite3
from multiprocessing import Pool as ProcessPool
from drqa.retriever import utils
from tqdm import tqdm
from common.util.log_helper import LogHelper
LogHelper.setup()
logger = LogHelper.get_logger("DrQA BuildDB")
# ------------------------------------------------------------------------------
# Preprocessing Function.
# ------------------------------------------------------------------------------
PREPROCESS_FN = None
def init(filename):
global PREPROCESS_FN
if filename:
PREPROCESS_FN = import_module(filename).preprocess
def main(mode: RTERunPhase, config=None, estimator=None):
LogHelper.setup()
logger = LogHelper.get_logger(
os.path.splitext(os.path.basename(__file__))[0] + "_" + str(mode))
if config is not None and isinstance(config, str):
logger.info("model: " + str(mode) + ", config: " + str(config))
Config.load_config(config)
logger.info("scorer type: " + Config.estimator_name)
logger.info("random seed: " + str(Config.seed))
logger.info("ESIM arguments: " + str(Config.esim_hyper_param))
logger.info("this script is only for FEVER dataset")
if mode == RTERunPhase.train:
# # training mode
if hasattr(Config, 'training_dump') and os.path.exists(
Config.training_dump):
with open(Config.training_dump, 'rb') as f:
(X_dict, y_train) = pickle.load(f)
else:
training_set, vocab, embeddings, _, _ = embed_data_set_with_glove_2(
Config.training_set_file,
Config.db_path,
glove_path=Config.glove_path,
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
threshold_h_sent_size=Config.max_claim_size)
h_sent_sizes = training_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
training_set['data']['h_sent_sizes'] = np.expand_dims(
h_sent_sizes, 1)
training_set['data']['h_sizes'] = h_sizes
training_set['data']['h_np'] = np.expand_dims(
training_set['data']['h_np'], 1)
training_set['data']['scores'] = load_scores(
Config.training_set_file, Config.max_sentences)
valid_set, _, _, _, _ = embed_data_set_with_glove_2(
Config.dev_set_file,
Config.db_path,
vocab_dict=vocab,
glove_embeddings=embeddings,
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
threshold_h_sent_size=Config.max_claim_size)
h_sent_sizes = valid_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
valid_set['data']['h_sent_sizes'] = np.expand_dims(h_sent_sizes, 1)
valid_set['data']['h_sizes'] = h_sizes
valid_set['data']['h_np'] = np.expand_dims(
valid_set['data']['h_np'], 1)
valid_set['data']['scores'] = load_scores(Config.dev_set_file,
Config.max_sentences)
X_dict = {
'X_train': training_set['data'],
'X_valid': valid_set['data'],
'y_valid': valid_set['label'],
'embedding': embeddings
}
y_train = training_set['label']
if hasattr(Config, 'training_dump'):
with open(Config.training_dump, 'wb') as f:
pickle.dump((X_dict, y_train),
f,
protocol=pickle.HIGHEST_PROTOCOL)
if estimator is None:
estimator = get_estimator(Config.estimator_name,
Config.ckpt_folder)
if 'CUDA_VISIBLE_DEVICES' not in os.environ or not str(
os.environ['CUDA_VISIBLE_DEVICES']).strip():
os.environ['CUDA_VISIBLE_DEVICES'] = str(
GPUtil.getFirstAvailable(maxLoad=1.0,
maxMemory=1.0 -
Config.max_gpu_memory)[0])
estimator.fit(X_dict, y_train)
save_model(estimator, Config.model_folder, Config.pickle_name, logger)
else:
# testing mode
restore_param_required = estimator is None
if estimator is None:
estimator = load_model(Config.model_folder, Config.pickle_name)
if estimator is None:
estimator = get_estimator(Config.estimator_name,
Config.ckpt_folder)
vocab, embeddings = load_whole_glove(Config.glove_path)
vocab = vocab_map(vocab)
test_set, _, _, _, _ = embed_data_set_with_glove_2(
Config.test_set_file,
Config.db_path,
vocab_dict=vocab,
glove_embeddings=embeddings,
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
threshold_h_sent_size=Config.max_claim_size)
h_sent_sizes = test_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
test_set['data']['h_sent_sizes'] = np.expand_dims(h_sent_sizes, 1)
test_set['data']['h_sizes'] = h_sizes
test_set['data']['h_np'] = np.expand_dims(test_set['data']['h_np'], 1)
test_set['data']['scores'] = load_scores(Config.test_set_file,
Config.max_sentences)
x_dict = {'X_test': test_set['data'], 'embedding': embeddings}
if 'CUDA_VISIBLE_DEVICES' not in os.environ or not str(
os.environ['CUDA_VISIBLE_DEVICES']).strip():
os.environ['CUDA_VISIBLE_DEVICES'] = str(
GPUtil.getFirstAvailable(maxLoad=1.0,
maxMemory=1.0 -
Config.max_gpu_memory)[0])
predictions = estimator.predict(
x_dict, restore_param_required=restore_param_required)
generate_submission(predictions, test_set['id'], Config.test_set_file,
Config.submission_file)
if 'label' in test_set:
print_metrics(test_set['label'], predictions, logger)
return estimator
def main(mode, config, estimator=None):
LogHelper.setup()
logger = LogHelper.get_logger(os.path.splitext(os.path.basename(__file__))[0] + "_" + mode)
logger.info("model: " + mode + ", config: " + str(config))
logger.info("scorer type: " + Config.estimator_name)
logger.info("random seed: " + str(Config.seed))
logger.info("ESIM arguments: " + str(Config.esim_hyper_param))
# loading FastText takes a long time, so better pickle the loaded FastText model
if os.path.splitext(Config.fasttext_path)[1] == '.p':
with open(Config.fasttext_path, "rb") as ft_file:
fasttext_model = pickle.load(ft_file)
else:
fasttext_model = Config.fasttext_path
if mode == 'train':
# # training mode
training_set, fasttext_model, vocab, embeddings, _, _ = embed_data_set_with_glove_and_fasttext(
Config.training_set_file, Config.db_path(), fasttext_model, glove_path=Config.glove_path,
threshold_b_sent_num=Config.max_sentences, threshold_b_sent_size=Config.max_sentence_size,
threshold_h_sent_size=Config.max_claim_size)
h_sent_sizes = training_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
training_set['data']['h_sent_sizes'] = np.expand_dims(h_sent_sizes, 1)
training_set['data']['h_sizes'] = h_sizes
training_set['data']['h_np'] = np.expand_dims(training_set['data']['h_np'], 1)
training_set['data']['h_ft_np'] = np.expand_dims(training_set['data']['h_ft_np'], 1)
valid_set, _, _, _, _, _ = embed_data_set_with_glove_and_fasttext(Config.dev_set_file, Config.db_path(),
fasttext_model, vocab_dict=vocab,
glove_embeddings=embeddings,
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
threshold_h_sent_size=Config.max_claim_size)
del fasttext_model
h_sent_sizes = valid_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
valid_set['data']['h_sent_sizes'] = np.expand_dims(h_sent_sizes, 1)
valid_set['data']['h_sizes'] = h_sizes
valid_set['data']['h_np'] = np.expand_dims(valid_set['data']['h_np'], 1)
valid_set['data']['h_ft_np'] = np.expand_dims(valid_set['data']['h_ft_np'], 1)
X_dict = {
'X_train': training_set['data'],
'X_valid': valid_set['data'],
'y_valid': valid_set['label'],
'embedding': embeddings
}
if estimator is None:
estimator = get_estimator(Config.estimator_name, Config.ckpt_folder)
estimator.fit(X_dict, training_set['label'])
save_model(estimator, Config.model_folder, Config.pickle_name, logger)
elif mode == 'test':
# testing mode
restore_param_required = estimator is None
if estimator is None:
estimator = load_model(Config.model_folder, Config.pickle_name)
if estimator is None:
estimator = get_estimator(Config.estimator_name, Config.ckpt_folder)
vocab, embeddings = load_whole_glove(Config.glove_path)
vocab = vocab_map(vocab)
test_set, _, _, _, _, _ = embed_data_set_with_glove_and_fasttext(Config.test_set_file, Config.db_path(),
fasttext_model, vocab_dict=vocab,
glove_embeddings=embeddings,
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
threshold_h_sent_size=Config.max_claim_size)
del fasttext_model
h_sent_sizes = test_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
test_set['data']['h_sent_sizes'] = np.expand_dims(h_sent_sizes, 1)
test_set['data']['h_sizes'] = h_sizes
test_set['data']['h_np'] = np.expand_dims(test_set['data']['h_np'], 1)
test_set['data']['h_ft_np'] = np.expand_dims(test_set['data']['h_ft_np'], 1)
x_dict = {
'X_test': test_set['data'],
'embedding': embeddings
}
predictions = estimator.predict(x_dict, restore_param_required)
generate_submission(predictions, test_set['id'], Config.test_set_file, Config.submission_file())
if 'label' in test_set:
print_metrics(test_set['label'], predictions, logger)
else:
logger.error("Invalid argument --mode: " + mode + " Argument --mode should be either 'train’ or ’test’")
def main(mode: RTERunPhase, config=None, estimator=None):
LogHelper.setup()
logger = LogHelper.get_logger(
os.path.splitext(os.path.basename(__file__))[0] + "_" + str(mode))
if config is not None and isinstance(config, str):
logger.info("model: " + str(mode) + ", config: " + str(config))
Config.load_config(config)
if hasattr(Config, 'use_inter_evidence_comparison'):
use_inter_evidence_comparison = Config.use_inter_evidence_comparison
else:
use_inter_evidence_comparison = False
# 'esim_inter_evidence' model and 'esim_inter_evidence_claim_evidences_comparison' models need inter evidence inputs
use_inter_evidence_comparison = use_inter_evidence_comparison or Config.estimator_name in {
'esim_inter_evidence', 'esim_inter_evidence_claim_evidences_comparison'
}
if hasattr(Config, 'use_claim_evidences_comparison'):
use_claim_evidences_comparison = Config.use_claim_evidences_comparison
else:
use_claim_evidences_comparison = False
# 'esim_inter_evidence_claim_evidences_comparison' model needs claim-evidence inputs
use_claim_evidences_comparison = use_claim_evidences_comparison or Config.estimator_name in {
'esim_inter_evidence_claim_evidences_comparison'
}
if hasattr(Config, 'use_extra_features'):
use_extra_features = Config.use_extra_features
else:
use_extra_features = False
if hasattr(Config, 'use_numeric_feature'):
use_numeric_feature = Config.use_numeric_feature
else:
use_numeric_feature = False
# 'esim_num_feature' model needs numeric feature inputs
use_numeric_feature = use_numeric_feature or Config.estimator_name in {
'esim_num_feature'
}
if hasattr(Config, 'is_snopes'):
is_snopes = Config.is_snopes
else:
is_snopes = False
logger.debug("is_snopes: " + str(is_snopes))
logger.info("scorer type: " + Config.estimator_name)
logger.info("random seed: " + str(Config.seed))
logger.info("ESIM arguments: " + str(Config.esim_end_2_end_hyper_param))
logger.info("use_inter_sentence_comparison: " +
str(use_inter_evidence_comparison))
logger.info("use_extra_features: " + str(use_extra_features))
logger.info("use_numeric_feature: " + str(use_numeric_feature))
logger.info("use_claim_evidences_comparison: " +
str(use_claim_evidences_comparison))
if mode == RTERunPhase.train:
# # training mode
if hasattr(Config, 'training_dump') and os.path.exists(
Config.training_dump):
with open(Config.training_dump, 'rb') as f:
(X_dict, y_train) = pickle.load(f)
else:
training_set, vocab, embeddings, _, _ = embed_data_set_with_glove_2(
Config.training_set_file,
Config.db_path,
glove_path=Config.glove_path,
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
threshold_h_sent_size=Config.max_claim_size,
is_snopes=is_snopes)
h_sent_sizes = training_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
training_set['data']['h_sent_sizes'] = np.expand_dims(
h_sent_sizes, 1)
training_set['data']['h_sizes'] = h_sizes
training_set['data']['h_np'] = np.expand_dims(
training_set['data']['h_np'], 1)
valid_set, _, _, _, _ = embed_data_set_with_glove_2(
Config.dev_set_file,
Config.db_path,
vocab_dict=vocab,
glove_embeddings=embeddings,
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
threshold_h_sent_size=Config.max_claim_size,
is_snopes=is_snopes)
h_sent_sizes = valid_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
valid_set['data']['h_sent_sizes'] = np.expand_dims(h_sent_sizes, 1)
valid_set['data']['h_sizes'] = h_sizes
valid_set['data']['h_np'] = np.expand_dims(
valid_set['data']['h_np'], 1)
if use_extra_features:
assert hasattr(
Config, 'feature_path'
), "Config should has feature_path if Config.use_feature is True"
training_claim_features, training_evidence_features = load_feature_by_data_set(
Config.training_set_file, Config.feature_path,
Config.max_sentences)
valid_claim_features, valid_evidence_features = load_feature_by_data_set(
Config.dev_set_file, Config.feature_path,
Config.max_sentences)
training_set['data']['h_feats'] = training_claim_features
training_set['data']['b_feats'] = training_evidence_features
valid_set['data']['h_feats'] = valid_claim_features
valid_set['data']['b_feats'] = valid_evidence_features
if use_numeric_feature:
training_num_feat = number_feature(Config.training_set_file,
Config.db_path,
Config.max_sentences,
is_snopes)
valid_num_feat = number_feature(Config.dev_set_file,
Config.db_path,
Config.max_sentences,
is_snopes)
training_set['data']['num_feat'] = training_num_feat
valid_set['data']['num_feat'] = valid_num_feat
if use_inter_evidence_comparison:
training_concat_sent_indices, training_concat_sent_sizes = generate_concat_indices_for_inter_evidence(
training_set['data']['b_np'],
training_set['data']['b_sent_sizes'],
Config.max_sentence_size, Config.max_sentences)
training_set['data'][
'b_concat_indices'] = training_concat_sent_indices
training_set['data'][
'b_concat_sizes'] = training_concat_sent_sizes
valid_concat_sent_indices, valid_concat_sent_sizes = generate_concat_indices_for_inter_evidence(
valid_set['data']['b_np'],
valid_set['data']['b_sent_sizes'],
Config.max_sentence_size, Config.max_sentences)
valid_set['data'][
'b_concat_indices'] = valid_concat_sent_indices
valid_set['data']['b_concat_sizes'] = valid_concat_sent_sizes
if use_claim_evidences_comparison:
training_all_evidences_indices, training_all_evidences_sizes = generate_concat_indices_for_claim(
training_set['data']['b_np'],
training_set['data']['b_sent_sizes'],
Config.max_sentence_size, Config.max_sentences)
training_set['data'][
'b_concat_indices_for_h'] = training_all_evidences_indices
training_set['data'][
'b_concat_sizes_for_h'] = training_all_evidences_sizes
valid_all_evidences_indices, valid_all_evidences_sizes = generate_concat_indices_for_claim(
valid_set['data']['b_np'],
valid_set['data']['b_sent_sizes'],
Config.max_sentence_size, Config.max_sentences)
valid_set['data'][
'b_concat_indices_for_h'] = valid_all_evidences_indices
valid_set['data'][
'b_concat_sizes_for_h'] = valid_all_evidences_sizes
X_dict = {
'X_train': training_set['data'],
'X_valid': valid_set['data'],
'y_valid': valid_set['label'],
'embedding': embeddings
}
y_train = training_set['label']
if hasattr(Config, 'training_dump'):
with open(Config.training_dump, 'wb') as f:
pickle.dump((X_dict, y_train),
f,
protocol=pickle.HIGHEST_PROTOCOL)
if estimator is None:
estimator = get_estimator(Config.estimator_name,
Config.ckpt_folder)
if 'CUDA_VISIBLE_DEVICES' not in os.environ or not str(
os.environ['CUDA_VISIBLE_DEVICES']).strip():
os.environ['CUDA_VISIBLE_DEVICES'] = str(
GPUtil.getFirstAvailable(maxLoad=1.0,
maxMemory=1.0 -
Config.max_gpu_memory)[0])
estimator.fit(X_dict, y_train)
save_model(estimator, Config.model_folder, Config.pickle_name, logger)
else:
# testing mode
restore_param_required = estimator is None
if estimator is None:
estimator = load_model(Config.model_folder, Config.pickle_name)
if estimator is None:
estimator = get_estimator(Config.estimator_name,
Config.ckpt_folder)
vocab, embeddings = load_whole_glove(Config.glove_path)
vocab = vocab_map(vocab)
test_set, _, _, _, _ = embed_data_set_with_glove_2(
Config.test_set_file,
Config.db_path,
vocab_dict=vocab,
glove_embeddings=embeddings,
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
threshold_h_sent_size=Config.max_claim_size,
is_snopes=is_snopes)
h_sent_sizes = test_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
test_set['data']['h_sent_sizes'] = np.expand_dims(h_sent_sizes, 1)
test_set['data']['h_sizes'] = h_sizes
test_set['data']['h_np'] = np.expand_dims(test_set['data']['h_np'], 1)
if use_extra_features:
assert hasattr(
Config, 'feature_path'
), "Config should has feature_path if Config.use_feature is True"
test_claim_features, test_evidence_features = load_feature_by_data_set(
Config.test_set_file, Config.feature_path,
Config.max_sentences)
test_set['data']['h_feats'] = test_claim_features
test_set['data']['b_feats'] = test_evidence_features
if use_numeric_feature:
test_num_feat = number_feature(Config.test_set_file,
Config.db_path,
Config.max_sentences, is_snopes)
test_set['data']['num_feat'] = test_num_feat
x_dict = {'X_test': test_set['data'], 'embedding': embeddings}
if use_inter_evidence_comparison:
test_concat_sent_indices, test_concat_sent_sizes = generate_concat_indices_for_inter_evidence(
test_set['data']['b_np'], test_set['data']['b_sent_sizes'],
Config.max_sentence_size, Config.max_sentences)
test_set['data']['b_concat_indices'] = test_concat_sent_indices
test_set['data']['b_concat_sizes'] = test_concat_sent_sizes
if use_claim_evidences_comparison:
test_all_evidences_indices, test_all_evidences_sizes = generate_concat_indices_for_claim(
test_set['data']['b_np'], test_set['data']['b_sent_sizes'],
Config.max_sentence_size, Config.max_sentences)
test_set['data'][
'b_concat_indices_for_h'] = test_all_evidences_indices
test_set['data']['b_concat_sizes_for_h'] = test_all_evidences_sizes
if 'CUDA_VISIBLE_DEVICES' not in os.environ or not str(
os.environ['CUDA_VISIBLE_DEVICES']).strip():
os.environ['CUDA_VISIBLE_DEVICES'] = str(
GPUtil.getFirstAvailable(maxLoad=1.0,
maxMemory=1.0 -
Config.max_gpu_memory)[0])
predictions = estimator.predict(
x_dict, restore_param_required=restore_param_required)
generate_submission(predictions, test_set['id'], Config.test_set_file,
Config.submission_file)
if 'label' in test_set:
print_metrics(test_set['label'], predictions, logger)
return estimator
def fever_app(caller):
#parser = ArgumentParser()
#parser.add_argument("--db-path", default="/local/fever-common/data/fever/fever.db")
#parser.add_argument("--random-seed", default=1234)
#parser.add_argument("--sentence-model", default="model/esim_0/sentence_retrieval_ensemble")
#parser.add_argument("--words-cache", default="model/sentence")
#parser.add_argument("--c-max-length", default=20)
#parser.add_argument("--s-max-length", default=60)
#parser.add_argument("--fasttext-path", default="data/fasttext/wiki.en.bin")
#parser.add_argument("--train-data", default="data/fever/train.wiki7.jsonl")
#parser.add_argument("--dev-data", default="data/fever/dev.wiki7.jsonl")
#parser.add_argument("--test-data", default="data/fever/test.wiki7.jsonl")
#parser.add_argument("--add-claim", default=True)
args = Struct(
**{
"db_path": "/local/fever-common/data/fever/fever.db",
"random_seed": 1234,
"sentence_model": "model/esim_0/sentence_retrieval_ensemble",
"words_cache": "model/sentence",
"c_max_length": 20,
"s_max_length": 60,
"fasttext_path": "data/fasttext/wiki.en.bin",
"train_data": "data/fever/train.wiki7.jsonl",
"dev_data": "data/fever/dev.wiki7.jsonl",
"test_data": "data/fever/test.wiki7.jsonl",
"add_claim": True
})
# Setup logging
LogHelper.setup()
logger = LogHelper.get_logger("setup")
logger.info("Logging started")
# Set seeds
logger.info("Set Seeds")
np.random.seed(args.random_seed)
random.seed(args.random_seed)
# Load GLove
logger.info("Load GloVe")
vocab, embeddings = load_whole_glove(Config.glove_path)
vocab = vocab_map(vocab)
# Document Retrieval
logger.info("Setup document retrieval")
retrieval = Doc_Retrieval(database_path=args.db_path,
add_claim=args.add_claim,
k_wiki_results=k_wiki)
# Sentence Selection
logger.info("Setup sentence loader")
#words, iwords = get_iwords(args, retrieval)
sentence_loader = SentenceDataLoader(fasttext_path=args.fasttext_path,
db_filepath=args.db_path,
h_max_length=args.c_max_length,
s_max_length=args.s_max_length,
reserve_embed=True)
sentence_loader.load_models(vocab,
sentence_loader.inverse_word_dict(vocab))
sargs = Config.sentence_retrieval_ensemble_param
sargs.update(vars(args))
sargs = Struct(**sargs)
logger.info("Sentence ESIM ensemble")
selections = [
SentenceESIM(h_max_length=sargs.c_max_length,
s_max_length=sargs.s_max_length,
learning_rate=sargs.learning_rate,
batch_size=sargs.batch_size,
num_epoch=sargs.num_epoch,
model_store_dir=sargs.sentence_model,
embedding=sentence_loader.embed,
word_dict=sentence_loader.word_dict,
dropout_rate=sargs.dropout_rate,
num_units=sargs.num_lstm_units,
share_rnn=False,
activation=tf.nn.tanh,
namespace="model_{}".format(i))
for i in range(sargs.num_model)
]
for i in range(sargs.num_model):
logger.info("Restore Model {}".format(i))
model_store_path = os.path.join(args.sentence_model,
"model{}".format(i + 1))
if not os.path.exists(model_store_path):
raise Exception("model must be trained before testing")
selections[i].restore_model(
os.path.join(model_store_path, "best_model.ckpt"))
logger.info("Load FastText")
fasttext_model = FastText.load_fasttext_format(Config.fasttext_path)
# RTE
logger.info("Setup RTE")
rte_predictor = get_estimator(Config.estimator_name, Config.ckpt_folder)
rte_predictor.embedding = embeddings
logger.info("Restore RTE Model")
rte_predictor.restore_model(rte_predictor.ckpt_path)
def get_docs_line(line):
nps, wiki_results, pages = retrieval.exact_match(line)
line['noun_phrases'] = nps
line['predicted_pages'] = pages
line['wiki_results'] = wiki_results
return line
def get_docs(lines):
return list(map(get_docs_line, lines))
def get_sents(lines):
indexes, location_indexes = sentence_loader.get_indexes(lines)
all_predictions = []
for i in range(sargs.num_model):
predictions = []
selection_model = selections[i]
for test_index in indexes:
prediction = selection_model.predict(test_index)
predictions.append(prediction)
all_predictions.append(predictions)
ensembled_predicitons = scores_processing(all_predictions, args)
processed_predictions, scores = post_processing(
ensembled_predicitons, location_indexes)
final_predictions = prediction_processing_no_reload(
lines, processed_predictions)
return final_predictions
def run_rte(lines):
test_set, _, _, _, _, _ = embed_claims(
lines,
args.db_path,
fasttext_model,
vocab_dict=vocab,
glove_embeddings=embeddings,
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
threshold_h_sent_size=Config.max_claim_size)
h_sent_sizes = test_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
test_set['data']['h_sent_sizes'] = np.expand_dims(h_sent_sizes, 1)
test_set['data']['h_sizes'] = h_sizes
test_set['data']['h_np'] = np.expand_dims(test_set['data']['h_np'], 1)
test_set['data']['h_ft_np'] = np.expand_dims(
test_set['data']['h_ft_np'], 1)
x_dict = {'X_test': test_set['data'], 'embedding': embeddings}
predictions = rte_predictor.predict(x_dict, False)
return predictions
def process_claims(claims):
print("CLAIMS LEN {}".format(len(claims)))
claims = get_docs(claims)
print("CLAIMS LEN {}".format(len(claims)))
claims = get_sents(claims)
print("CLAIMS LEN {}".format(len(claims)))
predictions = run_rte(claims)
print("PREDICTIONS LEN {}".format(len(predictions)))
ret = []
for idx in range(len(claims)):
claim = claims[idx]
prediction = predictions[idx]
return_line = {
"predicted_label": prediction_2_label(prediction),
"predicted_evidence": claim["predicted_evidence"]
}
ret.append(return_line)
return ret
return caller(process_claims)
def main(mode: RTERunPhase, config=None, estimator=None):
LogHelper.setup()
logger = LogHelper.get_logger(
os.path.splitext(os.path.basename(__file__))[0] + "_" + str(mode))
if config is not None and isinstance(config, str):
logger.info("model: " + str(mode) + ", config: " + str(config))
Config.load_config(config)
logger.info("scorer type: " + Config.estimator_name)
logger.info("random seed: " + str(Config.seed))
logger.info("ESIM arguments: " + str(Config.esim_hyper_param))
if hasattr(Config, 'is_snopes'):
is_snopes = Config.is_snopes
else:
is_snopes = False
logger.debug("is_snopes: " + str(is_snopes))
if mode == RTERunPhase.train:
# training mode
if hasattr(Config, 'training_dump') and os.path.exists(
Config.training_dump):
with open(Config.training_dump, 'rb') as f:
dataset_list = pickle.load(f)
else:
# process training JSONL file
training_set, _, _ = embed_data_set_for_elmo(
Config.training_set_file,
Config.db_path,
threshold_b_sent_num=Config.max_sentences,
threshold_h_sent_size=Config.max_claim_size,
threshold_b_sent_size=Config.max_sentence_size,
is_snopes=is_snopes)
h_sent_sizes = training_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
training_set['data']['h_sent_sizes'] = np.expand_dims(
h_sent_sizes, 1)
training_set['data']['h_sizes'] = h_sizes
training_set['data']['h_tokens'] = np.expand_dims(
training_set['data']['h_tokens'], 1)
# training_set['data']['h_ft_np'] = np.expand_dims(training_set['data']['h_ft_np'], 1)
valid_set, _, _ = embed_data_set_for_elmo(
Config.dev_set_file,
Config.db_path,
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
threshold_h_sent_size=Config.max_claim_size,
is_snopes=is_snopes)
h_sent_sizes = valid_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
valid_set['data']['h_sent_sizes'] = np.expand_dims(h_sent_sizes, 1)
valid_set['data']['h_sizes'] = h_sizes
valid_set['data']['h_tokens'] = np.expand_dims(
valid_set['data']['h_tokens'], 1)
dataset_list = [training_set, valid_set]
# save processed training data
if hasattr(Config, 'training_dump'):
with open(Config.training_dump, 'wb') as f:
pickle.dump(dataset_list,
f,
protocol=pickle.HIGHEST_PROTOCOL)
if estimator is None:
estimator = get_estimator(Config.estimator_name,
Config.ckpt_folder)
if 'CUDA_VISIBLE_DEVICES' not in os.environ or not str(
os.environ['CUDA_VISIBLE_DEVICES']).strip():
os.environ['CUDA_VISIBLE_DEVICES'] = str(
GPUtil.getFirstAvailable(maxLoad=1.0,
maxMemory=1.0 -
Config.max_gpu_memory)[0])
estimator.fit(dataset_list[0]['data'], dataset_list[0]['label'],
dataset_list[1]['data'], dataset_list[1]['label'])
save_model(estimator, Config.model_folder, Config.pickle_name, logger)
else:
# testing mode
restore_param_required = estimator is None
if estimator is None:
estimator = load_model(Config.model_folder, Config.pickle_name)
if estimator is None:
estimator = get_estimator(Config.estimator_name,
Config.ckpt_folder)
test_set, _, _ = embed_data_set_for_elmo(
Config.test_set_file,
Config.db_path,
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
threshold_h_sent_size=Config.max_claim_size,
is_snopes=is_snopes)
h_sent_sizes = test_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
test_set['data']['h_sent_sizes'] = np.expand_dims(h_sent_sizes, 1)
test_set['data']['h_sizes'] = h_sizes
test_set['data']['h_tokens'] = np.expand_dims(
test_set['data']['h_tokens'], 1)
if 'CUDA_VISIBLE_DEVICES' not in os.environ or not str(
os.environ['CUDA_VISIBLE_DEVICES']).strip():
os.environ['CUDA_VISIBLE_DEVICES'] = str(
GPUtil.getFirstAvailable(maxLoad=1.0,
maxMemory=1.0 -
Config.max_gpu_memory)[0])
logger.debug("CUDA_VISIBLE_DEVICES: " +
os.environ['CUDA_VISIBLE_DEVICES'])
predictions = estimator.predict(
test_set['data'], restore_param_required=restore_param_required)
generate_submission(predictions, test_set['id'], Config.test_set_file,
Config.submission_file)
if 'label' in test_set:
print_metrics(test_set['label'], predictions, logger)
return estimator
class TopNDocsTopNSents(RetrievalMethod):
class RankArgs:
def __init__(self):
self.ngram = 2
self.hash_size = int(math.pow(2, 24))
self.tokenizer = "simple"
self.num_workers = None
def __init__(self, db, n_docs, n_sents, model):
super().__init__(db)
self.n_docs = n_docs
self.n_sents = n_sents
self.ranker = retriever.get_class('tfidf')(tfidf_path=model)
self.onlineranker_args = self.RankArgs()
def get_docs_for_claim(self, claim_text):
doc_names, doc_scores = self.ranker.closest_docs(
claim_text, self.n_docs)
return zip(doc_names, doc_scores)
def tf_idf_sim(self, claim, lines, freqs=None):
tfidf = OnlineTfidfDocRanker(self.onlineranker_args,
[line["sentence"] for line in lines],
freqs)
line_ids, scores = tfidf.closest_docs(claim, self.n_sents)
ret_lines = []
for idx, line in enumerate(line_ids):
ret_lines.append(lines[line])
ret_lines[-1]["score"] = scores[idx]
return ret_lines
LogHelper.setup()
logger = LogHelper.get_logger(__name__)
def get_sentences_given_claim(self, page, logger, line_no):
lines = self.db.get_doc_lines(page)
lines = [
line.split("\t")[1] if len(line.split("\t")[1]) > 1 else ""
for line in lines.split("\n")
]
sent = lines[line_no]
return sent
def get_sentences_for_claim(self, claim_text, include_text=False):
#given a claim get a bunch of documents that might be relevant for it
pages = self.get_docs_for_claim(claim_text)
sorted_p = list(sorted(pages, reverse=True, key=lambda elem: elem[1]))
pages = [p[0] for p in sorted_p[:self.n_docs]]
p_lines = []
for page in pages:
logger.info("page:" + page)
#query the db and get the list of sentences in a given wikipedia page
lines = self.db.get_doc_lines(page)
logger.info(lines)
sys.exit(1)
lines = [
line.split("\t")[1] if len(line.split("\t")[1]) > 1 else ""
for line in lines.split("\n")
]
p_lines.extend(zip(lines, [page] * len(lines), range(len(lines))))
lines = []
for p_line in p_lines:
logger.info("value of sentence in p_line is:" + p_line[0])
sys.exit(1)
lines.append({
"sentence": p_line[0],
"page": p_line[1],
"line_on_page": p_line[2]
})
scores = self.tf_idf_sim(claim_text, lines)
if include_text:
return scores
return [(s["page"], s["line_on_page"]) for s in scores]
def main(mode: RTERunPhase, config=None, estimator=None):
LogHelper.setup()
logger = LogHelper.get_logger(
os.path.splitext(os.path.basename(__file__))[0] + "_" + str(mode))
if config is not None and isinstance(config, str):
logger.info("model: " + str(mode) + ", config: " + str(config))
Config.load_config(config)
assert hasattr(
Config, 'page_source_file_path'
), "'page_source_file_path' field is needed in config file for this script"
logger.info("scorer type: " + Config.estimator_name)
logger.info("random seed: " + str(Config.seed))
logger.info("ESIM credibility MTL arguments: " +
str(Config.esim_credibility_mtl_hyper_param))
logger.info("this script is only for Snopes dataset")
if mode == RTERunPhase.train:
# training sets
# @formatter:off
claim_training_set, word_vocab, word_embeddings, domain_vocab, domain_embeddings, suffix_vocab, \
suffix_embeddings, protocol_vocab, protocol_embeddings, claim_stance_vocab, claim_stance_embeddings = \
embed_data_set_with_glove_with_credibility(
Config.esim_credibility_mtl_hyper_param['claim_training_set'],
Config.db_path,
Config.page_source_file_path,
glove_path=Config.glove_path,
domain_embedding_size=Config.esim_credibility_mtl_hyper_param['domain_embedding_size'],
suffix_embedding_size=Config.esim_credibility_mtl_hyper_param['suffix_embedding_size'],
protocol_embedding_size=Config.esim_credibility_mtl_hyper_param['protocol_embedding_size'],
stance_embedding_size=Config.esim_credibility_mtl_hyper_param['stance_embedding_size'],
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
threshold_h_sent_size=Config.max_sentence_size)
# @formatter:on
claim_h_sent_sizes = claim_training_set['data']['h_sent_sizes']
claim_h_sizes = np.ones(len(claim_h_sent_sizes), np.int32)
claim_training_set['data']['h_sent_sizes'] = np.expand_dims(
claim_h_sent_sizes, 1)
claim_training_set['data']['h_sizes'] = claim_h_sizes
claim_training_set['data']['h_np'] = np.expand_dims(
claim_training_set['data']['h_np'], 1)
logger.info("size of training set: " +
str(claim_training_set['data']['h_np'].shape[0]))
stance_training_set, _, _, _, _ = embed_data_set_with_glove_2(
Config.esim_credibility_mtl_hyper_param['stance_training_set'],
Config.db_path,
Config.glove_path,
vocab_dict=word_vocab,
glove_embeddings=word_embeddings,
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
threshold_h_sent_size=Config.max_claim_size,
is_snopes=True)
stance_h_sent_sizes = stance_training_set['data']['h_sent_sizes']
stance_h_sizes = np.ones(len(stance_h_sent_sizes), np.int32)
stance_training_set['data']['h_sent_sizes'] = np.expand_dims(
stance_h_sent_sizes, 1)
stance_training_set['data']['h_sizes'] = stance_h_sizes
stance_training_set['data']['h_np'] = np.expand_dims(
stance_training_set['data']['h_np'], 1)
# valid sets
claim_valid_set, _, _ = embed_data_set_with_glove_with_credibility(
Config.esim_credibility_mtl_hyper_param['claim_dev_set'],
Config.db_path,
Config.page_source_file_path,
vocab_dict=word_vocab,
glove_embeddings=word_embeddings,
domain_vocab=domain_vocab,
domain_embeddings=domain_embeddings,
suffix_vocab=suffix_vocab,
suffix_embeddings=suffix_embeddings,
protocol_vocab=protocol_vocab,
protocol_embeddings=protocol_embeddings,
stance_vocab=claim_stance_vocab,
stance_embeddings=claim_stance_embeddings,
domain_embedding_size=Config.
esim_credibility_mtl_hyper_param['domain_embedding_size'],
suffix_embedding_size=Config.
esim_credibility_mtl_hyper_param['suffix_embedding_size'],
protocol_embedding_size=Config.
esim_credibility_mtl_hyper_param['protocol_embedding_size'],
stance_embedding_size=Config.
esim_credibility_mtl_hyper_param['stance_embedding_size'],
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
threshold_h_sent_size=Config.max_sentence_size)
claim_h_sent_sizes = claim_valid_set['data']['h_sent_sizes']
claim_h_sizes = np.ones(len(claim_h_sent_sizes), np.int32)
claim_valid_set['data']['h_sent_sizes'] = np.expand_dims(
claim_h_sent_sizes, 1)
claim_valid_set['data']['h_sizes'] = claim_h_sizes
claim_valid_set['data']['h_np'] = np.expand_dims(
claim_valid_set['data']['h_np'], 1)
logger.info("size of dev set: " +
str(claim_valid_set['data']['h_np'].shape[0]))
stance_valid_set, _, _, _, _ = embed_data_set_with_glove_2(
Config.esim_credibility_mtl_hyper_param['stance_dev_set'],
Config.db_path,
Config.glove_path,
vocab_dict=word_vocab,
glove_embeddings=word_embeddings,
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
threshold_h_sent_size=Config.max_claim_size,
is_snopes=True)
stance_h_sent_sizes = stance_valid_set['data']['h_sent_sizes']
stance_h_sizes = np.ones(len(stance_h_sent_sizes), np.int32)
stance_valid_set['data']['h_sent_sizes'] = np.expand_dims(
stance_h_sent_sizes, 1)
stance_valid_set['data']['h_sizes'] = stance_h_sizes
stance_valid_set['data']['h_np'] = np.expand_dims(
stance_valid_set['data']['h_np'], 1)
X_dict_claim = {
'train': claim_training_set['data'],
'valid': claim_valid_set['data'],
}
y_claim = {
'train': claim_training_set['label'],
'valid': claim_valid_set['label']
}
X_dict_stance = {
'train': stance_training_set['data'],
'valid': stance_valid_set['data'],
}
y_stance = {
'train': stance_training_set['label'],
'valid': stance_valid_set['label']
}
X_dict = {
'claim': X_dict_claim,
'stance': X_dict_stance,
'word_embedding': word_embeddings,
'domain_embedding': domain_embeddings,
'suffix_embedding': suffix_embeddings,
'protocol_embedding': protocol_embeddings,
'stance_embedding': claim_stance_embeddings
}
y_dict = {'claim': y_claim, 'stance': y_stance}
if estimator is None:
estimator = get_estimator(Config.estimator_name,
Config.ckpt_folder)
if 'CUDA_VISIBLE_DEVICES' not in os.environ or not str(
os.environ['CUDA_VISIBLE_DEVICES']).strip():
os.environ['CUDA_VISIBLE_DEVICES'] = str(
GPUtil.getFirstAvailable(maxLoad=1.0,
maxMemory=1.0 -
Config.max_gpu_memory)[0])
estimator.fit(X_dict, y_dict)
save_model(estimator, Config.model_folder, Config.pickle_name, logger)
dump_source_features_embeddings(
Config.
esim_credibility_mtl_hyper_param['features_embeddings_path'],
domain_vocab, domain_embeddings, suffix_vocab, suffix_embeddings,
protocol_vocab, protocol_embeddings, claim_stance_vocab,
claim_stance_embeddings)
else:
# testing mode
restore_param_required = estimator is None
if estimator is None:
estimator = load_model(Config.model_folder, Config.pickle_name)
if estimator is None:
estimator = get_estimator(Config.estimator_name,
Config.ckpt_folder)
word_vocab, word_embeddings = load_whole_glove(Config.glove_path)
word_vocab = vocab_map(word_vocab)
# @formatter:off
domain_vocab, domain_embeddings, \
suffix_vocab, suffix_embeddings, \
protocol_vocab, protocol_embeddings, \
claim_stance_vocab, claim_stance_embeddings = load_source_features_embeddings(
Config.esim_credibility_mtl_hyper_param['features_embeddings_path'])
# @formatter:on
test_set, _, _ = embed_data_set_with_glove_with_credibility(
Config.esim_credibility_mtl_hyper_param['claim_test_set'],
Config.db_path,
Config.page_source_file_path,
vocab_dict=word_vocab,
glove_embeddings=word_embeddings,
domain_vocab=domain_vocab,
domain_embeddings=domain_embeddings,
suffix_vocab=suffix_vocab,
suffix_embeddings=suffix_embeddings,
protocol_vocab=protocol_vocab,
protocol_embeddings=protocol_embeddings,
stance_vocab=claim_stance_vocab,
stance_embeddings=claim_stance_embeddings,
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
threshold_h_sent_size=Config.max_sentence_size)
claim_h_sent_sizes = test_set['data']['h_sent_sizes']
claim_h_sizes = np.ones(len(claim_h_sent_sizes), np.int32)
test_set['data']['h_sent_sizes'] = np.expand_dims(
claim_h_sent_sizes, 1)
test_set['data']['h_sizes'] = claim_h_sizes
test_set['data']['h_np'] = np.expand_dims(test_set['data']['h_np'], 1)
logger.info("size of test set: " +
str(test_set['data']['h_np'].shape[0]))
x_dict = {
'X_test': test_set['data'],
'word_embedding': word_embeddings,
'domain_embedding': domain_embeddings,
'suffix_embedding': suffix_embeddings,
'protocol_embedding': protocol_embeddings,
'stance_embedding': claim_stance_embeddings
}
if 'CUDA_VISIBLE_DEVICES' not in os.environ or not str(
os.environ['CUDA_VISIBLE_DEVICES']).strip():
os.environ['CUDA_VISIBLE_DEVICES'] = str(
GPUtil.getFirstAvailable(maxLoad=1.0,
maxMemory=1.0 -
Config.max_gpu_memory)[0])
predictions = estimator.predict(x_dict, restore_param_required)
generate_submission(predictions, test_set['id'], Config.test_set_file,
Config.submission_file)
if 'label' in test_set:
print_metrics(test_set['label'], predictions, logger)
return estimator
def fever_app(caller):
global db, tokenizer, text_encoder, encoder, X_train, M_train, X, M, Y_train, Y,params,sess, n_batch_train, db_file, \
drqa_index, max_page, max_sent, encoder_path, bpe_path, n_ctx, n_batch, model_file
global n_vocab,n_special,n_y,max_len,clf_token,eval_lm_losses,eval_clf_losses,eval_mgpu_clf_losses,eval_logits, \
eval_mgpu_logits,eval_logits
LogHelper.setup()
logger = LogHelper.get_logger("papelo")
logger.info("Load config")
config = json.load(open(os.getenv("CONFIG_FILE","configs/config-docker.json")))
globals().update(config)
print(globals())
logger.info("Set Seeds")
random.seed(42)
np.random.seed(42)
tf.set_random_seed(42)
logger.info("Load FEVER DB")
db = FeverDocDB(db_file)
retrieval = TopNDocsTopNSents(db, max_page, max_sent, True, False, drqa_index)
logger.info("Init word tokenizer")
tokenizer = SimpleWordSplitter()
# Prepare text encoder
logger.info("Load BPE Text Encoder")
text_encoder = TextEncoder(encoder_path, bpe_path)
encoder = text_encoder.encoder
n_vocab = len(text_encoder.encoder)
n_y = 3
encoder['_start_'] = len(encoder)
encoder['_delimiter_'] = len(encoder)
encoder['_classify_'] = len(encoder)
clf_token = encoder['_classify_']
n_special = 3
max_len = n_ctx // 2 - 2
n_batch_train = n_batch
logger.info("Create TF Placeholders")
X_train = tf.placeholder(tf.int32, [n_batch, 1, n_ctx, 2])
M_train = tf.placeholder(tf.float32, [n_batch, 1, n_ctx])
X = tf.placeholder(tf.int32, [None, 1, n_ctx, 2])
M = tf.placeholder(tf.float32, [None, 1, n_ctx])
Y_train = tf.placeholder(tf.int32, [n_batch])
Y = tf.placeholder(tf.int32, [None])
logger.info("Model Setup")
eval_logits, eval_clf_losses, eval_lm_losses = model(X, M, Y, train=False, reuse=None)
eval_mgpu_logits, eval_mgpu_clf_losses, eval_mgpu_lm_losses = mgpu_predict(X_train, M_train, Y_train)
logger.info("Create TF Session")
params = find_trainable_variables('model')
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=float(os.getenv("TF_GPU_MEMORY_FRACTION","0.5")))
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True, gpu_options=gpu_options))
sess.run(tf.global_variables_initializer())
sess.run([p.assign(ip) for p, ip in zip(params, joblib.load(model_file))])
logger.info("Ready")
def predict(instances):
predictions = []
for instance in tqdm(instances):
sents = retrieval.get_sentences_for_claim(instance["claim"])
found_evidence = resolve_evidence(sents)
instance["tokenized_claim"] = " ".join(map(lambda x: x.text, tokenizer.split_words(instance["claim"])))
sub_instances = make_instances(instance, found_evidence)
sub_predictions = predict_sub_instances(text_encoder, sub_instances)
refute_evidence = [i for i, x in enumerate(sub_predictions) if x == 2]
support_evidence = [i for i, x in enumerate(sub_predictions) if x == 0]
if len(support_evidence):
predicted_label = "SUPPORTS"
predicted_evidence = [[found_evidence[i]["title"], found_evidence[i]["line_number"]] for i in support_evidence]
elif len(refute_evidence):
predicted_label = "REFUTES"
predicted_evidence = [[found_evidence[i]["title"], found_evidence[i]["line_number"]] for i in refute_evidence]
else:
predicted_label = "NOT ENOUGH INFO"
predicted_evidence = []
predictions.append({"predicted_label":predicted_label,
"predicted_evidence": predicted_evidence})
return predictions
return caller(predict)
def main(mode: RTERunPhase, config=None, estimator=None):
LogHelper.setup()
logger = LogHelper.get_logger(
os.path.splitext(os.path.basename(__file__))[0] + "_" + str(mode))
if config is not None and isinstance(config, str):
logger.info("model: " + str(mode) + ", config: " + str(config))
Config.load_config(config)
logger.info("scorer type: " + Config.estimator_name)
logger.info("random seed: " + str(Config.seed))
logger.info("ESIM arguments: " + str(Config.esim_hyper_param))
# loading FastText takes long time, so better pickle the loaded FastText model
if os.path.splitext(Config.fasttext_path)[1] == '.p':
with open(Config.fasttext_path, "rb") as ft_file:
fasttext_model = pickle.load(ft_file)
else:
fasttext_model = Config.fasttext_path
if mode == RTERunPhase.train:
# # training mode
training_set, fasttext_model, vocab, embeddings = embed_data_set_with_glove_and_fasttext_claim_only(
Config.training_set_file,
fasttext_model,
glove_path=Config.glove_path,
threshold_h_sent_size=Config.max_sentence_size)
h_sent_sizes = training_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
training_set['data']['h_sent_sizes'] = np.expand_dims(h_sent_sizes, 1)
training_set['data']['h_sizes'] = h_sizes
training_set['data']['h_np'] = np.expand_dims(
training_set['data']['h_np'], 1)
training_set['data']['h_ft_np'] = np.expand_dims(
training_set['data']['h_ft_np'], 1)
valid_set, _, _, _ = embed_data_set_with_glove_and_fasttext_claim_only(
Config.dev_set_file,
fasttext_model,
vocab_dict=vocab,
glove_embeddings=embeddings,
threshold_h_sent_size=Config.max_sentence_size)
del fasttext_model
h_sent_sizes = valid_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
valid_set['data']['h_sent_sizes'] = np.expand_dims(h_sent_sizes, 1)
valid_set['data']['h_sizes'] = h_sizes
valid_set['data']['h_np'] = np.expand_dims(valid_set['data']['h_np'],
1)
valid_set['data']['h_ft_np'] = np.expand_dims(
valid_set['data']['h_ft_np'], 1)
X_dict = {
'X_train': training_set['data'],
'X_valid': valid_set['data'],
'y_valid': valid_set['label'],
'embedding': embeddings
}
if estimator is None:
estimator = get_estimator(Config.estimator_name,
Config.ckpt_folder)
if 'CUDA_VISIBLE_DEVICES' not in os.environ or not str(
os.environ['CUDA_VISIBLE_DEVICES']).strip():
os.environ['CUDA_VISIBLE_DEVICES'] = str(
GPUtil.getFirstAvailable(maxLoad=1.0,
maxMemory=1.0 -
Config.max_gpu_memory)[0])
estimator.fit(X_dict, training_set['label'])
save_model(estimator, Config.model_folder, Config.pickle_name, logger)
elif mode == 'test':
# testing mode
restore_param_required = estimator is None
if estimator is None:
estimator = load_model(Config.model_folder, Config.pickle_name)
if estimator is None:
estimator = get_estimator(Config.estimator_name,
Config.ckpt_folder)
vocab, embeddings = load_whole_glove(Config.glove_path)
vocab = vocab_map(vocab)
test_set, _, _, _ = embed_data_set_with_glove_and_fasttext_claim_only(
Config.test_set_file,
fasttext_model,
vocab_dict=vocab,
glove_embeddings=embeddings,
threshold_h_sent_size=Config.max_sentence_size)
del fasttext_model
h_sent_sizes = test_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
test_set['data']['h_sent_sizes'] = np.expand_dims(h_sent_sizes, 1)
test_set['data']['h_sizes'] = h_sizes
test_set['data']['h_np'] = np.expand_dims(test_set['data']['h_np'], 1)
test_set['data']['h_ft_np'] = np.expand_dims(
test_set['data']['h_ft_np'], 1)
x_dict = {'X_test': test_set['data'], 'embedding': embeddings}
if 'CUDA_VISIBLE_DEVICES' not in os.environ or not str(
os.environ['CUDA_VISIBLE_DEVICES']).strip():
os.environ['CUDA_VISIBLE_DEVICES'] = str(
GPUtil.getFirstAvailable(maxLoad=1.0,
maxMemory=1.0 -
Config.max_gpu_memory)[0])
predictions = estimator.predict(x_dict, restore_param_required)
generate_submission(predictions, test_set['id'], Config.test_set_file,
Config.submission_file)
if 'label' in test_set:
print_metrics(test_set['label'], predictions, logger)
return estimator
def main(mode: RTERunPhase, config=None, estimator=None):
LogHelper.setup()
logger = LogHelper.get_logger(os.path.splitext(os.path.basename(__file__))[0] + "_" + str(mode))
if config is not None and isinstance(config, str):
logger.info("model: " + str(mode) + ", config: " + str(config))
Config.load_config(config)
logger.info("scorer type: " + Config.estimator_name)
logger.info("random seed: " + str(Config.seed))
logger.info("ESIM arguments: " + str(Config.esim_hyper_param))
if hasattr(Config, 'is_snopes'):
is_snopes = Config.is_snopes
else:
is_snopes = False
logger.debug("is_snopes: " + str(is_snopes))
if mode == RTERunPhase.train:
# training mode
training_set = embed_data_set_with_bert(Config.training_set_file, Config.db_path,
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
is_snopes=is_snopes,
port=Config.bert_port,
port_out=Config.bert_port_out)
h_sent_sizes = training_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
training_set['data']['h_sent_sizes'] = np.expand_dims(h_sent_sizes, 1)
training_set['data']['h_sizes'] = h_sizes
training_set['data']['h_bert_np'] = np.expand_dims(training_set['data']['h_bert_np'], 1)
valid_set = embed_data_set_with_bert(Config.dev_set_file, Config.db_path,
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
is_snopes=is_snopes,
port=Config.bert_port,
port_out=Config.bert_port_out)
h_sent_sizes = valid_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
valid_set['data']['h_sent_sizes'] = np.expand_dims(h_sent_sizes, 1)
valid_set['data']['h_sizes'] = h_sizes
valid_set['data']['h_bert_np'] = np.expand_dims(valid_set['data']['h_bert_np'], 1)
X_dict = {
'X_train': training_set['data'],
'X_valid': valid_set['data'],
'y_valid': valid_set['label']
}
if estimator is None:
estimator = get_estimator(Config.estimator_name, Config.ckpt_folder)
if 'CUDA_VISIBLE_DEVICES' not in os.environ or not str(os.environ['CUDA_VISIBLE_DEVICES']).strip():
os.environ['CUDA_VISIBLE_DEVICES'] = str(
GPUtil.getFirstAvailable(maxLoad=1.0, maxMemory=1.0 - Config.max_gpu_memory)[0])
estimator.fit(X_dict, training_set['label'])
save_model(estimator, Config.model_folder, Config.pickle_name, logger)
else:
# testing mode
restore_param_required = estimator is None
if estimator is None:
estimator = load_model(Config.model_folder, Config.pickle_name)
if estimator is None:
estimator = get_estimator(Config.estimator_name, Config.ckpt_folder)
test_set = embed_data_set_with_bert(Config.test_set_file, Config.db_path,
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
is_snopes=is_snopes,
port=Config.bert_port,
port_out=Config.bert_port_out)
h_sent_sizes = test_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
test_set['data']['h_sent_sizes'] = np.expand_dims(h_sent_sizes, 1)
test_set['data']['h_sizes'] = h_sizes
test_set['data']['h_bert_np'] = np.expand_dims(test_set['data']['h_bert_np'], 1)
x_dict = {
'X_test': test_set['data']
}
if 'CUDA_VISIBLE_DEVICES' not in os.environ or not str(os.environ['CUDA_VISIBLE_DEVICES']).strip():
os.environ['CUDA_VISIBLE_DEVICES'] = str(
GPUtil.getFirstAvailable(maxLoad=1.0, maxMemory=1.0 - Config.max_gpu_memory)[0])
predictions = estimator.predict(x_dict, restore_param_required)
generate_submission(predictions, test_set['id'], Config.test_set_file, Config.submission_file)
if 'label' in test_set:
print_metrics(test_set['label'], predictions, logger)
return estimator
def main(mode, config, estimator=None):
LogHelper.setup()
logger = LogHelper.get_logger(
os.path.splitext(os.path.basename(__file__))[0] + "_" + mode)
logger.info("model: " + mode + ", config: " + str(config))
if hasattr(Config, 'use_inter_evidence_comparison'):
use_inter_evidence_comparison = Config.use_inter_evidence_comparison
else:
use_inter_evidence_comparison = False
if hasattr(Config, 'use_claim_evidences_comparison'):
use_claim_evidences_comparison = Config.use_claim_evidences_comparison
else:
use_claim_evidences_comparison = False
if hasattr(Config, 'use_extra_features'):
use_extra_features = Config.use_extra_features
else:
use_extra_features = False
if hasattr(Config, 'use_numeric_feature'):
use_numeric_feature = Config.use_numeric_feature
else:
use_numeric_feature = False
logger.info("scorer type: " + Config.estimator_name)
logger.info("random seed: " + str(Config.seed))
logger.info("ESIM arguments: " + str(Config.esim_end_2_end_hyper_param))
logger.info("use_inter_sentence_comparison: " +
str(use_inter_evidence_comparison))
logger.info("use_extra_features: " + str(use_extra_features))
logger.info("use_numeric_feature: " + str(use_numeric_feature))
logger.info("use_claim_evidences_comparison: " +
str(use_claim_evidences_comparison))
if mode == 'train':
# # training mode
if hasattr(Config, 'training_dump') and os.path.exists(
Config.training_dump):
with open(Config.training_dump, 'rb') as f:
(X_dict, y_train) = pickle.load(f)
else:
training_set, vocab, embeddings, _, _ = embed_data_set_with_glove_2(
Config.training_set_file,
Config.db_path,
glove_path=Config.glove_path,
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
threshold_h_sent_size=Config.max_sentence_size)
h_sent_sizes = training_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
training_set['data']['h_sent_sizes'] = np.expand_dims(
h_sent_sizes, 1)
training_set['data']['h_sizes'] = h_sizes
training_set['data']['h_np'] = np.expand_dims(
training_set['data']['h_np'], 1)
valid_set, _, _, _, _ = embed_data_set_with_glove_2(
Config.dev_set_file,
Config.db_path,
vocab_dict=vocab,
glove_embeddings=embeddings,
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
threshold_h_sent_size=Config.max_sentence_size)
h_sent_sizes = valid_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
valid_set['data']['h_sent_sizes'] = np.expand_dims(h_sent_sizes, 1)
valid_set['data']['h_sizes'] = h_sizes
valid_set['data']['h_np'] = np.expand_dims(
valid_set['data']['h_np'], 1)
if use_extra_features:
assert hasattr(
Config, 'feature_path'
), "Config should has feature_path if Config.use_feature is True"
training_claim_features, training_evidence_features = load_feature_by_data_set(
Config.training_set_file, Config.feature_path,
Config.max_sentences)
valid_claim_features, valid_evidence_features = load_feature_by_data_set(
Config.dev_set_file, Config.feature_path,
Config.max_sentences)
training_set['data']['h_feats'] = training_claim_features
training_set['data']['b_feats'] = training_evidence_features
valid_set['data']['h_feats'] = valid_claim_features
valid_set['data']['b_feats'] = valid_evidence_features
if use_numeric_feature:
training_num_feat = number_feature(Config.training_set_file,
Config.db_path,
Config.max_sentences)
valid_num_feat = number_feature(Config.dev_set_file,
Config.db_path,
Config.max_sentences)
training_set['data']['num_feat'] = training_num_feat
valid_set['data']['num_feat'] = valid_num_feat
if use_inter_evidence_comparison:
training_concat_sent_indices, training_concat_sent_sizes = generate_concat_indices_for_inter_evidence(
training_set['data']['b_np'],
training_set['data']['b_sent_sizes'],
Config.max_sentence_size, Config.max_sentences)
training_set['data'][
'b_concat_indices'] = training_concat_sent_indices
training_set['data'][
'b_concat_sizes'] = training_concat_sent_sizes
valid_concat_sent_indices, valid_concat_sent_sizes = generate_concat_indices_for_inter_evidence(
valid_set['data']['b_np'],
valid_set['data']['b_sent_sizes'],
Config.max_sentence_size, Config.max_sentences)
valid_set['data'][
'b_concat_indices'] = valid_concat_sent_indices
valid_set['data']['b_concat_sizes'] = valid_concat_sent_sizes
if use_claim_evidences_comparison:
training_all_evidences_indices, training_all_evidences_sizes = generate_concat_indices_for_claim(
training_set['data']['b_np'],
training_set['data']['b_sent_sizes'],
Config.max_sentence_size, Config.max_sentences)
training_set['data'][
'b_concat_indices_for_h'] = training_all_evidences_indices
training_set['data'][
'b_concat_sizes_for_h'] = training_all_evidences_sizes
valid_all_evidences_indices, valid_all_evidences_sizes = generate_concat_indices_for_claim(
valid_set['data']['b_np'],
valid_set['data']['b_sent_sizes'],
Config.max_sentence_size, Config.max_sentences)
valid_set['data'][
'b_concat_indices_for_h'] = valid_all_evidences_indices
valid_set['data'][
'b_concat_sizes_for_h'] = valid_all_evidences_sizes
X_dict = {
'X_train': training_set['data'],
'X_valid': valid_set['data'],
'y_valid': valid_set['label'],
'embedding': embeddings
}
y_train = training_set['label']
if hasattr(Config, 'training_dump'):
with open(Config.training_dump, 'wb') as f:
pickle.dump((X_dict, y_train),
f,
protocol=pickle.HIGHEST_PROTOCOL)
if estimator is None:
estimator = get_estimator(Config.estimator_name,
Config.ckpt_folder)
estimator.fit(X_dict, y_train)
save_model(estimator, Config.model_folder, Config.pickle_name, logger)
elif mode == 'test':
# testing mode
restore_param_required = estimator is None
if estimator is None:
estimator = load_model(Config.model_folder, Config.pickle_name)
vocab, embeddings = load_whole_glove(Config.glove_path)
vocab = vocab_map(vocab)
test_set, _, _, _, _ = embed_data_set_with_glove_2(
Config.test_set_file,
Config.db_path,
vocab_dict=vocab,
glove_embeddings=embeddings,
threshold_b_sent_num=Config.max_sentences,
threshold_b_sent_size=Config.max_sentence_size,
threshold_h_sent_size=Config.max_sentence_size)
h_sent_sizes = test_set['data']['h_sent_sizes']
h_sizes = np.ones(len(h_sent_sizes), np.int32)
test_set['data']['h_sent_sizes'] = np.expand_dims(h_sent_sizes, 1)
test_set['data']['h_sizes'] = h_sizes
test_set['data']['h_np'] = np.expand_dims(test_set['data']['h_np'], 1)
if use_extra_features:
assert hasattr(
Config, 'feature_path'
), "Config should has feature_path if Config.use_feature is True"
test_claim_features, test_evidence_features = load_feature_by_data_set(
Config.test_set_file, Config.feature_path,
Config.max_sentences)
test_set['data']['h_feats'] = test_claim_features
test_set['data']['b_feats'] = test_evidence_features
if use_numeric_feature:
test_num_feat = number_feature(Config.test_set_file,
Config.db_path,
Config.max_sentences)
test_set['data']['num_feat'] = test_num_feat
x_dict = {'X_test': test_set['data'], 'embedding': embeddings}
if use_inter_evidence_comparison:
test_concat_sent_indices, test_concat_sent_sizes = generate_concat_indices_for_inter_evidence(
test_set['data']['b_np'], test_set['data']['b_sent_sizes'],
Config.max_sentence_size, Config.max_sentences)
test_set['data']['b_concat_indices'] = test_concat_sent_indices
test_set['data']['b_concat_sizes'] = test_concat_sent_sizes
if use_claim_evidences_comparison:
test_all_evidences_indices, test_all_evidences_sizes = generate_concat_indices_for_claim(
test_set['data']['b_np'], test_set['data']['b_sent_sizes'],
Config.max_sentence_size, Config.max_sentences)
test_set['data'][
'b_concat_indices_for_h'] = test_all_evidences_indices
test_set['data']['b_concat_sizes_for_h'] = test_all_evidences_sizes
predictions = estimator.predict(x_dict, restore_param_required)
generate_submission(predictions, Config.test_set_file,
Config.submission_file)
if 'label' in test_set:
print_metrics(test_set['label'], predictions, logger)
else:
logger.error("Invalid argument --mode: " + mode +
" Argument --mode should be either 'train’ or ’test’")
return estimator
def main(mode: RTERunPhase, config=None, estimator=None):
LogHelper.setup()
logger = LogHelper.get_logger(os.path.splitext(os.path.basename(__file__))[0] + "_" + str(mode))
if config is not None and isinstance(config, str):
logger.info("model: " + str(mode) + ", config: " + str(config))
Config.load_config(config)
if hasattr(Config, 'is_snopes'):
is_snopes = Config.is_snopes
else:
is_snopes = False
logger.debug("is_snopes: " + str(is_snopes))
logger.info("scorer type: " + Config.estimator_name)
logger.info("random seed: " + str(Config.seed))
logger.info("BERT sentence embedding arguments: " + str(Config.bert_sent_hyper_parameter))
if mode == RTERunPhase.train:
# training mode
if hasattr(Config, 'training_dump') and os.path.exists(Config.training_dump):
with open(Config.training_dump, 'rb') as f:
(X_train, Y_labels_train, X_valid, Y_labels_valid) = pickle.load(f)
else:
# process training JSONL file
X_train, Y_labels_train = read_data_set_from_jsonl(Config.training_set_file,
Config.db_path,
num_sentences=Config.max_sentences,
is_snopes=is_snopes)
X_valid, Y_labels_valid = read_data_set_from_jsonl(Config.dev_set_file,
Config.db_path,
num_sentences=Config.max_sentences,
is_snopes=is_snopes)
X_train['b_sizes'] = get_num_sents_of_bodies(X_train['b'])
X_valid['b_sizes'] = get_num_sents_of_bodies(X_valid['b'])
b_train = X_train['b']
b_encoded_train = encode_multi_sentence_set_with_bert(b_train, Config.max_sentences, port=Config.bert_port,
port_out=Config.bert_port_out)
X_train['b'] = b_encoded_train
logger.debug("b_encoded_train.shape: " + str(b_encoded_train.shape))
h_train = X_train['h']
h_encoded_train = encode_single_sentence_set_with_bert(h_train, port=Config.bert_port,
port_out=Config.bert_port_out)
X_train['h'] = h_encoded_train
logger.debug("h_encoded_train.shape: " + str(h_encoded_train.shape))
b_valid = X_valid['b']
b_encoded_valid = encode_multi_sentence_set_with_bert(b_valid, Config.max_sentences, port=Config.bert_port,
port_out=Config.bert_port_out)
X_valid['b'] = b_encoded_valid
logger.debug("b_encoded_valid.shape: " + str(b_encoded_valid.shape))
h_valid = X_valid['h']
h_encoded_valid = encode_single_sentence_set_with_bert(h_valid, port=Config.bert_port,
port_out=Config.bert_port_out)
X_valid['h'] = h_encoded_valid
logger.debug("h_encoded_valid.shape: " + str(h_encoded_valid.shape))
if hasattr(Config, 'training_dump'):
with open(Config.training_dump, 'wb') as f:
pickle.dump((X_train, Y_labels_train, X_valid, Y_labels_valid), f, protocol=pickle.HIGHEST_PROTOCOL)
if estimator is None:
estimator = get_estimator(Config.estimator_name, Config.ckpt_folder)
if 'CUDA_VISIBLE_DEVICES' not in os.environ or not str(os.environ['CUDA_VISIBLE_DEVICES']).strip():
os.environ['CUDA_VISIBLE_DEVICES'] = str(
GPUtil.getFirstAvailable(maxLoad=1.0, maxMemory=1.0 - Config.max_gpu_memory)[0])
estimator.fit(X_train, Y_labels_train, X_valid, Y_labels_valid)
save_model(estimator, Config.model_folder, Config.pickle_name, logger)
else:
# testing mode
restore_param_required = estimator is None
if estimator is None:
estimator = load_model(Config.model_folder, Config.pickle_name)
if estimator is None:
estimator = get_estimator(Config.estimator_name, Config.ckpt_folder)
X_test, Y_labels_test = read_data_set_from_jsonl(Config.test_set_file,
Config.db_path,
num_sentences=Config.max_sentences,
is_snopes=is_snopes)
X_test['b_sizes'] = get_num_sents_of_bodies(X_test['b'])
b_test = X_test['b']
b_encoded_test = encode_multi_sentence_set_with_bert(b_test, Config.max_sentences, port=Config.bert_port,
port_out=Config.bert_port_out)
X_test['b'] = b_encoded_test
logger.debug("b_encoded_test.shape: " + str(b_encoded_test.shape))
h_test = X_test['h']
h_encoded_test = encode_single_sentence_set_with_bert(h_test, port=Config.bert_port,
port_out=Config.bert_port_out)
X_test['h'] = h_encoded_test
logger.debug("h_encoded_test.shape: " + str(h_encoded_test.shape))
if 'CUDA_VISIBLE_DEVICES' not in os.environ or not str(os.environ['CUDA_VISIBLE_DEVICES']).strip():
os.environ['CUDA_VISIBLE_DEVICES'] = str(
GPUtil.getFirstAvailable(maxLoad=1.0, maxMemory=1.0 - Config.max_gpu_memory)[0])
predictions = estimator.predict(X_test, restore_param_required)
generate_submission(predictions, X_test['id'], Config.test_set_file, Config.submission_file)
if Y_labels_test is not None:
print_metrics(Y_labels_test, predictions, logger)
return estimator