def test_model_train():
trainablemodel = models.BagOfWordsModel(parameters=config['model'],
logger=logger)
assert type(trainablemodel._model) == linear_model.LogisticRegression
webquestions = webquestions_io.WebQuestions(config['webquestions'],
logger=logger)
input_set, targets = webquestions.get_training_samples()
input_set, targets = input_set[:200], targets[:200]
trainablemodel.train(
(input_set, targets),
validation_with_targets=webquestions.get_validation_samples()
if 'train_validation' in config['webquestions']['path.to.dataset'] else
None)
print('Training finished')
def test_model_multiclass():
config['model']['epochs'] = 10
config['model']['loss'] = "mlml"
config['webquestions']['target.dist'] = "multiclass"
webquestions = webquestions_io.WebQuestions(config['webquestions'],
logger=logger)
_, silver_test_targets = webquestions.get_full_validation()
print(silver_test_targets[:10])
trainablemodel = models.CNNLabelsTorchModel(parameters=config['model'],
logger=logger)
trainablemodel.prepare_model(webquestions.get_training_tokens(),
webquestions.get_training_properties_tokens())
trainablemodel.train(
webquestions,
validation_with_targets=webquestions.get_validation_samples())
print('Training finished')
def test_model(path_to_model, config_file_path):
"""
:param path_to_model:
:param config_file_path:
:return:
"""
config = utils.load_config(config_file_path)
config_global = config.get('global', {})
np.random.seed(config_global.get('random_seed', 1))
logger = logging.getLogger(__name__)
logger.setLevel(config['logger']['level'])
ch = logging.StreamHandler()
ch.setLevel(config['logger']['level'])
logger.addHandler(ch)
results_logger = None
if 'log.results' in config['training']:
results_logger = logging.getLogger("results_logger")
results_logger.setLevel(logging.INFO)
fh = logging.FileHandler(filename=config['training']['log.results'])
fh.setLevel(logging.INFO)
results_logger.addHandler(fh)
results_logger.info(str(config))
webquestions = webquestions_io.WebQuestions(config['webquestions'], logger=logger)
config['model']['samples.per.epoch'] = webquestions.get_train_sample_size()
config['model']['graph.choices'] = config['webquestions'].get("max.negative.samples", 30)
trainablemodel = getattr(models, config['model']['class'])(parameters=config['model'], logger=logger)
trainablemodel.load_from_file(path_to_model)
print("Testing the model on silver data.")
if 'train_validation' in config['webquestions']['path.to.dataset']:
silver_test_set, silver_test_targets = webquestions.get_full_validation()
else:
silver_test_set, silver_test_targets = webquestions.get_full_training()
accuracy_on_silver, predicted_targets = trainablemodel.test_on_silver((silver_test_set, silver_test_targets), verbose=True)
print("Accuracy on silver data: {}".format(accuracy_on_silver))
with open(config['training']['log.results'].replace(".log", "_silver_predictions.log"), "w") as out:
if len(silver_test_targets) > 0 and not issubclass(type(silver_test_targets[0]), np.integer):
silver_test_targets = np.argmax(silver_test_targets, axis=-1)
json.dump((silver_test_set, predicted_targets, [int(t) for t in silver_test_targets]), out)
if results_logger:
results_logger.info("Accuracy on silver data: {}".format(accuracy_on_silver))
def train(config_file_path):
"""
:param config_file_path:
:return:
"""
config = utils.load_config(config_file_path)
if "training" not in config:
print("Training parameters not in the config file!")
sys.exit()
config_global = config.get('global', {})
np.random.seed(config_global.get('random.seed', 1))
torch.manual_seed(config_global.get('random.seed', 1))
if torch.cuda.is_available():
print("using your CUDA device")
torch.cuda.manual_seed(config_global.get('random.seed', 1))
logger = logging.getLogger(__name__)
logger.setLevel(config['logger']['level'])
ch = logging.StreamHandler()
ch.setLevel(config['logger']['level'])
logger.addHandler(ch)
logger.debug(str(datetime.datetime.now()))
results_logger = None
if 'log.results' in config['training']:
results_logger = logging.getLogger("results_logger")
results_logger.setLevel(logging.INFO)
fh = logging.FileHandler(filename=config['training']['log.results'])
fh.setLevel(logging.INFO)
results_logger.addHandler(fh)
results_logger.info(str(config))
config['webquestions']['max.entity.options'] = config['evaluation'].get(
'max.entity.options', 3)
webquestions = webquestions_io.WebQuestions(config['webquestions'],
logger=logger)
config['model']['samples.per.epoch'] = webquestions.get_train_sample_size()
config['model']['graph.choices'] = config['webquestions'].get(
"max.negative.samples", 30)
if config['training'].get('train.generator', False):
config['training']['train.mode'] = "generator"
trainablemodel = getattr(models, config['model']['class'])(
parameters=config['model'], logger=logger)
trainablemodel.prepare_model(webquestions.get_training_tokens(),
webquestions.get_training_properties_tokens())
if results_logger:
results_logger.info("Model save to: {}".format(
trainablemodel._model_file_name))
if config['training'].get('train.mode', "base") == "generator":
trainablemodel.train_on_generator(
webquestions,
validation_with_targets=webquestions.get_validation_samples()
if 'train_validation' in config['webquestions']['path.to.dataset']
else None)
elif config['training'].get('train.mode', "base") == "model":
trainablemodel.train(
webquestions,
validation_with_targets=webquestions.get_validation_samples()
if 'train_validation' in config['webquestions']['path.to.dataset']
else None,
model_sampling=True)
else:
trainablemodel.train(
webquestions,
validation_with_targets=webquestions.get_validation_samples()
if 'train_validation' in config['webquestions']['path.to.dataset']
else None)
print("Loading the best model")
trainablemodel.load_last_saved()
if 'train_validation' in config['webquestions']['path.to.dataset']:
silver_test_set, silver_test_targets = webquestions.get_full_validation(
)
else:
silver_test_set, silver_test_targets = webquestions.get_full_training()
accuracy_on_silver, predicted_targets = trainablemodel.test_on_silver(
(silver_test_set, silver_test_targets), verbose=True)
print("Accuracy on silver data: {}".format(accuracy_on_silver))
with open(
config['training']['log.results'].replace(
".log", "_silver_predictions.log"), "w") as out:
if len(silver_test_targets) > 0 and not issubclass(
type(silver_test_targets[0]), np.integer):
silver_test_targets = np.argmax(silver_test_targets, axis=-1)
json.dump((silver_test_set, predicted_targets,
[int(t) for t in silver_test_targets]), out)
if results_logger:
results_logger.info(
"Accuracy on silver data: {}".format(accuracy_on_silver))
with open("../questionanswering/default_config.yaml", 'r') as config_file:
config = yaml.load(config_file.read())
logger = logging.getLogger(__name__)
logger.setLevel(config['logger']['level'])
ch = logging.StreamHandler()
ch.setLevel(config['logger']['level'])
logger.addHandler(ch)
config['webquestions']['extensions'] = []
config['webquestions']['max.entity.options'] = 1
config['model']['graph.choices'] = config['webquestions'].get(
"max.negative.samples", 30)
config['model']['epochs'] = 6
config['model']['threshold'] = 10
webquestions = webquestions_io.WebQuestions(config['webquestions'],
logger=logger)
config['model']["batch.size"] = 10
def test_pool_selfatt():
config['model']['sibling.pool.mode'] = "selfatt"
config['model']["model.checkpoint"] = True
trainablemodel = models.pytorchmodel_impl.CNNLabelsHashesModel(
parameters=config['model'], logger=logger)
trainablemodel.prepare_model(webquestions)
trainablemodel.train(
webquestions,
validation_with_targets=webquestions.get_validation_samples())
print('Training finished')
trainablemodel.load_last_saved()
def generate(path_to_model, config_file_path):
config = utils.load_config(config_file_path)
if "evaluation" not in config:
print("Evaluation parameters not in the config file!")
sys.exit()
config_global = config.get('global', {})
np.random.seed(config_global.get('random.seed', 1))
logger = logging.getLogger(__name__)
logger.setLevel(config['logger']['level'])
ch = logging.StreamHandler()
ch.setLevel(config['logger']['level'])
logger.addHandler(ch)
# logging.basicConfig(level=config['logger']['level'])
wdaccess.wdaccess_p["timeout"] = config['wikidata'].get("timeout", 20)
wdaccess.wdaccess_p['wikidata_url'] = config['wikidata'].get("backend", "http://knowledgebase:8890/sparql")
wdaccess.sparql_init()
assert not(config['webquestions']["no.ne.tags"] and config['evaluation']["only.named.entities"])
entity_linking.entity_linking_p["no.ne.tags"] = config['webquestions']["no.ne.tags"]
assert not(entity_linking.entity_linking_p["no.ne.tags"] and config['evaluation']["only.named.entities"])
entity_linking.entity_linking_p["max.entity.options"] = config['evaluation']["max.entity.options"]
entity_linking.entity_linking_p["global.entity.grouping"] = not config['webquestions']["no.ne.tags"] and entity_linking.entity_linking_p["overlaping.nn.ne"]
entity_linking.entity_linking_p["np.parser"] = config['evaluation']["np.parser"]
wdaccess.wdaccess_p["restrict.hop"] = config['wikidata'].get("restrict.hop", False)
wdaccess.update_sparql_clauses()
staged_generation.generation_p["use.whitelist"] = config['evaluation'].get("use.whitelist", False)
staged_generation.generation_p["strict.structure"] = config['evaluation'].get("strict.structure", False)
staged_generation.generation_p["v.structure"] = config['evaluation'].get("v.structure", True)
staged_generation.generation_p["max.num.entities"] = config['evaluation'].get("max.num.entities", 3)
staged_generation.generation_p["topics.at.test"] = config['evaluation'].get("topics.at.test", False)
if not config['wikidata'].get("addclass.action", False):
stages.RESTRICT_ACTIONS = [stages.add_entity_and_relation, stages.last_relation_temporal,
stages.add_temporal_relation, stages.last_relation_numeric]
logger.debug("max.entity.options: {}".format(entity_linking.entity_linking_p["max.entity.options"]))
if 'hop.types' in config['wikidata']:
stages.HOP_TYPES = set(config['wikidata']['hop.types'])
if 'arg.types' in config['wikidata']:
stages.ARG_TYPES = set(config['wikidata']['arg.types'])
if 'filter.out.relation.classes' in config['wikidata']:
wdaccess.FILTER_RELATION_CLASSES = set(config['wikidata']['filter.out.relation.classes'])
logger.debug("entity_linking: {}".format(entity_linking.entity_linking_p))
with open(config['evaluation']['questions']) as f:
webquestions_questions = json.load(f)
webquestions = webquestions_io.WebQuestions(config['webquestions'], logger=logger)
logger.debug('Extracting entities.')
webquestions_entities = webquestions.extract_question_entities()
if 'class' not in config['model']:
config['model']['class'] = path_to_model.split('/')[-1].split('_')[0]
logger.debug('Loading the model from: {}'.format(path_to_model))
qa_model = getattr(models, config['model']['class'])(parameters=config['model'], logger=logger)
qa_model.load_from_file(path_to_model)
logger.debug('Testing')
global_answers = []
avg_metrics = np.zeros(3)
len_webquestion = webquestions.get_dataset_size()
for i in tqdm.trange(len_webquestion, ncols=100, ascii=True):
question_entities = webquestions_entities[i]
nes = [e for e in question_entities if e[1] != "NN"]
if config['evaluation'].get('only.named.entities', False) and len(nes) > 0:
question_entities = nes
ungrounded_graph = {'tokens': webquestions.get_original_question_tokens(i),
'edgeSet': [],
'fragments': question_entities}
chosen_graphs = staged_generation.generate_with_model(ungrounded_graph, qa_model, beam_size=config['evaluation'].get("beam.size", 10))
model_answers = []
g = ({},)
if chosen_graphs:
j = 0
while not model_answers and j < len(chosen_graphs):
g = chosen_graphs[j]
model_answers = wdaccess.query_graph_denotations(g[0]).get("e1")
j += 1
if config['evaluation'].get('label.answers', False):
gold_answers = [e.lower() for e in webquestions_io.get_answers_from_question(webquestions_questions[i])]
model_answers_labels = wdaccess.label_query_results(model_answers)
model_answers_labels = staged_generation.post_process_answers_given_graph(model_answers_labels, g[0])
metrics = evaluation.retrieval_prec_rec_f1_with_altlabels(gold_answers, model_answers_labels)
global_answers.append((i, list(metrics), model_answers, model_answers_labels,
[(c_g[0], float(c_g[1])) for c_g in chosen_graphs[:10]]))
else:
gold_answers = webquestions_io.get_answers_from_question(webquestions_questions[i])
metrics = evaluation.retrieval_prec_rec_f1(gold_answers, model_answers)
global_answers.append((i, list(metrics), model_answers,
[(c_g[0], float(c_g[1])) for c_g in chosen_graphs[:10]]))
avg_metrics += metrics
if i % 100 == 0:
logger.debug("Average f1 so far: {}".format((avg_metrics/(i+1))))
with open(config['evaluation']["save.answers.to"], 'w') as answers_out:
json.dump(global_answers, answers_out, sort_keys=True, indent=4)
print("Average metrics: {}".format((avg_metrics/(len_webquestion))))
logger.debug('Testing is finished')
with open(config['evaluation']["save.answers.to"], 'w') as answers_out:
json.dump(global_answers, answers_out, sort_keys=True, indent=4)
def generate(config_file_path):
config = utils.load_config(config_file_path)
if "generation" not in config:
print("Generation parameters not in the config file!")
sys.exit()
config_global = config.get('global', {})
np.random.seed(config_global.get('random.seed', 1))
logger = logging.getLogger(__name__)
logger.setLevel(config['logger']['level'])
ch = logging.StreamHandler()
ch.setLevel(config['logger']['level'])
logger.addHandler(ch)
# logging.basicConfig(level=logging.ERROR)
logger.debug(str(datetime.datetime.now()))
staged_generation.generation_p['label.query.results'] = config[
'generation'].get('label.query.results', False)
staged_generation.generation_p["use.whitelist"] = config['generation'].get(
"use.whitelist", False)
staged_generation.generation_p["min.fscore.to.stop"] = config[
'generation'].get("min.fscore.to.stop", 0.9)
entity_linking.entity_linking_p[
"global.entity.grouping"] = not config['webquestions']["no.ne.tags"]
entity_linking.entity_linking_p["max.entity.options"] = config[
'generation']["max.entity.options"]
wdaccess.wdaccess_p['wikidata_url'] = config['wikidata'].get(
"backend", "http://knowledgebase:8890/sparql")
wdaccess.wdaccess_p["restrict.hop"] = config['wikidata'].get(
"restrict.hop", False)
wdaccess.wdaccess_p["timeout"] = config['wikidata'].get("timeout", 20)
wdaccess.sparql_init()
wdaccess.update_sparql_clauses()
logger.debug("entity_linking: {}".format(entity_linking.entity_linking_p))
if 'hop.types' in config['wikidata']:
stages.HOP_TYPES = set(config['wikidata']['hop.types'])
logger.debug("Hop types set to: {}".format(stages.HOP_TYPES))
if 'arg.types' in config['wikidata']:
stages.ARG_TYPES = set(config['wikidata']['arg.types'])
if 'filter.out.relation.classes' in config['wikidata']:
wdaccess.FILTER_RELATION_CLASSES = set(
config['wikidata']['filter.out.relation.classes'])
logger.debug("Arg types set to: {}".format(stages.ARG_TYPES))
if wdaccess.wdaccess_p["restrict.hop"]:
logger.debug("HOP UP relations set to: {}".format(
wdaccess.HOP_UP_RELATIONS))
logger.debug("HOP UP relations set to: {}".format(
wdaccess.sparql_hopup_values))
logger.debug("HOP DOWN relations set to: {}".format(
wdaccess.HOP_DOWN_RELATIONS))
logger.debug("HOP UP relations set to: {}".format(
wdaccess.sparql_hopdown_values))
webquestions = webquestions_io.WebQuestions(config['webquestions'],
logger=logger)
logger.debug('Loaded WebQuestions, size: {}'.format(
webquestions.get_dataset_size()))
with open(config['generation']['questions']) as f:
webquestions_questions = json.load(f)
logger.debug(
'Loaded WebQuestions original training questions, size: {}'.format(
len(webquestions_questions)))
assert len(webquestions_questions) == webquestions.get_dataset_size()
logger.debug('Extracting entities.')
webquestions_entities = webquestions.extract_question_entities()
silver_dataset = []
previous_silver = []
if 'previous' in config['generation']:
logger.debug("Loading the previous result")
with open(config['generation']['previous']) as f:
previous_silver = json.load(f)
logger.debug("Previous length: {}".format(len(previous_silver)))
logger.debug("Previous number of answers covered: {}".format(
len([
1 for graphs in previous_silver if len(graphs) > 0
and any([len(g) > 1 and g[1][2] > 0.0 for g in graphs])
]) / len(previous_silver)))
logger.debug("Previous average f1: {}".format(
np.average([
np.max([g[1][2] if len(g) > 1 else 0.0
for g in graphs]) if len(graphs) > 0 else 0.0
for graphs in previous_silver
])))
previous_silver = [[
g for g in graph_set
if graph.if_graph_adheres(g[0],
allowed_extensions=config['webquestions']
.get("extensions", set()))
] for graph_set in previous_silver]
logger.debug(
"Previous average f1 (after extensions removed): {}".format(
np.average([
np.max([g[1][2] if len(g) > 1 else 0.0
for g in graphs]) if len(graphs) > 0 else 0.0
for graphs in previous_silver
])))
logger.debug("Reusable: {}".format(
len([
1 for graphs in previous_silver if len(graphs) > 0
and any([len(g) > 1 and g[1][2] > 0.9 for g in graphs])
]) / len(previous_silver)))
len_webquestion = webquestions.get_dataset_size()
start_with = 0
if 'start.with' in config['generation']:
start_with = config['generation']['start.with']
print("Starting with {}.".format(start_with))
if 'take_first' in config['generation']:
print("Taking the first {} questions.".format(
config['generation']['take_first']))
len_webquestion = config['generation']['take_first']
logger.debug("First question: {} {}\n {}".format(
start_with, webquestions_questions[start_with],
webquestions_io.get_answers_from_question(
webquestions_questions[start_with])))
for i in tqdm.tqdm(range(start_with, len_webquestion), ncols=100):
if len(previous_silver) > i and max(
g[1][2] if len(g) > 1 and len(g[1]) > 2 else 0.0
for g in previous_silver[i]) > 0.9:
silver_dataset.append(previous_silver[i])
else:
question_entities = webquestions_entities[i]
if "max.num.entities" in config['generation']:
question_entities = question_entities[:config['generation']
["max.num.entities"]]
if config['generation'].get('include_url_entities', False):
url_entity = webquestions_io.get_main_entity_from_question(
webquestions_questions[i])
if not any(e == url_entity[0] for e, t in question_entities):
# question_entities = [url_entity] + [(e, t) for e, t in question_entities if e != url_entity[0]]
question_entities = [url_entity] + question_entities
ungrounded_graph = {
'tokens': webquestions.get_original_question_tokens(i),
'edgeSet': [],
'fragments': question_entities
}
logger.log(level=0,
msg="Generating from: {}".format(ungrounded_graph))
gold_answers = webquestions_io.get_answers_from_question(
webquestions_questions[i])
if staged_generation.generation_p['label.query.results']:
gold_answers = [e.lower() for e in gold_answers]
generated_graphs = staged_generation.generate_with_gold(
ungrounded_graph, gold_answers)
silver_dataset.append(generated_graphs)
if i % 100 == 0:
logger.debug("Cov., avg. f1: {}, {}".format(
(len([
1 for graphs in silver_dataset if len(graphs) > 0
and any([len(g) > 1 and g[1][2] > 0.0 for g in graphs])
]) / (i + 1)),
np.average([
np.max([g[1][2] if len(g) > 1 else 0.0
for g in graphs]) if len(graphs) > 0 else 0.0
for graphs in silver_dataset
])))
# Dump the data set once in while
with open(config['generation']["save.silver.to"], 'w') as out:
json.dump(silver_dataset, out, sort_keys=True, indent=4)
logger.debug("Generation finished. Silver dataset size: {}".format(
len(silver_dataset)))
with open(config['generation']["save.silver.to"], 'w') as out:
json.dump(silver_dataset, out, sort_keys=True, indent=4)
print("Query cache: {}".format(len(wdaccess.query_cache)))
print("Number of answers covered: {}".format(
len([
1 for graphs in silver_dataset if len(graphs) > 0
and any([len(g) > 1 and g[1][2] > 0.0 for g in graphs])
]) / len_webquestion))
print("Average f1 of the silver data: {}".format(
np.average([
np.max([g[1][2] if len(g) > 1 else 0.0
for g in graphs]) if len(graphs) > 0 else 0.0
for graphs in silver_dataset
])))
