def test():
# This piece of code was used to print index of the external entity that produces the best candidate.
for dataset in datasets:
queries = get_evaluated_queries(dataset, True, parameters)
for index, query in enumerate(queries):
external_entities = dict()
max_f1 = 0
for candidate in query.eval_candidates:
f1 = candidate.evaluation_result.f1
max_f1 = max(max_f1, f1)
for entity in candidate.query_candidate.matched_entities:
if entity.entity.external_entity:
count = entity.entity.external_entity_count
if entity.entity.name not in external_entities:
external_entities[entity.entity.name] = (count, f1)
if external_entities[entity.entity.name][1] < f1:
external_entities[entity.entity.name] = (count, f1)
external_entities = [(entry[0], entry[1][1]) for entry in sorted(external_entities.iteritems(), key=lambda e: e[1][0], reverse=True)]
if external_entities:
max_external_f1 = max(f1 for _, f1 in external_entities)
if max_external_f1 == max_f1:
position = [f1 for _, f1 in external_entities].index(max_external_f1)
positions.append((query.utterance, external_entities[position][0], position))
print "\n".join(map(str, positions))
positions = [pos for q, n, pos in positions]
print positions, min(positions), avg(positions), max(positions)
def print_sparql_queries():
import argparse
parser = argparse.ArgumentParser(description="Dump qa entity pairs.")
parser.add_argument("--config",
default="config.cfg",
help="The configuration file to use.")
parser.add_argument("--output",
help="The file to dump results to.")
args = parser.parse_args()
globals.read_configuration(args.config)
scorer_globals.init()
parameters = translator.TranslatorParameters()
parameters.require_relation_match = False
parameters.restrict_answer_type = False
dataset = "webquestions_test_filter"
sparql_backend = globals.get_sparql_backend(globals.config)
queries = get_evaluated_queries(dataset, True, parameters)
for index, query in enumerate(queries):
print "--------------------------------------------"
print query.utterance
print "\n".join([str((entity.__class__, entity.entity)) for entity in query.eval_candidates[0].query_candidate.query.identified_entities])
for eval_candidate in query.eval_candidates:
query_candidate = eval_candidate.query_candidate
query_candidate.sparql_backend = sparql_backend
notable_types = query_candidate.get_answers_notable_types()
if notable_types:
print notable_types
print query_candidate.graph_as_simple_string().encode("utf-8")
print query_candidate.to_sparql_query().encode("utf-8")
print "\n\n"
def train_type_model():
globals.read_configuration('config.cfg')
parser = globals.get_parser()
scorer_globals.init()
datasets = ["webquestions_split_train", ]
parameters = translator.TranslatorParameters()
parameters.require_relation_match = False
parameters.restrict_answer_type = False
feature_extractor = FeatureExtractor(False, False, n_gram_types_features=True)
features = []
labels = []
for dataset in datasets:
queries = get_evaluated_queries(dataset, True, parameters)
for index, query in enumerate(queries):
tokens = [token.lemma for token in parser.parse(query.utterance).tokens]
n_grams = get_grams_feats(tokens)
answer_entities = [mid for answer in query.target_result
for mid in KBEntity.get_entityid_by_name(answer, keep_most_triples=True)]
correct_notable_types = set(filter(lambda x: x,
[KBEntity.get_notable_type(entity_mid)
for entity_mid in answer_entities]))
other_notable_types = set()
for candidate in query.eval_candidates:
entities = [mid for entity_name in candidate.prediction
for mid in KBEntity.get_entityid_by_name(entity_name, keep_most_triples=True)]
other_notable_types.update(set([KBEntity.get_notable_type(entity_mid) for entity_mid in entities]))
incorrect_notable_types = other_notable_types.difference(correct_notable_types)
for type in correct_notable_types.union(incorrect_notable_types):
if type in correct_notable_types:
labels.append(1)
else:
labels.append(0)
features.append(feature_extractor.extract_ngram_features(n_grams, [type, ], "type"))
with open("type_model_data.pickle", 'wb') as out:
pickle.dump((features, labels), out)
label_encoder = LabelEncoder()
labels = label_encoder.fit_transform(labels)
vec = DictVectorizer(sparse=True)
X = vec.fit_transform(features)
feature_selector = SelectPercentile(chi2, percentile=5).fit(X, labels)
vec.restrict(feature_selector.get_support())
X = feature_selector.transform(X)
type_scorer = SGDClassifier(loss='log', class_weight='auto',
n_iter=1000,
alpha=1.0,
random_state=999,
verbose=5)
type_scorer.fit(X, labels)
with open("type-model.pickle", 'wb') as out:
pickle.dump((vec, type_scorer), out)
scorer_globals.init()
parameters = translator.TranslatorParameters()
parameters.require_relation_match = False
parameters.restrict_answer_type = False
# datasets = ["webquestions_split_train", "webquestions_split_dev",]
# datasets = ["webquestions_split_train_externalentities", "webquestions_split_dev_externalentities",]
# datasets = ["webquestions_split_train_externalentities3", "webquestions_split_dev_externalentities3",]
datasets = ["webquestions_train_externalentities_all", "webquestions_test_externalentities_all", ]
count = 0
correct_relations = set()
positions = []
for dataset in datasets:
queries = get_evaluated_queries(dataset, True, parameters)
for index, query in enumerate(queries):
# Correct answer
# entity_names.update(query.target_result)
# if query.oracle_position != -1:
# if dataset == datasets[0]:
# correct_relations.update([r.name for r in query.eval_candidates[query.oracle_position - 1].query_candidate.relations])
# else:
# for relation in query.eval_candidates[query.oracle_position - 1].query_candidate.relations:
# if relation.name not in correct_relations:
# print query.utterance
# print relation.name
# print query.eval_candidates[query.oracle_position - 1].query_candidate
# print "-----"
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description="Console based translation.")
parser.add_argument("--config", default="config.cfg", help="The configuration file to use.")
args = parser.parse_args()
globals.read_configuration(args.config)
scorer_globals.init()
question_serps = get_questions_serps()
parameters = translator.TranslatorParameters()
parameters.require_relation_match = False
parameters.restrict_answer_type = False
queries = get_evaluated_queries("webquestions_small_train", True, parameters)
urls = set()
entity_mids = set()
entity_names = set()
for query in queries:
entity_names.update(query.target_result)
for candidate in query.eval_candidates:
entity_names.update(candidate.prediction)
for entity in candidate.query_candidate.matched_entities:
entity_mids.add(entity.entity.entity.id)
entity_names.add(entity.entity.name)
# Go through search results.
question = query.utterance
for document in question_serps[question][: globals.SEARCH_RESULTS_TOPN]:
urls.add(document.url)
def extract_npmi_ngram_type_pairs():
globals.read_configuration('config.cfg')
scorer_globals.init()
datasets = ["webquestions_split_train", ]
parameters = translator.TranslatorParameters()
parameters.require_relation_match = False
parameters.restrict_answer_type = False
n_gram_type_counts = dict()
type_counts = dict()
n_gram_counts = dict()
total = 0
year_pattern = re.compile("[0-9]+")
for dataset in datasets:
queries = get_evaluated_queries(dataset, True, parameters)
for index, query in enumerate(queries):
if query.oracle_position != -1 and query.oracle_position <= len(query.eval_candidates):
correct_candidate = query.eval_candidates[query.oracle_position - 1]
logger.info(query.utterance)
logger.info(correct_candidate.query_candidate)
n_grams = set(get_n_grams_features(correct_candidate.query_candidate))
answer_entities = [mid for answer in query.target_result
if year_pattern.match(answer) is None
for mid in KBEntity.get_entityid_by_name(answer, keep_most_triples=True)]
correct_notable_types = set(filter(lambda x: x,
[KBEntity.get_notable_type(entity_mid)
for entity_mid in answer_entities]))
for notable_type in correct_notable_types:
if notable_type not in type_counts:
type_counts[notable_type] = 0
type_counts[notable_type] += 1
for n_gram in n_grams:
if n_gram not in n_gram_counts:
n_gram_counts[n_gram] = 0
n_gram_counts[n_gram] += 1
for notable_type in correct_notable_types:
pair = (n_gram, notable_type)
if pair not in n_gram_type_counts:
n_gram_type_counts[pair] = 0
n_gram_type_counts[pair] += 1
total += 1
npmi = dict()
from math import log
for n_gram_type_pair, n_gram_type_count in n_gram_type_counts.iteritems():
if n_gram_type_count > 4:
n_gram, type = n_gram_type_pair
npmi[n_gram_type_pair] = (log(n_gram_type_count) - log(n_gram_counts[n_gram]) - log(type_counts[type]) +
log(total)) / (-log(n_gram_type_count) + log(total))
with open("type_model_npmi.pickle", 'wb') as out:
pickle.dump(npmi, out)
import operator
npmi = sorted(npmi.items(), key=operator.itemgetter(1), reverse=True)
print "\n".join(map(str, npmi[:50]))
