def train_type_model(): globals.read_configuration('config.cfg') parser = globals.get_parser() scorer_globals.init() datasets = ["webquestions_split_train", ] parameters = translator.TranslatorParameters() parameters.require_relation_match = False parameters.restrict_answer_type = False feature_extractor = FeatureExtractor(False, False, n_gram_types_features=True) features = [] labels = [] for dataset in datasets: queries = get_evaluated_queries(dataset, True, parameters) for index, query in enumerate(queries): tokens = [token.lemma for token in parser.parse(query.utterance).tokens] n_grams = get_grams_feats(tokens) answer_entities = [mid for answer in query.target_result for mid in KBEntity.get_entityid_by_name(answer, keep_most_triples=True)] correct_notable_types = set(filter(lambda x: x, [KBEntity.get_notable_type(entity_mid) for entity_mid in answer_entities])) other_notable_types = set() for candidate in query.eval_candidates: entities = [mid for entity_name in candidate.prediction for mid in KBEntity.get_entityid_by_name(entity_name, keep_most_triples=True)] other_notable_types.update(set([KBEntity.get_notable_type(entity_mid) for entity_mid in entities])) incorrect_notable_types = other_notable_types.difference(correct_notable_types) for type in correct_notable_types.union(incorrect_notable_types): if type in correct_notable_types: labels.append(1) else: labels.append(0) features.append(feature_extractor.extract_ngram_features(n_grams, [type, ], "type")) with open("type_model_data.pickle", 'wb') as out: pickle.dump((features, labels), out) label_encoder = LabelEncoder() labels = label_encoder.fit_transform(labels) vec = DictVectorizer(sparse=True) X = vec.fit_transform(features) feature_selector = SelectPercentile(chi2, percentile=5).fit(X, labels) vec.restrict(feature_selector.get_support()) X = feature_selector.transform(X) type_scorer = SGDClassifier(loss='log', class_weight='auto', n_iter=1000, alpha=1.0, random_state=999, verbose=5) type_scorer.fit(X, labels) with open("type-model.pickle", 'wb') as out: pickle.dump((vec, type_scorer), out)
def get_answer_notable_types(self): """ Returns a list of notable types of each of the result entities. :return: """ if self.answer_notable_types is None: self.answer_notable_types = [] for mid, answer in zip(self.get_results_mids(), self.get_results_text()): if _year_pattern.match(answer) is not None: continue self.answer_notable_types.append(KBEntity.get_notable_type(mid)) return self.answer_notable_types
def filter_answers_by_type(self, type_filter, score): assert self.type_filter is None text_results = self.get_results_text() mid_results = self.get_results_mids() assert len(text_results) == len(mid_results) new_results_text = [] new_results_mids = [] for mid, answer in zip(mid_results, text_results): if KBEntity.get_notable_type(mid) == type_filter: new_results_mids.append(mid) new_results_text.append(answer) self.query_results = new_results_text self.query_results_mids = new_results_mids self.type_filter = type_filter self.type_filter_max_npmi = score[0] self.type_filter_avg_npmi = score[0] self.cached_result_count = len(self.query_results) return self.get_results_text()
def extract_npmi_ngram_type_pairs(): globals.read_configuration('config.cfg') scorer_globals.init() datasets = ["webquestions_split_train", ] parameters = translator.TranslatorParameters() parameters.require_relation_match = False parameters.restrict_answer_type = False n_gram_type_counts = dict() type_counts = dict() n_gram_counts = dict() total = 0 year_pattern = re.compile("[0-9]+") for dataset in datasets: queries = get_evaluated_queries(dataset, True, parameters) for index, query in enumerate(queries): if query.oracle_position != -1 and query.oracle_position <= len(query.eval_candidates): correct_candidate = query.eval_candidates[query.oracle_position - 1] logger.info(query.utterance) logger.info(correct_candidate.query_candidate) n_grams = set(get_n_grams_features(correct_candidate.query_candidate)) answer_entities = [mid for answer in query.target_result if year_pattern.match(answer) is None for mid in KBEntity.get_entityid_by_name(answer, keep_most_triples=True)] correct_notable_types = set(filter(lambda x: x, [KBEntity.get_notable_type(entity_mid) for entity_mid in answer_entities])) for notable_type in correct_notable_types: if notable_type not in type_counts: type_counts[notable_type] = 0 type_counts[notable_type] += 1 for n_gram in n_grams: if n_gram not in n_gram_counts: n_gram_counts[n_gram] = 0 n_gram_counts[n_gram] += 1 for notable_type in correct_notable_types: pair = (n_gram, notable_type) if pair not in n_gram_type_counts: n_gram_type_counts[pair] = 0 n_gram_type_counts[pair] += 1 total += 1 npmi = dict() from math import log for n_gram_type_pair, n_gram_type_count in n_gram_type_counts.iteritems(): if n_gram_type_count > 4: n_gram, type = n_gram_type_pair npmi[n_gram_type_pair] = (log(n_gram_type_count) - log(n_gram_counts[n_gram]) - log(type_counts[type]) + log(total)) / (-log(n_gram_type_count) + log(total)) with open("type_model_npmi.pickle", 'wb') as out: pickle.dump(npmi, out) import operator npmi = sorted(npmi.items(), key=operator.itemgetter(1), reverse=True) print "\n".join(map(str, npmi[:50]))
with open("type-model.pickle", 'wb') as out: pickle.dump((vec, type_scorer), out) if __name__ == "__main__": extract_npmi_ngram_type_pairs() exit() globals.read_configuration('config.cfg') parser = globals.get_parser() scorer_globals.init() datasets = ["webquestions_split_train", ] # datasets = ["webquestions_split_train_externalentities", "webquestions_split_dev_externalentities",] # datasets = ["webquestions_split_train_externalentities3", "webquestions_split_dev_externalentities3",] data = [] for dataset in datasets: queries = load_eval_queries(dataset) for index, query in enumerate(queries): tokens = [token.token for token in parser.parse(query.utterance).tokens] answer_entities = [mid for answer in query.target_result for mid in KBEntity.get_entityid_by_name(answer, keep_most_triples=True)] notable_types = [KBEntity.get_notable_type(entity_mid) for entity_mid in answer_entities] data.append((tokens, notable_types)) logger.info(tokens) logger.info([KBEntity.get_entity_name(notable_type) for notable_type in notable_types]) with open("question_tokens_notable_types.pickle", 'wb') as out: pickle.dump(data, out)