def __init__(self):
self.persons_ids = {}
self.ids_persons = {}
for line in FileUtils.read_file_as_list(PERSONS_IDS_F):
person, id = line.split("\t", maxsplit=1)
self.persons_ids[person] = id
self.ids_persons[id] = person
self.professions_ids = {}
self.ids_professions = {}
for line in FileUtils.read_file_as_list(PROFESSIONS_IDS_F):
prof, id = line.split("\t", maxsplit=1)
self.professions_ids[prof] = id
self.ids_professions[id] = prof
self.nationalities_ids = {}
self.ids_nationalities = {}
for line in FileUtils.read_file_as_list(NATIONALITIES_IDS_F):
nation, id = line.split("\t", maxsplit=1)
self.nationalities_ids[nation] = id
self.ids_nationalities[id] = nation
self.nationalities_countries = {}
self.countries_nationalities = {}
for line in FileUtils.read_file_as_list(COUNTRIES_NATIONALITIES_F):
country, nation = line.split("\t", maxsplit=1)
self.nationalities_countries[nation] = country
self.countries_nationalities[country] = nation
def __load_items_stats(self, items_tfidf_fpath):
"""Loads pre-computed tf-idf stats for items.
:param items_tfidf_fpath:
:return:
"""
item_term_weights = {}
for line in FileUtils.read_file_as_list(items_tfidf_fpath):
item, term, _, _, weight = line.split("\t", maxsplit=4)
if len(term) < 4: # avoid short terms
continue
# some cleanings
if term.startswith("_"):
term = term[1:]
if term.endswith("_"):
term = term[:-1]
if term.startswith(WP_PREFIX): # it's a person name
term = term.split(WP_PREFIX)[-1] # remove prefix
term = "_".join([
word[0].upper() + word[1:] for word in term.split("_")
]) # capitalize every word
item_d = item_term_weights.get(item, {})
item_d[term] = float(weight)
item_term_weights[item] = item_d
return item_term_weights
def __load_entity_abstracts(self, filename):
prefix = URIPrefix()
t = Triple()
p = NTriplesParser(t)
lines_counter = 0
PLOGGER.info("Loading entity abstracts from {}".format(filename))
for line in FileUtils.read_file_as_list(filename):
# basic line parsing
line = line.decode("utf-8") if isinstance(line, bytes) else line
try:
p.parsestring(line)
except ParseError: # skip lines that couldn't be parsed
continue
if t.subject() is None: # only if parsed as a triple
continue
# Subject and object identification
subj = prefix.get_prefixed(t.subject())
obj = ""
if type(t.object()) is URIRef:
# PLOGGER.error("Error: it is URIRef the parsed obj")
pass
else:
obj = t.object().encode("utf-8")
if len(obj) == 0:
continue # skip empty objects
self.__entity_abstracts[subj] = obj
lines_counter += 1
if lines_counter % 10000 == 0:
PLOGGER.info("\t{}K lines processed".format(lines_counter // 1000))
pass
PLOGGER.info("\n### Loading entity abstracts... Done.")
def make_persons_fb_ids(persons_fpath):
"""Our person ID is the Freebase ID where the prefix 'm.' is replaced with 'fb_'.
:param persons_fpath: 'persons'-named file path.
"""
with open(PERSONS_IDS_F, "w") as f_out:
for line in FileUtils.read_file_as_list(persons_fpath):
person, raw_fb_id = line.split("\t", maxsplit=1)
fb_id = "fb_" + raw_fb_id[2:]
f_out.write("{}\t{}\n".format(person, fb_id))
def make_professions_kb_translation(dest_translation_fpath, persons_ids_fpath, professions_ids_fpath,
person_items_fpath):
"""A person ID -to- item ID translation schema is convenient.
:param dest_translation_fpath: destination file path of IDs translation.
:param persons_ids_fpath: person IDs file path.
:param professions_ids_fpath: profession IDs file path.
:param person_items_fpath: path to the file with person items (a '.kb'-extension file).
:return:
"""
persons_ids = {}
for line in FileUtils.read_file_as_list(persons_ids_fpath):
person, id = line.split("\t", maxsplit=1)
persons_ids[person] = id
professions_ids = {}
for line in FileUtils.read_file_as_list(professions_ids_fpath):
prof, id = line.split("\t", maxsplit=1)
professions_ids[prof] = id
person_items = {}
for line in FileUtils.read_file_as_list(person_items_fpath):
person, item = line.split("\t", maxsplit=1)
person_items.get(person, []).append(item)
translations = {}
for person, items in person_items.items():
if person not in persons_ids:
continue
person_id = persons_ids[person]
items_ids = []
for item in items:
if item not in professions_ids:
continue
items_ids.append(professions_ids[item])
translations[person_id] = items_ids
with open(dest_translation_fpath, "w") as f_out:
for person_id, items_ids in translations.items():
for item_id in items_ids:
f_out.write("{}\t{}\n".format(person_id, item_id))
def make_relation_item_ids(rel_items_fpath):
"""Our relation ID is the original relation where the prefix any dash or blank is replaced with underscore.
:param rel_items_fpath: items file path.
"""
basename = os.path.basename(rel_items_fpath) # professions or nationalities
with open(os.sep.join([DATA_DIR, "{}_ids.tsv".format(basename)]), "w") as f_out:
for item in FileUtils.read_file_as_list(rel_items_fpath):
id = item.lower().replace(" ", "_").replace("-", "_")
f_out.write("{}\t{}\n".format(item, id))
def __load_entity_abstracts(self):
num_lines = 0
filename = os.sep.join([self.__dbpedia_path, ENTITY_ABSTRACTS_FILE])
PLOGGER.info("Loading entity abstracts from {}".format(filename))
for line in FileUtils.read_file_as_list(filename):
entity, abstract = self.__parse_line(line)
if abstract and len(abstract) > 0: # skip empty objects
self.__entity_abstracts[entity] = abstract
num_lines += 1
if num_lines % 10000 == 0:
PLOGGER.info(" {}K lines processed".format(num_lines // 1000))
PLOGGER.info(" Done.")
def generate_features(self, kb_file, output_file):
"""Core function for generating into output_file the features, with person-item data from kb_file.
:param kb_file: path to the file with person items (a '.kb'-extension file).
:param output_file:
:return:
"""
feat_w2v_approx = FeaturesW2VSimApprox()
with open(output_file, "w") as f_out:
# write tsv header
header = ["person_id", "prof_id"]
for k in self.K_VALUES:
header.append("simCos_w2v_" + str(k))
f_out.write("\t".join(header) + "\n")
for line in FileUtils.read_file_as_list(kb_file):
person_id, prof_id = line.split(
"\t") # strip() done in read_file_as_list()
values = [person_id, prof_id]
person_tf, num_sent = self.get_person_tf(person_id)
for k in self.K_VALUES:
# we take top-K profession terms
# compute simCosK
# where K is the top-K terms for the profession
term_weights_pr = {
} # dict from top-K profession terms to their tfidf weights
term_weights_pe = {
} # dict from top-K person terms to their tfidf weights
if prof_id in self.__stats:
for term, s in self.__stats[prof_id].items():
if s["rank"] <= k:
term_weights_pr[term] = float(s["tfidf"])
idf = s["tfidf"] / s[
"tf"] # we back-generate IDF from profession's TF-IDF
term_weights_pe[term] = person_tf.get(term,
0) * idf
vec_pr = feat_w2v_approx.get_vector(term_weights_pr)
vec_pe = feat_w2v_approx.get_vector(term_weights_pe)
cos = cos_sim(vec_pr, vec_pe)
else:
cos = 0 # in some exceptional cases the profession does not have any sentences
values.append(str(cos))
f_out.write("\t".join(values) + "\n")
def __load_entity_types(self):
num_lines = 0
for types_file in ENTITY_TYPES_FILES:
filename = os.sep.join([self.__dbpedia_path, types_file])
PLOGGER.info("Loading entity types from {}".format(filename))
for line in FileUtils.read_file_as_list(filename):
entity, entity_type = self.__parse_line(line)
if type(entity_type) != str: # Likely result of parsing error
continue
if not entity_type.startswith("<dbo:"):
PLOGGER.info(" Non-DBpedia type: {}".format(entity_type))
continue
if not entity.startswith("<dbpedia:"):
PLOGGER.info(" Invalid entity: {}".format(entity))
continue
self.__types_entities[entity_type].append(entity)
num_lines += 1
if num_lines % 10000 == 0:
PLOGGER.info(" {}K lines processed".format(num_lines //
1000))
PLOGGER.info(" Done.")
def create_pdf(self,
diff_file,
pdf_file,
title="",
xlabel="",
ylabel="",
aspect_ratio="equal",
separator="\t"):
"""Create bar plot for differences in pdf.
This function is used to load difference .csv file,
create bar plot and store as a pfd file.
:pdf: created and saved pdf file
"""
data = FileUtils.read_file_as_list(diff_file)
scores = []
for item in data:
if "diff" in item: # ignore the first line(title)
continue
scores.append(float(item.split(separator)[3]))
scores = sorted(scores, reverse=True)
with PdfPages(pdf_file) as pdf:
n = len(scores)
x = range(n)
plt.figure(figsize=(4, 4))
plt.title(title)
plt.xlabel(xlabel)
plt.ylabel(ylabel)
plt.bar(x, scores, self.width, color=self.color)
plt.tight_layout() # warning,still working
pdf.savefig()
plt.close()
def __load_sample_entities(self):
"""Loads the set of entities to be sampled from file."""
self.sample_entities = FileUtils.read_file_as_list(self.entities_file)
def get_all_features_approx(self, all_items_fpath, items_tfidf_fpath,
first_snippets_fpath, person_items_fpath,
dest_fpath):
"""Core function for generating into output_file the features, with person-item data from kb_file.
:param all_items_fpath: path to the file with all items (e.g. 'professions' named file).
:param items_tfidf_fpath: path to the file with item tf-idf stats.
:param first_snippets_fpath: path to the file with first snippets.
:param person_items_fpath: path to the file with person items (a '.kb'-extension file).
:param dest_fpath: destination file path.
:return:
"""
wcup_ids = WSDMCupIDs()
# -------
# Get persons vectors
# Load person-to-snippet mapping
persons_to_vec = {}
for line in FileUtils.read_file_as_list(first_snippets_fpath):
person, snippet = line.split("\t")
person = person.split(
"<{}".format(DBPEDIA_URI_PREFIX))[-1].split(">")[0].replace(
"_", " ")
c = {
k: v
for k, v in Counter(snippet.split()).items()
if k not in STOPWORDS and v >= 2
} # min freq = 2
total = sum([v for k, v in c.items()])
c = {k: round(v / total, 4) for k, v in c.items()}
v = self.get_vector(c)
persons_to_vec[person] = v
print("Get persons vectors... OK.")
# -------
# Get profession vectors
# Load professions tf-idf
# dict from each item (e.g. professions) to its term-to-weight dict
item_term_weights = self.__load_items_stats(items_tfidf_fpath)
prof_to_vec = {}
for raw_item in FileUtils.read_file_as_list(all_items_fpath):
# for professions
item = wcup_ids.get_id_from_prof(raw_item) # DON'T FORGET!
v = self.get_vector(item_term_weights.get(item, {}))
# ---
# Alternative vector version, using the w2v vector when it exists, without top-K terms
#
# if not self.word2vec.contains_word(item): # build our own vector with top-K terms for that item
# v = self.get_vector(item_term_weights.get(item, {}))
# else: # take advantage of an already well-represented vector present in the w2v collection
# v = self.word2vec.get_vector(item)
# ---
prof_to_vec[item] = v
print("Get profession vectors... OK.")
# -------
# Load persons-to-professions and get cos sims
person_to_prof_to_cos = {} # e.g. person-to-profession-to-cos_sim
for line in FileUtils.read_file_as_list(person_items_fpath):
person, item = line.split("\t", maxsplit=1)
item = wcup_ids.get_id_from_prof(item) # DON'T FORGET!
if person not in person_to_prof_to_cos:
person_to_prof_to_cos[person] = {}
person_vec = persons_to_vec.get(
person,
self.word2vec.get_zeros_vector(self.word2vec.dimension))
prof_vec = prof_to_vec.get(
item, self.word2vec.get_zeros_vector(self.word2vec.dimension))
feat_value = cos_sim(person_vec, prof_vec)
person_to_prof_to_cos[person][item] = feat_value
pass
with open(dest_fpath, "w") as f_out:
f_out.write("person\tprofession\tsimCos_w2v_aggr_100\n")
for person, data in sorted(person_to_prof_to_cos.items()
): # sorted by person full name
for item, sim in sorted(
data.items(), key=itemgetter(1),
reverse=True): # sorted decreasingly by sim
f_out.write("{}\t{}\t{}\n".format(
wcup_ids.get_id_from_person(person), item, sim))
def main(all_items_fpath, first_snippets_fpath, person_items_fpath, relation):
"""Extracts features about 1st Wikipedia sentences and paragraphs.
:param all_items_fpath: path to the file with all items (e.g. 'professions' named file).
:param first_snippets_fpath:path to the file with first snippets.
:param person_items_fpath: path to the file with person items (a '.kb'-extension file).
:param relation: a string in {REL_PROFESSION, REL_NATIONALITY}.
:return:
"""
snippet_basename = os.path.basename(first_snippets_fpath).split(".txt")[
0] # for creating dump filename
print("Processing for {}".format(snippet_basename))
# Load all items, e.g. all professions
all_rel_items = {
item
for item in FileUtils.read_file_as_list(all_items_fpath)
}
# Load person-to-snippet mapping
persons_snippets = {}
for line in FileUtils.read_file_as_list(first_snippets_fpath):
person, snippet = line.split("\t")
person = person.split(
"<{}".format(DBPEDIA_URI_PREFIX))[-1].split(">")[0].replace(
"_", " ")
persons_snippets[person] = snippet
print("\t Loaded snippets... OK.")
# Make dict: person to set of (all possible) items in snippet
person_to_all_snippet_items = {}
# person_counter = 0
for person, snippet in persons_snippets.items():
for item in all_rel_items:
# important to lowercase when searching for item
match = re.search(r"[,\s\.]({})[,\s\.]".format(item.lower()),
snippet)
if match:
person_to_all_snippet_items.get(person, set()).add(item)
print("\t Made 1st dict... OK.")
# Make dict: person to first item in snippet (among all possible items)
person_to_first_snippet_item = {}
# person_counter = 0
for person, snippet in persons_snippets.items():
fst_item_index = len(snippet) - 1
fst_item = ""
for item in person_to_all_snippet_items.get(person, set()):
# in this case, we use re.search() since we need the string index where the found occurrence starts
# important to lowercase when searching for item
match = re.search(r"[,\s\.]({})[,\s\.]".format(item.lower()),
snippet)
if match:
# item (e.g. profession) occurs not only as substring, but all its words (sometimes it's multiword)
# appear as words in snippet
# otherwise, it will count, e.g., "orator" as a profession when it's only a substring of "oratorio"
item_index = match.start(1)
if item_index < fst_item_index:
fst_item_index = item_index
fst_item = item
person_to_first_snippet_item[person] = fst_item
print("\t Made 2nd dict... OK.")
# Make final feature mappings
is_item_in = {} # e.g. person-to-profession-to-yes_or_no mapping
is_1st_item_in = {} # e.g. person-to-profession-to-yes_or_no mapping
for line in FileUtils.read_file_as_list(person_items_fpath):
person, item = line.split("\t", maxsplit=1)
if person not in is_item_in:
is_item_in[person] = {}
feat_value = "1" if item in person_to_all_snippet_items.get(
person, set()) else "0"
is_item_in[person][item] = feat_value
if person not in is_1st_item_in:
is_1st_item_in[person] = {}
feat_value = "1" if item in person_to_first_snippet_item.get(
person, set()) else "0"
is_1st_item_in[person][item] = feat_value
print("\t Main for-loop running... OK.")
# -------
# Dumping tsv
persons_snippets.clear()
wcup_ids = WSDMCupIDs()
with open(
os.sep.join([
OUTPUT_DIR, "is_{}_in_{}.tsv".format(relation,
snippet_basename)
]), "w") as f_is_item_in:
f_is_item_in.write("person\t{}\tis_{}_in_{}\n".format(
relation, relation, snippet_basename))
for person, item_val in sorted(is_item_in.items()):
for item, val in sorted(item_val.items()):
f_is_item_in.write("{}\t{}\t{}\n".format(
wcup_ids.get_id_from_person(person),
wcup_ids.get_id_from_prof(item), val))
with open(
os.sep.join([
OUTPUT_DIR,
"is_1st_{}_in_{}.tsv".format(relation, snippet_basename)
]), "w") as f_is_item_in:
f_is_item_in.write("person\t{}\tis_the_1st_{}_in_{}\n".format(
relation, relation, snippet_basename))
for person, item_val in sorted(is_1st_item_in.items()):
for item, val in sorted(item_val.items()):
f_is_item_in.write("{}\t{}\t{}\n".format(
wcup_ids.get_id_from_person(person),
wcup_ids.get_id_from_prof(item), val))
print("\t Dumping both tsv files... OK.")