def run_umbel(input_dir, output_file, sw_map_file):
"""
Read N-Triples files containing Umbel data, outputting a file of
ConceptNet edges and a file of mappings between the Semantic Web and
ConceptNet.
"""
out = MsgpackStreamWriter(output_file)
map_out = NTriplesWriter(sw_map_file)
reader = NTriplesReader()
labels = {}
label_sets = defaultdict(set)
# There are two files we want to parse:
# - umbel.nt, a transformation of umbel.n3, which is available from
# https://github.com/structureddynamics/UMBEL/.
# - umbel_links.nt, distributed with DBPedia 3.9.
#
# We parse them both in this file so that umbel_links can reuse the
# concept names extracted from umbel.nt.
main_file = os.path.join(input_dir, 'umbel.nt')
dbpedia_link_file = os.path.join(input_dir, 'umbel_links.nt')
# Read through umbel.nt once, finding all the "preferred labels". We will
# use these as the surface texts for the nodes.
for web_subj, web_rel, web_obj, objtag in reader.parse_file(main_file):
if resource_name(web_rel) == 'prefLabel':
# 'CW' and 'PCW' are Cyc jargon for 'conceptual works'. If a node
# cannot be described except as a CW, we're probably not
# interested in it.
if 'CW' not in web_obj.split() and 'PCW' not in web_obj.split():
labels[web_subj] = web_obj
if resource_name(web_rel).endswith('Label'):
text = standardize_text(web_obj)
label_sets[text].add(web_subj)
# Read through umbel.nt again and extract ConceptNet edges.
for web_subj, web_rel, web_obj, objtag in reader.parse_file(main_file):
if objtag == 'URL' and acceptable_node(web_obj) and acceptable_node(
web_subj):
# Only use nodes for which we've seen preferred labels.
# (This skips some anonymous OWL-cruft nodes.)
if web_subj in labels and web_obj in labels:
subj_uri = standardized_concept_uri('en', labels[web_subj])
obj_uri = standardized_concept_uri('en', labels[web_obj])
rel_name = resource_name(web_rel)
# Check if this is a relation we want to handle.
if rel_name in REL_MAPPING:
# Write the ConceptNet edges and the mappings to Semantic Web URLs.
rel_uri, frame = REL_MAPPING[rel_name]
surface = frame % (labels[web_subj], labels[web_obj])
out.write(
umbel_edge(rel_uri, subj_uri, obj_uri, surface,
SOURCE))
map_out.write_link(web_rel, full_conceptnet_url(rel_uri))
map_out.write_link(web_subj, full_conceptnet_url(subj_uri))
map_out.write_link(web_obj, full_conceptnet_url(obj_uri))
# altLabel relations assign different texts to the same node. We'll
# represent those in ConceptNet with Synonym relations.
elif web_rel.endswith('altLabel'):
# Make sure we know what's being labeled.
if web_subj in labels:
name = web_obj
words = name.split(' ')
if standardized_concept_name(
'en', name) != standardized_concept_name(
'en', labels[web_subj]):
if not set(words) & IGNORED_WORDS:
main_label = standardized_concept_uri(
'en', labels[web_subj])
name_text = standardize_text(name)
if len(label_sets[name_text]) >= 2 or len(
name_text) <= 3:
disambig = un_camel_case(resource_name(web_subj))
# Cyc does not distinguish texts by their part of speech, so use
# '_' as the part of speech symbol.
alt_label = standardized_concept_uri(
'en', name, '_', disambig)
else:
alt_label = standardized_concept_uri('en', name)
surface = SYN_FRAME % (name, labels[web_subj])
out.write(
umbel_edge('/r/Synonym', alt_label, main_label,
surface, SOURCE))
for web_subj, web_rel, web_obj, objtag in reader.parse_file(
dbpedia_link_file):
if objtag == 'URL' and acceptable_node(web_obj) and acceptable_node(
web_subj):
if web_obj in labels:
subj_label = resource_name(web_subj).replace('_', ' ')
subj_uri = translate_dbpedia_url(web_subj)
obj_label = labels[web_obj]
obj_uri = standardized_concept_uri('en', obj_label)
rel_name = resource_name(web_rel)
if rel_name in REL_MAPPING:
rel_uri, frame = REL_MAPPING[rel_name]
surface = frame % (subj_label, obj_label)
out.write(
umbel_edge(rel_uri, subj_uri, obj_uri, surface,
LINK_SOURCE))
map_out.write_link(web_rel, full_conceptnet_url(rel_uri))
map_out.write_link(web_subj, full_conceptnet_url(subj_uri))
map_out.write_link(web_obj, full_conceptnet_url(obj_uri))
def handle_file(filename, output_file):
"""
JMdict is a Japanese translation dictionary, targeting multiple languages,
released under a Creative Commons Attribution-ShareAlike license. That's
awesome.
It's released as a kind of annoying XML structure, using fancy XML features
like entities, so in order to read it we need a full-blown XML parser. Python's
built-in XML parsers are horrifying, so here we use the 'xmltodict' module, which
is also horrifying but gets the job done.
The majorly weird thing about xmltodict that we have to work around is that
it gives results of different *types* when you get 0, 1, or many child nodes.
This is what get_list is for.
"""
# Read the XML file as UTF-8, and parse it into a dictionary.
file = codecs.open(filename, encoding='utf-8')
out = MsgpackStreamWriter(output_file)
data = file.read()
file.close()
xml = xmltodict.parse(data, disable_entities=False)
# The dictionary contains a list of <entry> tags.
root_node = xml['JMdict']
for entry in get_list(root_node, 'entry'):
# From JMdict's documentation: "The kanji element, or in its absence,
# the reading element, is the defining component of each entry."
#
# Quick summary of what's going on here: most Japanese words can be
# written using kanji or kana.
#
# Kana are phonetic characters. Every word can be written in kana, in
# one of two alphabets (hiragana or katakana). Words that are homonyms
# have the same kana, unless they're written in different alphabets.
#
# Kanji are Chinese-based characters that are related to the meaning of
# the word. They're compact and good at disambiguating homonyms, so
# kanji are usually used as the canonical representation of a word.
# However, some words have no kanji.
#
# The kana version of a word written in kanji is called its 'reading'.
# Words that are pronounced differently in different contexts have
# multiple readings.
#
# Okay, done with the intro to Japanese orthography. In JMdict, if
# a word can be written in kanji, it has a <k_ele> element, containing
# a <keb> element that contains the text. Every word also has an
# <r_ele> element, containing one or more <reb> elements that are phonetic
# readings of the word.
#
# We get the "defining text" of a word by taking its <keb> if it exists,
# or all of its <reb>s if not. There's no way to tell which <reb> is the
# most "defining" in the case where there's no <keb>.
headwords = [word['keb'] for word in get_list(entry, 'k_ele')]
if not headwords:
headwords = [word['reb'] for word in get_list(entry, 'r_ele')]
# An entry might have different word senses that are translated
# differently to other languages. Ideally, we'd remember that they're
# different senses. However, we have no way to refer to the different
# senses. So for now, we disregard word senses. One day we might have
# a better overall plan for word senses in ConceptNet.
senses = get_list(entry, 'sense')
for sense_num, sense in enumerate(senses):
# Glosses are translations of the word to different languages.
# If the word is a loan-word, the foreign word it was derived from
# will be marked with the <lsource> tag instead of <gloss>.
#
# Get all the glosses, including the lsource if it's there.
glosses = get_list(sense, 'gloss') + get_list(sense, 'lsource')
contexts = [
fix_context(context)
for context in get_list(sense, 'field')
]
pos = '_'
for pos_tag in get_list(sense, 'pos'):
if pos_tag[:10] in NOUN_TYPES:
pos = 'n'
elif pos_tag[:10] in VERB_TYPES:
pos = 'v'
elif pos_tag[:10] in ADJ_TYPES:
pos = 'a'
elif pos_tag[:10] in ADV_TYPES:
pos = 'r'
for gloss in glosses:
if '#text' in gloss:
# A gloss node might be marked with a 'lang' attribute. If so,
# xmltodict represents it as a dictionary with '#text' and
# '@xml:lang' elements.
text = parse_gloss(gloss['#text'])
lang = convert_lang_code(gloss['@xml:lang'])
elif isinstance(gloss, STRING_TYPE):
# If there's no 'lang' attribute, the gloss is in English,
# and xmltodict gives it to us as a plain Unicode string.
lang = 'en'
text = parse_gloss(gloss)
# If we parsed the node at all and the text looks good, then we can
# add edges to ConceptNet.
#
# We don't want to deal with texts with periods (these might be
# dictionary-style abbreviations, which are sort of unhelpful when
# we can't expand them), and we also don't want to deal with texts
# that are more than five words long.
if (
text is not None and '.' not in text and
text.count(' ') <= 4 and valid_concept_name(text)
):
for head in headwords:
if len(senses) >= 2:
sensekey = '%d' % (sense_num + 1)
ja_concept = standardized_concept_uri('ja', head, pos, 'jmdict', sensekey)
else:
ja_concept = standardized_concept_uri('ja', head, pos)
other_concept = standardized_concept_uri(lang, text)
output_edge(out, '/r/Synonym', ja_concept, other_concept)
for context in contexts:
context_node = standardized_concept_uri('en', context)
output_edge(out, '/r/HasContext', ja_concept, context_node)
def transform_file(self, input_filename, output_file):
out = MsgpackStreamWriter(output_file)
for obj in read_json_stream(input_filename):
for new_obj in self.handle_assertion(obj):
out.write(new_obj)
def handle_file(filename, output_file):
out = MsgpackStreamWriter(output_file)
for line in gzip.open(filename, 'rt'):
# skip the intro information
if line.startswith('#'):
continue
# parse the data to extract the traditional form, simplified form and the English definition
traditional, simplified, definitions = re.match(LINE_REGEX, line).groups()
# Make an edge between the traditional and simplified version
edge = make_edge(rel='/r/Synonym',
start=standardized_concept_uri('zh-Hant', traditional),
end=standardized_concept_uri('zh-Hans', simplified),
dataset=DATASET,
license=LICENSE,
sources=SOURCE)
out.write(edge)
definitions = re.split(r'\/|;', definitions)
for definition in definitions:
# Skip pronunciation information
if 'Taiwan pr.' in definition or 'also pr.' in definition:
continue
# Check if it's the definition matches a person syntax, i.e. includes a date range
person_match = re.match(DATE_RANGE_REGEX, definition)
if person_match:
persons = extract_person(person_match)
for person in persons:
edge = make_edge(rel='/r/Synonym',
start=standardized_concept_uri('zh-Hant', traditional),
end=standardized_concept_uri('en', person),
dataset=DATASET,
license=LICENSE,
sources=SOURCE)
out.write(edge)
edge = make_edge(rel='/r/Synonym',
start=standardized_concept_uri('zh-Hans', simplified),
end=standardized_concept_uri('en', person),
dataset=DATASET,
license=LICENSE,
sources=SOURCE)
out.write(edge)
continue
# Remove clarifying information in parenthesis
definition = PAREN_REGEX.sub('', definition)
# Check if a word is a measure word
if definition.startswith('CL:'):
related_words = extract_measure_words(definition)
for word in related_words:
edge = make_edge(rel='/r/RelatedTo',
start=standardized_concept_uri('zh-Hant', traditional),
end=standardized_concept_uri('zh', word),
dataset=DATASET,
license=LICENSE,
sources=SOURCE)
out.write(edge)
edge = make_edge(rel='/r/RelatedTo',
start=standardized_concept_uri('zh-Hans', simplified),
end=standardized_concept_uri('zh', word),
dataset=DATASET,
license=LICENSE,
sources=SOURCE)
out.write(edge)
continue
# Check if a word is a form/variant of a different word
variant_match = re.match(VARIANT_REGEX, definition)
if variant_match:
variants = extract_variants(definition)
for variant in variants:
edge = make_edge(rel='/r/RelatedTo',
start=standardized_concept_uri('zh-Hant', traditional),
end=standardized_concept_uri('zh', variant),
dataset=DATASET,
license=LICENSE,
sources=SOURCE)
out.write(edge)
edge = make_edge(rel='/r/RelatedTo',
start=standardized_concept_uri('zh-Hans', simplified),
end=standardized_concept_uri('zh', variant),
dataset=DATASET,
license=LICENSE,
sources=SOURCE)
out.write(edge)
continue
# Handle abbreviations
if re.match(ABBR_REGEX, definition):
abbreviations = extract_abbreviations(definition)
if abbreviations:
for abbr in abbreviations:
edge = make_edge(rel='/r/RelatedTo',
start=standardized_concept_uri('zh-Hant', traditional),
end=standardized_concept_uri('zh', abbr),
dataset=DATASET,
license=LICENSE,
sources=SOURCE)
out.write(edge)
edge = make_edge(rel='/r/RelatedTo',
start=standardized_concept_uri('zh-Hans', simplified),
end=standardized_concept_uri('zh', abbr),
dataset=DATASET,
license=LICENSE,
sources=SOURCE)
out.write(edge)
continue
# Remove 'lit.', 'fig.'
definition = LIT_FIG_REGEX.sub('', definition)
# Expand sth and sb
definition = definition.replace('sth', 'something')
definition = definition.replace('sb', 'someone')
definition = remove_reference_syntax(definition)
definition = remove_additional_info(definition)
# Skip long definitions
if len(definition.split()) < 6:
edge = make_edge(rel='/r/Synonym',
start=standardized_concept_uri('zh-Hant', traditional),
end=standardized_concept_uri('en', definition),
dataset=DATASET,
license=LICENSE,
sources=SOURCE)
out.write(edge)
edge = make_edge(rel='/r/RelatedTo',
start=standardized_concept_uri('zh-Hans', simplified),
end=standardized_concept_uri('en', definition),
dataset=DATASET,
license=LICENSE,
sources=SOURCE)
out.write(edge)
def handle_file(infile, outfile):
count = 0
outcomes = defaultdict(int)
writer = MsgpackStreamWriter(outfile)
for line in open(infile):
parts = line.strip().split('\t')
if not parts:
outcomes['blank'] += 1
continue
# The first 5 columns of the Verbosity output file are:
#
# left: the word being clued
# relation: the relation between the word and the clue that the
# clue-giver chose, in a form such as "it is part of"
# right: the one or two words used as the clue
# freq: the number of different times this clue was given
# orderscore: the average position in the list of clues
#
# 'orderscore' is a number from 0 to 999, representing the average
# quantile of its position in the list of clues. (It's like a
# percentile, except there are 1000 of them, not 100.)
#
# A clue that's always given first has an orderscore of 0. A clue
# that always appears halfway through the list has an orderscore of
# 500.
#
# This may seem like a strange thing to measure, and I didn't come up
# with it, but it actually turns out to be somewhat informative.
# A clue with an orderscore of 0 is probably a good common-sense
# relation, representing the first thing that comes to mind. A clue
# with a high order score may be a move of desperation after several
# other clues have failed. It causes the guesser to get the answer
# soon afterward, but perhaps because it's a "cheating" move. So,
# low orderscores represent better common sense relations.
left, relation, right, freq, orderscore = parts[:5]
freq = int(freq)
orderscore = int(orderscore)
# Test each word
flagged = False
for rword in right.split():
if BAD_CLUE_REGEX.match(rword):
flagged = True
break
if flagged:
outcomes['flag word'] += 1
continue
if len(right) < 3:
outcomes['clue too short'] += 1
continue
if len(right.split()[-1]) == 1:
outcomes['letter'] += 1
continue
# The Verbosity interface and gameplay did not particularly encourage
# players to choose an appropriate relation. In practice, players seem
# to have used them all interchangeably, except for the negative
# relation "it is the opposite of", expressing /r/Antonym.
#
# Another way that players expressed negative relations was to use
# 'not' as the first word of their clue; we make that into an instance
# of /r/Antonym as well.
#
# In other cases, the relation is a positive relation, so we replace it
# with the most general positive relation, /r/RelatedTo.
rel = '/r/RelatedTo'
reltext = 'is related to'
if right.startswith('not '):
rel = '/r/DistinctFrom'
right = right[4:]
reltext = 'is not'
if relation == 'it is the opposite of':
rel = '/r/Antonym'
reltext = 'is the opposite of'
# The "sounds-like score" determines whether this clue seems to be a
# pun or rhyme, rather than an actual common-sense relationship. If
# the sounds-like score is over 0.35, skip the assertion.
sls = sounds_like_score(left, right)
if sls > 0.35:
outcomes['text similarity'] += 1
continue
# Calculate a score for the assertion:
#
# - The number of times it's been used as a clue
# - ...with a linear penalty for a high sounds-like score
# - ...and a linear penalty for high orderscores
#
# The penalties are multiplicative factors from 0 to 1, which decrease
# linearly as the relevant penalties increase. If a clue is given N
# times, with a sounds-like score of 0 and an orderscore of 0, it will
# get an overall score of 2N - 1. This is a formula we should probably
# revisit.
#
# The weight is the score divided by 100. All divisions are floating
# point, as defined by the __future__ import at the top of this module.
score = (freq * 2 - 1) * (1 - sls) * (1 - orderscore / 1000)
if score <= 1.:
outcomes['low score'] += 1
continue
weight = score**.5 / 10
# If the clue on the right is a two-word phrase, we make additional
# connections to both words individually. We label them with the
# rule-based source '/s/process/split_words' to track that this
# happened.
rightwords = [right]
if ' ' in right:
morewords = [
word for word in right.split(' ') if word not in STOPWORDS
]
rightwords.extend(morewords)
for i, rightword in enumerate(rightwords):
source = {'contributor': '/s/resource/verbosity'}
if i > 0:
source['process'] = '/s/process/split_words'
# Build the natural-language-ish surface text for this clue
text = '[[%s]] %s [[%s]]' % (left, reltext, rightword)
count += 1
outcomes['success'] += 1
leftc = standardized_concept_uri('en', left)
rightc = standardized_concept_uri('en', rightword)
edge = make_edge(rel,
leftc,
rightc,
dataset='/d/verbosity',
license=Licenses.cc_attribution,
sources=[source],
surfaceText=text,
weight=weight)
writer.write(edge)
def read_wiktionary(input_file, db_file, output_file):
"""
Convert a stream of parsed Wiktionary data into ConceptNet edges.
A `db_file` containing all known words in all languages must have already
been prepared from the same data.
"""
db = sqlite3.connect(db_file)
out = MsgpackStreamWriter(output_file)
for heading, items in segmented_stream(input_file):
language = heading['language']
title = heading['title']
dataset = '/d/wiktionary/{}'.format(language)
url_title = heading['title'].replace(' ', '_')
web_url = 'http://{}.wiktionary.org/wiki/{}'.format(
language, url_title)
web_source = '/s/resource/wiktionary/{}'.format(language)
source = {'contributor': web_source, 'process': PARSER_RULE}
# Scan through the 'from' items, such as the start nodes of
# translations, looking for distinct etymologies. If we get more than
# one etymology for a language, we need to distinguish them as
# different senses in that language.
all_etyms = {
(item['from']['language'], etym_label(language, item['from']))
for item in items
if 'language' in item['from'] and item['from']['text'] == title
and etym_label(language, item['from']) is not None
}
word_languages = {wlang for (wlang, _) in all_etyms}
for wlang in sorted(word_languages):
cpage = standardized_concept_uri(wlang, title)
ld_edge = make_edge('/r/ExternalURL',
cpage,
web_url,
dataset=dataset,
weight=0.25,
sources=[source],
license=Licenses.cc_sharealike)
out.write(ld_edge)
etym_to_translation_sense = {}
language_etym_counts = Counter(lang for (lang, etym) in all_etyms)
polysemous_languages = {
lang
for lang in language_etym_counts if language_etym_counts[lang] > 1
}
for item in items:
tfrom = item['from']
tto = item['to']
assumed_languages = [language]
lang1 = tfrom.get('language')
lang2 = tto.get('language')
if lang1 and (lang1
not in assumed_languages) and valid_language(lang1):
assumed_languages.append(lang1)
if lang2 and (lang2
not in assumed_languages) and valid_language(lang2):
assumed_languages.append(lang2)
cfrom = transform_term(language,
tfrom,
assumed_languages,
db,
use_etyms=(lang1 in polysemous_languages))
cpage = cfrom
cto = transform_term(language,
tto,
assumed_languages,
db,
use_etyms=(lang2 in polysemous_languages))
if cfrom is None or cto is None:
continue
if uri_prefix(cfrom, 3) == uri_prefix(cto, 3):
continue
rel, switch = transform_relation(item['rel'])
if rel is None:
continue
if switch:
cfrom, cto = cto, cfrom
# When translations are separated by sense, use only the first
# sense we see for each etymology. That will have the most
# representative translations.
if item['rel'] == 'translation':
etym_key = (tfrom['language'], etym_label(language, tfrom))
sense = tfrom.get('sense', '')
if etym_key in etym_to_translation_sense:
if etym_to_translation_sense[etym_key] != sense:
continue
else:
etym_to_translation_sense[etym_key] = sense
weight = 1.
if rel == '/r/EtymologicallyRelatedTo':
weight = 0.25
edge = make_edge(rel,
cfrom,
cto,
dataset=dataset,
weight=weight,
sources=[source],
surfaceStart=tfrom['text'],
surfaceEnd=tto['text'],
license=Licenses.cc_sharealike)
out.write(edge)
out.close()
def run_wordnet(input_file, output_file):
out = MsgpackStreamWriter(output_file)
synset_senses = defaultdict(list)
sense_synsets = {}
synset_labels = defaultdict(list)
synset_canonical_labels = {}
synset_categories = {}
synset_domains = {}
synset_glosses = {}
synset_disambig = {}
synset_uris = {}
# First pass: find data about synsets
quads = parse_nquads(open(input_file, encoding="utf-8"))
for subj_dict, rel_dict, obj_dict, _graph in quads:
if "url" not in subj_dict or "url" not in rel_dict:
continue
subj = subj_dict["url"]
rel = rel_dict["url"]
obj = obj_dict.get("url")
objtext = obj_dict.get("text")
relname = resource_name(rel)
if relname == "label":
if obj_dict["lang"] == "en":
synset_labels[subj].append(objtext)
elif relname == "sameAs":
if obj.startswith(WN20_URL):
# If we have a link to RDF WordNet 2.0, the URL (URI? IRI?)
# will contain a standardized label for this concept, which
# we should use when we want to use this synset as the name of
# a disambiguation category. RDF WordNet 3.1 assigns synsets
# a number of labels in no particular order, making it hard to
# determine from 3.1 alone what to name a category.
objname = resource_name(obj)
parts = objname.split("-")[1:-2]
# Handle missing apostrophes
label = (
"-".join(parts)
.replace("_s_", "'s_")
.replace("_s-", "'s_")
.replace("s__", "s'_")
.replace("s_-", "s'-")
.replace("_", " ")
)
synset_canonical_labels[subj] = label
elif relname == "domain_category":
synset_categories[subj] = obj
elif relname == "lexical_domain":
target = resource_name(obj)
if "." in target:
domain = target.split(".")[1]
synset_domains[subj] = domain
elif relname == "gloss":
synset_glosses[subj] = objtext
elif relname == "reference":
lemma = resource_name(subj)
synset = obj
synset_senses[synset].append(lemma)
sense_synsets[lemma] = synset
used_labels = set(synset_canonical_labels.values())
for synset, values in synset_labels.items():
values.sort(key=lambda label: (label in used_labels,) + label_sort_key(label))
if (
synset not in synset_canonical_labels
or synset_canonical_labels[synset][0].isupper()
and synset_domains.get(synset) == "person"
):
label = values[0]
synset_canonical_labels[synset] = label
used_labels.add(label)
for synset, labels in synset_labels.items():
if synset in synset_categories:
category_name = synset_canonical_labels[synset_categories[synset]]
else:
category_name = synset_domains.get(synset, None)
synset_no_fragment = synset.split("#")[0]
pos = synset_no_fragment[-1].lower()
assert pos in "nvarsp", synset
if pos == "s":
pos = "a"
elif pos == "p":
pos = "-"
if category_name in ("pert", "all", "tops"):
category_name = None
synset_disambig[synset] = (pos, category_name)
canon = synset_canonical_labels[synset]
canon_uri = standardized_concept_uri("en", canon, pos, "wn", category_name)
synset_uris[synset] = canon_uri
for label in labels:
if label != canon:
other_uri = standardized_concept_uri(
"en", label, pos, "wn", category_name
)
rel_uri = "/r/Synonym"
surface = "[[{0}]] is a synonym of [[{1}]]".format(label, canon)
edge = make_edge(
rel_uri,
other_uri,
canon_uri,
dataset=DATASET,
surfaceText=surface,
license=Licenses.cc_attribution,
sources=[SOURCE],
weight=2.0,
)
out.write(edge)
quads = parse_nquads(open(input_file, encoding="utf-8"))
for subj_dict, rel_dict, obj_dict, _graph in quads:
if "url" not in subj_dict or "url" not in rel_dict:
continue
subj = subj_dict["url"]
rel = rel_dict["url"]
obj = obj_dict.get("url")
relname = resource_name(rel)
if relname in REL_MAPPING:
pos, sense = synset_disambig.get(subj, (None, None))
if relname == "hypernym" and pos == "v":
relname = "hypernym-v"
rel, frame = REL_MAPPING[relname]
reversed_frame = False
if rel.startswith("~"):
rel = rel[1:]
reversed_frame = True
rel_uri = "/r/" + rel
if obj is not None:
obj_uri = synset_uris.get(obj)
if obj not in synset_canonical_labels:
continue
obj_label = synset_canonical_labels[obj]
else:
text = obj_dict["text"]
# Some WordNets use strings with "!" in them to indicate
# out-of-band information, such as a missing translation
if (not text) or "!" in text:
continue
lang = obj_dict["lang"]
obj_uri = standardized_concept_uri(lang, text, pos, "wn", sense)
obj_label = text
if subj not in synset_uris or subj not in synset_canonical_labels:
continue
subj_uri = synset_uris[subj]
subj_label = synset_canonical_labels[subj]
license = Licenses.cc_attribution
langcode = subj_uri.split("/")[2]
if langcode in SHAREALIKE_LANGUAGES:
license = Licenses.cc_sharealike
if reversed_frame:
subj_uri, obj_uri = obj_uri, subj_uri
subj_label, obj_label = obj_label, subj_label
surface = frame.format("[[%s]]" % subj_label, "[[%s]]" % obj_label)
edge = make_edge(
rel_uri,
subj_uri,
obj_uri,
dataset=DATASET,
surfaceText=surface,
license=license,
sources=[SOURCE],
weight=2.0,
)
out.write(edge)
for wn_url in sorted(synset_uris):
cn_uri = synset_uris[wn_url]
edge = make_edge(
"/r/ExternalURL",
cn_uri,
wn_url,
dataset=DATASET,
license=Licenses.cc_sharealike,
sources=[SOURCE],
weight=1.0,
)
out.write(edge)
out.close()
def build_from_dir(dirname, output_file):
"""
Read a GlobalMind database exported in YAML files, translate
it into ConceptNet 5 edges, and write those edges to disk using
a MsgpackStreamWriter.
"""
out = MsgpackStreamWriter(output_file)
userdata = yaml.load_all(open(dirname + '/GMUser.yaml'))
users = {}
for userinfo in userdata:
users[userinfo['pk']] = userinfo
frame_data = yaml.load_all(open(dirname + '/GMFrame.yaml'))
frames = {}
for frame in frame_data:
frames[frame['pk']] = frame['fields']
assertiondata = yaml.load_all(open(dirname + '/GMAssertion.yaml'))
assertions = {}
for assertion in assertiondata:
obj = assertion['fields']
frame = frames[obj['frame']]
frametext = frame['text']
userinfo = users[obj['author']]
username = userinfo['fields']['username']
# As far as I can tell, GlobalMind used the same namespace of
# usernames as the original Open Mind.
user_source = "/s/contributor/omcs/%s" % username
sources = [user_source, "/s/activity/globalmind/assert"]
lang = LANG_CODES[obj['lcode']]
start = normalized_concept_uri(lang, obj['node1'])
end = normalized_concept_uri(lang, obj['node2'])
rel = '/r/' + RELATION_MAP.get(frame['relation'], frame['relation'])
# fix messy english "around in"
if ' around ' in frametext:
if obj['node2'].startswith('in '):
frametext = frametext.replace(' around ', ' in ')
obj['node2'] = obj['node2'][3:]
else:
frametext = frametext.replace(' around ', ' near ')
rel = '/r/LocatedNear'
# fix more awkward English. I wonder how bad the other languages are.
frametext = frametext.replace('hits your head', 'comes to mind')
frametext = frametext.replace(': [node1], [node2]',
' [node1] and [node2]')
node1 = u'[[' + obj['node1'] + u']]'
node2 = u'[[' + obj['node2'] + u']]'
surfaceText = frametext.replace('//', '').replace('[node1]',
node1).replace(
'[node2]', node2)
edge = make_edge(rel,
start,
end,
dataset='/d/globalmind',
license='/l/CC/By',
sources=sources,
surfaceText=surfaceText,
weight=1)
# Avoid duplication with the ConceptNet reader, but still save every edge so that we can
# handle translations.
if username != 'openmind':
out.write(edge)
assertions[assertion['pk']] = edge
translationdata = yaml.load_all(open(dirname + '/GMTranslation.yaml'))
for translation in translationdata:
obj = translation['fields']
assertion1 = assertions[obj['assertion1']]
assertion2 = assertions[obj['assertion2']]
start = assertion1['uri']
end = assertion2['uri']
rel = '/r/TranslationOf'
text1 = assertion1['surfaceText'].replace('[[', '').replace(']]', '')
text2 = assertion2['surfaceText'].replace('[[', '').replace(']]', '')
lang1 = LANG_NAMES[get_lang(assertion1)]
lang2 = LANG_NAMES[get_lang(assertion2)]
surfaceText = u"[[%s]] in %s means [[%s]] in %s." % (text1, lang1,
text2, lang2)
userinfo = users[obj['author']]
username = userinfo['fields']['username']
userlocale = userinfo['fields']['ccode'].lower()
if userlocale:
user_source = "/s/contributor/globalmind/%s/%s" % (userlocale,
username)
else:
user_source = "/s/contributor/globalmind/%s" % username
sources = [user_source, "/s/activity/globalmind/translate"]
edge = make_edge(rel,
start,
end,
dataset='/d/globalmind',
license=Licenses.cc_attribution,
sources=sources,
surfaceText=surfaceText,
weight=1)
out.write(edge)
def process_dbpedia(input_dir, output_file, concept_file):
"""
Read through multiple DBPedia files and output filtered assertions to
`output_file`.
"""
ok_concepts = read_concept_file(concept_file)
input_path = pathlib.Path(input_dir)
interlang_path = input_path / 'interlanguage_links_en.tql.bz2'
mapped_urls = interlanguage_mapping(interlang_path, ok_concepts)
out = MsgpackStreamWriter(output_file)
types_path = input_path / 'instance_types_en.tql.bz2'
quads = parse_nquads(bz2.open(str(types_path), 'rt'))
for subj, pred, obj, _graph in quads:
subj_url = subj['url']
if ('Category:' in subj_url or 'File:' in subj_url
or 'List_of' in subj_url or '__' in subj_url
or 'Template:' in subj_url):
continue
if subj_url in mapped_urls:
subj_concept = translate_dbpedia_url(subj_url)
obj_type = un_camel_case(resource_name(obj['url']))
if obj_type not in TYPE_BLACKLIST:
obj_concept = standardized_concept_uri('en', obj_type, 'n')
if obj_concept not in CONCEPT_BLACKLIST:
edge = make_edge('/r/IsA',
subj_concept,
obj_concept,
dataset='/d/dbpedia/en',
license=Licenses.cc_sharealike,
sources=[{
'contributor':
'/s/resource/dbpedia/2015/en'
}],
weight=0.5,
surfaceStart=url_to_label(subj['url']),
surfaceEnd=url_to_label(obj['url']))
out.write(edge)
for other_url in mapped_urls[subj_url]:
if other_url.startswith('http://wikidata.dbpedia.org/'):
urledge = make_edge('/r/ExternalURL',
subj_concept,
other_url,
dataset='/d/dbpedia/en',
license=Licenses.cc_sharealike,
sources=[{
'contributor':
'/s/resource/dbpedia/2015/en'
}],
weight=1.0)
out.write(urledge)
else:
other_concept = translate_dbpedia_url(other_url)
if other_concept:
urledge = make_edge(
'/r/ExternalURL',
other_concept,
other_url,
dataset='/d/dbpedia/en',
license=Licenses.cc_sharealike,
sources=[{
'contributor':
'/s/resource/dbpedia/2015/en'
}],
weight=1.0)
out.write(urledge)
edge = make_edge(
'/r/Synonym',
other_concept,
subj_concept,
dataset='/d/dbpedia/en',
license=Licenses.cc_sharealike,
sources=[{
'contributor':
'/s/resource/dbpedia/2015/en'
}],
weight=0.5,
surfaceStart=url_to_label(other_url),
surfaceEnd=url_to_label(subj_url))
out.write(edge)
relations_path = input_path / 'mappingbased_objects_en.tql.bz2'
quads = parse_nquads(bz2.open(str(relations_path), 'rt'))
for subj, pred, obj, _graph in quads:
subj_concept = translate_dbpedia_url(subj['url'])
obj_concept = translate_dbpedia_url(obj['url'])
rel_name = resource_name(pred['url'])
if (subj_concept and obj_concept and subj['url'] in mapped_urls
and obj['url'] in mapped_urls):
if rel_name in RELATIONS:
rel = RELATIONS[rel_name]
edge = make_edge(rel,
subj_concept,
obj_concept,
dataset='/d/dbpedia/en',
license=Licenses.cc_sharealike,
sources=[{
'contributor':
'/s/resource/dbpedia/2015/en'
}],
weight=0.5,
surfaceStart=url_to_label(subj['url']),
surfaceEnd=url_to_label(obj['url']))
out.write(edge)
out.close()
def run_wordnet(input_file, output_file):
out = MsgpackStreamWriter(output_file)
synset_senses = defaultdict(list)
sense_synsets = {}
synset_labels = defaultdict(list)
synset_canonical_labels = {}
synset_categories = {}
synset_domains = {}
synset_glosses = {}
synset_disambig = {}
synset_uris = {}
# First pass: find data about synsets
quads = parse_nquads(open(input_file, encoding='utf-8'))
for subj_dict, rel_dict, obj_dict, _graph in quads:
if 'url' not in subj_dict or 'url' not in rel_dict:
continue
subj = subj_dict['url']
rel = rel_dict['url']
obj = obj_dict.get('url')
objtext = obj_dict.get('text')
relname = resource_name(rel)
if relname == 'label':
if obj_dict['lang'] == 'en':
synset_labels[subj].append(objtext)
elif relname == 'sameAs':
if obj.startswith(WN20_URL):
# If we have a link to RDF WordNet 2.0, the URL (URI? IRI?)
# will contain a standardized label for this concept, which
# we should use when we want to use this synset as the name of
# a disambiguation category. RDF WordNet 3.1 assigns synsets
# a number of labels in no particular order, making it hard to
# determine from 3.1 alone what to name a category.
objname = resource_name(obj)
parts = objname.split('-')[1:-2]
# Handle missing apostrophes
label = '-'.join(parts).replace('_s_', "'s_").replace(
'_s-',
"'s_").replace("s__",
"s'_").replace("s_-",
"s'-").replace('_', ' ')
synset_canonical_labels[subj] = label
elif relname == 'domain_category':
synset_categories[subj] = obj
elif relname == 'lexical_domain':
target = resource_name(obj)
if '.' in target:
domain = target.split('.')[1]
synset_domains[subj] = domain
elif relname == 'gloss':
synset_glosses[subj] = objtext
elif relname == 'reference':
lemma = resource_name(subj)
synset = obj
synset_senses[synset].append(lemma)
sense_synsets[lemma] = synset
used_labels = set(synset_canonical_labels.values())
for synset, values in synset_labels.items():
values.sort(
key=lambda label: (label in used_labels, ) + label_sort_key(label))
if (synset not in synset_canonical_labels
or synset_canonical_labels[synset][0].isupper()
and synset_domains.get(synset) == 'person'):
label = values[0]
synset_canonical_labels[synset] = label
used_labels.add(label)
for synset, labels in synset_labels.items():
if synset in synset_categories:
category_name = synset_canonical_labels[synset_categories[synset]]
else:
category_name = synset_domains.get(synset, None)
synset_no_fragment = synset.split('#')[0]
pos = synset_no_fragment[-1].lower()
assert pos in 'nvarsp', synset
if pos == 's':
pos = 'a'
elif pos == 'p':
pos = '-'
if category_name in ('pert', 'all', 'tops'):
category_name = None
synset_disambig[synset] = (pos, category_name)
canon = synset_canonical_labels[synset]
canon_uri = standardized_concept_uri('en', canon, pos, 'wn',
category_name)
synset_uris[synset] = canon_uri
for label in labels:
if label != canon:
other_uri = standardized_concept_uri('en', label, pos, 'wn',
category_name)
rel_uri = '/r/Synonym'
surface = '[[{0}]] is a synonym of [[{1}]]'.format(
label, canon)
edge = make_edge(rel_uri,
other_uri,
canon_uri,
dataset=DATASET,
surfaceText=surface,
license=Licenses.cc_attribution,
sources=[SOURCE],
weight=2.0)
out.write(edge)
quads = parse_nquads(open(input_file, encoding='utf-8'))
for subj_dict, rel_dict, obj_dict, _graph in quads:
if 'url' not in subj_dict or 'url' not in rel_dict:
continue
subj = subj_dict['url']
rel = rel_dict['url']
obj = obj_dict.get('url')
relname = resource_name(rel)
if relname in REL_MAPPING:
rel, frame = REL_MAPPING[relname]
reversed_frame = False
if rel.startswith('~'):
rel = rel[1:]
reversed_frame = True
rel_uri = '/r/' + rel
if obj is not None:
obj_uri = synset_uris.get(obj)
if obj not in synset_canonical_labels:
continue
obj_label = synset_canonical_labels[obj]
else:
text = obj_dict['text']
# Some WordNets use strings with "!" in them to indicate
# out-of-band information, such as a missing translation
if (not text) or '!' in text:
continue
lang = obj_dict['lang']
pos, sense = synset_disambig.get(subj, (None, None))
obj_uri = standardized_concept_uri(lang, text, pos, 'wn',
sense)
obj_label = text
if subj not in synset_uris or subj not in synset_canonical_labels:
continue
subj_uri = synset_uris[subj]
subj_label = synset_canonical_labels[subj]
license = Licenses.cc_attribution
langcode = subj_uri.split('/')[2]
if langcode in SHAREALIKE_LANGUAGES:
license = Licenses.cc_sharealike
if reversed_frame:
subj_uri, obj_uri = obj_uri, subj_uri
subj_label, obj_label = obj_label, subj_label
surface = frame.format('[[%s]]' % subj_label, '[[%s]]' % obj_label)
edge = make_edge(rel_uri,
subj_uri,
obj_uri,
dataset=DATASET,
surfaceText=surface,
license=license,
sources=[SOURCE],
weight=2.0)
out.write(edge)
for wn_url in sorted(synset_uris):
cn_uri = synset_uris[wn_url]
edge = make_edge('/r/ExternalURL',
cn_uri,
wn_url,
dataset=DATASET,
license=Licenses.cc_sharealike,
sources=[SOURCE],
weight=1.0)
out.write(edge)
out.close()
def convert_to_msgpack(input_filename, output_filename):
out_stream = MsgpackStreamWriter(output_filename)
for obj in read_json_stream(input_filename):
out_stream.write(obj)
out_stream.close()
