def output_sense_translation(self, lang, foreign, english, sense):
pos, disambiguation = sense
if 'Wik' in foreign or 'Wik' in english or term_is_bad(
foreign) or term_is_bad(english):
return
# Quick fix that drops definitions written in Lojban syntax
if lang == 'jbo' and re.search(r'x[1-5]', english):
return
if lang == 'zh-cn':
lang = 'zh_CN'
elif lang == 'zh-tw':
lang = 'zh_TW'
source = normalized_concept_uri(lang,
unicodedata.normalize('NFKC', foreign))
target = normalized_concept_uri('en', english, pos, disambiguation)
relation = '/r/TranslationOf'
try:
surfaceRel = "is %s for" % (
CODE_TO_ENGLISH_NAME[lang.split('_')[0]])
except KeyError:
surfaceRel = "is [language %s] for" % lang
surfaceText = "[[%s]] %s [[%s (%s)]]" % (
foreign, surfaceRel, english,
disambiguation.split('/')[-1].replace('_', ' '))
edge = make_edge(relation,
source,
target,
'/d/wiktionary/en/%s' % lang,
license=Licenses.cc_sharealike,
sources=[SOURCE, TRANSLATE],
weight=1.0,
surfaceText=surfaceText)
self.writer.write(edge)
def output_sense_translation(self, lang, foreign, english, sense):
pos, disambiguation = sense
if 'Wik' in foreign or 'Wik' in english or term_is_bad(foreign) or term_is_bad(english):
return
# Quick fix that drops definitions written in Lojban syntax
if lang == 'jbo' and re.search(r'x[1-5]', english):
return
if lang == 'zh-cn':
lang = 'zh_CN'
elif lang == 'zh-tw':
lang = 'zh_TW'
source = normalized_concept_uri(
lang, unicodedata.normalize('NFKC', foreign)
)
target = normalized_concept_uri(
'en', english, pos, disambiguation
)
relation = '/r/TranslationOf'
try:
surfaceRel = "is %s for" % (CODE_TO_ENGLISH_NAME[lang.split('_')[0]])
except KeyError:
surfaceRel = "is [language %s] for" % lang
surfaceText = "[[%s]] %s [[%s (%s)]]" % (foreign, surfaceRel, english, disambiguation.split('/')[-1].replace('_', ' '))
edge = make_edge(relation, source, target, '/d/wiktionary/en/%s' % lang,
license=Licenses.cc_sharealike,
sources=[SOURCE, TRANSLATE],
weight=1.0,
surfaceText=surfaceText)
self.writer.write(edge)
def output_translation(self, foreign, english, locale=''):
if term_is_bad(foreign) or term_is_bad(english):
return
# Quick fix that drops definitions written in Lojban syntax
if self.langcode == 'jbo' and re.search(r'x[1-5]', english):
return
source = normalized_concept_uri(
self.langcode + locale,
foreign
)
target = normalized_concept_uri(
'en', english
)
relation = '/r/TranslationOf'
try:
surfaceRel = "is %s for" % (CODE_TO_ENGLISH_NAME[self.langcode.split('_')[0]])
except KeyError:
surfaceRel = "is [language %s] for" % self.langcode
surfaceText = "[[%s]] %s [[%s]]" % (foreign, surfaceRel, english)
edge = make_edge(relation, source, target, '/d/wiktionary/en/%s' % self.langcode,
license=Licenses.cc_sharealike,
sources=[SOURCE, INTERLINGUAL],
weight=1.0,
surfaceText=surfaceText)
self.writer.write(edge)
def handle_raw_assertion(line):
parts = line.split(', ')
user, frame_id, concept1, concept2 = parts
fdata = FRAME_DATA[frame_id]
ftext = fdata['text']
rel = fdata['relation']
surfaceText = ftext.replace('{1}', '[[' + concept1 + ']]').replace('{2}', '[[' + concept2 + ']]')
start = normalized_concept_uri('zh_TW', concept1)
end = normalized_concept_uri('zh_TW', concept2)
sources = ['/s/activity/ptt/petgame', '/s/contributor/petgame/' + user]
yield make_edge(rel, start, end, dataset='/d/conceptnet/4/zh',
license='/l/CC/By', sources=sources,
surfaceText=surfaceText, weight=1)
def output_monolingual(self, lang, relation, term1, term2):
if term_is_bad(term1) or term_is_bad(term2):
return
source = normalized_concept_uri(lang, term1)
if self.pos:
target = normalized_concept_uri(lang, term2, self.pos)
else:
target = normalized_concept_uri(lang, term2)
surfaceText = "[[%s]] %s [[%s]]" % (term1, relation, term2)
edge = make_edge('/r/'+relation, source, target, '/d/wiktionary/%s/%s' % (lang, lang),
license=Licenses.cc_sharealike,
sources=[SOURCE, MONOLINGUAL],
weight=1.0,
surfaceText=surfaceText)
self.writer.write(edge)
def normalize():
language = flask.request.args.get('language')
term = flask.request.args.get('term').replace('_', ' ')
if term is None or language is None:
return flask.jsonify({
'error': 'Invalid request',
'details': "You should include the 'term' and 'language' parameters"
}), 400
uri = normalized_concept_uri(language, term)
return flask.jsonify(uri=uri)
def output_monolingual(self, lang, relation, term1, term2):
if term_is_bad(term1) or term_is_bad(term2):
return
source = normalized_concept_uri(lang, term1)
if self.pos:
target = normalized_concept_uri(lang, term2, self.pos)
else:
target = normalized_concept_uri(lang, term2)
surfaceText = "[[%s]] %s [[%s]]" % (term1, relation, term2)
edge = make_edge('/r/' + relation,
source,
target,
'/d/wiktionary/%s/%s' % (lang, lang),
license=Licenses.cc_sharealike,
sources=[SOURCE, MONOLINGUAL],
weight=1.0,
surfaceText=surfaceText)
self.writer.write(edge)
def handle_raw_assertion(line):
parts = line.split(', ')
user, frame_id, concept1, concept2 = parts
fdata = FRAME_DATA[frame_id]
ftext = fdata['text']
rel = fdata['relation']
surfaceText = ftext.replace('{1}', '[[' + concept1 + ']]').replace(
'{2}', '[[' + concept2 + ']]')
start = normalized_concept_uri('zh_TW', concept1)
end = normalized_concept_uri('zh_TW', concept2)
sources = ['/s/activity/ptt/petgame', '/s/contributor/petgame/' + user]
yield make_edge(rel,
start,
end,
dataset='/d/conceptnet/4/zh',
license='/l/CC/By',
sources=sources,
surfaceText=surfaceText,
weight=1)
def normalize():
language = flask.request.args.get('language')
term = flask.request.args.get('term').replace('_', ' ')
if term is None or language is None:
return flask.jsonify({
'error':
'Invalid request',
'details':
"You should include the 'term' and 'language' parameters"
}), 400
uri = normalized_concept_uri(language, term)
return flask.jsonify(uri=uri)
def output_translation(self, foreign, english, locale=''):
if term_is_bad(foreign) or term_is_bad(english):
return
# Quick fix that drops definitions written in Lojban syntax
if self.langcode == 'jbo' and re.search(r'x[1-5]', english):
return
source = normalized_concept_uri(self.langcode + locale, foreign)
target = normalized_concept_uri('en', english)
relation = '/r/TranslationOf'
try:
surfaceRel = "is %s for" % (
CODE_TO_ENGLISH_NAME[self.langcode.split('_')[0]])
except KeyError:
surfaceRel = "is [language %s] for" % self.langcode
surfaceText = "[[%s]] %s [[%s]]" % (foreign, surfaceRel, english)
edge = make_edge(relation,
source,
target,
'/d/wiktionary/en/%s' % self.langcode,
license=Licenses.cc_sharealike,
sources=[SOURCE, INTERLINGUAL],
weight=1.0,
surfaceText=surfaceText)
self.writer.write(edge)
def translate_dbpedia_url(url, lang='en'):
"""
Convert an object that's defined by a DBPedia URL to a ConceptNet
URI. We do this by finding the part of the URL that names the object,
and using that as surface text for ConceptNet.
This is, in some ways, abusing a naming convention in the Semantic Web.
The URL of an object doesn't have to mean anything at all. The
human-readable name is supposed to be a string, specified by the "name"
relation.
The problem here is that the "name" relation is not unique in either
direction. A URL can have many names, and the same name can refer to
many URLs, and some of these names are the result of parsing glitches.
The URL itself is a stable thing that we can build a ConceptNet URI from,
on the other hand.
"""
# Some Semantic Web URLs are camel-cased. ConceptNet URIs use underscores
# between words.
pieces = parse_topic_name(resource_name(url))
pieces[0] = un_camel_case(pieces[0])
return normalized_concept_uri(lang, *pieces)
def build_from_dir(dirname, output_file):
"""
Read a GlobalMind database exported in YAML files, translate
it into ConceptNet 5 edges, and write those edges to disk using
a JSONStreamWriter.
"""
out = JSONStreamWriter(output_file)
userdata = yaml.load_all(open(dirname + '/GMUser.yaml'))
users = {}
for userinfo in userdata:
users[userinfo['pk']] = userinfo
frame_data = yaml.load_all(open(dirname + '/GMFrame.yaml'))
frames = {}
for frame in frame_data:
frames[frame['pk']] = frame['fields']
assertiondata = yaml.load_all(open(dirname + '/GMAssertion.yaml'))
assertions = {}
for assertion in assertiondata:
obj = assertion['fields']
frame = frames[obj['frame']]
frametext = frame['text']
userinfo = users[obj['author']]
username = userinfo['fields']['username']
# GlobalMind provides information about what country the user is from, which
# we can preserve in the contributor URI.
#
# If I got to re-choose these URIs, I would distinguish usernames with
# a country code from those without a country code by something more
# than the number of slashes, and I would write the country code in
# capital letters.
userlocale = userinfo['fields']['ccode'].lower()
if userlocale:
user_source = "/s/contributor/globalmind/%s/%s" % (userlocale, username)
else:
user_source = "/s/contributor/globalmind/%s" % username
sources = [
user_source,
"/s/activity/globalmind/assert"
]
lang = LANG_CODES[obj['lcode']]
start = normalized_concept_uri(lang, obj['node1'])
end = normalized_concept_uri(lang, obj['node2'])
rel = '/r/' + RELATION_MAP.get(frame['relation'], frame['relation'])
# fix messy english "around in"
if ' around ' in frametext:
if obj['node2'].startswith('in '):
frametext = frametext.replace(' around ', ' in ')
obj['node2'] = obj['node2'][3:]
else:
frametext = frametext.replace(' around ', ' near ')
rel = '/r/LocatedNear'
# fix more awkward English. I wonder how bad the other languages are.
frametext = frametext.replace('hits your head', 'comes to mind')
frametext = frametext.replace(': [node1], [node2]', ' [node1] and [node2]')
node1 = u'[[' + obj['node1'] + u']]'
node2 = u'[[' + obj['node2'] + u']]'
surfaceText = frametext.replace('//', '').replace('[node1]', node1).replace('[node2]', node2)
edge = make_edge(rel, start, end,
dataset='/d/globalmind',
license='/l/CC/By',
sources=sources,
surfaceText=surfaceText,
weight=1)
out.write(edge)
assertions[assertion['pk']] = edge
translationdata = yaml.load_all(open(dirname + '/GMTranslation.yaml'))
for translation in translationdata:
obj = translation['fields']
assertion1 = assertions[obj['assertion1']]
assertion2 = assertions[obj['assertion2']]
start = assertion1['uri']
end = assertion2['uri']
rel = '/r/TranslationOf'
text1 = assertion1['surfaceText'].replace('[[', '').replace(']]', '')
text2 = assertion2['surfaceText'].replace('[[', '').replace(']]', '')
lang1 = LANG_NAMES[get_lang(assertion1)]
lang2 = LANG_NAMES[get_lang(assertion2)]
surfaceText = u"[[%s]] in %s means [[%s]] in %s." % (text1, lang1, text2, lang2)
userinfo = users[obj['author']]
username = userinfo['fields']['username']
userlocale = userinfo['fields']['ccode'].lower()
if userlocale:
user_source = "/s/contributor/globalmind/%s/%s" % (userlocale, username)
else:
user_source = "/s/contributor/globalmind/%s" % username
sources = [
user_source,
"/s/activity/globalmind/translate"
]
edge = make_edge(rel, start, end,
dataset='/d/globalmind',
license=Licenses.cc_attribution,
sources=sources,
surfaceText=surfaceText,
weight=1)
out.write(edge)
def build_start(parts_dict):
lang = parts_dict['lang']
startText = parts_dict["startText"]
start = normalized_concept_uri(lang, startText)
return start
def build_end(parts_dict):
lang = parts_dict['lang']
endText = parts_dict["endText"]
end = normalized_concept_uri(lang, endText)
return end
def run_wordnet(input_dir, output_file, sw_map_file):
out = MsgpackStreamWriter(output_file)
map_out = NTriplesWriter(sw_map_file)
reader = NTriplesReader()
synset_senses = defaultdict(list)
sense_synsets = {}
labels = {}
glossary = {}
concept_map = {}
sense_to_synset = {}
# Parse lines such as:
# wn30:synset-Aeolian-noun-2 rdfs:label "Aeolian"@en-us .
for subj, rel, obj, objtag in reader.parse_file(os.path.join(input_dir, 'wordnet-synset.ttl')):
if resource_name(rel) == 'label':
# Everything in WordNet is in English
assert objtag == 'en'
labels[subj] = obj
for subj, rel, obj, objtag in reader.parse_file(os.path.join(input_dir, 'wordnet-glossary.ttl')):
if resource_name(rel) == 'gloss':
assert objtag == 'en'
# Take the definition up to the first semicolon
text = obj.split(';')[0]
# Remove introductory phrases with a colon
text = text.split(': ', 1)[-1]
# Remove parenthesized expressions
while True:
newtext = re.sub(r'\(.+?\) ?', '', text).strip()
if newtext == text or newtext == '':
break
else:
text = newtext
glossary[subj] = text.replace('/', '_')
# Get the list of word senses in each synset, and make a bidirectional mapping.
#
# Example line:
# wn30:synset-Aeolian-noun-2 wn20schema:containsWordSense wn30:wordsense-Aeolian-noun-2 .
for subj, rel, obj, objtag in reader.parse_file(os.path.join(input_dir, 'full/wordnet-wordsense-synset-relations.ttl')):
if resource_name(rel) == 'containsWordSense':
synset_senses[subj].append(obj)
sense_synsets[obj] = subj
# Assign every synset to a disambiguated concept.
for synset in synset_senses:
synset_name = labels[synset]
synset_pos = synset.split('-')[-2]
pos = PARTS_OF_SPEECH[synset_pos]
disambig = glossary[synset]
concept = normalized_concept_uri('en', synset_name, pos, disambig)
concept_map[synset] = concept
# Map senses to their synsets.
for sense, synset in sense_synsets.items():
sense_to_synset[sense] = synset
for filename in (
'wordnet-attribute.ttl', 'wordnet-causes.ttl',
'wordnet-classifiedby.ttl', 'wordnet-entailment.ttl',
'wordnet-hyponym.ttl', 'wordnet-instances.ttl',
'wordnet-membermeronym.ttl', 'wordnet-partmeronym.ttl',
'wordnet-sameverbgroupas.ttl', 'wordnet-similarity.ttl',
'wordnet-substancemeronym.ttl', 'full/wordnet-antonym.ttl',
'full/wordnet-derivationallyrelated.ttl',
'full/wordnet-participleof.ttl',
'full/wordnet-pertainsto.ttl',
'full/wordnet-seealso.ttl'
):
filepath = os.path.join(input_dir, filename)
if os.path.exists(filepath):
for web_subj, web_rel, web_obj, objtag in reader.parse_file(filepath):
# If this relation involves word senses, map them to their synsets
# first.
if web_subj in sense_to_synset:
web_subj = sense_to_synset[web_subj]
if web_obj in sense_to_synset:
web_obj = sense_to_synset[web_obj]
subj = concept_map[web_subj]
obj = concept_map[web_obj]
pred_label = resource_name(web_rel)
if pred_label in REL_MAPPING:
mapped_rel = REL_MAPPING[pred_label]
# Handle WordNet relations that are the reverse of ConceptNet
# relations. Change the word 'meronym' to 'holonym' if
# necessary.
if mapped_rel.startswith('~'):
subj, obj = obj, subj
web_subj, web_obj = web_obj, web_subj
web_rel = web_rel.replace('meronym', 'holonym')
mapped_rel = mapped_rel[1:]
rel = join_uri('r', mapped_rel)
else:
rel = join_uri('r', 'wordnet', pred_label)
map_out.write_link(web_rel, full_conceptnet_url(rel))
map_out.write_link(web_subj, full_conceptnet_url(subj))
map_out.write_link(web_obj, full_conceptnet_url(obj))
edge = make_edge(
rel, subj, obj, dataset='/d/wordnet/3.0',
license='/l/CC/By', sources=SOURCE, weight=2.0
)
out.write(edge)
def web_search():
keyword = request.form.get('keyword')
lang = request.form.get('language')
return redirect(WEB_ROOT + normalized_concept_uri(lang, keyword))
def build_from_dir(dirname, output_file):
"""
Read a GlobalMind database exported in YAML files, translate
it into ConceptNet 5 edges, and write those edges to disk using
a MsgpackStreamWriter.
"""
out = MsgpackStreamWriter(output_file)
userdata = yaml.load_all(open(dirname + '/GMUser.yaml'))
users = {}
for userinfo in userdata:
users[userinfo['pk']] = userinfo
frame_data = yaml.load_all(open(dirname + '/GMFrame.yaml'))
frames = {}
for frame in frame_data:
frames[frame['pk']] = frame['fields']
assertiondata = yaml.load_all(open(dirname + '/GMAssertion.yaml'))
assertions = {}
for assertion in assertiondata:
obj = assertion['fields']
frame = frames[obj['frame']]
frametext = frame['text']
userinfo = users[obj['author']]
username = userinfo['fields']['username']
# As far as I can tell, GlobalMind used the same namespace of
# usernames as the original Open Mind.
user_source = "/s/contributor/omcs/%s" % username
sources = [
user_source,
"/s/activity/globalmind/assert"
]
lang = LANG_CODES[obj['lcode']]
start = normalized_concept_uri(lang, obj['node1'])
end = normalized_concept_uri(lang, obj['node2'])
rel = '/r/' + RELATION_MAP.get(frame['relation'], frame['relation'])
# fix messy english "around in"
if ' around ' in frametext:
if obj['node2'].startswith('in '):
frametext = frametext.replace(' around ', ' in ')
obj['node2'] = obj['node2'][3:]
else:
frametext = frametext.replace(' around ', ' near ')
rel = '/r/LocatedNear'
# fix more awkward English. I wonder how bad the other languages are.
frametext = frametext.replace('hits your head', 'comes to mind')
frametext = frametext.replace(': [node1], [node2]', ' [node1] and [node2]')
node1 = u'[[' + obj['node1'] + u']]'
node2 = u'[[' + obj['node2'] + u']]'
surfaceText = frametext.replace('//', '').replace('[node1]', node1).replace('[node2]', node2)
edge = make_edge(rel, start, end,
dataset='/d/globalmind',
license='/l/CC/By',
sources=sources,
surfaceText=surfaceText,
weight=1)
# Avoid duplication with the ConceptNet reader, but still save every edge so that we can
# handle translations.
if username != 'openmind':
out.write(edge)
assertions[assertion['pk']] = edge
translationdata = yaml.load_all(open(dirname + '/GMTranslation.yaml'))
for translation in translationdata:
obj = translation['fields']
assertion1 = assertions[obj['assertion1']]
assertion2 = assertions[obj['assertion2']]
start = assertion1['uri']
end = assertion2['uri']
rel = '/r/TranslationOf'
text1 = assertion1['surfaceText'].replace('[[', '').replace(']]', '')
text2 = assertion2['surfaceText'].replace('[[', '').replace(']]', '')
lang1 = LANG_NAMES[get_lang(assertion1)]
lang2 = LANG_NAMES[get_lang(assertion2)]
surfaceText = u"[[%s]] in %s means [[%s]] in %s." % (text1, lang1, text2, lang2)
userinfo = users[obj['author']]
username = userinfo['fields']['username']
userlocale = userinfo['fields']['ccode'].lower()
if userlocale:
user_source = "/s/contributor/globalmind/%s/%s" % (userlocale, username)
else:
user_source = "/s/contributor/globalmind/%s" % username
sources = [
user_source,
"/s/activity/globalmind/translate"
]
edge = make_edge(rel, start, end,
dataset='/d/globalmind',
license=Licenses.cc_attribution,
sources=sources,
surfaceText=surfaceText,
weight=1)
out.write(edge)
def build_from_dir(dirname, output_file):
"""
Read a GlobalMind database exported in YAML files, translate
it into ConceptNet 5 edges, and write those edges to disk using
a JSONStreamWriter.
"""
out = JSONStreamWriter(output_file)
userdata = yaml.load_all(open(dirname + '/GMUser.yaml'))
users = {}
for userinfo in userdata:
users[userinfo['pk']] = userinfo
frame_data = yaml.load_all(open(dirname + '/GMFrame.yaml'))
frames = {}
for frame in frame_data:
frames[frame['pk']] = frame['fields']
assertiondata = yaml.load_all(open(dirname + '/GMAssertion.yaml'))
assertions = {}
for assertion in assertiondata:
obj = assertion['fields']
frame = frames[obj['frame']]
frametext = frame['text']
userinfo = users[obj['author']]
username = userinfo['fields']['username']
# GlobalMind provides information about what country the user is from, which
# we can preserve in the contributor URI.
#
# If I got to re-choose these URIs, I would distinguish usernames with
# a country code from those without a country code by something more
# than the number of slashes, and I would write the country code in
# capital letters.
userlocale = userinfo['fields']['ccode'].lower()
if userlocale:
user_source = "/s/contributor/globalmind/%s/%s" % (userlocale,
username)
else:
user_source = "/s/contributor/globalmind/%s" % username
sources = [user_source, "/s/activity/globalmind/assert"]
lang = LANG_CODES[obj['lcode']]
start = normalized_concept_uri(lang, obj['node1'])
end = normalized_concept_uri(lang, obj['node2'])
rel = '/r/' + RELATION_MAP.get(frame['relation'], frame['relation'])
# fix messy english "around in"
if ' around ' in frametext:
if obj['node2'].startswith('in '):
frametext = frametext.replace(' around ', ' in ')
obj['node2'] = obj['node2'][3:]
else:
frametext = frametext.replace(' around ', ' near ')
rel = '/r/LocatedNear'
# fix more awkward English. I wonder how bad the other languages are.
frametext = frametext.replace('hits your head', 'comes to mind')
frametext = frametext.replace(': [node1], [node2]',
' [node1] and [node2]')
node1 = u'[[' + obj['node1'] + u']]'
node2 = u'[[' + obj['node2'] + u']]'
surfaceText = frametext.replace('//', '').replace('[node1]',
node1).replace(
'[node2]', node2)
edge = make_edge(rel,
start,
end,
dataset='/d/globalmind',
license='/l/CC/By',
sources=sources,
surfaceText=surfaceText,
weight=1)
out.write(edge)
assertions[assertion['pk']] = edge
translationdata = yaml.load_all(open(dirname + '/GMTranslation.yaml'))
for translation in translationdata:
obj = translation['fields']
assertion1 = assertions[obj['assertion1']]
assertion2 = assertions[obj['assertion2']]
start = assertion1['uri']
end = assertion2['uri']
rel = '/r/TranslationOf'
text1 = assertion1['surfaceText'].replace('[[', '').replace(']]', '')
text2 = assertion2['surfaceText'].replace('[[', '').replace(']]', '')
lang1 = LANG_NAMES[get_lang(assertion1)]
lang2 = LANG_NAMES[get_lang(assertion2)]
surfaceText = u"[[%s]] in %s means [[%s]] in %s." % (text1, lang1,
text2, lang2)
userinfo = users[obj['author']]
username = userinfo['fields']['username']
userlocale = userinfo['fields']['ccode'].lower()
if userlocale:
user_source = "/s/contributor/globalmind/%s/%s" % (userlocale,
username)
else:
user_source = "/s/contributor/globalmind/%s" % username
sources = [user_source, "/s/activity/globalmind/translate"]
edge = make_edge(rel,
start,
end,
dataset='/d/globalmind',
license=Licenses.cc_attribution,
sources=sources,
surfaceText=surfaceText,
weight=1)
out.write(edge)
def handle_file(filename, output_file):
"""
JMdict is a Japanese translation dictionary, targeting multiple languages,
released under a Creative Commons Attribution-ShareAlike license. That's
awesome.
It's released as a kind of annoying XML structure, using fancy XML features
like entities, so in order to read it we need a full-blown XML parser. Python's
built-in XML parsers are horrifying, so here we use the 'xmltodict' module, which
is also horrifying but gets the job done.
The majorly weird thing about xmltodict that we have to work around is that
it gives results of different *types* when you get 0, 1, or many child nodes.
This is what get_list is for.
"""
# Read the XML file as UTF-8, and parse it into a dictionary.
file = codecs.open(filename, encoding="utf-8")
out = JSONStreamWriter(output_file)
data = file.read()
file.close()
xml = xmltodict.parse(data)
# The dictionary contains a list of <entry> tags.
root_node = xml["JMdict"]
for entry in get_list(root_node, "entry"):
# From JMdict's documentation: "The kanji element, or in its absence,
# the reading element, is the defining component of each entry."
#
# Quick summary of what's going on here: most Japanese words can be
# written using kanji or kana.
#
# Kana are phonetic characters. Every word can be written in kana, in
# one of two alphabets (hiragana or katakana). Words that are homonyms
# have the same kana, unless they're written in different alphabets.
#
# Kanji are Chinese-based characters that are related to the meaning of
# the word. They're compact and good at disambiguating homonyms, so
# kanji are usually used as the canonical representation of a word.
# However, some words have no kanji.
#
# The kana version of a word written in kanji is called its 'reading'.
# Words that are pronounced differently in different contexts have
# multiple readings.
#
# Okay, done with the intro to Japanese orthography. In JMdict, if
# a word can be written in kanji, it has a <k_ele> element, containing
# a <keb> element that contains the text. Every word also has an
# <r_ele> element, containing one or more <reb> elements that are phonetic
# readings of the word.
#
# We get the "defining text" of a word by taking its <keb> if it exists,
# or all of its <reb>s if not. There's no way to tell which <reb> is the
# most "defining" in the case where there's no <keb>.
headwords = [word["keb"] for word in get_list(entry, "k_ele")]
if not headwords:
headwords = [word["reb"] for word in get_list(entry, "r_ele")]
# An entry might have different word senses that are translated
# differently to other languages. Ideally, we'd remember that they're
# different senses. However, we have no way to refer to the different
# senses. So for now, we disregard word senses. One day we might have
# a better overall plan for word senses in ConceptNet.
for sense in get_list(entry, "sense"):
# Glosses are translations of the word to different languages.
# If the word is a loan-word, the foreign word it was derived from
# will be marked with the <lsource> tag instead of <gloss>.
#
# Get all the glosses, including the lsource if it's there.
glosses = get_list(sense, "gloss") + get_list(sense, "lsource")
for gloss in glosses:
text = lang = None
if "#text" in gloss:
# A gloss node might be marked with a 'lang' attribute. If so,
# xmltodict represents it as a dictionary with '#text' and
# '@xml:lang' elements.
text = parse_gloss(gloss["#text"])
lang = convert_lang_code(gloss["@xml:lang"])
elif isinstance(gloss, STRING_TYPE):
# If there's no 'lang' attribute, the gloss is in English,
# and xmltodict gives it to us as a plain Unicode string.
lang = "en"
text = parse_gloss(gloss)
# If we parsed the node at all and the text looks good, then we can
# add edges to ConceptNet.
#
# We don't want to deal with texts with periods (these might be
# dictionary-style abbreviations, which are sort of unhelpful when
# we can't expand them), and we also don't want to deal with texts
# that are more than five words long.
if text is not None and "." not in text and text.count(" ") <= 4 and text not in BAD_NAMES_FOR_THINGS:
for head in headwords:
ja_concept = normalized_concept_uri("ja", head)
other_concept = normalized_concept_uri(lang, text)
output_edge(out, ja_concept, other_concept)
def build_from_dir(dirname, output_file):
"""
Read a GlobalMind database exported in YAML files, translate
it into ConceptNet 5 edges, and write those edges to disk using
a MsgpackStreamWriter.
"""
out = MsgpackStreamWriter(output_file)
userdata = yaml.load_all(open(dirname + '/GMUser.yaml'))
users = {}
for userinfo in userdata:
users[userinfo['pk']] = userinfo
frame_data = yaml.load_all(open(dirname + '/GMFrame.yaml'))
frames = {}
for frame in frame_data:
frames[frame['pk']] = frame['fields']
assertiondata = yaml.load_all(open(dirname + '/GMAssertion.yaml'))
assertions = {}
for assertion in assertiondata:
obj = assertion['fields']
frame = frames[obj['frame']]
frametext = frame['text']
userinfo = users[obj['author']]
username = userinfo['fields']['username']
# As far as I can tell, GlobalMind used the same namespace of
# usernames as the original Open Mind.
user_source = "/s/contributor/omcs/%s" % username
sources = [user_source, "/s/activity/globalmind/assert"]
lang = LANG_CODES[obj['lcode']]
start = normalized_concept_uri(lang, obj['node1'])
end = normalized_concept_uri(lang, obj['node2'])
rel = '/r/' + RELATION_MAP.get(frame['relation'], frame['relation'])
# fix messy english "around in"
if ' around ' in frametext:
if obj['node2'].startswith('in '):
frametext = frametext.replace(' around ', ' in ')
obj['node2'] = obj['node2'][3:]
else:
frametext = frametext.replace(' around ', ' near ')
rel = '/r/LocatedNear'
# fix more awkward English. I wonder how bad the other languages are.
frametext = frametext.replace('hits your head', 'comes to mind')
frametext = frametext.replace(': [node1], [node2]',
' [node1] and [node2]')
node1 = u'[[' + obj['node1'] + u']]'
node2 = u'[[' + obj['node2'] + u']]'
surfaceText = frametext.replace('//', '').replace('[node1]',
node1).replace(
'[node2]', node2)
edge = make_edge(rel,
start,
end,
dataset='/d/globalmind',
license='/l/CC/By',
sources=sources,
surfaceText=surfaceText,
weight=1)
# Avoid duplication with the ConceptNet reader, but still save every edge so that we can
# handle translations.
if username != 'openmind':
out.write(edge)
assertions[assertion['pk']] = edge
translationdata = yaml.load_all(open(dirname + '/GMTranslation.yaml'))
for translation in translationdata:
obj = translation['fields']
assertion1 = assertions[obj['assertion1']]
assertion2 = assertions[obj['assertion2']]
start = assertion1['uri']
end = assertion2['uri']
rel = '/r/TranslationOf'
text1 = assertion1['surfaceText'].replace('[[', '').replace(']]', '')
text2 = assertion2['surfaceText'].replace('[[', '').replace(']]', '')
lang1 = LANG_NAMES[get_lang(assertion1)]
lang2 = LANG_NAMES[get_lang(assertion2)]
surfaceText = u"[[%s]] in %s means [[%s]] in %s." % (text1, lang1,
text2, lang2)
userinfo = users[obj['author']]
username = userinfo['fields']['username']
userlocale = userinfo['fields']['ccode'].lower()
if userlocale:
user_source = "/s/contributor/globalmind/%s/%s" % (userlocale,
username)
else:
user_source = "/s/contributor/globalmind/%s" % username
sources = [user_source, "/s/activity/globalmind/translate"]
edge = make_edge(rel,
start,
end,
dataset='/d/globalmind',
license=Licenses.cc_attribution,
sources=sources,
surfaceText=surfaceText,
weight=1)
out.write(edge)
def search():
keyword = request.form.get('keyword')
lang = request.form.get('language')
return redirect(site + web_root + normalized_concept_uri(lang, keyword))
def handle_file(filename, output_file):
"""
JMdict is a Japanese translation dictionary, targeting multiple languages,
released under a Creative Commons Attribution-ShareAlike license. That's
awesome.
It's released as a kind of annoying XML structure, using fancy XML features
like entities, so in order to read it we need a full-blown XML parser. Python's
built-in XML parsers are horrifying, so here we use the 'xmltodict' module, which
is also horrifying but gets the job done.
The majorly weird thing about xmltodict that we have to work around is that
it gives results of different *types* when you get 0, 1, or many child nodes.
This is what get_list is for.
"""
# Read the XML file as UTF-8, and parse it into a dictionary.
file = codecs.open(filename, encoding='utf-8')
out = JSONStreamWriter(output_file)
data = file.read()
file.close()
xml = xmltodict.parse(data)
# The dictionary contains a list of <entry> tags.
root_node = xml['JMdict']
for entry in get_list(root_node, 'entry'):
# From JMdict's documentation: "The kanji element, or in its absence,
# the reading element, is the defining component of each entry."
#
# Quick summary of what's going on here: most Japanese words can be
# written using kanji or kana.
#
# Kana are phonetic characters. Every word can be written in kana, in
# one of two alphabets (hiragana or katakana). Words that are homonyms
# have the same kana, unless they're written in different alphabets.
#
# Kanji are Chinese-based characters that are related to the meaning of
# the word. They're compact and good at disambiguating homonyms, so
# kanji are usually used as the canonical representation of a word.
# However, some words have no kanji.
#
# The kana version of a word written in kanji is called its 'reading'.
# Words that are pronounced differently in different contexts have
# multiple readings.
#
# Okay, done with the intro to Japanese orthography. In JMdict, if
# a word can be written in kanji, it has a <k_ele> element, containing
# a <keb> element that contains the text. Every word also has an
# <r_ele> element, containing one or more <reb> elements that are phonetic
# readings of the word.
#
# We get the "defining text" of a word by taking its <keb> if it exists,
# or all of its <reb>s if not. There's no way to tell which <reb> is the
# most "defining" in the case where there's no <keb>.
headwords = [word['keb'] for word in get_list(entry, 'k_ele')]
if not headwords:
headwords = [word['reb'] for word in get_list(entry, 'r_ele')]
# An entry might have different word senses that are translated
# differently to other languages. Ideally, we'd remember that they're
# different senses. However, we have no way to refer to the different
# senses. So for now, we disregard word senses. One day we might have
# a better overall plan for word senses in ConceptNet.
for sense in get_list(entry, 'sense'):
# Glosses are translations of the word to different languages.
# If the word is a loan-word, the foreign word it was derived from
# will be marked with the <lsource> tag instead of <gloss>.
#
# Get all the glosses, including the lsource if it's there.
glosses = get_list(sense, 'gloss') + get_list(sense, 'lsource')
for gloss in glosses:
text = lang = None
if '#text' in gloss:
# A gloss node might be marked with a 'lang' attribute. If so,
# xmltodict represents it as a dictionary with '#text' and
# '@xml:lang' elements.
text = parse_gloss(gloss['#text'])
lang = convert_lang_code(gloss['@xml:lang'])
elif isinstance(gloss, STRING_TYPE):
# If there's no 'lang' attribute, the gloss is in English,
# and xmltodict gives it to us as a plain Unicode string.
lang = 'en'
text = parse_gloss(gloss)
# If we parsed the node at all and the text looks good, then we can
# add edges to ConceptNet.
#
# We don't want to deal with texts with periods (these might be
# dictionary-style abbreviations, which are sort of unhelpful when
# we can't expand them), and we also don't want to deal with texts
# that are more than five words long.
if (text is not None and '.' not in text
and text.count(' ') <= 4
and text not in BAD_NAMES_FOR_THINGS):
for head in headwords:
ja_concept = normalized_concept_uri('ja', head)
other_concept = normalized_concept_uri(lang, text)
output_edge(out, ja_concept, other_concept)
def handle_file(infile, outfile):
count = 0
outcomes = defaultdict(int)
sources = ['/s/site/verbosity']
writer = MsgpackStreamWriter(outfile)
for line in open(infile):
parts = line.strip().split('\t')
if not parts:
outcomes['blank'] += 1
continue
# The first 5 columns of the Verbosity output file are:
#
# left: the word being clued
# relation: the relation between the word and the clue that the
# clue-giver chose, in a form such as "it is part of"
# right: the one or two words used as the clue
# freq: the number of different times this clue was given
# orderscore: the average position in the list of clues
#
# 'orderscore' is a number from 0 to 999, representing the average
# quantile of its position in the list of clues. (It's like a
# percentile, except there are 1000 of them, not 100.)
#
# A clue that's always given first has an orderscore of 0. A clue
# that always appears halfway through the list has an orderscore of
# 500.
#
# This may seem like a strange thing to measure, and I didn't come up
# with it, but it actually turns out to be somewhat informative.
# A clue with an orderscore of 0 is probably a good common-sense
# relation, representing the first thing that comes to mind. A clue
# with a high order score may be a move of desperation after several
# other clues have failed. It causes the guesser to get the answer
# soon afterward, but perhaps because it's a "cheating" move. So,
# low orderscores represent better common sense relations.
left, relation, right, freq, orderscore = parts[:5]
freq = int(freq)
orderscore = int(orderscore)
# Test each word
flagged = False
for rword in right.split():
if BAD_CLUE_REGEX.match(rword):
flagged = True
break
if flagged:
outcomes['flag word'] += 1
continue
if len(right) < 3:
outcomes['clue too short'] += 1
continue
if len(right.split()[-1]) == 1:
outcomes['letter'] += 1
continue
# The Verbosity interface and gameplay did not particularly encourage
# players to choose an appropriate relation. In practice, players seem
# to have used them all interchangeably, except for the negative
# relation "it is the opposite of", expressing /r/Antonym.
#
# Another way that players expressed negative relations was to use
# 'not' as the first word of their clue; we make that into an instance
# of /r/Antonym as well.
#
# In other cases, the relation is a positive relation, so we replace it
# with the most general positive relation, /r/RelatedTo.
rel = '/r/RelatedTo'
reltext = 'is related to'
if right.startswith('not '):
rel = '/r/Antonym'
right = right[4:]
reltext = 'is not'
if relation == 'it is the opposite of':
rel = '/r/Antonym'
reltext = 'is the opposite of'
# The "sounds-like score" determines whether this clue seems to be a
# pun or rhyme, rather than an actual common-sense relationship. If
# the sounds-like score is over 0.35, skip the assertion.
sls = sounds_like_score(left, right)
if sls > 0.35:
outcomes['text similarity'] += 1
continue
# Calculate a score for the assertion:
#
# - The number of times it's been used as a clue
# - ...with a linear penalty for a high sounds-like score
# - ...and a linear penalty for high orderscores
#
# The penalties are multiplicative factors from 0 to 1, which decrease
# linearly as the relevant penalties increase. If a clue is given N
# times, with a sounds-like score of 0 and an orderscore of 0, it will
# get an overall score of 2N - 1. This is a formula we should probably
# revisit.
#
# The weight is the score divided by 100. All divisions are floating
# point, as defined by the __future__ import at the top of this module.
score = (freq * 2 - 1) * (1 - sls) * (1 - orderscore / 1000)
if score <= 0.5:
outcomes['low score'] += 1
continue
weight = score / 100
# If the clue on the right is a two-word phrase, we make additional
# connections to both words individually. We label them with the
# rule-based source '/s/rule/split_words' to track that this happened.
rightwords = [right]
if ' ' in right:
morewords = [word for word in right.split(' ') if word not in STOPWORDS]
rightwords.extend(morewords)
for i, rightword in enumerate(rightwords):
edge_sources = list(sources)
if i > 0:
edge_sources.append('/s/rule/split_words')
# Build the natural-language-ish surface text for this clue
text = '[[%s]] %s [[%s]]' % (left, reltext, rightword)
count += 1
outcomes['success'] += 1
leftc = normalized_concept_uri('en', left)
rightc = normalized_concept_uri('en', rightword)
edge = make_edge(rel, leftc, rightc, dataset='/d/verbosity',
license=Licenses.cc_attribution,
sources=sources, surfaceText=text,
weight=weight)
writer.write(edge)
# Count the various outcomes. This can be used as a sanity-check. It
# also was used for a graph in a ConceptNet 5 paper.
print("Verbosity outcomes: %s" % outcomes)
def web_search():
keyword = request.form.get('keyword')
lang = request.form.get('language')
return redirect(WEB_ROOT + normalized_concept_uri(lang, keyword))
def handle_file(infile, outfile):
count = 0
outcomes = defaultdict(int)
sources = ['/s/site/verbosity']
writer = JSONStreamWriter(outfile)
for line in open(infile):
parts = line.strip().split('\t')
if not parts:
outcomes['blank'] += 1
continue
# The first 5 columns of the Verbosity output file are:
#
# left: the word being clued
# relation: the relation between the word and the clue that the
# clue-giver chose, in a form such as "it is part of"
# right: the one or two words used as the clue
# freq: the number of different times this clue was given
# orderscore: the average position in the list of clues
#
# 'orderscore' is a number from 0 to 999, representing the average
# quantile of its position in the list of clues. (It's like a
# percentile, except there are 1000 of them, not 100.)
#
# A clue that's always given first has an orderscore of 0. A clue
# that always appears halfway through the list has an orderscore of
# 500.
#
# This may seem like a strange thing to measure, and I didn't come up
# with it, but it actually turns out to be somewhat informative.
# A clue with an orderscore of 0 is probably a good common-sense
# relation, representing the first thing that comes to mind. A clue
# with a high order score may be a move of desperation after several
# other clues have failed. It causes the guesser to get the answer
# soon afterward, but perhaps because it's a "cheating" move. So,
# low orderscores represent better common sense relations.
left, relation, right, freq, orderscore = parts[:5]
freq = int(freq)
orderscore = int(orderscore)
# Test each word
flagged = False
for rword in right.split():
if BAD_CLUE_REGEX.match(rword):
flagged = True
break
if flagged:
outcomes['flag word'] += 1
continue
if len(right) < 3:
outcomes['clue too short'] += 1
continue
if len(right.split()[-1]) == 1:
outcomes['letter'] += 1
continue
# The Verbosity interface and gameplay did not particularly encourage
# players to choose an appropriate relation. In practice, players seem
# to have used them all interchangeably, except for the negative
# relation "it is the opposite of", expressing /r/Antonym.
#
# Another way that players expressed negative relations was to use
# 'not' as the first word of their clue; we make that into an instance
# of /r/Antonym as well.
#
# In other cases, the relation is a positive relation, so we replace it
# with the most general positive relation, /r/RelatedTo.
rel = '/r/RelatedTo'
reltext = 'is related to'
if right.startswith('not '):
rel = '/r/Antonym'
right = right[4:]
reltext = 'is not'
if relation == 'it is the opposite of':
rel = '/r/Antonym'
reltext = 'is the opposite of'
# The "sounds-like score" determines whether this clue seems to be a
# pun or rhyme, rather than an actual common-sense relationship. If
# the sounds-like score is over 0.35, skip the assertion.
sls = sounds_like_score(left, right)
if sls > 0.35:
outcomes['text similarity'] += 1
continue
# Calculate a score for the assertion:
#
# - The number of times it's been used as a clue
# - ...with a linear penalty for a high sounds-like score
# - ...and a linear penalty for high orderscores
#
# The penalties are multiplicative factors from 0 to 1, which decrease
# linearly as the relevant penalties increase. If a clue is given N
# times, with a sounds-like score of 0 and an orderscore of 0, it will
# get an overall score of 2N - 1. This is a formula we should probably
# revisit.
#
# The weight is the score divided by 100. All divisions are floating
# point, as defined by the __future__ import at the top of this module.
score = (freq * 2 - 1) * (1 - sls) * (1 - orderscore / 1000)
if score <= 0.5:
outcomes['low score'] += 1
continue
weight = score / 100
# If the clue on the right is a two-word phrase, we make additional
# connections to both words individually. We label them with the
# rule-based source '/s/rule/split_words' to track that this happened.
rightwords = [right]
if ' ' in right:
morewords = [
word for word in right.split(' ') if word not in STOPWORDS
]
rightwords.extend(morewords)
for i, rightword in enumerate(rightwords):
edge_sources = list(sources)
if i > 0:
edge_sources.append('/s/rule/split_words')
# Build the natural-language-ish surface text for this clue
text = '[[%s]] %s [[%s]]' % (left, reltext, rightword)
count += 1
outcomes['success'] += 1
leftc = normalized_concept_uri('en', left)
rightc = normalized_concept_uri('en', rightword)
edge = make_edge(rel,
leftc,
rightc,
dataset='/d/verbosity',
license=Licenses.cc_attribution,
sources=sources,
surfaceText=text,
weight=weight)
writer.write(edge)
# Count the various outcomes. This can be used as a sanity-check. It
# also was used for a graph in a ConceptNet 5 paper.
print("Verbosity outcomes: %s" % outcomes)
def run_umbel(input_dir, output_file, sw_map_file):
"""
Read N-Triples files containing Umbel data, outputting a file of
ConceptNet edges and a file of mappings between the Semantic Web and
ConceptNet.
"""
out = MsgpackStreamWriter(output_file)
map_out = NTriplesWriter(sw_map_file)
reader = NTriplesReader()
labels = {}
label_sets = defaultdict(set)
# There are two files we want to parse:
# - umbel.nt, a transformation of umbel.n3, which is available from
# https://github.com/structureddynamics/UMBEL/.
# - umbel_links.nt, distributed with DBPedia 3.9.
#
# We parse them both in this file so that umbel_links can reuse the
# concept names extracted from umbel.nt.
main_file = os.path.join(input_dir, 'umbel.nt')
dbpedia_link_file = os.path.join(input_dir, 'umbel_links.nt')
# Read through umbel.nt once, finding all the "preferred labels". We will
# use these as the surface texts for the nodes.
for web_subj, web_rel, web_obj, objtag in reader.parse_file(main_file):
if resource_name(web_rel) == 'prefLabel':
# 'CW' and 'PCW' are Cyc jargon for 'conceptual works'. If a node
# cannot be described except as a CW, we're probably not
# interested in it.
if 'CW' not in web_obj.split() and 'PCW' not in web_obj.split():
labels[web_subj] = web_obj
if resource_name(web_rel).endswith('Label'):
text = normalize_text(web_obj)
label_sets[text].add(web_subj)
# Read through umbel.nt again and extract ConceptNet edges.
for web_subj, web_rel, web_obj, objtag in reader.parse_file(main_file):
if objtag == 'URL' and acceptable_node(web_obj) and acceptable_node(web_subj):
# Only use nodes for which we've seen preferred labels.
# (This skips some anonymous OWL-cruft nodes.)
if web_subj in labels and web_obj in labels:
subj_uri = normalized_concept_uri('en', labels[web_subj])
obj_uri = normalized_concept_uri('en', labels[web_obj])
rel_name = resource_name(web_rel)
# Check if this is a relation we want to handle.
if rel_name in REL_MAPPING:
# Write the ConceptNet edges and the mappings to Semantic Web URLs.
rel_uri, frame = REL_MAPPING[rel_name]
surface = frame % (labels[web_subj], labels[web_obj])
out.write(umbel_edge(rel_uri, subj_uri, obj_uri, surface, SOURCE))
map_out.write_link(web_rel, full_conceptnet_url(rel_uri))
map_out.write_link(web_subj, full_conceptnet_url(subj_uri))
map_out.write_link(web_obj, full_conceptnet_url(obj_uri))
# altLabel relations assign different texts to the same node. We'll
# represent those in ConceptNet with Synonym relations.
elif web_rel.endswith('altLabel'):
# Make sure we know what's being labeled.
if web_subj in labels:
name = web_obj
words = name.split(' ')
if normalized_concept_name('en', name) != normalized_concept_name('en', labels[web_subj]):
if not set(words) & IGNORED_WORDS:
main_label = normalized_concept_uri('en', labels[web_subj])
name_text = normalize_text(name)
if len(label_sets[name_text]) >= 2 or len(name_text) <= 3:
disambig = un_camel_case(resource_name(web_subj))
# Cyc does not distinguish texts by their part of speech, so use
# '_' as the part of speech symbol.
alt_label = normalized_concept_uri('en', name, '_', disambig)
else:
alt_label = normalized_concept_uri('en', name)
surface = SYN_FRAME % (name, labels[web_subj])
out.write(umbel_edge('/r/Synonym', alt_label, main_label, surface, SOURCE))
for web_subj, web_rel, web_obj, objtag in reader.parse_file(dbpedia_link_file):
if objtag == 'URL' and acceptable_node(web_obj) and acceptable_node(web_subj):
if web_obj in labels:
subj_label = resource_name(web_subj).replace('_', ' ')
subj_uri = translate_dbpedia_url(web_subj)
obj_label = labels[web_obj]
obj_uri = normalized_concept_uri('en', obj_label)
rel_name = resource_name(web_rel)
if rel_name in REL_MAPPING:
rel_uri, frame = REL_MAPPING[rel_name]
surface = frame % (subj_label, obj_label)
out.write(umbel_edge(rel_uri, subj_uri, obj_uri, surface, LINK_SOURCE))
map_out.write_link(web_rel, full_conceptnet_url(rel_uri))
map_out.write_link(web_subj, full_conceptnet_url(subj_uri))
map_out.write_link(web_obj, full_conceptnet_url(obj_uri))
