def cyc_to_conceptnet_uri(labels, unlabels, uri):
"""
Convert a Cyc URI to a ConceptNet URI, with the following rules:
- Use the RDF label as the text. (Alternate labels appear to provide
synonyms, but these are generally automatically generated and aren't
particularly accurate.)
- The part of speech is always 'n'. Cyc describes its concepts in a
noun-like way. At worst, they're gerunds -- instead of "to eat", Cyc
would define an event of "Eating".
- If two different Cyc URIs have the same text, we will attempt to
disambiguate them using the last component of the Cyc URI.
- Remove the camel-casing from the Cyc URI component. If the phrase we
get is the same as the natural-language label, disregard it as an
uninformative disambiguation. Otherwise, that is the disambiguation text.
A possible objection: Our disambiguation doesn't distinguish Cyc URIs that
differ in capitalization, or differ by using underscores instead of
camel-case. However, I've noticed that such URIs are usually
*unintentional* duplicates that are okay to merge. If they were really
unrelated concepts that needed to be distinguished, someone would have
given them different names.
Even so, we end up with some unnecessary word senses, such as different
senses for "mens clothing", "men's clothing", and "men s clothing".
"""
label = filter_stopwords(labels[uri])
if len(unlabels[label]) >= 2:
disambig = filter_stopwords(un_camel_case(resource_name(uri)))
if simple_tokenize(disambig) != simple_tokenize(label):
return standardized_concept_uri('en', label, 'n', 'opencyc', disambig)
return standardized_concept_uri('en', label, 'n')
def transform_term(data_language, termdata, assumed_languages, db, use_etyms=True):
text = termdata['text']
# Sometimes - is used to fill a slot in a Wiktionary template where the
# term would usually be. It typically means "don't show this part", with
# the implication "the term in question is obvious from context".
#
# Context is hard, so let's just cope with a hyphen as the term by
# discarding it.
if text == '-':
return None
language = termdata.get('language')
if language is None:
language = disambiguate_language(text, assumed_languages, db)
if not valid_language(language):
return None
# Remove unnecessary subtags from the Wiktionary language
if '-' in language and language not in ALL_LANGUAGES:
language = language.split('-')[0]
if 'pos' not in termdata:
return standardized_concept_uri(language, text)
else:
pos = termdata['pos']
etym_sense = None
if use_etyms:
etym_sense = etym_label(data_language, termdata)
if etym_sense is not None:
return standardized_concept_uri(language, text, pos, 'wikt', etym_sense)
else:
return standardized_concept_uri(language, text, pos)
def transform_term(data_language, termdata, assumed_languages, db, use_etyms=True):
text = termdata['text']
# Sometimes - is used to fill a slot in a Wiktionary template where the
# term would usually be. It typically means "don't show this part", with
# the implication "the term in question is obvious from context".
#
# Context is hard, so let's just cope with a hyphen as the term by
# discarding it.
if text == '-':
return None
language = termdata.get('language')
if language is None:
language = disambiguate_language(text, assumed_languages, db)
if not valid_language(language):
return None
# Remove unnecessary subtags from the Wiktionary language
if '-' in language and language not in ALL_LANGUAGES:
language = language.split('-')[0]
if 'pos' not in termdata:
return standardized_concept_uri(language, text)
else:
pos = termdata['pos']
etym_sense = None
if use_etyms:
etym_sense = etym_label(data_language, termdata)
if etym_sense is not None:
return standardized_concept_uri(language, text, pos, 'wikt', etym_sense)
else:
return standardized_concept_uri(language, text, pos)
def transform_term(data_language,
termdata,
assumed_languages,
db,
use_etyms=True):
text = termdata['text']
language = termdata.get('language')
if language is None:
language = disambiguate_language(text, assumed_languages, db)
if not valid_language(language):
return None
# Remove unnecessary subtags from the Wiktionary language
if '-' in language and language not in ALL_LANGUAGES:
language = language.split('-')[0]
if 'pos' not in termdata:
return standardized_concept_uri(language, text)
else:
pos = termdata['pos']
etym_sense = None
if use_etyms:
etym_sense = etym_label(data_language, termdata)
if etym_sense is not None:
return standardized_concept_uri(language, text, pos, 'wikt',
etym_sense)
else:
return standardized_concept_uri(language, text, pos)
def handle_raw_assertion(line):
parts = line.split(', ')
user, frame_id, concept1, concept2 = parts
fdata = FRAME_DATA[frame_id]
ftext = fdata['text']
rel = fdata['relation']
surfaceText = ftext.replace('{1}', '[[' + concept1 + ']]').replace(
'{2}', '[[' + concept2 + ']]'
)
# We mark surface texts with * if {2} comes before {1}.
if ftext.find('{2}') < ftext.find('{1}'):
surfaceText = '*' + surfaceText
start = standardized_concept_uri('zh_TW', concept1)
end = standardized_concept_uri('zh_TW', concept2)
source = {
'contributor': '/s/contributor/petgame/' + user,
'activity': '/s/activity/ptt/petgame',
}
yield make_edge(
rel,
start,
end,
dataset='/d/conceptnet/4/zh',
license=Licenses.cc_attribution,
sources=[source],
surfaceText=surfaceText,
weight=1,
)
def handle_raw_assertion(line):
parts = line.split(', ')
user, frame_id, concept1, concept2 = parts
fdata = FRAME_DATA[frame_id]
ftext = fdata['text']
rel = fdata['relation']
surfaceText = ftext.replace('{1}', '[[' + concept1 + ']]').replace(
'{2}', '[[' + concept2 + ']]')
# We mark surface texts with * if {2} comes before {1}.
if ftext.find('{2}') < ftext.find('{1}'):
surfaceText = '*' + surfaceText
start = standardized_concept_uri('zh_TW', concept1)
end = standardized_concept_uri('zh_TW', concept2)
source = {
'contributor': '/s/contributor/petgame/' + user,
'activity': '/s/activity/ptt/petgame'
}
yield make_edge(rel,
start,
end,
dataset='/d/conceptnet/4/zh',
license=Licenses.cc_attribution,
sources=[source],
surfaceText=surfaceText,
weight=1)
def sparse_from_parallel_text(input_path, languages):
vocabs = {language: get_vocab(language) for language in languages}
labels = OrderedSet()
pairs = OrderedSet()
rows = []
cols = []
values = []
for lang1 in languages:
for lang2 in languages:
if lang1 < lang2:
print(lang1, lang2)
filename = input_path / "{}-{}.txt".format(lang1, lang2)
with open(str(filename), encoding='utf-8') as infile:
lines = list(infile.readlines())
random.shuffle(lines)
for i, line in enumerate(lines):
text1, text2 = line.rstrip('\n').split('\t')
terms1 = [
replace_numbers(
standardized_concept_uri(lang1, word))
for word in text1.split(' ')
if word in vocabs[lang1]
]
terms2 = [
replace_numbers(
standardized_concept_uri(lang2, word))
for word in text2.split(' ')
if word in vocabs[lang2]
]
terms = terms1 + terms2
if i > 0 and i % 100000 == 0:
print('\t', i, '\t', len(values), terms)
if i == 1000000:
break
for t1 in terms:
index1 = labels.add(t1)
for t2 in terms:
index2 = labels.add(t2)
pair_index = pairs.add((index1, index2))
assert pair_index <= len(values)
if pair_index == len(values):
rows.append(index1)
cols.append(index2)
values.append(1 / len(terms))
else:
values[pair_index] += 1 / len(terms)
shape = (len(labels), len(labels))
index = pd.Index(labels)
mat = sparse.coo_matrix((values, (rows, cols)), shape=shape,
dtype='f').tocsr()
return mat, index
def handle_file(input_file, output_file):
tree = ET.parse(input_file)
out = MsgpackStreamWriter(output_file)
root = tree.getroot()
lang = root[0][1].attrib['type']
for annotation in root[1]:
for word in strip_words(annotation.text):
start = standardized_concept_uri('mul', annotation.attrib['cp'])
end = standardized_concept_uri(lang, word)
edge = make_edge(REL, start, end, DATASET, LICENSE, SOURCE)
out.write(edge)
def handle_raw_assertion(line):
parts = line.split(', ')
user, frame_id, concept1, concept2 = parts
fdata = FRAME_DATA[frame_id]
ftext = fdata['text']
rel = fdata['relation']
surfaceText = ftext.replace('{1}', '[[' + concept1 + ']]').replace('{2}', '[[' + concept2 + ']]')
start = standardized_concept_uri('zh_TW', concept1)
end = standardized_concept_uri('zh_TW', concept2)
sources = ['/s/activity/ptt/petgame', '/s/contributor/petgame/' + user]
yield make_edge(rel, start, end, dataset='/d/conceptnet/4/zh',
license='/l/CC/By', sources=sources,
surfaceText=surfaceText, weight=1)
def run_opencyc(input_file, output_file):
"""
Read an .nq file containing OpenCyc data, outputting a file of
ConceptNet edges and a file of mappings between the Semantic Web and
ConceptNet.
"""
out = MsgpackStreamWriter(output_file)
labels = {}
unlabels = defaultdict(set)
seen_external_urls = set()
# Read through the file once, finding all the "preferred labels". We will
# use these as the surface texts for the nodes.
for subj, pred, obj, _graph in parse_nquads(
open(input_file, encoding='utf-8')):
if pred['url'] == RDF_LABEL:
labels[subj['url']] = obj['text']
unlabels[obj['text']].add(subj['url'])
# Read through the file again and extract ConceptNet edges.
for subj, pred, obj, _graph in parse_nquads(
open(input_file, encoding='utf-8')):
rel_name = resource_name(pred['url'])
web_subj = subj.get('url')
web_obj = obj.get('url')
if rel_name == 'subClassOf' and web_obj is not None and web_subj in labels and web_obj in labels:
subj_label = labels[web_subj]
obj_label = labels[web_obj]
if '_' in subj_label or '_' in obj_label:
continue
if subj_label.startswith('xsd:') or obj_label.startswith('xsd:'):
continue
subj_words = set(simple_tokenize(subj_label))
obj_words = set(simple_tokenize(obj_label))
if (subj_words & BLACKLIST_WORDS) or (obj_words & BLACKLIST_WORDS):
continue
if len(subj_words) > 4 or len(obj_words) > 4:
continue
subj_uri = cyc_to_conceptnet_uri(labels, unlabels, web_subj)
obj_uri = cyc_to_conceptnet_uri(labels, unlabels, web_obj)
out.write(
opencyc_edge('/r/IsA', subj_uri, obj_uri, subj_label,
obj_label))
if (subj_uri, web_subj) not in seen_external_urls:
out.write(external_url_edge(subj_uri, web_subj))
seen_external_urls.add((subj_uri, web_subj))
if (obj_uri, web_obj) not in seen_external_urls:
out.write(external_url_edge(obj_uri, web_obj))
seen_external_urls.add((obj_uri, web_obj))
elif rel_name == 'sameAs' and web_subj in labels and web_obj.startswith(
'http://umbel.org/'):
subj_label = labels[web_subj]
subj_uri = standardized_concept_uri('en', subj_label)
if (subj_uri, web_obj) not in seen_external_urls:
out.write(external_url_edge(subj_uri, web_obj))
seen_external_urls.add((subj_uri, web_obj))
out.close()
def main(args):
global verbose
verbose = args.verbose
r_term = re.compile(r'\[\[(?P<word>.+?)\]\]')
CHUNKSIZE = 100000
cols = ['uri', 'rel', 'start', 'end', 'meta']
reader = pd.read_table(args.path_input, encoding='utf_8', names=cols,
chunksize=CHUNKSIZE)
df = pd.concat(filter_by_lang(r, args.lang) for r in reader)
if verbose:
logger.info('Read {} rows from {}'.format(len(df), args.path_input))
df['surfaceText'] = df['meta'].apply(extract_surfaceText)
df.drop(['uri', 'meta'], axis=1, inplace=True)
indices = df[df['surfaceText'].apply(lambda s: len(s) == 0)].index
df.drop(indices, axis=0, inplace=True)
if verbose:
logger.info('Remained: {} rows'.format(len(df)))
df.index = np.arange(len(df))
indices = []
buff = {'rel': [], 'start': [], 'end': [],
'surfaceText': [], 'surfaceText-org': []}
pb = ProgressBar(maxval=len(df))
for idx, row in df.iterrows():
surface = row['surfaceText']
if len(surface) == 0:
continue
pb.update(idx+1)
done = {'start': False, 'end': False}
_surface = surface
for m in r_term.finditer(_surface):
term = standardized_concept_uri(args.lang, m.group('word'))
for col in ['start', 'end']:
if term == row[col]:
surface = surface.replace('[[{}]]'.format(m.group('word')),
'[[{}]]'.format(col))
done[col] = True
break
if done['start'] is False or done['end'] is False:
indices.append(idx)
continue
buff['rel'].append(row['rel'])
buff['start'].append(row['start'])
buff['end'].append(row['end'])
buff['surfaceText'].append(surface)
buff['surfaceText-org'].append(_surface)
result = pd.DataFrame.from_dict(buff)
result.to_csv(args.path_output, sep='\t', encoding='utf_8',
index=False)
if verbose:
logger.info('Write {} lines to {}'.format(len(result),
args.path_output))
return 0
def lookup(term):
EdgeList = []
query = standardized_concept_uri('en', term)
results = list(FINDER.lookup(query=query, limit=1000, offset=0))
print(len(results))
for result in results:
EdgeList.append(Edge(result))
return EdgeList
def wikipedia_connected_conceptnet_nodes(db, start):
nodes = set()
c = db.cursor()
c.execute("SELECT lemma FROM words WHERE page=? LIMIT 1000",
(start.casefold(), ))
for row in c.fetchall():
lemma = row[0]
nodes.add(standardized_concept_uri('en', lemma))
return list(nodes)
def handle_file(input_file, output_file):
tree = ET.parse(input_file)
out = MsgpackStreamWriter(output_file)
root = tree.getroot()
lang = root[0][1].attrib[
'type'
] # language is at position [1] within the child node [0]
if len(root) >= 2:
for annotation in root[1]:
for word in strip_words(annotation.text):
start = standardized_concept_uri('mul', annotation.attrib['cp'])
end = standardized_concept_uri(lang, word)
edge = make_edge(REL, start, end, DATASET, LICENSE, SOURCE)
out.write(edge)
else:
print("No emoji data in {!r}".format(input_file))
out.close()
def run_opencyc(input_file, output_file):
"""
Read an .nq file containing OpenCyc data, outputting a file of
ConceptNet edges and a file of mappings between the Semantic Web and
ConceptNet.
"""
out = MsgpackStreamWriter(output_file)
labels = {}
unlabels = defaultdict(set)
seen_external_urls = set()
# Read through the file once, finding all the "preferred labels". We will
# use these as the surface texts for the nodes.
for subj, pred, obj, _graph in parse_nquads(open(input_file, encoding='utf-8')):
if pred['url'] == RDF_LABEL:
labels[subj['url']] = obj['text']
unlabels[obj['text']].add(subj['url'])
# Read through the file again and extract ConceptNet edges.
for subj, pred, obj, _graph in parse_nquads(open(input_file, encoding='utf-8')):
rel_name = resource_name(pred['url'])
web_subj = subj.get('url')
web_obj = obj.get('url')
if rel_name == 'subClassOf' and web_obj is not None and web_subj in labels and web_obj in labels:
subj_label = labels[web_subj]
obj_label = labels[web_obj]
if '_' in subj_label or '_' in obj_label:
continue
if subj_label.startswith('xsd:') or obj_label.startswith('xsd:'):
continue
subj_words = set(simple_tokenize(subj_label))
obj_words = set(simple_tokenize(obj_label))
if (subj_words & BLACKLIST_WORDS) or (obj_words & BLACKLIST_WORDS):
continue
if len(subj_words) > 4 or len(obj_words) > 4:
continue
subj_uri = cyc_to_conceptnet_uri(labels, unlabels, web_subj)
obj_uri = cyc_to_conceptnet_uri(labels, unlabels, web_obj)
out.write(opencyc_edge('/r/IsA', subj_uri, obj_uri, subj_label, obj_label))
if (subj_uri, web_subj) not in seen_external_urls:
out.write(external_url_edge(subj_uri, web_subj))
seen_external_urls.add((subj_uri, web_subj))
if (obj_uri, web_obj) not in seen_external_urls:
out.write(external_url_edge(obj_uri, web_obj))
seen_external_urls.add((obj_uri, web_obj))
elif rel_name == 'sameAs' and web_subj in labels and web_obj.startswith('http://umbel.org/'):
subj_label = labels[web_subj]
subj_uri = standardized_concept_uri('en', subj_label)
if (subj_uri, web_obj) not in seen_external_urls:
out.write(external_url_edge(subj_uri, web_obj))
seen_external_urls.add((subj_uri, web_obj))
out.close()
def query_standardize_uri():
"""
Look up the URI for a given piece of text. 'text' and 'language' should be
given as parameters.
"""
language = flask.request.args.get('language')
text = flask.request.args.get('text') or flask.request.args.get('term')
return jsonify({
'@context': responses.CONTEXT,
'@id': standardized_concept_uri(language, text)
})
def handle_raw_assertion(line):
parts = line.split(', ')
user, frame_id, concept1, concept2 = parts
fdata = FRAME_DATA[frame_id]
ftext = fdata['text']
rel = fdata['relation']
surfaceText = ftext.replace('{1}', '[[' + concept1 + ']]').replace(
'{2}', '[[' + concept2 + ']]')
start = standardized_concept_uri('zh_TW', concept1)
end = standardized_concept_uri('zh_TW', concept2)
sources = ['/s/activity/ptt/petgame', '/s/contributor/petgame/' + user]
yield make_edge(rel,
start,
end,
dataset='/d/conceptnet/4/zh',
license='/l/CC/By',
sources=sources,
surfaceText=surfaceText,
weight=1)
def standardized_uri(language, term):
"""
Get a URI that is suitable to label a row of a vector space, by making sure
that both ConceptNet's and word2vec's normalizations are applied to it.
If the term already looks like a ConceptNet URI, it will only have its
sequences of digits replaced by #. Otherwise, it will be turned into a
ConceptNet URI in the given language, and then have its sequences of digits
replaced.
"""
if not (term.startswith('/') and term.count('/') >= 2):
term = standardized_concept_uri(language, term)
return replace_numbers(term)
def standardize_uri(language, text):
"""
Look up the URI for a given piece of text.
"""
if text is None or language is None:
return error(
{}, 400,
"You should include the 'text' and 'language' parameters.")
text = text.replace('_', ' ')
uri = standardized_concept_uri(language, text)
response = {'@id': uri}
return success(response)
def standardized_uri(language, term):
"""
Get a URI that is suitable to label a row of a vector space, by making sure
that both ConceptNet's and word2vec's normalizations are applied to it.
If the term already looks like a ConceptNet URI, it will only have its
sequences of digits replaced by #. Otherwise, it will be turned into a
ConceptNet URI in the given language, and then have its sequences of digits
replaced.
"""
if not (term.startswith('/') and term.count('/') >= 2):
term = standardized_concept_uri(language, term)
return replace_numbers(term)
def standardize_uri(language, text):
"""
Look up the URI for a given piece of text.
"""
if text is None or language is None:
return error(
{}, 400, "You should include the 'text' and 'language' parameters."
)
text = text.replace('_', ' ')
uri = standardized_concept_uri(language, text)
response = {'@id': uri}
return success(response)
def transform_term(data_language, termdata, assumed_languages, db, use_etyms=True):
text = termdata['text']
language = termdata.get('language')
if language is None:
language = disambiguate_language(text, assumed_languages, db)
if not valid_language(language):
return None
# Remove unnecessary subtags from the Wiktionary language
if '-' in language and language not in ALL_LANGUAGES:
language = language.split('-')[0]
if 'pos' not in termdata:
return standardized_concept_uri(language, text)
else:
pos = termdata['pos']
etym_sense = None
if use_etyms:
etym_sense = etym_label(data_language, termdata)
if etym_sense is not None:
return standardized_concept_uri(language, text, pos, 'wikt', etym_sense)
else:
return standardized_concept_uri(language, text, pos)
def query_standardize_uri():
"""
Look up the URI for a given piece of text. 'text' and 'language' should be
given as parameters.
"""
language = flask.request.args.get('language')
text = flask.request.args.get('text') or flask.request.args.get('term')
if not language:
return render_error(400, "Please specify a 'language' parameter.")
if not text:
return render_error(400, "Please specify a 'text' parameter.")
return jsonify({
'@context': responses.CONTEXT,
'@id': standardized_concept_uri(language, text)
})
def standardize_uri():
"""
Look up the URI for a given piece of text. 'text' and 'language' should be
given as parameters.
"""
language = flask.request.args.get('language')
text = flask.request.args.get('text') or flask.request.args.get('term')
if text is None or language is None:
return flask.jsonify({
'error': 'Invalid request',
'details': "You should include the 'text' and 'language' parameters"
}), 400
text = text.replace('_', ' ')
uri = standardized_concept_uri(language, text)
return flask.jsonify(uri=uri)
def query_standardize_uri():
"""
Look up the URI for a given piece of text. 'text' and 'language' should be
given as parameters.
"""
language = flask.request.args.get('language')
text = flask.request.args.get('text') or flask.request.args.get('term')
if not language:
return render_error(400, "Please specify a 'language' parameter.")
if not text:
return render_error(400, "Please specify a 'text' parameter.")
return jsonify({
'@context': responses.CONTEXT,
'@id': standardized_concept_uri(language, text)
})
def _augment_knowledge(self):
try:
with open("conceptnetquerycache.pkl", "rb") as f:
cache = pickle.load(f)
except:
cache = {}
# use conceptnet to augment knowledge
def get_results(feature):
feature = feature.lower()
if feature in cache:
ret = cache[feature]
else:
with urlopen(
'http://conceptnet5.media.mit.edu/data/5.4%s?limit=1000'
% quote('/c/en/' + feature)) as response:
html = response.read().decode('utf8')
result = json.loads(html)
ret = []
for x in result['edges']:
r = pattern.match(x['uri'][3:])
if r:
ret.append(r.groups())
cache[feature] = ret
return ret
current_features = list(self.features)
for feature in current_features:
# convert dbpedia entry to conceptnet uri
pieces = dbpedia.parse_topic_name(feature)
pieces[0] = un_camel_case(pieces[0])
cneturi = standardized_concept_uri('en', *pieces)
ret = get_results(cneturi)
for (rtype, src, dest) in ret:
if src not in self.features:
self.features[src] = Feature(src, self)
if dest not in self.features:
self.features[dest] = Feature(dest, self)
self.usage_map.setdefault(rtype, set()).add((src, dest))
self.features[src].add_relation(rtype, dest)
self.features[dest].add_predecessor(rtype, src)
with open("conceptnetquerycache.pkl", "wb") as f:
pickle.dump(cache, f)
def standardize_uri():
"""
Look up the URI for a given piece of text. 'text' and 'language' should be
given as parameters.
"""
language = flask.request.args.get('language')
text = flask.request.args.get('text') or flask.request.args.get('term')
if text is None or language is None:
return flask.jsonify({
'error':
'Invalid request',
'details':
"You should include the 'text' and 'language' parameters"
}), 400
text = text.replace('_', ' ')
uri = standardized_concept_uri(language, text)
return flask.jsonify(uri=uri)
def translate_dbpedia_url(url):
"""
Convert an object that's defined by a DBPedia URL to a ConceptNet
URI. We do this by finding the part of the URL that names the object,
and using that as surface text for ConceptNet.
This is, in some ways, abusing a naming convention in the Semantic Web.
The URL of an object doesn't have to mean anything at all. The
human-readable name is supposed to be a string, specified by the "name"
relation.
The problem here is that the "name" relation is not unique in either
direction. A URL can have many names, and the same name can refer to
many URLs, and some of these names are the result of parsing glitches.
The URL itself is a stable thing that we can build a ConceptNet URI from,
on the other hand.
"""
parsed = parse_url(url)
domain = parsed.netloc
if domain == 'dbpedia.org':
# Handle old DBPedia URLs that had no language code
lang = 'en'
else:
domain_parts = domain.split('.', 1)
if domain_parts[1] == 'dbpedia.org':
lang = domain_parts[0]
# If we can't name this language in English, it's probably
# not really a language.
if langcodes.get(lang).language_name('en') == lang:
return None
else:
return None
# Some Semantic Web URLs are camel-cased. ConceptNet URIs use underscores
# between words.
pieces = parse_topic_name(resource_name(url))
pieces[0] = un_camel_case(pieces[0])
return standardized_concept_uri(lang, *pieces)
def translate_dbpedia_url(url):
"""
Convert an object that's defined by a DBPedia URL to a ConceptNet
URI. We do this by finding the part of the URL that names the object,
and using that as surface text for ConceptNet.
This is, in some ways, abusing a naming convention in the Semantic Web.
The URL of an object doesn't have to mean anything at all. The
human-readable name is supposed to be a string, specified by the "name"
relation.
The problem here is that the "name" relation is not unique in either
direction. A URL can have many names, and the same name can refer to
many URLs, and some of these names are the result of parsing glitches.
The URL itself is a stable thing that we can build a ConceptNet URI from,
on the other hand.
"""
parsed = parse_url(url)
domain = parsed.netloc
if domain == "dbpedia.org":
# Handle old DBPedia URLs that had no language code
lang = "en"
else:
domain_parts = domain.split(".", 1)
if domain_parts[1] == "dbpedia.org":
lang = domain_parts[0]
# If we can't name this language in English, it's probably
# not really a language.
if langcodes.get(lang).language_name("en") == lang:
return None
else:
return None
# Some Semantic Web URLs are camel-cased. ConceptNet URIs use underscores
# between words.
pieces = parse_topic_name(resource_name(url))
pieces[0] = un_camel_case(pieces[0])
return standardized_concept_uri(lang, *pieces)
def evaluate(standard, assoc, verbose=True, standardize_text=False):
correct = 0
total = 0
for a, b, c, d in standard:
if total % 10 == 0:
print(correct, total)
av = text_to_vector(a, assoc, standardize=standardize_text)
bv = text_to_vector(b, assoc, standardize=standardize_text)
cv = text_to_vector(c, assoc, standardize=standardize_text)
term = assoc.most_similar_to_vector(normalize(bv - av + cv)[0])[0]
print(a, b, c, d, term)
if standardize_text:
if term == standardized_concept_uri('en', d):
correct += 1
else:
if term == d:
correct += 1
total += 1
if verbose:
print(100 * correct / total)
return (correct, total)
def evaluate(standard, assoc, verbose=True, standardize_text=False):
correct = 0
total = 0
for a,b,c,d in standard:
if total % 10 == 0:
print(correct, total)
av = text_to_vector(a, assoc, standardize=standardize_text)
bv = text_to_vector(b, assoc, standardize=standardize_text)
cv = text_to_vector(c, assoc, standardize=standardize_text)
term = assoc.most_similar_to_vector(normalize(bv-av+cv)[0])[0]
print(a,b,c,d,term)
if standardize_text:
if term == standardized_concept_uri('en', d):
correct += 1
else:
if term == d:
correct += 1
total += 1
if verbose:
print(100*correct/total)
return (correct, total)
def build_start(parts_dict):
lang = parts_dict['lang']
startText = parts_dict["startText"]
start = standardized_concept_uri(lang, startText)
return start
def run_wordnet(input_file, output_file):
out = MsgpackStreamWriter(output_file)
synset_senses = defaultdict(list)
sense_synsets = {}
synset_labels = defaultdict(list)
synset_canonical_labels = {}
synset_categories = {}
synset_domains = {}
synset_glosses = {}
synset_disambig = {}
synset_uris = {}
# First pass: find data about synsets
quads = parse_nquads(open(input_file, encoding="utf-8"))
for subj_dict, rel_dict, obj_dict, _graph in quads:
if "url" not in subj_dict or "url" not in rel_dict:
continue
subj = subj_dict["url"]
rel = rel_dict["url"]
obj = obj_dict.get("url")
objtext = obj_dict.get("text")
relname = resource_name(rel)
if relname == "label":
if obj_dict["lang"] == "en":
synset_labels[subj].append(objtext)
elif relname == "sameAs":
if obj.startswith(WN20_URL):
# If we have a link to RDF WordNet 2.0, the URL (URI? IRI?)
# will contain a standardized label for this concept, which
# we should use when we want to use this synset as the name of
# a disambiguation category. RDF WordNet 3.1 assigns synsets
# a number of labels in no particular order, making it hard to
# determine from 3.1 alone what to name a category.
objname = resource_name(obj)
parts = objname.split("-")[1:-2]
# Handle missing apostrophes
label = (
"-".join(parts)
.replace("_s_", "'s_")
.replace("_s-", "'s_")
.replace("s__", "s'_")
.replace("s_-", "s'-")
.replace("_", " ")
)
synset_canonical_labels[subj] = label
elif relname == "domain_category":
synset_categories[subj] = obj
elif relname == "lexical_domain":
target = resource_name(obj)
if "." in target:
domain = target.split(".")[1]
synset_domains[subj] = domain
elif relname == "gloss":
synset_glosses[subj] = objtext
elif relname == "reference":
lemma = resource_name(subj)
synset = obj
synset_senses[synset].append(lemma)
sense_synsets[lemma] = synset
used_labels = set(synset_canonical_labels.values())
for synset, values in synset_labels.items():
values.sort(key=lambda label: (label in used_labels,) + label_sort_key(label))
if (
synset not in synset_canonical_labels
or synset_canonical_labels[synset][0].isupper()
and synset_domains.get(synset) == "person"
):
label = values[0]
synset_canonical_labels[synset] = label
used_labels.add(label)
for synset, labels in synset_labels.items():
if synset in synset_categories:
category_name = synset_canonical_labels[synset_categories[synset]]
else:
category_name = synset_domains.get(synset, None)
synset_no_fragment = synset.split("#")[0]
pos = synset_no_fragment[-1].lower()
assert pos in "nvarsp", synset
if pos == "s":
pos = "a"
elif pos == "p":
pos = "-"
if category_name in ("pert", "all", "tops"):
category_name = None
synset_disambig[synset] = (pos, category_name)
canon = synset_canonical_labels[synset]
canon_uri = standardized_concept_uri("en", canon, pos, "wn", category_name)
synset_uris[synset] = canon_uri
for label in labels:
if label != canon:
other_uri = standardized_concept_uri(
"en", label, pos, "wn", category_name
)
rel_uri = "/r/Synonym"
surface = "[[{0}]] is a synonym of [[{1}]]".format(label, canon)
edge = make_edge(
rel_uri,
other_uri,
canon_uri,
dataset=DATASET,
surfaceText=surface,
license=Licenses.cc_attribution,
sources=[SOURCE],
weight=2.0,
)
out.write(edge)
quads = parse_nquads(open(input_file, encoding="utf-8"))
for subj_dict, rel_dict, obj_dict, _graph in quads:
if "url" not in subj_dict or "url" not in rel_dict:
continue
subj = subj_dict["url"]
rel = rel_dict["url"]
obj = obj_dict.get("url")
relname = resource_name(rel)
if relname in REL_MAPPING:
pos, sense = synset_disambig.get(subj, (None, None))
if relname == "hypernym" and pos == "v":
relname = "hypernym-v"
rel, frame = REL_MAPPING[relname]
reversed_frame = False
if rel.startswith("~"):
rel = rel[1:]
reversed_frame = True
rel_uri = "/r/" + rel
if obj is not None:
obj_uri = synset_uris.get(obj)
if obj not in synset_canonical_labels:
continue
obj_label = synset_canonical_labels[obj]
else:
text = obj_dict["text"]
# Some WordNets use strings with "!" in them to indicate
# out-of-band information, such as a missing translation
if (not text) or "!" in text:
continue
lang = obj_dict["lang"]
obj_uri = standardized_concept_uri(lang, text, pos, "wn", sense)
obj_label = text
if subj not in synset_uris or subj not in synset_canonical_labels:
continue
subj_uri = synset_uris[subj]
subj_label = synset_canonical_labels[subj]
license = Licenses.cc_attribution
langcode = subj_uri.split("/")[2]
if langcode in SHAREALIKE_LANGUAGES:
license = Licenses.cc_sharealike
if reversed_frame:
subj_uri, obj_uri = obj_uri, subj_uri
subj_label, obj_label = obj_label, subj_label
surface = frame.format("[[%s]]" % subj_label, "[[%s]]" % obj_label)
edge = make_edge(
rel_uri,
subj_uri,
obj_uri,
dataset=DATASET,
surfaceText=surface,
license=license,
sources=[SOURCE],
weight=2.0,
)
out.write(edge)
for wn_url in sorted(synset_uris):
cn_uri = synset_uris[wn_url]
edge = make_edge(
"/r/ExternalURL",
cn_uri,
wn_url,
dataset=DATASET,
license=Licenses.cc_sharealike,
sources=[SOURCE],
weight=1.0,
)
out.write(edge)
out.close()
def read_wiktionary(input_file, db_file, output_file):
"""
Convert a stream of parsed Wiktionary data into ConceptNet edges.
A `db_file` containing all known words in all languages must have already
been prepared from the same data.
"""
db = sqlite3.connect(db_file)
out = MsgpackStreamWriter(output_file)
for heading, items in segmented_stream(input_file):
language = heading['language']
title = heading['title']
dataset = '/d/wiktionary/{}'.format(language)
url_title = heading['title'].replace(' ', '_')
web_url = 'http://{}.wiktionary.org/wiki/{}'.format(
language, url_title)
web_source = '/s/resource/wiktionary/{}'.format(language)
source = {'contributor': web_source, 'process': PARSER_RULE}
# Scan through the 'from' items, such as the start nodes of
# translations, looking for distinct etymologies. If we get more than
# one etymology for a language, we need to distinguish them as
# different senses in that language.
all_etyms = {
(item['from']['language'], etym_label(language, item['from']))
for item in items
if 'language' in item['from'] and item['from']['text'] == title
and etym_label(language, item['from']) is not None
}
word_languages = {wlang for (wlang, _) in all_etyms}
for wlang in sorted(word_languages):
if valid_language(wlang):
cpage = standardized_concept_uri(wlang, title)
ld_edge = make_edge('/r/ExternalURL',
cpage,
web_url,
dataset=dataset,
weight=0.25,
sources=[source],
license=Licenses.cc_sharealike)
out.write(ld_edge)
etym_to_translation_sense = {}
language_etym_counts = Counter(lang for (lang, etym) in all_etyms)
polysemous_languages = {
lang
for lang in language_etym_counts if language_etym_counts[lang] > 1
}
for item in items:
tfrom = item['from']
tto = item['to']
assumed_languages = [language]
lang1 = tfrom.get('language')
lang2 = tto.get('language')
if lang1 and (lang1
not in assumed_languages) and valid_language(lang1):
assumed_languages.append(lang1)
if lang2 and (lang2
not in assumed_languages) and valid_language(lang2):
assumed_languages.append(lang2)
cfrom = transform_term(language,
tfrom,
assumed_languages,
db,
use_etyms=(lang1 in polysemous_languages))
cpage = cfrom
cto = transform_term(language,
tto,
assumed_languages,
db,
use_etyms=(lang2 in polysemous_languages))
if cfrom is None or cto is None:
continue
if uri_prefix(cfrom, 3) == uri_prefix(cto, 3):
continue
rel, switch = transform_relation(item['rel'])
if rel is None:
continue
if switch:
cfrom, cto = cto, cfrom
# When translations are separated by sense, use only the first
# sense we see for each etymology. That will have the most
# representative translations.
if item['rel'] == 'translation':
etym_key = (tfrom['language'], etym_label(language, tfrom))
sense = tfrom.get('sense', '')
if etym_key in etym_to_translation_sense:
if etym_to_translation_sense[etym_key] != sense:
continue
else:
etym_to_translation_sense[etym_key] = sense
weight = 1.
if rel == '/r/EtymologicallyRelatedTo':
weight = 0.25
edge = make_edge(rel,
cfrom,
cto,
dataset=dataset,
weight=weight,
sources=[source],
surfaceStart=tfrom['text'],
surfaceEnd=tto['text'],
license=Licenses.cc_sharealike)
out.write(edge)
out.close()
def handle_file(infile, outfile):
count = 0
outcomes = defaultdict(int)
writer = MsgpackStreamWriter(outfile)
for line in open(infile):
parts = line.strip().split('\t')
if not parts:
outcomes['blank'] += 1
continue
# The first 5 columns of the Verbosity output file are:
#
# left: the word being clued
# relation: the relation between the word and the clue that the
# clue-giver chose, in a form such as "it is part of"
# right: the one or two words used as the clue
# freq: the number of different times this clue was given
# orderscore: the average position in the list of clues
#
# 'orderscore' is a number from 0 to 999, representing the average
# quantile of its position in the list of clues. (It's like a
# percentile, except there are 1000 of them, not 100.)
#
# A clue that's always given first has an orderscore of 0. A clue
# that always appears halfway through the list has an orderscore of
# 500.
#
# This may seem like a strange thing to measure, and I didn't come up
# with it, but it actually turns out to be somewhat informative.
# A clue with an orderscore of 0 is probably a good common-sense
# relation, representing the first thing that comes to mind. A clue
# with a high order score may be a move of desperation after several
# other clues have failed. It causes the guesser to get the answer
# soon afterward, but perhaps because it's a "cheating" move. So,
# low orderscores represent better common sense relations.
left, relation, right, freq, orderscore = parts[:5]
freq = int(freq)
orderscore = int(orderscore)
# Test each word
flagged = False
for rword in right.split():
if BAD_CLUE_REGEX.match(rword):
flagged = True
break
if flagged:
outcomes['flag word'] += 1
continue
if len(right) < 3:
outcomes['clue too short'] += 1
continue
if len(right.split()[-1]) == 1:
outcomes['letter'] += 1
continue
# The Verbosity interface and gameplay did not particularly encourage
# players to choose an appropriate relation. In practice, players seem
# to have used them all interchangeably, except for the negative
# relation "it is the opposite of", expressing /r/Antonym.
#
# Another way that players expressed negative relations was to use
# 'not' as the first word of their clue; we make that into an instance
# of /r/Antonym as well.
#
# In other cases, the relation is a positive relation, so we replace it
# with the most general positive relation, /r/RelatedTo.
rel = '/r/RelatedTo'
reltext = 'is related to'
if right.startswith('not '):
rel = '/r/DistinctFrom'
right = right[4:]
reltext = 'is not'
if relation == 'it is the opposite of':
rel = '/r/Antonym'
reltext = 'is the opposite of'
# The "sounds-like score" determines whether this clue seems to be a
# pun or rhyme, rather than an actual common-sense relationship. If
# the sounds-like score is over 0.35, skip the assertion.
sls = sounds_like_score(left, right)
if sls > 0.35:
outcomes['text similarity'] += 1
continue
# Calculate a score for the assertion:
#
# - The number of times it's been used as a clue
# - ...with a linear penalty for a high sounds-like score
# - ...and a linear penalty for high orderscores
#
# The penalties are multiplicative factors from 0 to 1, which decrease
# linearly as the relevant penalties increase. If a clue is given N
# times, with a sounds-like score of 0 and an orderscore of 0, it will
# get an overall score of 2N - 1. This is a formula we should probably
# revisit.
#
# The weight is the score divided by 100. All divisions are floating
# point.
score = (freq * 2 - 1) * (1 - sls) * (1 - orderscore / 1000)
if score <= 1.:
outcomes['low score'] += 1
continue
weight = score ** .5 / 10
# If the clue on the right is a two-word phrase, we make additional
# connections to both words individually. We label them with the
# rule-based source '/s/process/split_words' to track that this
# happened.
rightwords = [right]
if ' ' in right:
morewords = [word for word in right.split(' ') if word not in STOPWORDS]
rightwords.extend(morewords)
for i, rightword in enumerate(rightwords):
source = {'contributor': '/s/resource/verbosity'}
if i > 0:
source['process'] = '/s/process/split_words'
# Build the natural-language-ish surface text for this clue
text = '[[%s]] %s [[%s]]' % (left, reltext, rightword)
count += 1
outcomes['success'] += 1
leftc = standardized_concept_uri('en', left)
rightc = standardized_concept_uri('en', rightword)
edge = make_edge(
rel,
leftc,
rightc,
dataset='/d/verbosity',
license=Licenses.cc_attribution,
sources=[source],
surfaceText=text,
weight=weight,
)
writer.write(edge)
def build_from_dir(dirname, output_file):
"""
Read a GlobalMind database exported in YAML files, translate
it into ConceptNet 5 edges, and write those edges to disk using
a MsgpackStreamWriter.
"""
out = MsgpackStreamWriter(output_file)
userdata = yaml.load_all(open(dirname + '/GMUser.yaml'))
users = {}
for userinfo in userdata:
users[userinfo['pk']] = userinfo
frame_data = yaml.load_all(open(dirname + '/GMFrame.yaml'))
frames = {}
for frame in frame_data:
frames[frame['pk']] = frame['fields']
assertiondata = yaml.load_all(open(dirname + '/GMAssertion.yaml'))
assertions = {}
for assertion in assertiondata:
obj = assertion['fields']
frame = frames[obj['frame']]
frametext = frame['text']
userinfo = users[obj['author']]
username = userinfo['fields']['username']
# As far as I can tell, GlobalMind used the same namespace of
# usernames as the original Open Mind.
user_source = "/s/contributor/omcs/%s" % username
sources = [
user_source,
"/s/activity/globalmind/assert"
]
lang = LANG_CODES[obj['lcode']]
start = standardized_concept_uri(lang, obj['node1'])
end = standardized_concept_uri(lang, obj['node2'])
rel = '/r/' + RELATION_MAP.get(frame['relation'], frame['relation'])
# fix messy english "around in"
if ' around ' in frametext:
if obj['node2'].startswith('in '):
frametext = frametext.replace(' around ', ' in ')
obj['node2'] = obj['node2'][3:]
else:
frametext = frametext.replace(' around ', ' near ')
rel = '/r/LocatedNear'
# fix more awkward English. I wonder how bad the other languages are.
frametext = frametext.replace('hits your head', 'comes to mind')
frametext = frametext.replace(': [node1], [node2]', ' [node1] and [node2]')
node1 = u'[[' + obj['node1'] + u']]'
node2 = u'[[' + obj['node2'] + u']]'
surfaceText = frametext.replace('//', '').replace('[node1]', node1).replace('[node2]', node2)
edge = make_edge(rel, start, end,
dataset='/d/globalmind',
license='/l/CC/By',
sources=sources,
surfaceText=surfaceText,
weight=1)
# Avoid duplication with the ConceptNet reader, but still save every edge so that we can
# handle translations.
if username != 'openmind':
out.write(edge)
assertions[assertion['pk']] = edge
translationdata = yaml.load_all(open(dirname + '/GMTranslation.yaml'))
for translation in translationdata:
obj = translation['fields']
assertion1 = assertions[obj['assertion1']]
assertion2 = assertions[obj['assertion2']]
start = assertion1['uri']
end = assertion2['uri']
rel = '/r/TranslationOf'
text1 = assertion1['surfaceText'].replace('[[', '').replace(']]', '')
text2 = assertion2['surfaceText'].replace('[[', '').replace(']]', '')
lang1 = LANG_NAMES[get_lang(assertion1)]
lang2 = LANG_NAMES[get_lang(assertion2)]
surfaceText = u"[[%s]] in %s means [[%s]] in %s." % (text1, lang1, text2, lang2)
userinfo = users[obj['author']]
username = userinfo['fields']['username']
userlocale = userinfo['fields']['ccode'].lower()
if userlocale:
user_source = "/s/contributor/globalmind/%s/%s" % (userlocale, username)
else:
user_source = "/s/contributor/globalmind/%s" % username
sources = [
user_source,
"/s/activity/globalmind/translate"
]
edge = make_edge(rel, start, end,
dataset='/d/globalmind',
license=Licenses.cc_attribution,
sources=sources,
surfaceText=surfaceText,
weight=1)
out.write(edge)
def run_wordnet(input_file, output_file):
out = MsgpackStreamWriter(output_file)
synset_senses = defaultdict(list)
sense_synsets = {}
synset_labels = defaultdict(list)
synset_canonical_labels = {}
synset_categories = {}
synset_domains = {}
synset_glosses = {}
synset_disambig = {}
synset_uris = {}
# First pass: find data about synsets
quads = parse_nquads(open(input_file, encoding='utf-8'))
for subj_dict, rel_dict, obj_dict, _graph in quads:
if 'url' not in subj_dict or 'url' not in rel_dict:
continue
subj = subj_dict['url']
rel = rel_dict['url']
obj = obj_dict.get('url')
objtext = obj_dict.get('text')
relname = resource_name(rel)
if relname == 'label':
if obj_dict['lang'] == 'en':
synset_labels[subj].append(objtext)
elif relname == 'sameAs':
if obj.startswith(WN20_URL):
# If we have a link to RDF WordNet 2.0, the URL (URI? IRI?)
# will contain a standardized label for this concept, which
# we should use when we want to use this synset as the name of
# a disambiguation category. RDF WordNet 3.1 assigns synsets
# a number of labels in no particular order, making it hard to
# determine from 3.1 alone what to name a category.
objname = resource_name(obj)
parts = objname.split('-')[1:-2]
# Handle missing apostrophes
label = '-'.join(parts).replace('_s_', "'s_").replace('_s-', "'s_").replace("s__", "s'_").replace("s_-", "s'-").replace('_', ' ')
synset_canonical_labels[subj] = label
elif relname == 'domain_category':
synset_categories[subj] = obj
elif relname == 'lexical_domain':
target = resource_name(obj)
if '.' in target:
domain = target.split('.')[1]
synset_domains[subj] = domain
elif relname == 'gloss':
synset_glosses[subj] = objtext
elif relname == 'reference':
lemma = resource_name(subj)
synset = obj
synset_senses[synset].append(lemma)
sense_synsets[lemma] = synset
used_labels = set(synset_canonical_labels.values())
for synset, values in synset_labels.items():
values.sort(key=lambda label: (label in used_labels,) + label_sort_key(label))
if (
synset not in synset_canonical_labels or
synset_canonical_labels[synset][0].isupper() and synset_domains.get(synset) == 'person'
):
label = values[0]
synset_canonical_labels[synset] = label
used_labels.add(label)
for synset, labels in synset_labels.items():
if synset in synset_categories:
category_name = synset_canonical_labels[synset_categories[synset]]
else:
category_name = synset_domains.get(synset, None)
synset_no_fragment = synset.split('#')[0]
pos = synset_no_fragment[-1].lower()
assert pos in 'nvarsp', synset
if pos == 's':
pos = 'a'
elif pos == 'p':
pos = '-'
if category_name in ('pert', 'all', 'tops'):
category_name = None
synset_disambig[synset] = (pos, category_name)
canon = synset_canonical_labels[synset]
canon_uri = standardized_concept_uri('en', canon, pos, 'wn', category_name)
synset_uris[synset] = canon_uri
for label in labels:
if label != canon:
other_uri = standardized_concept_uri('en', label, pos, 'wn', category_name)
rel_uri = '/r/Synonym'
surface = '[[{0}]] is a synonym of [[{1}]]'.format(label, canon)
edge = make_edge(
rel_uri, other_uri, canon_uri, dataset=DATASET, surfaceText=surface,
license=Licenses.cc_attribution, sources=[SOURCE], weight=2.0
)
out.write(edge)
quads = parse_nquads(open(input_file, encoding='utf-8'))
for subj_dict, rel_dict, obj_dict, _graph in quads:
if 'url' not in subj_dict or 'url' not in rel_dict:
continue
subj = subj_dict['url']
rel = rel_dict['url']
obj = obj_dict.get('url')
relname = resource_name(rel)
if relname in REL_MAPPING:
rel, frame = REL_MAPPING[relname]
reversed_frame = False
if rel.startswith('~'):
rel = rel[1:]
reversed_frame = True
rel_uri = '/r/' + rel
if obj is not None:
obj_uri = synset_uris.get(obj)
if obj not in synset_canonical_labels:
continue
obj_label = synset_canonical_labels[obj]
else:
text = obj_dict['text']
# Some WordNets use strings with "!" in them to indicate
# out-of-band information, such as a missing translation
if (not text) or '!' in text:
continue
lang = obj_dict['lang']
pos, sense = synset_disambig.get(subj, (None, None))
obj_uri = standardized_concept_uri(lang, text, pos, 'wn', sense)
obj_label = text
if subj not in synset_uris or subj not in synset_canonical_labels:
continue
subj_uri = synset_uris[subj]
subj_label = synset_canonical_labels[subj]
license = Licenses.cc_attribution
langcode = subj_uri.split('/')[2]
if langcode in SHAREALIKE_LANGUAGES:
license = Licenses.cc_sharealike
if reversed_frame:
subj_uri, obj_uri = obj_uri, subj_uri
subj_label, obj_label = obj_label, subj_label
surface = frame.format('[[%s]]' % subj_label, '[[%s]]' % obj_label)
edge = make_edge(
rel_uri, subj_uri, obj_uri, dataset=DATASET, surfaceText=surface,
license=license, sources=[SOURCE], weight=2.0
)
out.write(edge)
for wn_url in sorted(synset_uris):
cn_uri = synset_uris[wn_url]
edge = make_edge(
'/r/ExternalURL', cn_uri, wn_url, dataset=DATASET,
license=Licenses.cc_sharealike, sources=[SOURCE], weight=1.0
)
out.write(edge)
out.close()
def build_end(parts_dict):
lang = parts_dict['lang']
endText = parts_dict["endText"]
end = standardized_concept_uri(lang, endText)
return end
import numpy as np
import pandas as pd
from conceptnet5.nodes import standardized_concept_uri
from tools import find_closest_in_dataset
if __name__ == '__main__':
numberbatch = pd.read_hdf("../retroembeddings.h5","mat",encoding="utf-8")
testwords = ["dog","doggo","pink elephant","elephant"]
for idx, word in enumerate(testwords):
print(word)
retro_representation = np.array(list(numberbatch.loc[standardized_concept_uri("en",word)]))
find_closest_in_dataset(retro_representation, "../retroembeddings.h5",n_top=20)
def web_search():
keyword = request.form.get('keyword')
lang = request.form.get('language')
return redirect(WEB_ROOT + standardized_concept_uri(lang, keyword))
def text_to_vector(text, assoc, standardize=False):
if standardize:
text = standardized_concept_uri('en', text)
return assoc.vector_from_terms([(text, 1.)])
def build_from_dir(dirname, output_file):
"""
Read a GlobalMind database exported in YAML files, translate
it into ConceptNet 5 edges, and write those edges to disk using
a MsgpackStreamWriter.
"""
out = MsgpackStreamWriter(output_file)
userdata = yaml.load_all(open(dirname + '/GMUser.yaml'))
users = {}
for userinfo in userdata:
users[userinfo['pk']] = userinfo
frame_data = yaml.load_all(open(dirname + '/GMFrame.yaml'))
frames = {}
for frame in frame_data:
frames[frame['pk']] = frame['fields']
assertiondata = yaml.load_all(open(dirname + '/GMAssertion.yaml'))
assertions = {}
for assertion in assertiondata:
obj = assertion['fields']
frame = frames[obj['frame']]
frametext = frame['text']
userinfo = users[obj['author']]
username = userinfo['fields']['username']
# As far as I can tell, GlobalMind used the same namespace of
# usernames as the original Open Mind.
user_source = "/s/contributor/omcs/%s" % username
sources = [user_source, "/s/activity/globalmind/assert"]
lang = LANG_CODES[obj['lcode']]
start = standardized_concept_uri(lang, obj['node1'])
end = standardized_concept_uri(lang, obj['node2'])
rel = '/r/' + RELATION_MAP.get(frame['relation'], frame['relation'])
# fix messy english "around in"
if ' around ' in frametext:
if obj['node2'].startswith('in '):
frametext = frametext.replace(' around ', ' in ')
obj['node2'] = obj['node2'][3:]
else:
frametext = frametext.replace(' around ', ' near ')
rel = '/r/LocatedNear'
# fix more awkward English. I wonder how bad the other languages are.
frametext = frametext.replace('hits your head', 'comes to mind')
frametext = frametext.replace(': [node1], [node2]',
' [node1] and [node2]')
node1 = u'[[' + obj['node1'] + u']]'
node2 = u'[[' + obj['node2'] + u']]'
surfaceText = frametext.replace('//', '').replace('[node1]',
node1).replace(
'[node2]', node2)
edge = make_edge(rel,
start,
end,
dataset='/d/globalmind',
license='/l/CC/By',
sources=sources,
surfaceText=surfaceText,
weight=1)
# Avoid duplication with the ConceptNet reader, but still save every edge so that we can
# handle translations.
if username != 'openmind':
out.write(edge)
assertions[assertion['pk']] = edge
translationdata = yaml.load_all(open(dirname + '/GMTranslation.yaml'))
for translation in translationdata:
obj = translation['fields']
assertion1 = assertions[obj['assertion1']]
assertion2 = assertions[obj['assertion2']]
start = assertion1['uri']
end = assertion2['uri']
rel = '/r/TranslationOf'
text1 = assertion1['surfaceText'].replace('[[', '').replace(']]', '')
text2 = assertion2['surfaceText'].replace('[[', '').replace(']]', '')
lang1 = LANG_NAMES[get_lang(assertion1)]
lang2 = LANG_NAMES[get_lang(assertion2)]
surfaceText = u"[[%s]] in %s means [[%s]] in %s." % (text1, lang1,
text2, lang2)
userinfo = users[obj['author']]
username = userinfo['fields']['username']
userlocale = userinfo['fields']['ccode'].lower()
if userlocale:
user_source = "/s/contributor/globalmind/%s/%s" % (userlocale,
username)
else:
user_source = "/s/contributor/globalmind/%s" % username
sources = [user_source, "/s/activity/globalmind/translate"]
edge = make_edge(rel,
start,
end,
dataset='/d/globalmind',
license=Licenses.cc_attribution,
sources=sources,
surfaceText=surfaceText,
weight=1)
out.write(edge)
def build_end(parts_dict):
lang = parts_dict['lang']
endText = parts_dict["endText"]
end = standardized_concept_uri(lang, endText)
return end
def run_wordnet(input_dir, output_file, sw_map_file):
out = MsgpackStreamWriter(output_file)
map_out = NTriplesWriter(sw_map_file)
reader = NTriplesReader()
synset_senses = defaultdict(list)
sense_synsets = {}
labels = {}
glossary = {}
concept_map = {}
sense_to_synset = {}
# Parse lines such as:
# wn30:synset-Aeolian-noun-2 rdfs:label "Aeolian"@en-us .
for subj, rel, obj, objtag in reader.parse_file(os.path.join(input_dir, 'wordnet-synset.ttl')):
if resource_name(rel) == 'label':
# Everything in WordNet is in English
assert objtag == 'en'
labels[subj] = obj
for subj, rel, obj, objtag in reader.parse_file(os.path.join(input_dir, 'wordnet-glossary.ttl')):
if resource_name(rel) == 'gloss':
assert objtag == 'en'
# Take the definition up to the first semicolon
text = obj.split(';')[0]
# Remove introductory phrases with a colon
text = text.split(': ', 1)[-1]
# Remove parenthesized expressions
while True:
newtext = re.sub(r'\(.+?\) ?', '', text).strip()
if newtext == text or newtext == '':
break
else:
text = newtext
glossary[subj] = text.replace('/', '_')
# Get the list of word senses in each synset, and make a bidirectional mapping.
#
# Example line:
# wn30:synset-Aeolian-noun-2 wn20schema:containsWordSense wn30:wordsense-Aeolian-noun-2 .
for subj, rel, obj, objtag in reader.parse_file(os.path.join(input_dir, 'full/wordnet-wordsense-synset-relations.ttl')):
if resource_name(rel) == 'containsWordSense':
synset_senses[subj].append(obj)
sense_synsets[obj] = subj
# Assign every synset to a disambiguated concept.
for synset in synset_senses:
synset_name = labels[synset]
synset_pos = synset.split('-')[-2]
pos = PARTS_OF_SPEECH[synset_pos]
disambig = glossary[synset]
concept = standardized_concept_uri('en', synset_name, pos, disambig)
concept_map[synset] = concept
# Map senses to their synsets.
for sense, synset in sense_synsets.items():
sense_to_synset[sense] = synset
for filename in (
'wordnet-attribute.ttl', 'wordnet-causes.ttl',
'wordnet-classifiedby.ttl', 'wordnet-entailment.ttl',
'wordnet-hyponym.ttl', 'wordnet-instances.ttl',
'wordnet-membermeronym.ttl', 'wordnet-partmeronym.ttl',
'wordnet-sameverbgroupas.ttl', 'wordnet-similarity.ttl',
'wordnet-substancemeronym.ttl', 'full/wordnet-antonym.ttl',
'full/wordnet-derivationallyrelated.ttl',
'full/wordnet-participleof.ttl',
'full/wordnet-pertainsto.ttl',
'full/wordnet-seealso.ttl'
):
filepath = os.path.join(input_dir, filename)
if os.path.exists(filepath):
for web_subj, web_rel, web_obj, objtag in reader.parse_file(filepath):
# If this relation involves word senses, map them to their synsets
# first.
if web_subj in sense_to_synset:
web_subj = sense_to_synset[web_subj]
if web_obj in sense_to_synset:
web_obj = sense_to_synset[web_obj]
subj = concept_map[web_subj]
obj = concept_map[web_obj]
pred_label = resource_name(web_rel)
if pred_label in REL_MAPPING:
mapped_rel = REL_MAPPING[pred_label]
# Handle WordNet relations that are the reverse of ConceptNet
# relations. Change the word 'meronym' to 'holonym' if
# necessary.
if mapped_rel.startswith('~'):
subj, obj = obj, subj
web_subj, web_obj = web_obj, web_subj
web_rel = web_rel.replace('meronym', 'holonym')
mapped_rel = mapped_rel[1:]
rel = join_uri('r', mapped_rel)
else:
rel = join_uri('r', 'wordnet', pred_label)
map_out.write_link(web_rel, full_conceptnet_url(rel))
map_out.write_link(web_subj, full_conceptnet_url(subj))
map_out.write_link(web_obj, full_conceptnet_url(obj))
edge = make_edge(
rel, subj, obj, dataset='/d/wordnet/3.0',
license='/l/CC/By', sources=SOURCE, weight=2.0
)
out.write(edge)
def query_maker(term):
query = standardized_concept_uri('en', term)
return query
def read_wiktionary(input_file, db_file, output_file):
"""
Convert a stream of parsed Wiktionary data into ConceptNet edges.
A `db_file` containing all known words in all languages must have already
been prepared from the same data.
"""
db = sqlite3.connect(db_file)
out = MsgpackStreamWriter(output_file)
for heading, items in segmented_stream(input_file):
language = heading['language']
title = heading['title']
dataset = '/d/wiktionary/{}'.format(language)
url_title = heading['title'].replace(' ', '_')
web_url = 'http://{}.wiktionary.org/wiki/{}'.format(language, url_title)
web_source = '/s/resource/wiktionary/{}'.format(language)
source = {
'contributor': web_source,
'process': PARSER_RULE
}
# Scan through the 'from' items, such as the start nodes of
# translations, looking for distinct etymologies. If we get more than
# one etymology for a language, we need to distinguish them as
# different senses in that language.
all_etyms = {
(item['from']['language'], etym_label(language, item['from']))
for item in items
if 'language' in item['from'] and item['from']['text'] == title
and etym_label(language, item['from']) is not None
}
word_languages = {wlang for (wlang, _) in all_etyms}
for wlang in sorted(word_languages):
cpage = standardized_concept_uri(wlang, title)
ld_edge = make_edge(
'/r/ExternalURL', cpage, web_url,
dataset=dataset, weight=0.25, sources=[source],
license=Licenses.cc_sharealike
)
out.write(ld_edge)
etym_to_translation_sense = {}
language_etym_counts = Counter(lang for (lang, etym) in all_etyms)
polysemous_languages = {
lang for lang in language_etym_counts
if language_etym_counts[lang] > 1
}
for item in items:
tfrom = item['from']
tto = item['to']
assumed_languages = [language]
lang1 = tfrom.get('language')
lang2 = tto.get('language')
if lang1 and (lang1 not in assumed_languages) and valid_language(lang1):
assumed_languages.append(lang1)
if lang2 and (lang2 not in assumed_languages) and valid_language(lang2):
assumed_languages.append(lang2)
cfrom = transform_term(
language, tfrom, assumed_languages, db,
use_etyms=(lang1 in polysemous_languages)
)
cpage = cfrom
cto = transform_term(
language, tto, assumed_languages, db,
use_etyms=(lang2 in polysemous_languages)
)
if cfrom is None or cto is None:
continue
if uri_prefix(cfrom, 3) == uri_prefix(cto, 3):
continue
rel, switch = transform_relation(item['rel'])
if rel is None:
continue
if switch:
cfrom, cto = cto, cfrom
# When translations are separated by sense, use only the first
# sense we see for each etymology. That will have the most
# representative translations.
if item['rel'] == 'translation':
etym_key = (tfrom['language'], etym_label(language, tfrom))
sense = tfrom.get('sense', '')
if etym_key in etym_to_translation_sense:
if etym_to_translation_sense[etym_key] != sense:
continue
else:
etym_to_translation_sense[etym_key] = sense
weight = 1.
if rel == '/r/EtymologicallyRelatedTo':
weight = 0.25
edge = make_edge(rel, cfrom, cto, dataset=dataset, weight=weight,
sources=[source],
surfaceStart=tfrom['text'],
surfaceEnd=tto['text'],
license=Licenses.cc_sharealike)
out.write(edge)
out.close()
def run_wordnet(input_file, output_file):
out = MsgpackStreamWriter(output_file)
synset_senses = defaultdict(list)
sense_synsets = {}
synset_labels = defaultdict(list)
synset_canonical_labels = {}
synset_categories = {}
synset_domains = {}
synset_glosses = {}
synset_disambig = {}
synset_uris = {}
# First pass: find data about synsets
quads = parse_nquads(open(input_file, encoding='utf-8'))
for subj_dict, rel_dict, obj_dict, _graph in quads:
if 'url' not in subj_dict or 'url' not in rel_dict:
continue
subj = subj_dict['url']
rel = rel_dict['url']
obj = obj_dict.get('url')
objtext = obj_dict.get('text')
relname = resource_name(rel)
if relname == 'label':
if obj_dict['lang'] == 'en':
synset_labels[subj].append(objtext)
elif relname == 'sameAs':
if obj.startswith(WN20_URL):
# If we have a link to RDF WordNet 2.0, the URL (URI? IRI?)
# will contain a standardized label for this concept, which
# we should use when we want to use this synset as the name of
# a disambiguation category. RDF WordNet 3.1 assigns synsets
# a number of labels in no particular order, making it hard to
# determine from 3.1 alone what to name a category.
objname = resource_name(obj)
parts = objname.split('-')[1:-2]
# Handle missing apostrophes
label = '-'.join(parts).replace('_s_', "'s_").replace(
'_s-',
"'s_").replace("s__",
"s'_").replace("s_-",
"s'-").replace('_', ' ')
synset_canonical_labels[subj] = label
elif relname == 'domain_category':
synset_categories[subj] = obj
elif relname == 'lexical_domain':
target = resource_name(obj)
if '.' in target:
domain = target.split('.')[1]
synset_domains[subj] = domain
elif relname == 'gloss':
synset_glosses[subj] = objtext
elif relname == 'reference':
lemma = resource_name(subj)
synset = obj
synset_senses[synset].append(lemma)
sense_synsets[lemma] = synset
used_labels = set(synset_canonical_labels.values())
for synset, values in synset_labels.items():
values.sort(
key=lambda label: (label in used_labels, ) + label_sort_key(label))
if (synset not in synset_canonical_labels
or synset_canonical_labels[synset][0].isupper()
and synset_domains.get(synset) == 'person'):
label = values[0]
synset_canonical_labels[synset] = label
used_labels.add(label)
for synset, labels in synset_labels.items():
if synset in synset_categories:
category_name = synset_canonical_labels[synset_categories[synset]]
else:
category_name = synset_domains.get(synset, None)
synset_no_fragment = synset.split('#')[0]
pos = synset_no_fragment[-1].lower()
assert pos in 'nvarsp', synset
if pos == 's':
pos = 'a'
elif pos == 'p':
pos = '-'
if category_name in ('pert', 'all', 'tops'):
category_name = None
synset_disambig[synset] = (pos, category_name)
canon = synset_canonical_labels[synset]
canon_uri = standardized_concept_uri('en', canon, pos, 'wn',
category_name)
synset_uris[synset] = canon_uri
for label in labels:
if label != canon:
other_uri = standardized_concept_uri('en', label, pos, 'wn',
category_name)
rel_uri = '/r/Synonym'
surface = '[[{0}]] is a synonym of [[{1}]]'.format(
label, canon)
edge = make_edge(rel_uri,
other_uri,
canon_uri,
dataset=DATASET,
surfaceText=surface,
license=Licenses.cc_attribution,
sources=[SOURCE],
weight=2.0)
out.write(edge)
quads = parse_nquads(open(input_file, encoding='utf-8'))
for subj_dict, rel_dict, obj_dict, _graph in quads:
if 'url' not in subj_dict or 'url' not in rel_dict:
continue
subj = subj_dict['url']
rel = rel_dict['url']
obj = obj_dict.get('url')
relname = resource_name(rel)
if relname in REL_MAPPING:
pos, sense = synset_disambig.get(subj, (None, None))
if relname == 'hypernym' and pos == 'v':
relname = 'hypernym-v'
rel, frame = REL_MAPPING[relname]
reversed_frame = False
if rel.startswith('~'):
rel = rel[1:]
reversed_frame = True
rel_uri = '/r/' + rel
if obj is not None:
obj_uri = synset_uris.get(obj)
if obj not in synset_canonical_labels:
continue
obj_label = synset_canonical_labels[obj]
else:
text = obj_dict['text']
# Some WordNets use strings with "!" in them to indicate
# out-of-band information, such as a missing translation
if (not text) or '!' in text:
continue
lang = obj_dict['lang']
obj_uri = standardized_concept_uri(lang, text, pos, 'wn',
sense)
obj_label = text
if subj not in synset_uris or subj not in synset_canonical_labels:
continue
subj_uri = synset_uris[subj]
subj_label = synset_canonical_labels[subj]
license = Licenses.cc_attribution
langcode = subj_uri.split('/')[2]
if langcode in SHAREALIKE_LANGUAGES:
license = Licenses.cc_sharealike
if reversed_frame:
subj_uri, obj_uri = obj_uri, subj_uri
subj_label, obj_label = obj_label, subj_label
surface = frame.format('[[%s]]' % subj_label, '[[%s]]' % obj_label)
edge = make_edge(rel_uri,
subj_uri,
obj_uri,
dataset=DATASET,
surfaceText=surface,
license=license,
sources=[SOURCE],
weight=2.0)
out.write(edge)
for wn_url in sorted(synset_uris):
cn_uri = synset_uris[wn_url]
edge = make_edge('/r/ExternalURL',
cn_uri,
wn_url,
dataset=DATASET,
license=Licenses.cc_sharealike,
sources=[SOURCE],
weight=1.0)
out.write(edge)
out.close()
def filtered_uri(lang, text):
if lang == 'en':
text = filter_stopwords(text)
return standardized_concept_uri(lang, text)
def run_umbel(input_dir, output_file, sw_map_file):
"""
Read N-Triples files containing Umbel data, outputting a file of
ConceptNet edges and a file of mappings between the Semantic Web and
ConceptNet.
"""
out = MsgpackStreamWriter(output_file)
map_out = NTriplesWriter(sw_map_file)
reader = NTriplesReader()
labels = {}
label_sets = defaultdict(set)
# There are two files we want to parse:
# - umbel.nt, a transformation of umbel.n3, which is available from
# https://github.com/structureddynamics/UMBEL/.
# - umbel_links.nt, distributed with DBPedia 3.9.
#
# We parse them both in this file so that umbel_links can reuse the
# concept names extracted from umbel.nt.
main_file = os.path.join(input_dir, 'umbel.nt')
dbpedia_link_file = os.path.join(input_dir, 'umbel_links.nt')
# Read through umbel.nt once, finding all the "preferred labels". We will
# use these as the surface texts for the nodes.
for web_subj, web_rel, web_obj, objtag in reader.parse_file(main_file):
if resource_name(web_rel) == 'prefLabel':
# 'CW' and 'PCW' are Cyc jargon for 'conceptual works'. If a node
# cannot be described except as a CW, we're probably not
# interested in it.
if 'CW' not in web_obj.split() and 'PCW' not in web_obj.split():
labels[web_subj] = web_obj
if resource_name(web_rel).endswith('Label'):
text = standardize_text(web_obj)
label_sets[text].add(web_subj)
# Read through umbel.nt again and extract ConceptNet edges.
for web_subj, web_rel, web_obj, objtag in reader.parse_file(main_file):
if objtag == 'URL' and acceptable_node(web_obj) and acceptable_node(web_subj):
# Only use nodes for which we've seen preferred labels.
# (This skips some anonymous OWL-cruft nodes.)
if web_subj in labels and web_obj in labels:
subj_uri = standardized_concept_uri('en', labels[web_subj])
obj_uri = standardized_concept_uri('en', labels[web_obj])
rel_name = resource_name(web_rel)
# Check if this is a relation we want to handle.
if rel_name in REL_MAPPING:
# Write the ConceptNet edges and the mappings to Semantic Web URLs.
rel_uri, frame = REL_MAPPING[rel_name]
surface = frame % (labels[web_subj], labels[web_obj])
out.write(umbel_edge(rel_uri, subj_uri, obj_uri, surface, SOURCE))
map_out.write_link(web_rel, full_conceptnet_url(rel_uri))
map_out.write_link(web_subj, full_conceptnet_url(subj_uri))
map_out.write_link(web_obj, full_conceptnet_url(obj_uri))
# altLabel relations assign different texts to the same node. We'll
# represent those in ConceptNet with Synonym relations.
elif web_rel.endswith('altLabel'):
# Make sure we know what's being labeled.
if web_subj in labels:
name = web_obj
words = name.split(' ')
if standardized_concept_name('en', name) != standardized_concept_name('en', labels[web_subj]):
if not set(words) & IGNORED_WORDS:
main_label = standardized_concept_uri('en', labels[web_subj])
name_text = standardize_text(name)
if len(label_sets[name_text]) >= 2 or len(name_text) <= 3:
disambig = un_camel_case(resource_name(web_subj))
# Cyc does not distinguish texts by their part of speech, so use
# '_' as the part of speech symbol.
alt_label = standardized_concept_uri('en', name, '_', disambig)
else:
alt_label = standardized_concept_uri('en', name)
surface = SYN_FRAME % (name, labels[web_subj])
out.write(umbel_edge('/r/Synonym', alt_label, main_label, surface, SOURCE))
for web_subj, web_rel, web_obj, objtag in reader.parse_file(dbpedia_link_file):
if objtag == 'URL' and acceptable_node(web_obj) and acceptable_node(web_subj):
if web_obj in labels:
subj_label = resource_name(web_subj).replace('_', ' ')
subj_uri = translate_dbpedia_url(web_subj)
obj_label = labels[web_obj]
obj_uri = standardized_concept_uri('en', obj_label)
rel_name = resource_name(web_rel)
if rel_name in REL_MAPPING:
rel_uri, frame = REL_MAPPING[rel_name]
surface = frame % (subj_label, obj_label)
out.write(umbel_edge(rel_uri, subj_uri, obj_uri, surface, LINK_SOURCE))
map_out.write_link(web_rel, full_conceptnet_url(rel_uri))
map_out.write_link(web_subj, full_conceptnet_url(subj_uri))
map_out.write_link(web_obj, full_conceptnet_url(obj_uri))
def handle_file(filename, output_file):
"""
JMdict is a Japanese translation dictionary, targeting multiple languages,
released under a Creative Commons Attribution-ShareAlike license. That's
awesome.
It's released as a kind of annoying XML structure, using fancy XML features
like entities, so in order to read it we need a full-blown XML parser. Python's
built-in XML parsers are horrifying, so here we use the 'xmltodict' module, which
is also horrifying but gets the job done.
The majorly weird thing about xmltodict that we have to work around is that
it gives results of different *types* when you get 0, 1, or many child nodes.
This is what get_list is for.
"""
# Read the XML file as UTF-8, and parse it into a dictionary.
file = codecs.open(filename, encoding='utf-8')
out = MsgpackStreamWriter(output_file)
data = file.read()
file.close()
xml = xmltodict.parse(data)
# The dictionary contains a list of <entry> tags.
root_node = xml['JMdict']
for entry in get_list(root_node, 'entry'):
# From JMdict's documentation: "The kanji element, or in its absence,
# the reading element, is the defining component of each entry."
#
# Quick summary of what's going on here: most Japanese words can be
# written using kanji or kana.
#
# Kana are phonetic characters. Every word can be written in kana, in
# one of two alphabets (hiragana or katakana). Words that are homonyms
# have the same kana, unless they're written in different alphabets.
#
# Kanji are Chinese-based characters that are related to the meaning of
# the word. They're compact and good at disambiguating homonyms, so
# kanji are usually used as the canonical representation of a word.
# However, some words have no kanji.
#
# The kana version of a word written in kanji is called its 'reading'.
# Words that are pronounced differently in different contexts have
# multiple readings.
#
# Okay, done with the intro to Japanese orthography. In JMdict, if
# a word can be written in kanji, it has a <k_ele> element, containing
# a <keb> element that contains the text. Every word also has an
# <r_ele> element, containing one or more <reb> elements that are phonetic
# readings of the word.
#
# We get the "defining text" of a word by taking its <keb> if it exists,
# or all of its <reb>s if not. There's no way to tell which <reb> is the
# most "defining" in the case where there's no <keb>.
headwords = [word['keb'] for word in get_list(entry, 'k_ele')]
if not headwords:
headwords = [word['reb'] for word in get_list(entry, 'r_ele')]
# An entry might have different word senses that are translated
# differently to other languages. Ideally, we'd remember that they're
# different senses. However, we have no way to refer to the different
# senses. So for now, we disregard word senses. One day we might have
# a better overall plan for word senses in ConceptNet.
senses = get_list(entry, 'sense')
for sense_num, sense in enumerate(senses):
# Glosses are translations of the word to different languages.
# If the word is a loan-word, the foreign word it was derived from
# will be marked with the <lsource> tag instead of <gloss>.
#
# Get all the glosses, including the lsource if it's there.
glosses = get_list(sense, 'gloss') + get_list(sense, 'lsource')
contexts = [
fix_context(context)
for context in get_list(sense, 'field')
]
pos = '_'
for pos_tag in get_list(sense, 'pos'):
if pos_tag[:10] in NOUN_TYPES:
pos = 'n'
elif pos_tag[:10] in VERB_TYPES:
pos = 'v'
elif pos_tag[:10] in ADJ_TYPES:
pos = 'a'
elif pos_tag[:10] in ADV_TYPES:
pos = 'r'
for gloss in glosses:
if '#text' in gloss:
# A gloss node might be marked with a 'lang' attribute. If so,
# xmltodict represents it as a dictionary with '#text' and
# '@xml:lang' elements.
text = parse_gloss(gloss['#text'])
lang = convert_lang_code(gloss['@xml:lang'])
elif isinstance(gloss, STRING_TYPE):
# If there's no 'lang' attribute, the gloss is in English,
# and xmltodict gives it to us as a plain Unicode string.
lang = 'en'
text = parse_gloss(gloss)
# If we parsed the node at all and the text looks good, then we can
# add edges to ConceptNet.
#
# We don't want to deal with texts with periods (these might be
# dictionary-style abbreviations, which are sort of unhelpful when
# we can't expand them), and we also don't want to deal with texts
# that are more than five words long.
if (
text is not None and '.' not in text and
text.count(' ') <= 4 and valid_concept_name(text)
):
for head in headwords:
if len(senses) >= 2:
sensekey = '%d' % (sense_num + 1)
ja_concept = standardized_concept_uri('ja', head, pos, 'jmdict', sensekey)
else:
ja_concept = standardized_concept_uri('ja', head, pos)
other_concept = standardized_concept_uri(lang, text)
output_edge(out, '/r/Synonym', ja_concept, other_concept)
for context in contexts:
context_node = standardized_concept_uri('en', context)
output_edge(out, '/r/HasContext', ja_concept, context_node)
elif field_match(end(edge), term) and split_uri(start(edge))[1] == 'en':
neighbor = edge['start']
else:
continue
neighbor_weight = 1.0 * min(10, edge['weight'])
if edge['rel'].startswith('/r/Not'):
neighbor_weight *= -1
for prefix in uri_prefixes(neighbor):
uris.add(prefix)
results.append((neighbor, neighbor_weight))
total_weight = sum(abs(weight) for (term, weight) in results)
if total_weight == 0:
return []
return [(term, weight, weight / total_weight) for (term, weight) in results]
expand_terms([standardized_concept_uri('en', 'fetish')], 10)
def associations(self, terms, filter=None, limit=20):
vec = expanded_vector(terms)
similar = terms_similar_to_vector(vec, num=None)
similar = [item for item in similar if item[1] > SMALL and self.passes_filter(item[0], filter)][:limit]
return similar
from builtins import property as _property, tuple as _tuple
from operator import itemgetter as _itemgetter
from collections import OrderedDict
DB_PATH = '/Users/Kristen/.conceptnet5/data/db/assertions.db'
ASSERTION_DIR = '/Users/Kristen/.conceptnet5/data/assertions'
def filtered_uri(lang, text):
if lang == "en":
text = filter_stopwords(text)
return concept_uri("en", standardize_text(text, english_filter))
else:
return standardized_concept_uri(lang, text)
def filtered_uri(lang, text):
if lang == 'en':
text = filter_stopwords(text)
return standardized_concept_uri(lang, text)
def filtered_uri(lang, text):
if lang == 'en':
text = filter_stopwords(text)
return concept_uri('en', standardize_text(text, english_filter))
else:
return standardized_concept_uri(lang, text)
def handle_file(filename, output_file):
out = MsgpackStreamWriter(output_file)
for line in gzip.open(filename, 'rt'):
# skip the intro information
if line.startswith('#'):
continue
# parse the data to extract the traditional form, simplified form and the English definition
traditional, simplified, definitions = re.match(LINE_REGEX, line).groups()
# Make an edge between the traditional and simplified version
edge = make_edge(
rel='/r/Synonym',
start=standardized_concept_uri('zh-Hant', traditional),
end=standardized_concept_uri('zh-Hans', simplified),
dataset=DATASET,
license=LICENSE,
sources=SOURCE,
)
out.write(edge)
for definition in re.split(DEFINITIONS_REGEX, definitions):
# Skip pronunciation information
if 'Taiwan pr.' in definition or 'also pr.' in definition:
continue
# Check if it's the definition matches a person syntax, i.e. includes a date range
person_match = re.match(DATE_RANGE_REGEX, definition)
if person_match:
persons = extract_person(person_match)
for person in persons:
edge = make_edge(
rel='/r/Synonym',
start=standardized_concept_uri('zh-Hant', traditional),
end=standardized_concept_uri('en', person),
dataset=DATASET,
license=LICENSE,
sources=SOURCE,
)
out.write(edge)
edge = make_edge(
rel='/r/Synonym',
start=standardized_concept_uri('zh-Hans', simplified),
end=standardized_concept_uri('en', person),
dataset=DATASET,
license=LICENSE,
sources=SOURCE,
)
out.write(edge)
continue
# Check if a word is a measure word
if definition.startswith('CL:'):
related_words = extract_measure_words(definition)
for word in related_words:
edge = make_edge(
rel='/r/RelatedTo',
start=standardized_concept_uri('zh-Hant', traditional),
end=standardized_concept_uri('zh', word),
dataset=DATASET,
license=LICENSE,
sources=SOURCE,
)
out.write(edge)
edge = make_edge(
rel='/r/RelatedTo',
start=standardized_concept_uri('zh-Hans', simplified),
end=standardized_concept_uri('zh', word),
dataset=DATASET,
license=LICENSE,
sources=SOURCE,
)
out.write(edge)
continue
# Remove clarifying information in parenthesis
definition = PAREN_REGEX.sub('', definition)
# Handle variants/word forms and abbreviations
if re.match(VARIANT_REGEX, definition) or re.match(ABBR_REGEX, definition):
variants = extract_han_characters(definition)
for variant in variants:
edge = make_edge(
rel='/r/Synonym',
start=standardized_concept_uri('zh-Hant', traditional),
end=standardized_concept_uri('zh', variant),
dataset=DATASET,
license=LICENSE,
sources=SOURCE,
)
out.write(edge)
edge = make_edge(
rel='/r/Synonym',
start=standardized_concept_uri('zh-Hans', simplified),
end=standardized_concept_uri('zh', variant),
dataset=DATASET,
license=LICENSE,
sources=SOURCE,
)
out.write(edge)
continue
if re.match(SEE_ALSO_REGEX, definition):
references = extract_han_characters(definition)
for reference in references:
edge = make_edge(
rel='/r/RelatedTo',
start=standardized_concept_uri('zh-Hant', traditional),
end=standardized_concept_uri('zh', reference),
dataset=DATASET,
license=LICENSE,
sources=SOURCE,
)
out.write(edge)
edge = make_edge(
rel='/r/RelatedTo',
start=standardized_concept_uri('zh-Hans', simplified),
end=standardized_concept_uri('zh', reference),
dataset=DATASET,
license=LICENSE,
sources=SOURCE,
)
out.write(edge)
# Remove 'lit.', 'fig.'
definition = LIT_FIG_REGEX.sub('', definition)
# Expand sth and sb
definition = SB_REGEX.sub('someone', definition)
definition = STH_REGEX.sub('something', definition)
# Additional cleanups
definition = remove_reference_syntax(definition)
definition = remove_additional_info(definition)
# Skip long definitions and make an edge out of remaining information
if len(definition.split()) < 6:
edge = make_edge(
rel='/r/Synonym',
start=standardized_concept_uri('zh-Hant', traditional),
end=standardized_concept_uri('en', definition),
dataset=DATASET,
license=LICENSE,
sources=SOURCE,
)
out.write(edge)
edge = make_edge(
rel='/r/Synonym',
start=standardized_concept_uri('zh-Hans', simplified),
end=standardized_concept_uri('en', definition),
dataset=DATASET,
license=LICENSE,
sources=SOURCE,
)
out.write(edge)
def search_concept():
req_args = flask.request.args
lang = req_args.get('language', 'en')
text = req_args.get('text', '').strip()
uri = standardized_concept_uri(lang, text).rstrip('/')
return flask.redirect(uri)
def process_dbpedia(input_dir, output_file, concept_file):
"""
Read through multiple DBPedia files and output filtered assertions to
`output_file`.
"""
ok_concepts = read_concept_file(concept_file)
input_path = pathlib.Path(input_dir)
interlang_path = input_path / 'interlanguage_links_en.tql.bz2'
mapped_urls = interlanguage_mapping(interlang_path, ok_concepts)
out = MsgpackStreamWriter(output_file)
types_path = input_path / 'instance_types_en.tql.bz2'
quads = parse_nquads(bz2.open(str(types_path), 'rt'))
for subj, pred, obj, _graph in quads:
subj_url = subj['url']
if (
'Category:' in subj_url or 'File:' in subj_url or
'List_of' in subj_url or '__' in subj_url or
'Template:' in subj_url
):
continue
if subj_url in mapped_urls:
subj_concept = translate_dbpedia_url(subj_url)
obj_type = un_camel_case(resource_name(obj['url']))
if obj_type not in TYPE_BLACKLIST:
obj_concept = standardized_concept_uri('en', obj_type, 'n')
if obj_concept not in CONCEPT_BLACKLIST:
edge = make_edge(
'/r/IsA', subj_concept, obj_concept,
dataset='/d/dbpedia/en',
license=Licenses.cc_sharealike,
sources=[{'contributor': '/s/resource/dbpedia/2015/en'}],
weight=0.5,
surfaceStart=url_to_label(subj['url']),
surfaceEnd=url_to_label(obj['url'])
)
out.write(edge)
for other_url in mapped_urls[subj_url]:
if other_url.startswith('http://wikidata.dbpedia.org/'):
urledge = make_edge(
'/r/ExternalURL',
subj_concept, other_url,
dataset='/d/dbpedia/en',
license=Licenses.cc_sharealike,
sources=[{'contributor': '/s/resource/dbpedia/2015/en'}],
weight=1.0
)
out.write(urledge)
else:
other_concept = translate_dbpedia_url(other_url)
if other_concept:
urledge = make_edge(
'/r/ExternalURL',
other_concept, other_url,
dataset='/d/dbpedia/en',
license=Licenses.cc_sharealike,
sources=[{'contributor': '/s/resource/dbpedia/2015/en'}],
weight=1.0
)
out.write(urledge)
edge = make_edge(
'/r/Synonym',
other_concept, subj_concept,
dataset='/d/dbpedia/en',
license=Licenses.cc_sharealike,
sources=[{'contributor': '/s/resource/dbpedia/2015/en'}],
weight=0.5,
surfaceStart=url_to_label(other_url),
surfaceEnd=url_to_label(subj_url)
)
out.write(edge)
relations_path = input_path / 'mappingbased_objects_en.tql.bz2'
quads = parse_nquads(bz2.open(str(relations_path), 'rt'))
for subj, pred, obj, _graph in quads:
subj_concept = translate_dbpedia_url(subj['url'])
obj_concept = translate_dbpedia_url(obj['url'])
rel_name = resource_name(pred['url'])
if (
subj_concept and obj_concept and
subj['url'] in mapped_urls and obj['url'] in mapped_urls
):
if rel_name in RELATIONS:
rel = RELATIONS[rel_name]
edge = make_edge(
rel, subj_concept, obj_concept,
dataset='/d/dbpedia/en',
license=Licenses.cc_sharealike,
sources=[{'contributor': '/s/resource/dbpedia/2015/en'}],
weight=0.5,
surfaceStart=url_to_label(subj['url']),
surfaceEnd=url_to_label(obj['url'])
)
out.write(edge)
out.close()