def cyc_to_conceptnet_uri(labels, unlabels, uri):
"""
Convert a Cyc URI to a ConceptNet URI, with the following rules:
- Use the RDF label as the text. (Alternate labels appear to provide
synonyms, but these are generally automatically generated and aren't
particularly accurate.)
- The part of speech is always 'n'. Cyc describes its concepts in a
noun-like way. At worst, they're gerunds -- instead of "to eat", Cyc
would define an event of "Eating".
- If two different Cyc URIs have the same text, we will attempt to
disambiguate them using the last component of the Cyc URI.
- Remove the camel-casing from the Cyc URI component. If the phrase we
get is the same as the natural-language label, disregard it as an
uninformative disambiguation. Otherwise, that is the disambiguation text.
A possible objection: Our disambiguation doesn't distinguish Cyc URIs that
differ in capitalization, or differ by using underscores instead of
camel-case. However, I've noticed that such URIs are usually
*unintentional* duplicates that are okay to merge. If they were really
unrelated concepts that needed to be distinguished, someone would have
given them different names.
Even so, we end up with some unnecessary word senses, such as different
senses for "mens clothing", "men's clothing", and "men s clothing".
"""
label = filter_stopwords(labels[uri])
if len(unlabels[label]) >= 2:
disambig = filter_stopwords(un_camel_case(resource_name(uri)))
if simple_tokenize(disambig) != simple_tokenize(label):
return standardized_concept_uri('en', label, 'n', 'opencyc', disambig)
return standardized_concept_uri('en', label, 'n')
def _augment_knowledge(self):
try:
with open("conceptnetquerycache.pkl", "rb") as f:
cache = pickle.load(f)
except:
cache = {}
# use conceptnet to augment knowledge
def get_results(feature):
feature = feature.lower()
if feature in cache:
ret = cache[feature]
else:
with urlopen(
'http://conceptnet5.media.mit.edu/data/5.4%s?limit=1000'
% quote('/c/en/' + feature)) as response:
html = response.read().decode('utf8')
result = json.loads(html)
ret = []
for x in result['edges']:
r = pattern.match(x['uri'][3:])
if r:
ret.append(r.groups())
cache[feature] = ret
return ret
current_features = list(self.features)
for feature in current_features:
# convert dbpedia entry to conceptnet uri
pieces = dbpedia.parse_topic_name(feature)
pieces[0] = un_camel_case(pieces[0])
cneturi = standardized_concept_uri('en', *pieces)
ret = get_results(cneturi)
for (rtype, src, dest) in ret:
if src not in self.features:
self.features[src] = Feature(src, self)
if dest not in self.features:
self.features[dest] = Feature(dest, self)
self.usage_map.setdefault(rtype, set()).add((src, dest))
self.features[src].add_relation(rtype, dest)
self.features[dest].add_predecessor(rtype, src)
with open("conceptnetquerycache.pkl", "wb") as f:
pickle.dump(cache, f)
def translate_dbpedia_url(url):
"""
Convert an object that's defined by a DBPedia URL to a ConceptNet
URI. We do this by finding the part of the URL that names the object,
and using that as surface text for ConceptNet.
This is, in some ways, abusing a naming convention in the Semantic Web.
The URL of an object doesn't have to mean anything at all. The
human-readable name is supposed to be a string, specified by the "name"
relation.
The problem here is that the "name" relation is not unique in either
direction. A URL can have many names, and the same name can refer to
many URLs, and some of these names are the result of parsing glitches.
The URL itself is a stable thing that we can build a ConceptNet URI from,
on the other hand.
"""
parsed = parse_url(url)
domain = parsed.netloc
if domain == 'dbpedia.org':
# Handle old DBPedia URLs that had no language code
lang = 'en'
else:
domain_parts = domain.split('.', 1)
if domain_parts[1] == 'dbpedia.org':
lang = domain_parts[0]
# If we can't name this language in English, it's probably
# not really a language.
if langcodes.get(lang).language_name('en') == lang:
return None
else:
return None
# Some Semantic Web URLs are camel-cased. ConceptNet URIs use underscores
# between words.
pieces = parse_topic_name(resource_name(url))
pieces[0] = un_camel_case(pieces[0])
return standardized_concept_uri(lang, *pieces)
def translate_dbpedia_url(url):
"""
Convert an object that's defined by a DBPedia URL to a ConceptNet
URI. We do this by finding the part of the URL that names the object,
and using that as surface text for ConceptNet.
This is, in some ways, abusing a naming convention in the Semantic Web.
The URL of an object doesn't have to mean anything at all. The
human-readable name is supposed to be a string, specified by the "name"
relation.
The problem here is that the "name" relation is not unique in either
direction. A URL can have many names, and the same name can refer to
many URLs, and some of these names are the result of parsing glitches.
The URL itself is a stable thing that we can build a ConceptNet URI from,
on the other hand.
"""
parsed = parse_url(url)
domain = parsed.netloc
if domain == "dbpedia.org":
# Handle old DBPedia URLs that had no language code
lang = "en"
else:
domain_parts = domain.split(".", 1)
if domain_parts[1] == "dbpedia.org":
lang = domain_parts[0]
# If we can't name this language in English, it's probably
# not really a language.
if langcodes.get(lang).language_name("en") == lang:
return None
else:
return None
# Some Semantic Web URLs are camel-cased. ConceptNet URIs use underscores
# between words.
pieces = parse_topic_name(resource_name(url))
pieces[0] = un_camel_case(pieces[0])
return standardized_concept_uri(lang, *pieces)
def translate_dbpedia_url(url, lang='en'):
"""
Convert an object that's defined by a DBPedia URL to a ConceptNet
URI. We do this by finding the part of the URL that names the object,
and using that as surface text for ConceptNet.
This is, in some ways, abusing a naming convention in the Semantic Web.
The URL of an object doesn't have to mean anything at all. The
human-readable name is supposed to be a string, specified by the "name"
relation.
The problem here is that the "name" relation is not unique in either
direction. A URL can have many names, and the same name can refer to
many URLs, and some of these names are the result of parsing glitches.
The URL itself is a stable thing that we can build a ConceptNet URI from,
on the other hand.
"""
# Some Semantic Web URLs are camel-cased. ConceptNet URIs use underscores
# between words.
pieces = parse_topic_name(resource_name(url))
pieces[0] = un_camel_case(pieces[0])
return normalized_concept_uri(lang, *pieces)
def translate_dbpedia_url(url, lang='en'):
"""
Convert an object that's defined by a DBPedia URL to a ConceptNet
URI. We do this by finding the part of the URL that names the object,
and using that as surface text for ConceptNet.
This is, in some ways, abusing a naming convention in the Semantic Web.
The URL of an object doesn't have to mean anything at all. The
human-readable name is supposed to be a string, specified by the "name"
relation.
The problem here is that the "name" relation is not unique in either
direction. A URL can have many names, and the same name can refer to
many URLs, and some of these names are the result of parsing glitches.
The URL itself is a stable thing that we can build a ConceptNet URI from,
on the other hand.
"""
# Some Semantic Web URLs are camel-cased. ConceptNet URIs use underscores
# between words.
pieces = parse_topic_name(resource_name(url))
pieces[0] = un_camel_case(pieces[0])
return normalized_concept_uri(lang, *pieces)
def handle_triple(line, reader, out, map_out):
subj, pred, obj, tag = reader.parse_line(line)
if tag != "URL":
return
# Ignore types of edges that we don't care about:
# - Homepage links
# - GIS features
# - Assertions that something "is a thing"
# - Anonymous nodes identified with double-underscores, such as the node
# "Alfred_Nobel__1", which means "Alfred Nobel's occupation, whatever
# it is"
# - Nodes that are articles named "List of X" on Wikipedia
if (
"foaf/0.1/homepage" in pred
or "_Feature" in obj
or "#Thing" in obj
or "__" in subj
or "__" in obj
or "List_of" in subj
or "List_of" in obj
or "Wikidata:" in obj
):
return
# We don't try to parse URIs from outside of dbpedia.org's namespace.
if "dbpedia.org" not in obj:
return
subj_concept = translate_dbpedia_url(subj)
obj_concept = translate_dbpedia_url(obj)
subj_text = un_camel_case(parse_topic_name(resource_name(subj))[0])
obj_text = un_camel_case(parse_topic_name(resource_name(obj))[0])
if subj_concept is None or obj_concept is None:
return
# DBPedia categorizes a lot of things as 'works', which causes unnecessary
# ambiguity. Disregard these edges; there will almost always be a more
# specific edge calling it a 'creative work' anyway.
if obj_concept in CONCEPT_BLACKLIST:
return
rel = map_dbpedia_relation(pred)
if rel is None:
return
if rel in {"/r/IsA", "/r/TranslationOf"}:
obj_text = obj_text.lower()
# We've successfully converted this Semantic Web triple to ConceptNet URIs.
# Now write the results to the 'sw_map' file so others can follow this
# mapping.
mapped_pairs = [(pred, rel), (subj, subj_concept), (obj, obj_concept)]
for sw_url, conceptnet_uri in mapped_pairs:
conceptnet_url = full_conceptnet_url(conceptnet_uri)
map_out.write_link(conceptnet_url, sw_url)
edge = make_edge(
rel,
subj_concept,
obj_concept,
dataset="/d/dbpedia/en",
license=Licenses.cc_sharealike,
sources=["/s/dbpedia/2014"],
surfaceText=make_surface_text(rel, subj_text, obj_text),
weight=0.5,
)
out.write(edge)
def run_umbel(input_dir, output_file, sw_map_file):
"""
Read N-Triples files containing Umbel data, outputting a file of
ConceptNet edges and a file of mappings between the Semantic Web and
ConceptNet.
"""
out = MsgpackStreamWriter(output_file)
map_out = NTriplesWriter(sw_map_file)
reader = NTriplesReader()
labels = {}
label_sets = defaultdict(set)
# There are two files we want to parse:
# - umbel.nt, a transformation of umbel.n3, which is available from
# https://github.com/structureddynamics/UMBEL/.
# - umbel_links.nt, distributed with DBPedia 3.9.
#
# We parse them both in this file so that umbel_links can reuse the
# concept names extracted from umbel.nt.
main_file = os.path.join(input_dir, 'umbel.nt')
dbpedia_link_file = os.path.join(input_dir, 'umbel_links.nt')
# Read through umbel.nt once, finding all the "preferred labels". We will
# use these as the surface texts for the nodes.
for web_subj, web_rel, web_obj, objtag in reader.parse_file(main_file):
if resource_name(web_rel) == 'prefLabel':
# 'CW' and 'PCW' are Cyc jargon for 'conceptual works'. If a node
# cannot be described except as a CW, we're probably not
# interested in it.
if 'CW' not in web_obj.split() and 'PCW' not in web_obj.split():
labels[web_subj] = web_obj
if resource_name(web_rel).endswith('Label'):
text = standardize_text(web_obj)
label_sets[text].add(web_subj)
# Read through umbel.nt again and extract ConceptNet edges.
for web_subj, web_rel, web_obj, objtag in reader.parse_file(main_file):
if objtag == 'URL' and acceptable_node(web_obj) and acceptable_node(
web_subj):
# Only use nodes for which we've seen preferred labels.
# (This skips some anonymous OWL-cruft nodes.)
if web_subj in labels and web_obj in labels:
subj_uri = standardized_concept_uri('en', labels[web_subj])
obj_uri = standardized_concept_uri('en', labels[web_obj])
rel_name = resource_name(web_rel)
# Check if this is a relation we want to handle.
if rel_name in REL_MAPPING:
# Write the ConceptNet edges and the mappings to Semantic Web URLs.
rel_uri, frame = REL_MAPPING[rel_name]
surface = frame % (labels[web_subj], labels[web_obj])
out.write(
umbel_edge(rel_uri, subj_uri, obj_uri, surface,
SOURCE))
map_out.write_link(web_rel, full_conceptnet_url(rel_uri))
map_out.write_link(web_subj, full_conceptnet_url(subj_uri))
map_out.write_link(web_obj, full_conceptnet_url(obj_uri))
# altLabel relations assign different texts to the same node. We'll
# represent those in ConceptNet with Synonym relations.
elif web_rel.endswith('altLabel'):
# Make sure we know what's being labeled.
if web_subj in labels:
name = web_obj
words = name.split(' ')
if standardized_concept_name(
'en', name) != standardized_concept_name(
'en', labels[web_subj]):
if not set(words) & IGNORED_WORDS:
main_label = standardized_concept_uri(
'en', labels[web_subj])
name_text = standardize_text(name)
if len(label_sets[name_text]) >= 2 or len(
name_text) <= 3:
disambig = un_camel_case(resource_name(web_subj))
# Cyc does not distinguish texts by their part of speech, so use
# '_' as the part of speech symbol.
alt_label = standardized_concept_uri(
'en', name, '_', disambig)
else:
alt_label = standardized_concept_uri('en', name)
surface = SYN_FRAME % (name, labels[web_subj])
out.write(
umbel_edge('/r/Synonym', alt_label, main_label,
surface, SOURCE))
for web_subj, web_rel, web_obj, objtag in reader.parse_file(
dbpedia_link_file):
if objtag == 'URL' and acceptable_node(web_obj) and acceptable_node(
web_subj):
if web_obj in labels:
subj_label = resource_name(web_subj).replace('_', ' ')
subj_uri = translate_dbpedia_url(web_subj)
obj_label = labels[web_obj]
obj_uri = standardized_concept_uri('en', obj_label)
rel_name = resource_name(web_rel)
if rel_name in REL_MAPPING:
rel_uri, frame = REL_MAPPING[rel_name]
surface = frame % (subj_label, obj_label)
out.write(
umbel_edge(rel_uri, subj_uri, obj_uri, surface,
LINK_SOURCE))
map_out.write_link(web_rel, full_conceptnet_url(rel_uri))
map_out.write_link(web_subj, full_conceptnet_url(subj_uri))
map_out.write_link(web_obj, full_conceptnet_url(obj_uri))
def process_dbpedia(input_dir, output_file, concept_file):
"""
Read through multiple DBPedia files and output filtered assertions to
`output_file`.
"""
ok_concepts = read_concept_file(concept_file)
input_path = pathlib.Path(input_dir)
interlang_path = input_path / 'interlanguage_links_en.tql.bz2'
mapped_urls = interlanguage_mapping(interlang_path, ok_concepts)
out = MsgpackStreamWriter(output_file)
types_path = input_path / 'instance_types_en.tql.bz2'
quads = parse_nquads(bz2.open(str(types_path), 'rt'))
for subj, pred, obj, _graph in quads:
subj_url = subj['url']
if (
'Category:' in subj_url or 'File:' in subj_url or
'List_of' in subj_url or '__' in subj_url or
'Template:' in subj_url
):
continue
if subj_url in mapped_urls:
subj_concept = translate_dbpedia_url(subj_url)
obj_type = un_camel_case(resource_name(obj['url']))
if obj_type not in TYPE_BLACKLIST:
obj_concept = standardized_concept_uri('en', obj_type, 'n')
if obj_concept not in CONCEPT_BLACKLIST:
edge = make_edge(
'/r/IsA', subj_concept, obj_concept,
dataset='/d/dbpedia/en',
license=Licenses.cc_sharealike,
sources=[{'contributor': '/s/resource/dbpedia/2015/en'}],
weight=0.5,
surfaceStart=url_to_label(subj['url']),
surfaceEnd=url_to_label(obj['url'])
)
out.write(edge)
for other_url in mapped_urls[subj_url]:
if other_url.startswith('http://wikidata.dbpedia.org/'):
urledge = make_edge(
'/r/ExternalURL',
subj_concept, other_url,
dataset='/d/dbpedia/en',
license=Licenses.cc_sharealike,
sources=[{'contributor': '/s/resource/dbpedia/2015/en'}],
weight=1.0
)
out.write(urledge)
else:
other_concept = translate_dbpedia_url(other_url)
if other_concept:
urledge = make_edge(
'/r/ExternalURL',
other_concept, other_url,
dataset='/d/dbpedia/en',
license=Licenses.cc_sharealike,
sources=[{'contributor': '/s/resource/dbpedia/2015/en'}],
weight=1.0
)
out.write(urledge)
edge = make_edge(
'/r/Synonym',
other_concept, subj_concept,
dataset='/d/dbpedia/en',
license=Licenses.cc_sharealike,
sources=[{'contributor': '/s/resource/dbpedia/2015/en'}],
weight=0.5,
surfaceStart=url_to_label(other_url),
surfaceEnd=url_to_label(subj_url)
)
out.write(edge)
relations_path = input_path / 'mappingbased_objects_en.tql.bz2'
quads = parse_nquads(bz2.open(str(relations_path), 'rt'))
for subj, pred, obj, _graph in quads:
subj_concept = translate_dbpedia_url(subj['url'])
obj_concept = translate_dbpedia_url(obj['url'])
rel_name = resource_name(pred['url'])
if (
subj_concept and obj_concept and
subj['url'] in mapped_urls and obj['url'] in mapped_urls
):
if rel_name in RELATIONS:
rel = RELATIONS[rel_name]
edge = make_edge(
rel, subj_concept, obj_concept,
dataset='/d/dbpedia/en',
license=Licenses.cc_sharealike,
sources=[{'contributor': '/s/resource/dbpedia/2015/en'}],
weight=0.5,
surfaceStart=url_to_label(subj['url']),
surfaceEnd=url_to_label(obj['url'])
)
out.write(edge)
out.close()
def handle_triple(line, reader, out, map_out):
subj, pred, obj, tag = reader.parse_line(line)
if tag != 'URL':
return
# Ignore types of edges that we don't care about:
# - Homepage links
# - GIS features
# - Assertions that something "is a thing"
# - Anonymous nodes identified with double-underscores, such as the node
# "Alfred_Nobel__1", which means "Alfred Nobel's occupation, whatever
# it is"
# - Nodes that are articles named "List of X" on Wikipedia
if (
'foaf/0.1/homepage' in pred or '_Feature' in obj or '#Thing' in obj or
'__' in subj or '__' in obj or 'List_of' in subj or 'List_of' in obj
or 'Wikidata:' in obj
):
return
# We don't try to parse URIs from outside of dbpedia.org's namespace.
if 'dbpedia.org' not in obj:
return
subj_concept = translate_dbpedia_url(subj)
obj_concept = translate_dbpedia_url(obj)
subj_text = un_camel_case(parse_topic_name(resource_name(subj))[0])
obj_text = un_camel_case(parse_topic_name(resource_name(obj))[0])
if subj_concept is None or obj_concept is None:
return
# DBPedia categorizes a lot of things as 'works', which causes unnecessary
# ambiguity. Disregard these edges; there will almost always be a more
# specific edge calling it a 'creative work' anyway.
if obj_concept in CONCEPT_BLACKLIST:
return
rel = map_dbpedia_relation(pred)
if rel is None:
return
if rel in {'/r/IsA', '/r/TranslationOf'}:
obj_text = obj_text.lower()
# We've successfully converted this Semantic Web triple to ConceptNet URIs.
# Now write the results to the 'sw_map' file so others can follow this
# mapping.
mapped_pairs = [
(pred, rel),
(subj, subj_concept),
(obj, obj_concept)
]
for sw_url, conceptnet_uri in mapped_pairs:
conceptnet_url = full_conceptnet_url(conceptnet_uri)
map_out.write_link(conceptnet_url, sw_url)
edge = make_edge(rel, subj_concept, obj_concept,
dataset='/d/dbpedia/en',
license=Licenses.cc_sharealike,
sources=['/s/dbpedia/2014'],
surfaceText=make_surface_text(rel, subj_text, obj_text),
weight=0.5)
out.write(edge)
def run_umbel(input_dir, output_file, sw_map_file):
"""
Read N-Triples files containing Umbel data, outputting a file of
ConceptNet edges and a file of mappings between the Semantic Web and
ConceptNet.
"""
out = MsgpackStreamWriter(output_file)
map_out = NTriplesWriter(sw_map_file)
reader = NTriplesReader()
labels = {}
label_sets = defaultdict(set)
# There are two files we want to parse:
# - umbel.nt, a transformation of umbel.n3, which is available from
# https://github.com/structureddynamics/UMBEL/.
# - umbel_links.nt, distributed with DBPedia 3.9.
#
# We parse them both in this file so that umbel_links can reuse the
# concept names extracted from umbel.nt.
main_file = os.path.join(input_dir, 'umbel.nt')
dbpedia_link_file = os.path.join(input_dir, 'umbel_links.nt')
# Read through umbel.nt once, finding all the "preferred labels". We will
# use these as the surface texts for the nodes.
for web_subj, web_rel, web_obj, objtag in reader.parse_file(main_file):
if resource_name(web_rel) == 'prefLabel':
# 'CW' and 'PCW' are Cyc jargon for 'conceptual works'. If a node
# cannot be described except as a CW, we're probably not
# interested in it.
if 'CW' not in web_obj.split() and 'PCW' not in web_obj.split():
labels[web_subj] = web_obj
if resource_name(web_rel).endswith('Label'):
text = standardize_text(web_obj)
label_sets[text].add(web_subj)
# Read through umbel.nt again and extract ConceptNet edges.
for web_subj, web_rel, web_obj, objtag in reader.parse_file(main_file):
if objtag == 'URL' and acceptable_node(web_obj) and acceptable_node(web_subj):
# Only use nodes for which we've seen preferred labels.
# (This skips some anonymous OWL-cruft nodes.)
if web_subj in labels and web_obj in labels:
subj_uri = standardized_concept_uri('en', labels[web_subj])
obj_uri = standardized_concept_uri('en', labels[web_obj])
rel_name = resource_name(web_rel)
# Check if this is a relation we want to handle.
if rel_name in REL_MAPPING:
# Write the ConceptNet edges and the mappings to Semantic Web URLs.
rel_uri, frame = REL_MAPPING[rel_name]
surface = frame % (labels[web_subj], labels[web_obj])
out.write(umbel_edge(rel_uri, subj_uri, obj_uri, surface, SOURCE))
map_out.write_link(web_rel, full_conceptnet_url(rel_uri))
map_out.write_link(web_subj, full_conceptnet_url(subj_uri))
map_out.write_link(web_obj, full_conceptnet_url(obj_uri))
# altLabel relations assign different texts to the same node. We'll
# represent those in ConceptNet with Synonym relations.
elif web_rel.endswith('altLabel'):
# Make sure we know what's being labeled.
if web_subj in labels:
name = web_obj
words = name.split(' ')
if standardized_concept_name('en', name) != standardized_concept_name('en', labels[web_subj]):
if not set(words) & IGNORED_WORDS:
main_label = standardized_concept_uri('en', labels[web_subj])
name_text = standardize_text(name)
if len(label_sets[name_text]) >= 2 or len(name_text) <= 3:
disambig = un_camel_case(resource_name(web_subj))
# Cyc does not distinguish texts by their part of speech, so use
# '_' as the part of speech symbol.
alt_label = standardized_concept_uri('en', name, '_', disambig)
else:
alt_label = standardized_concept_uri('en', name)
surface = SYN_FRAME % (name, labels[web_subj])
out.write(umbel_edge('/r/Synonym', alt_label, main_label, surface, SOURCE))
for web_subj, web_rel, web_obj, objtag in reader.parse_file(dbpedia_link_file):
if objtag == 'URL' and acceptable_node(web_obj) and acceptable_node(web_subj):
if web_obj in labels:
subj_label = resource_name(web_subj).replace('_', ' ')
subj_uri = translate_dbpedia_url(web_subj)
obj_label = labels[web_obj]
obj_uri = standardized_concept_uri('en', obj_label)
rel_name = resource_name(web_rel)
if rel_name in REL_MAPPING:
rel_uri, frame = REL_MAPPING[rel_name]
surface = frame % (subj_label, obj_label)
out.write(umbel_edge(rel_uri, subj_uri, obj_uri, surface, LINK_SOURCE))
map_out.write_link(web_rel, full_conceptnet_url(rel_uri))
map_out.write_link(web_subj, full_conceptnet_url(subj_uri))
map_out.write_link(web_obj, full_conceptnet_url(obj_uri))
def process_dbpedia(input_dir, output_file, concept_file):
"""
Read through multiple DBPedia files and output filtered assertions to
`output_file`.
"""
ok_concepts = read_concept_file(concept_file)
input_path = pathlib.Path(input_dir)
interlang_path = input_path / 'interlanguage_links_en.tql.bz2'
mapped_urls = interlanguage_mapping(interlang_path, ok_concepts)
out = MsgpackStreamWriter(output_file)
types_path = input_path / 'instance_types_en.tql.bz2'
quads = parse_nquads(bz2.open(str(types_path), 'rt'))
for subj, pred, obj, _graph in quads:
subj_url = subj['url']
if ('Category:' in subj_url or 'File:' in subj_url
or 'List_of' in subj_url or '__' in subj_url
or 'Template:' in subj_url):
continue
if subj_url in mapped_urls:
subj_concept = translate_dbpedia_url(subj_url)
obj_type = un_camel_case(resource_name(obj['url']))
if obj_type not in TYPE_BLACKLIST:
obj_concept = standardized_concept_uri('en', obj_type, 'n')
if obj_concept not in CONCEPT_BLACKLIST:
edge = make_edge('/r/IsA',
subj_concept,
obj_concept,
dataset='/d/dbpedia/en',
license=Licenses.cc_sharealike,
sources=[{
'contributor':
'/s/resource/dbpedia/2015/en'
}],
weight=0.5,
surfaceStart=url_to_label(subj['url']),
surfaceEnd=url_to_label(obj['url']))
out.write(edge)
for other_url in mapped_urls[subj_url]:
if other_url.startswith('http://wikidata.dbpedia.org/'):
urledge = make_edge('/r/ExternalURL',
subj_concept,
other_url,
dataset='/d/dbpedia/en',
license=Licenses.cc_sharealike,
sources=[{
'contributor':
'/s/resource/dbpedia/2015/en'
}],
weight=1.0)
out.write(urledge)
else:
other_concept = translate_dbpedia_url(other_url)
if other_concept:
urledge = make_edge(
'/r/ExternalURL',
other_concept,
other_url,
dataset='/d/dbpedia/en',
license=Licenses.cc_sharealike,
sources=[{
'contributor':
'/s/resource/dbpedia/2015/en'
}],
weight=1.0)
out.write(urledge)
edge = make_edge(
'/r/Synonym',
other_concept,
subj_concept,
dataset='/d/dbpedia/en',
license=Licenses.cc_sharealike,
sources=[{
'contributor':
'/s/resource/dbpedia/2015/en'
}],
weight=0.5,
surfaceStart=url_to_label(other_url),
surfaceEnd=url_to_label(subj_url))
out.write(edge)
relations_path = input_path / 'mappingbased_objects_en.tql.bz2'
quads = parse_nquads(bz2.open(str(relations_path), 'rt'))
for subj, pred, obj, _graph in quads:
subj_concept = translate_dbpedia_url(subj['url'])
obj_concept = translate_dbpedia_url(obj['url'])
rel_name = resource_name(pred['url'])
if (subj_concept and obj_concept and subj['url'] in mapped_urls
and obj['url'] in mapped_urls):
if rel_name in RELATIONS:
rel = RELATIONS[rel_name]
edge = make_edge(rel,
subj_concept,
obj_concept,
dataset='/d/dbpedia/en',
license=Licenses.cc_sharealike,
sources=[{
'contributor':
'/s/resource/dbpedia/2015/en'
}],
weight=0.5,
surfaceStart=url_to_label(subj['url']),
surfaceEnd=url_to_label(obj['url']))
out.write(edge)
out.close()