def model_data() -> None:
"""
This function generates triples for user defined properties for uploading them to wikidata
:return:
"""
stream = open(Path.cwd().parent /
"Datasets/new-property-configuration.yaml",
'r',
encoding='utf8')
yaml_data = yaml.safe_load(stream)
# initialize
kg_schema = KGSchema()
kg_schema.add_schema('@prefix : <http://isi.edu/> .', 'ttl')
etk = ETK(kg_schema=kg_schema, modules=ETKModule)
doc = etk.create_document({}, doc_id="http://isi.edu/default-ns/projects")
# bind prefixes
doc.kg.bind('wikibase', 'http://wikiba.se/ontology#')
doc.kg.bind('wd', 'http://www.wikidata.org/entity/')
doc.kg.bind('wdt', 'http://www.wikidata.org/prop/direct/')
doc.kg.bind('wdtn', 'http://www.wikidata.org/prop/direct-normalized/')
doc.kg.bind('wdno', 'http://www.wikidata.org/prop/novalue/')
doc.kg.bind('wds', 'http://www.wikidata.org/entity/statement/')
doc.kg.bind('wdv', 'http://www.wikidata.org/value/')
doc.kg.bind('wdref', 'http://www.wikidata.org/reference/')
doc.kg.bind('p', 'http://www.wikidata.org/prop/')
doc.kg.bind('pr', 'http://www.wikidata.org/prop/reference/')
doc.kg.bind('prv', 'http://www.wikidata.org/prop/reference/value/')
doc.kg.bind('prn',
'http://www.wikidata.org/prop/reference/value-normalized/')
doc.kg.bind('ps', 'http://www.wikidata.org/prop/statement/')
doc.kg.bind('psv', 'http://www.wikidata.org/prop/statement/value/')
doc.kg.bind('psn',
'http://www.wikidata.org/prop/statement/value-normalized/')
doc.kg.bind('pq', 'http://www.wikidata.org/prop/qualifier/')
doc.kg.bind('pqv', 'http://www.wikidata.org/prop/qualifier/value/')
doc.kg.bind('pqn',
'http://www.wikidata.org/prop/qualifier/value-normalized/')
doc.kg.bind('skos', 'http://www.w3.org/2004/02/skos/core#')
doc.kg.bind('prov', 'http://www.w3.org/ns/prov#')
doc.kg.bind('schema', 'http://schema.org/')
sparql_endpoint = "https://query.wikidata.org/sparql"
type_map = {'quantity': Datatype.QuantityValue, 'url': URLValue}
property_type_cache = {}
for k, v in yaml_data.items():
p = WDProperty(k,
type_map[v['type']],
creator='http://www.isi.edu/t2wml')
for lang, value in v['label'].items():
for val in value:
p.add_label(val, lang=lang)
for lang, value in v['description'].items():
for val in value:
p.add_description(val, lang=lang)
for pnode, items in v['statements'].items():
for item in items:
try:
property_type = property_type_cache[pnode]
except KeyError:
property_type = get_property_type(pnode, sparql_endpoint)
property_type_cache[pnode] = property_type
if property_type == "WikibaseItem":
value = Item(str(item['value']))
elif property_type == "WikibaseProperty":
value = Property(item['value'])
elif property_type == "String":
value = StringValue(item['value'])
elif property_type == "Quantity":
value = QuantityValue(item['value'])
elif property_type == "Time":
value = TimeValue(
str(item['value']), Item(item["calendar"]),
translate_precision_to_integer(item["precision"]),
item["time_zone"])
elif property_type == "Url":
value = URLValue(item['value'])
elif property_type == "Monolingualtext":
value = MonolingualText(item['value'], item["lang"])
elif property_type == "ExternalId":
value = ExternalIdentifier(item['value'])
elif property_type == "GlobeCoordinate":
value = GlobeCoordinate(item["latitude"],
item["longitude"],
item["precision"])
p.add_statement(pnode, value)
doc.kg.add_subject(p)
with open(Path.cwd().parent / "new_properties/result.ttl", "w") as f:
data = doc.kg.serialize('ttl')
f.write(data)
def generate_normal_triple(self, node1: str, property: str, node2: str,
is_qualifier_edge: bool, e_id: str) -> bool:
if self.use_id:
e_id = TripleGenerator.replace_illegal_string(e_id)
entity = self._node_2_entity(node1)
edge_type = self.prop_types[property]
if edge_type == Item:
object = WDItem(TripleGenerator.replace_illegal_string(node2))
elif edge_type == WDProperty:
object = WDProperty(TripleGenerator.replace_illegal_string(node2),
self.prop_types[node2])
elif edge_type == TimeValue:
if self.yyyy_mm_dd_pattern.match(node2):
try:
dateTimeString = node2
object = TimeValue(
value=dateTimeString, # TODO
calendar=Item("Q1985727"),
precision=Precision.year,
time_zone=0,
)
except:
return False
elif self.yyyy_pattern.match(node2):
try:
dateTimeString = node2 + "-01-01"
object = TimeValue(
value=dateTimeString, # TODO
calendar=Item("Q1985727"),
precision=Precision.year,
time_zone=0,
)
except:
return False
else:
try:
# TODO, in future, the two cases above will be dropped in principle to comply with the iso format
# now it is iso format
assert (node2[0] == "^")
node2 = node2[1:] # remove ^
if node2.startswith("+"):
node2 = node2[1:]
dateTimeString, precision = node2.split("/")
dateTimeString = dateTimeString[:-1] # remove Z
object = TimeValue(
value=dateTimeString,
calendar=Item("Q1985727"),
precision=precision,
time_zone=0,
)
except:
return False
elif edge_type == GlobeCoordinate:
latitude, longitude = node2[1:].split("/")
latitude = float(latitude)
longitude = float(longitude)
object = GlobeCoordinate(latitude,
longitude,
0.0001,
globe=Item("Q2")) # earth
elif edge_type == QuantityValue:
# +70[+60,+80]Q743895
res = self.quantity_pattern.match(node2).groups()
amount, lower_bound, upper_bound, unit = res
amount = TripleGenerator.clean_number_string(amount)
num_type = self.xsd_number_type(amount)
lower_bound = TripleGenerator.clean_number_string(lower_bound)
upper_bound = TripleGenerator.clean_number_string(upper_bound)
if unit != None:
if upper_bound != None and lower_bound != None:
object = QuantityValue(amount,
unit=Item(unit),
upper_bound=upper_bound,
lower_bound=lower_bound,
type=num_type)
else:
object = QuantityValue(amount,
unit=Item(unit),
type=num_type)
else:
if upper_bound != None and lower_bound != None:
object = QuantityValue(amount,
upper_bound=upper_bound,
lower_bound=lower_bound,
type=num_type)
else:
object = QuantityValue(amount, type=num_type)
elif edge_type == MonolingualText:
text_string, lang = TripleGenerator.process_text_string(node2)
object = MonolingualText(text_string, lang)
elif edge_type == ExternalIdentifier:
object = ExternalIdentifier(node2)
elif edge_type == URLValue:
if TripleGenerator.is_valid_uri_with_scheme_and_host(node2):
object = URLValue(node2)
else:
return False
else:
# treat everything else as stringValue
object = StringValue(node2)
if type(object) == WDItem or type(object) == WDProperty:
self.doc.kg.add_subject(object)
if is_qualifier_edge:
# edge: e8 p9 ^2013-01-01T00:00:00Z/11
# create qualifier edge on previous STATEMENT and return the updated STATEMENT
self.to_append_statement.add_qualifier(property, object)
self.doc.kg.add_subject(self.to_append_statement)
else:
# edge: q1 p8 q2 e8
# create brand new property edge and replace STATEMENT
if self.truthy:
self.to_append_statement = entity.add_truthy_statement(
property, object, statement_id=e_id
) if self.use_id else entity.add_truthy_statement(
property, object)
else:
self.to_append_statement = entity.add_statement(
property, object, statement_id=e_id
) if self.use_id else entity.add_statement(property, object)
self.doc.kg.add_subject(entity)
return True
def generate_triples(user_id: str,
resolved_excel: list,
sparql_endpoint: str,
filetype: str = 'ttl',
created_by: str = 't2wml') -> str:
"""
This function uses ETK to generate the RDF triples
:param user_id:
:param resolved_excel:
:param sparql_endpoint:
:param filetype:
:return:
"""
# initialize
kg_schema = KGSchema()
kg_schema.add_schema('@prefix : <http://isi.edu/> .', 'ttl')
etk = ETK(kg_schema=kg_schema, modules=ETKModule)
doc = etk.create_document({}, doc_id="http://isi.edu/default-ns/projects")
property_type_map = property_type_dict
# bind prefixes
doc.kg.bind('wikibase', 'http://wikiba.se/ontology#')
doc.kg.bind('wd', 'http://www.wikidata.org/entity/')
doc.kg.bind('wdt', 'http://www.wikidata.org/prop/direct/')
doc.kg.bind('wdtn', 'http://www.wikidata.org/prop/direct-normalized/')
doc.kg.bind('wdno', 'http://www.wikidata.org/prop/novalue/')
doc.kg.bind('wds', 'http://www.wikidata.org/entity/statement/')
doc.kg.bind('wdv', 'http://www.wikidata.org/value/')
doc.kg.bind('wdref', 'http://www.wikidata.org/reference/')
doc.kg.bind('p', 'http://www.wikidata.org/prop/')
doc.kg.bind('pr', 'http://www.wikidata.org/prop/reference/')
doc.kg.bind('prv', 'http://www.wikidata.org/prop/reference/value/')
doc.kg.bind('prn',
'http://www.wikidata.org/prop/reference/value-normalized/')
doc.kg.bind('ps', 'http://www.wikidata.org/prop/statement/')
doc.kg.bind('psv', 'http://www.wikidata.org/prop/statement/value/')
doc.kg.bind('psn',
'http://www.wikidata.org/prop/statement/value-normalized/')
doc.kg.bind('pq', 'http://www.wikidata.org/prop/qualifier/')
doc.kg.bind('pqv', 'http://www.wikidata.org/prop/qualifier/value/')
doc.kg.bind('pqn',
'http://www.wikidata.org/prop/qualifier/value-normalized/')
doc.kg.bind('skos', 'http://www.w3.org/2004/02/skos/core#')
doc.kg.bind('prov', 'http://www.w3.org/ns/prov#')
doc.kg.bind('schema', 'http://schema.org/')
# property_type_cache = {}
is_error = False
for i in resolved_excel:
_item = i["statement"]["item"]
if _item is not None:
item = WDItem(_item,
creator='http://www.isi.edu/{}'.format(created_by))
try:
property_type = property_type_map[i["statement"]["property"]]
except KeyError:
property_type = get_property_type(i["statement"]["property"],
sparql_endpoint)
if property_type != "Property Not Found" and i["statement"][
"property"] not in property_type_map:
property_type_map[i["statement"]
["property"]] = property_type
if property_type == "WikibaseItem":
value = Item(str(i["statement"]["value"]))
elif property_type == "WikibaseProperty":
value = Property(i["statement"]["value"])
elif property_type == "String":
value = StringValue(i["statement"]["value"])
elif property_type == "Quantity":
_value = i["statement"]["value"]
_value = str(_value).replace(',', '')
value = QuantityValue(_value)
elif property_type == "Time":
value = TimeValue(
str(i["statement"]["value"]),
Item(i["statement"]["calendar"]),
translate_precision_to_integer(
i["statement"]["precision"]),
i["statement"]["time_zone"])
elif property_type == "Url":
value = URLValue(i["statement"]["value"])
elif property_type == "Monolingualtext":
value = MonolingualText(i["statement"]["value"],
i["statement"]["lang"])
elif property_type == "ExternalId":
value = ExternalIdentifier(i["statement"]["value"])
elif property_type == "GlobeCoordinate":
value = GlobeCoordinate(i["statement"]["latitude"],
i["statement"]["longitude"],
i["statement"]["precision"])
elif property_type == "Property Not Found":
is_error = True
break
s = item.add_statement(i["statement"]["property"], value)
doc.kg.add_subject(item)
if "qualifier" in i["statement"]:
for j in i["statement"]["qualifier"]:
try:
property_type = property_type_map[j["property"]]
except KeyError:
property_type = get_property_type(
j["property"], sparql_endpoint)
if property_type != "Property Not Found" and i[
"statement"][
"property"] not in property_type_map:
property_type_map[i["statement"]
["property"]] = property_type
if property_type == "WikibaseItem":
value = Item(str(j["value"]))
elif property_type == "WikibaseProperty":
value = Property(j["value"])
elif property_type == "String":
value = StringValue(j["value"])
elif property_type == "Quantity":
value = QuantityValue(j["value"])
elif property_type == "Time":
value = TimeValue(str(j["value"]), Item(j["calendar"]),
j["precision"], j["time_zone"])
elif property_type == "Url":
value = URLValue(j["value"])
elif property_type == "Monolingualtext":
value = MonolingualText(j["value"], j["lang"])
elif property_type == "ExternalId":
value = ExternalIdentifier(j["value"])
elif property_type == "GlobeCoordinate":
value = GlobeCoordinate(j["latitude"], j["longitude"],
j["precision"])
elif property_type == "Property Not Found":
is_error = True
if value is None:
continue
else:
s.add_qualifier(j["property"], value)
doc.kg.add_subject(s)
if not is_error:
data = doc.kg.serialize(filetype)
else:
# data = "Property Not Found"
raise Exception('data exception while generating triples')
return data
def generate_triples(user_id: str, resolved_excel: list, sparql_endpoint: str, filetype: str = 'ttl') -> str:
"""
This function uses ETK to generate the RDF triples
:param user_id:
:param resolved_excel:
:param sparql_endpoint:
:param filetype:
:return:
"""
# initialize
kg_schema = KGSchema()
kg_schema.add_schema('@prefix : <http://isi.edu/> .', 'ttl')
etk = ETK(kg_schema=kg_schema, modules=ETKModule)
doc = etk.create_document({}, doc_id="http://isi.edu/default-ns/projects")
# bind prefixes
doc.kg.bind('wikibase', 'http://wikiba.se/ontology#')
doc.kg.bind('wd', 'http://www.wikidata.org/entity/')
doc.kg.bind('wdt', 'http://www.wikidata.org/prop/direct/')
doc.kg.bind('wdtn', 'http://www.wikidata.org/prop/direct-normalized/')
doc.kg.bind('wdno', 'http://www.wikidata.org/prop/novalue/')
doc.kg.bind('wds', 'http://www.wikidata.org/entity/statement/')
doc.kg.bind('wdv', 'http://www.wikidata.org/value/')
doc.kg.bind('wdref', 'http://www.wikidata.org/reference/')
doc.kg.bind('p', 'http://www.wikidata.org/prop/')
doc.kg.bind('pr', 'http://www.wikidata.org/prop/reference/')
doc.kg.bind('prv', 'http://www.wikidata.org/prop/reference/value/')
doc.kg.bind('prn', 'http://www.wikidata.org/prop/reference/value-normalized/')
doc.kg.bind('ps', 'http://www.wikidata.org/prop/statement/')
doc.kg.bind('psv', 'http://www.wikidata.org/prop/statement/value/')
doc.kg.bind('psn', 'http://www.wikidata.org/prop/statement/value-normalized/')
doc.kg.bind('pq', 'http://www.wikidata.org/prop/qualifier/')
doc.kg.bind('pqv', 'http://www.wikidata.org/prop/qualifier/value/')
doc.kg.bind('pqn', 'http://www.wikidata.org/prop/qualifier/value-normalized/')
doc.kg.bind('skos', 'http://www.w3.org/2004/02/skos/core#')
doc.kg.bind('prov', 'http://www.w3.org/ns/prov#')
doc.kg.bind('schema', 'http://schema.org/')
# property_type_cache = {}
is_error = False
for i in resolved_excel:
item = WDItem(i["statement"]["item"], creator='http://www.isi.edu/t2wml')
try:
property_type = property_type_map[i["statement"]["property"]]
except KeyError:
property_type = get_property_type(i["statement"]["property"], sparql_endpoint)
property_type_map[i["statement"]["property"]] = property_type
if property_type == "WikibaseItem":
value = Item(str(i["statement"]["value"]))
elif property_type == "WikibaseProperty":
value = Property(i["statement"]["value"])
elif property_type == "String":
value = StringValue(i["statement"]["value"])
elif property_type == "Quantity":
value = QuantityValue(i["statement"]["value"])
elif property_type == "Time":
value = TimeValue(str(i["statement"]["value"]), Item(i["statement"]["calendar"]), translate_precision_to_integer(i["statement"]["precision"]), i["statement"]["time_zone"])
elif property_type == "Url":
value = URLValue(i["statement"]["value"])
elif property_type == "Monolingualtext":
value = MonolingualText(i["statement"]["value"], i["statement"]["lang"])
elif property_type == "ExternalId":
value = ExternalIdentifier(i["statement"]["value"])
elif property_type == "GlobeCoordinate":
value = GlobeCoordinate(i["statement"]["latitude"], i["statement"]["longitude"], i["statement"]["precision"])
elif property_type == "Property Not Found":
is_error = True
break
s = item.add_statement(i["statement"]["property"], value)
doc.kg.add_subject(item)
if "qualifier" in i["statement"]:
for j in i["statement"]["qualifier"]:
try:
property_type = property_type_map[j["property"]]
except KeyError:
property_type = get_property_type(j["property"], sparql_endpoint)
property_type_map[j["property"]] = property_type
if property_type == "WikibaseItem":
value = Item(str(j["value"]))
elif property_type == "WikibaseProperty":
value = Property(j["value"])
elif property_type == "String":
value = StringValue(j["value"])
elif property_type == "Quantity":
value = QuantityValue(j["value"])
elif property_type == "Time":
value = TimeValue(str(j["value"]), Item(j["calendar"]), j["precision"], j["time_zone"])
elif property_type == "Url":
value = URLValue(j["value"])
elif property_type == "Monolingualtext":
value = MonolingualText(j["value"], j["lang"])
elif property_type == "ExternalId":
value = ExternalIdentifier(j["value"])
elif property_type == "GlobeCoordinate":
value = GlobeCoordinate(j["latitude"], j["longitude"], j["precision"])
elif property_type == "Property Not Found":
is_error = True
s.add_qualifier(j["property"], value)
doc.kg.add_subject(s)
if not is_error:
data = doc.kg.serialize(filetype)
else:
data = "Property Not Found"
# os.makedirs(Path.cwd() / "new_properties", exist_ok=True)
# results_file_name = user_id + "_results.ttl"
# changes_file_name = user_id + "_changes.tsv"
# with open(Path(app.config['downloads']) / results_file_name, "w") as fp:
# fp.write(data)
# with open(Path(app.config['downloads']) / changes_file_name, "w") as fp:
# serialize_change_record(fp)
return data