def __createJSON(self):
try:
profile = {}
query = "PREFIX ual: <http://terms.library.ualberta.ca/> PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> SELECT * WHERE {GRAPH ual:%s {?property ?annotation ?value} }" % (self.ptype)
sparql.setReturnFormat(JSON)
sparql.setQuery(query)
results = sparql.query().convert()
print ('creating JSON')
for result in results['results']['bindings']:
if result['property']['value'] not in profile.keys():
profile[result['property']['value']] = {}
if result['annotation']['value'] == 'http://terms.library.ualberta.ca/acceptedValue':
if 'acceptedValues' not in profile[result['property']['value']]:
profile[result['property']['value']]['acceptedValues'] = []
query = "PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> PREFIX ual: <http://terms.library.ualberta.ca/> SELECT * WHERE { GRAPH ual:instances { <%s> rdfs:label ?label ; ual:onForm ?onForm } }" % (result['value']['value'])
sparql.setQuery(query)
annotations = sparql.query().convert()
for annotation in annotations['results']['bindings']:
profile[result['property']['value']]['acceptedValues'].append({'uri': result['value']['value'], 'onForm': annotation['onForm']['value'], 'label': annotation['label']['value']})
elif result['annotation']['value'] in profile[result['property']['value']]:
profile[result['property']['value']][result['annotation']['value']].append(result['value']['value'])
else:
profile[result['property']['value']][result['annotation']['value']] = []
profile[result['property']['value']][result['annotation']['value']].append(result['value']['value'])
directory = "data_dictionary/profiles/%s/" % (self.ptype)
if not os.path.exists(directory):
os.makedirs(directory)
filename = directory + 'profile.json'
with open(filename, 'w+') as p:
json.dump(profile, p, sort_keys=True, indent=4)
except:
PrintException()
def __createGithubMessage(self):
lines = ["Daily changes made to metadata profiles:"]
dateFilter = datetime.now() - timedelta(days=1)
query = "prefix dcterms: <http://purl.org/dc/terms/> prefix xsd: <http://www.w3.org/2001/XMLSchema#> prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> prefix schema: <http://schema.org/> prefix ual: <http://terms.library.ualberta.ca/> select ?username ?date ?type ?graph ?property ?annotation ?insertion ?deletion where { graph ual:audit { ?event schema:agent ?user ; rdf:type ?type ; schema:endTime ?date ; dcterms:isPartOf ?graph ; schema:targetCollection ?property ; schema:object ?annotation . OPTIONAL { ?event ual:deletion ?deletion} . OPTIONAL { ?event ual:insertion ?insertion } } } ORDER BY desc(?date)"
sparql.setReturnFormat(JSON)
sparql.setQuery(query)
results = sparql.query().convert()
for result in results['results']['bindings']:
line = ""
date = datetime.strptime(result['date']['value'], "%Y-%m-%d %H:%M:%S")
if date > dateFilter:
ptype = addPrefixes(result['graph']['value'])
predicate = addPrefixes(result['property']['value'])
annotation = addPrefixes(result['annotation']['value'])
line = """
{}
- change made to profile: '{}',
- to the property: '{}',
- to the annotation '{}',
- the following changes:""".format(date, ptype, predicate, annotation)
if ('insertion' in result) and (result['insertion']["value"] != ''):
line = """{}
- inserted: '{}'""".format(line, addPrefixes(result['insertion']['value']))
if ('deletion' in result) and (result['deletion']["value"] != ''):
line = """{}
- deleted: '{}'""".format(line, addPrefixes(result['deletion']['value']))
line = """{}
""".format(line)
lines.append(line)
with open('data_dictionary/scripts/lib/message.md', 'w+') as f:
f.writelines(lines)
def __createGithubMessage(self):
lines = ["Daily changes made to metadata profiles:"]
dateFilter = datetime.now() - timedelta(days=1)
query = "prefix dcterms: <http://purl.org/dc/terms/> prefix xsd: <http://www.w3.org/2001/XMLSchema#> prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> prefix schema: <http://schema.org/> prefix ual: <http://terms.library.ualberta.ca/> select ?username ?date ?type ?graph ?property ?annotation ?insertion ?deletion where { graph ual:audit { ?event schema:agent ?user ; rdf:type ?type ; schema:endTime ?date ; dcterms:isPartOf ?graph ; schema:targetCollection ?property ; schema:object ?annotation . OPTIONAL { ?event ual:deletion ?deletion} . OPTIONAL { ?event ual:insertion ?insertion } } } ORDER BY desc(?date)"
sparql.setReturnFormat(JSON)
sparql.setQuery(query)
results = sparql.query().convert()
for result in results['results']['bindings']:
line = ""
date = datetime.strptime(result['date']['value'],
"%Y-%m-%d %H:%M:%S")
if date > dateFilter:
ptype = addPrefixes(result['graph']['value'])
predicate = addPrefixes(result['property']['value'])
annotation = addPrefixes(result['annotation']['value'])
line = """
{}
- change made to profile: '{}',
- to the property: '{}',
- to the annotation '{}',
- the following changes:""".format(date, ptype, predicate, annotation)
if ('insertion'
in result) and (result['insertion']["value"] != ''):
line = """{}
- inserted: '{}'""".format(line, addPrefixes(result['insertion']['value']))
if ('deletion'
in result) and (result['deletion']["value"] != ''):
line = """{}
- deleted: '{}'""".format(line, addPrefixes(result['deletion']['value']))
line = """{}
""".format(line)
lines.append(line)
with open('data_dictionary/scripts/lib/message.md', 'w+') as f:
f.writelines(lines)
def __backupTriples(self):
sparql = SPARQLWrapper(
"http://206.167.181.124:7200/repositories/era-dd")
sparql.setReturnFormat(JSON)
query = "select * where { graph ?g {?s ?p ?o} }"
sparql.setQuery(query)
results = sparql.query().convert()
for result in sorted(results['results']['bindings'],
key=lambda k: k['o']['value']):
if result['o']['type'] == 'literal':
if result['s']['value'].startswith('t'):
print("_:%s <%s> \"%s\" <%s> ." %
(result['s']['value'], result['p']['value'],
re.sub("\"", "\\\"",
result['o']['value']), result['g']['value']))
else:
print("<%s> <%s> \"%s\" <%s> ." %
(result['s']['value'], result['p']['value'],
re.sub("\"", "\\\"",
result['o']['value']), result['g']['value']))
elif result['o']['type'] == 'uri':
if result['s']['value'].startswith('t'):
print("_:%s <%s> <%s> <%s> ." %
(result['s']['value'], result['p']['value'],
result['o']['value'], result['g']['value']))
else:
print("<%s> <%s> <%s> <%s> ." %
(result['s']['value'], result['p']['value'],
result['o']['value'], result['g']['value']))
def __createJSON(self):
try:
profile = {}
query = "PREFIX ual: <http://terms.library.ualberta.ca/> PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> SELECT * WHERE {GRAPH ual:%s {?property ?annotation ?value} }" % (
self.ptype)
sparql.setReturnFormat(JSON)
sparql.setQuery(query)
results = sparql.query().convert()
print('creating JSON')
for result in results['results']['bindings']:
if result['property']['value'] not in profile.keys():
profile[result['property']['value']] = {}
if result['annotation'][
'value'] == 'http://terms.library.ualberta.ca/acceptedValue':
if 'acceptedValues' not in profile[result['property']
['value']]:
profile[result['property']
['value']]['acceptedValues'] = []
query = "PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> PREFIX ual: <http://terms.library.ualberta.ca/> SELECT * WHERE { GRAPH ual:instances { <%s> rdfs:label ?label ; ual:onForm ?onForm } }" % (
result['value']['value'])
sparql.setQuery(query)
annotations = sparql.query().convert()
for annotation in annotations['results']['bindings']:
profile[result['property']
['value']]['acceptedValues'].append({
'uri':
result['value']['value'],
'onForm':
annotation['onForm']['value'],
'label':
annotation['label']['value']
})
elif result['annotation']['value'] in profile[
result['property']['value']]:
profile[result['property']['value']][
result['annotation']['value']].append(
result['value']['value'])
else:
profile[result['property']['value']][result['annotation']
['value']] = []
profile[result['property']['value']][
result['annotation']['value']].append(
result['value']['value'])
directory = "data_dictionary/profiles/%s/" % (self.ptype)
if not os.path.exists(directory):
os.makedirs(directory)
filename = directory + 'profile.json'
with open(filename, 'w+') as p:
json.dump(profile, p, sort_keys=True, indent=4)
except:
PrintException()
def __backupTriples(self):
sparql = SPARQLWrapper("http://206.167.181.124:7200/repositories/era-dd")
sparql.setReturnFormat(JSON)
query = "select * where { graph ?g {?s ?p ?o} }"
sparql.setQuery(query)
results = sparql.query().convert()
for result in sorted(results['results']['bindings'], key=lambda k: k['o']['value']):
if result['o']['type'] == 'literal':
if result['s']['value'].startswith('t'):
print("_:%s <%s> \"%s\" <%s> ." % (result['s']['value'], result['p']['value'], re.sub("\"", "\\\"", result['o']['value']), result['g']['value']))
else:
print("<%s> <%s> \"%s\" <%s> ." % (result['s']['value'], result['p']['value'], re.sub("\"", "\\\"", result['o']['value']), result['g']['value']))
elif result['o']['type'] == 'uri':
if result['s']['value'].startswith('t'):
print("_:%s <%s> <%s> <%s> ." % (result['s']['value'], result['p']['value'], result['o']['value'], result['g']['value']))
else:
print("<%s> <%s> <%s> <%s> ." % (result['s']['value'], result['p']['value'], result['o']['value'], result['g']['value']))
def pushOntologyToTripleStore():
""" only for the first implementation of the terms database (will destroy changes by issueing a fresh copy of properties from the ontology)"""
sparql.setMethod("POST")
for ptype in ["collection", "community", "generic", "thesis", "instances"]:
sparql.setQuery('DROP GRAPH <http://terms.library.ualberta.ca/%s>' % (ptype))
sparql.query()
for ptype in ["collection", "generic", "thesis"]:
directory = "data_dictionary/profiles/%s/" % (ptype)
if not os.path.exists(directory):
os.makedirs(directory)
filename = directory + 'profile.json'
with open(filename) as data:
data = json.load(data)
for item in data:
query = "PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> PREFIX ual: <http://terms.library.ualberta.ca/> INSERT DATA { GRAPH ual:%s { <%s> rdf:type rdf:Property" % (ptype, item)
for key in data[item].keys():
if 'acceptedValues' in key:
for triple in data[item][key]:
addValue = "PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> PREFIX ual: <http://terms.library.ualberta.ca/> INSERT DATA { GRAPH ual:instances { <%s> rdfs:label \"%s\" ; ual:onForm \"%s\" } } " % (triple['uri'], triple['label'], "true")
sparql.setQuery(addValue)
sparql.query()
query = query + "; ual:acceptedValue <%s>" % (triple['uri'])
elif isinstance(data[item][key], str) and ("http" in data[item][key]):
query = query + "; <%s> <%s> " % (key, data[item][key])
elif data[item][key] == "none":
query = query + "; <%s> \"%s\"" % (key, "")
else:
query = query + "; <%s> \"%s\"" % (key, str(data[item][key]).lower())
query = query + "} }"
sparql.setQuery(query)
sparql.query()
import re
sparqlData = "http://206.167.181.124:7200/repositories/era-dd"
sparqlData = SPARQLWrapper(sparqlData)
sparqlData.setReturnFormat(JSON)
with open ('modifyDD.txt' , 'w+') as out:
for subgraph in ['community', 'collection', 'generic', 'thesis', 'oai_pmh', 'oai_etdms', 'instances']:
#out.write("PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> prefix ual: <http://terms.library.ualberta.ca/> prefix ualid: <http://terms.library.ualberta.ca/id>" + "\n")
sparqlData.setQuery("prefix ual: <http://terms.library.ualberta.ca/> select * where {graph ual:%s {?s ?p ?o} }" %(subgraph))
r = sparqlData.query().convert()['results']['bindings']
for triple in r:
subject = triple['s']['value'].lower()
predicate = triple['p']['value']
objectt = triple['o']['value']
print (objectt)
sparql = "http://206.167.181.124:7200/repositories/test/statements"
sparql = SPARQLWrapper(sparql)
sparql.setMethod('POST')
if triple['o']['type'] == "uri":
print ("uri")
query = "PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> prefix ual: <http://terms.library.ualberta.ca/> prefix ualid: <http://terms.library.ualberta.ca/id> INSERT DATA {GRAPH ual:%s {<%s> <%s> <%s>} } " % (subgraph, subject, predicate, objectt)
if triple['o']['type'] == "literal":
print (subject, " is a literal")
query = 'PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> prefix ual: <http://terms.library.ualberta.ca/> prefix ualid: <http://terms.library.ualberta.ca/id> INSERT DATA {GRAPH ual:%s {<%s> <%s> "%s"} } ; ' % (subgraph, subject, predicate, objectt)
#out.write(query + "\n")
sparql.setQuery(query)
sparql.query()
#query = "INSERT DATA {GRAPH ual:%s {<%s> <http://terms.library.ualberta.ca/dataDictionaryLabel> <%s>} } ; " % (subgraph, subject, triple['s']['value'])
#out.write(query + "\n")
for subgraph in [
'community', 'collection', 'generic', 'thesis', 'oai_pmh',
'oai_etdms', 'instances'
]:
#out.write("PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> prefix ual: <http://terms.library.ualberta.ca/> prefix ualid: <http://terms.library.ualberta.ca/id>" + "\n")
sparqlData.setQuery(
"prefix ual: <http://terms.library.ualberta.ca/> select * where {graph ual:%s {?s ?p ?o} }"
% (subgraph))
r = sparqlData.query().convert()['results']['bindings']
for triple in r:
subject = triple['s']['value'].lower()
predicate = triple['p']['value']
objectt = triple['o']['value']
print(objectt)
sparql = "http://206.167.181.124:7200/repositories/test/statements"
sparql = SPARQLWrapper(sparql)
sparql.setMethod('POST')
if triple['o']['type'] == "uri":
print("uri")
query = "PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> prefix ual: <http://terms.library.ualberta.ca/> prefix ualid: <http://terms.library.ualberta.ca/id> INSERT DATA {GRAPH ual:%s {<%s> <%s> <%s>} } " % (
subgraph, subject, predicate, objectt)
if triple['o']['type'] == "literal":
print(subject, " is a literal")
query = 'PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> prefix ual: <http://terms.library.ualberta.ca/> prefix ualid: <http://terms.library.ualberta.ca/id> INSERT DATA {GRAPH ual:%s {<%s> <%s> "%s"} } ; ' % (
subgraph, subject, predicate, objectt)
#out.write(query + "\n")
sparql.setQuery(query)
sparql.query()
#query = "INSERT DATA {GRAPH ual:%s {<%s> <http://terms.library.ualberta.ca/dataDictionaryLabel> <%s>} } ; " % (subgraph, subject, triple['s']['value'])
#out.write(query + "\n")