def validate_fix_eol_objects(self):
""" Searches for EOL links in the
LinkAnnotations table, then fixes
badly URIs with cruft. Also
calls the EOL API to get labels
for URIs with no record in the LinkEntity
table.
"""
checked_uris = []
eol_las = LinkAnnotation.objects\
.filter(object_uri__icontains=self.EOL_URI_PREFIX)
for eol_la in eol_las:
eol_uri = eol_la.object_uri
leg = LinkEntityGeneration()
le_gen = LinkEntityGeneration()
eol_uri = le_gen.make_clean_uri(
eol_uri) # strip off any cruft in the URI
if eol_uri != eol_la.object_uri:
print('Has cruft: ' + str(eol_la.object_uri))
LinkAnnotation.objects\
.filter(hash_id=eol_la.hash_id)\
.delete() # delete the old
eol_la.object_uri = eol_uri
eol_la.save() # save the cleaned URI
if eol_uri not in checked_uris:
# only check on a given URI once
checked_uris.append(eol_uri)
try:
le = LinkEntity.objects.get(uri=eol_uri)
except LinkEntity.DoesNotExist:
le = False
if le is False:
print('Getting missing data for: ' + eol_uri)
self.get_save_entity_label(eol_uri)
def get_json_for_geonames_uri(self, geonames_uri):
"""
gets json data from a geonames_uri
"""
le_gen = LinkEntityGeneration()
geonames_uri = le_gen.make_clean_uri(geonames_uri) # strip off any cruft in the URI
geo_ex = geonames_uri.split('/')
geonames_id = geo_ex[-1]
url = self.json_base_url + str(geonames_id)
if self.delay_before_request > 0:
# default to sleep BEFORE a request is sent, to
# give the remote service a break.
sleep(self.delay_before_request)
try:
gapi = GeneralAPI()
r = requests.get(url,
timeout=240,
headers=gapi.client_headers)
r.raise_for_status()
self.request_url = r.url
json_r = r.json()
except:
json_r = False
self.json_data = json_r
return self.json_data
def validate_fix_eol_objects(self):
""" Searches for EOL links in the
LinkAnnotations table, then fixes
badly URIs with cruft. Also
calls the EOL API to get labels
for URIs with no record in the LinkEntity
table.
"""
checked_uris = []
eol_las = LinkAnnotation.objects\
.filter(object_uri__icontains=self.EOL_URI_PREFIX)
for eol_la in eol_las:
eol_uri = eol_la.object_uri
leg = LinkEntityGeneration()
le_gen = LinkEntityGeneration()
eol_uri = le_gen.make_clean_uri(eol_uri) # strip off any cruft in the URI
if eol_uri != eol_la.object_uri:
print('Has cruft: ' + str(eol_la.object_uri))
LinkAnnotation.objects\
.filter(hash_id=eol_la.hash_id)\
.delete() # delete the old
eol_la.object_uri = eol_uri
eol_la.save() # save the cleaned URI
if eol_uri not in checked_uris:
# only check on a given URI once
checked_uris.append(eol_uri)
try:
le = LinkEntity.objects.get(uri=eol_uri)
except LinkEntity.DoesNotExist:
le = False
if le is False:
print('Getting missing data for: ' + eol_uri)
self.get_save_entity_label(eol_uri)
def clean_uris(self):
"""
cleans URIs to keep them consistent and empty of 'cruft'
"""
le_gen = LinkEntityGeneration()
self.subject = le_gen.make_clean_uri(self.subject)
self.predicate_uri = le_gen.make_clean_uri(self.predicate_uri)
self.object_uri = le_gen.make_clean_uri(self.object_uri)
def clean_uris(self):
"""
cleans URIs to keep them consistent and empty of 'cruft'
"""
le_gen = LinkEntityGeneration()
self.subject = le_gen.make_clean_uri(self.subject)
self.predicate_uri = le_gen.make_clean_uri(self.predicate_uri)
self.object_uri = le_gen.make_clean_uri(self.object_uri)
def validate_fix_eol_objects(self):
""" Searches for EOL links in the
LinkAnnotations table, then fixes
badly URIs with cruft. Also
calls the EOL API to get labels
for URIs with no record in the LinkEntity
table.
"""
checked_uris = []
eol_las = LinkAnnotation.objects\
.filter(object_uri__icontains=self.EOL_URI_PREFIX)
for eol_la in eol_las:
eol_uri = eol_la.object_uri
leg = LinkEntityGeneration()
le_gen = LinkEntityGeneration()
eol_uri = le_gen.make_clean_uri(
eol_uri) # strip off any cruft in the URI
if eol_uri != eol_la.object_uri:
print('Has cruft: ' + str(eol_la.object_uri))
LinkAnnotation.objects\
.filter(hash_id=eol_la.hash_id)\
.delete() # delete the old
eol_la.object_uri = eol_uri
eol_la.save() # save the cleaned URI
if eol_uri not in checked_uris:
# only check on a given URI once
checked_uris.append(eol_uri)
try:
le = LinkEntity.objects.get(uri=eol_uri)
except LinkEntity.DoesNotExist:
le = False
if le is False:
print('Getting missing data for: ' + eol_uri)
label = False
eol_api = eolAPI()
eol_data = eol_api.get_basic_json_for_eol_uri(eol_uri)
if isinstance(eol_data, dict):
print('Reading data...')
if 'scientificName' in eol_data:
label = eol_data['scientificName']
else:
print('Failed to read data: ' + str(eol_data))
if label is not False:
print('Saving data for: ' + str(label) + ' (' +
eol_uri + ')')
le = LinkEntity()
le.uri = eol_uri
le.label = label
le.alt_label = label
le.ent_type = 'class'
le.vocab_uri = self.EOL_VOCAB_URI
le.save()
def validate_fix_eol_objects(self):
""" Searches for EOL links in the
LinkAnnotations table, then fixes
badly URIs with cruft. Also
calls the EOL API to get labels
for URIs with no record in the LinkEntity
table.
"""
checked_uris = []
eol_las = LinkAnnotation.objects.filter(object_uri__icontains=self.EOL_URI_PREFIX)
for eol_la in eol_las:
eol_uri = eol_la.object_uri
leg = LinkEntityGeneration()
le_gen = LinkEntityGeneration()
eol_uri = le_gen.make_clean_uri(eol_uri) # strip off any cruft in the URI
if eol_uri != eol_la.object_uri:
print("Has cruft: " + str(eol_la.object_uri))
LinkAnnotation.objects.filter(hash_id=eol_la.hash_id).delete() # delete the old
eol_la.object_uri = eol_uri
eol_la.save() # save the cleaned URI
if eol_uri not in checked_uris:
# only check on a given URI once
checked_uris.append(eol_uri)
try:
le = LinkEntity.objects.get(uri=eol_uri)
except LinkEntity.DoesNotExist:
le = False
if le is False:
print("Getting missing data for: " + eol_uri)
label = False
eol_api = eolAPI()
eol_data = eol_api.get_basic_json_for_eol_uri(eol_uri)
if isinstance(eol_data, dict):
print("Reading data...")
if "scientificName" in eol_data:
label = eol_data["scientificName"]
else:
print("Failed to read data: " + str(eol_data))
if label is not False:
print("Saving data for: " + str(label) + " (" + eol_uri + ")")
le = LinkEntity()
le.uri = eol_uri
le.label = label
le.alt_label = label
le.ent_type = "class"
le.vocab_uri = self.EOL_VOCAB_URI
le.save()
def validate_fix_uberon_objects(self):
""" Searches for UBERON links in the
LinkAnnotations table, then fixes
badly URIs with cruft. Also
calls the UBERON API to get labels
for URIs with no record in the LinkEntity
table.
"""
checked_uris = []
uberon_las = LinkAnnotation.objects\
.filter(object_uri__icontains=self.UBERON_URI_PREFIX)
for uberon_la in uberon_las:
uberon_uri = uberon_la.object_uri
le_gen = LinkEntityGeneration()
uberon_uri = le_gen.make_clean_uri(
uberon_uri) # strip off any cruft in the URI
if uberon_uri != uberon_la.object_uri:
print('Has cruft: ' + str(uberon_la.object_uri))
LinkAnnotation.objects\
.filter(hash_id=uberon_la.hash_id)\
.delete() # delete the old
uberon_la.object_uri = uberon_uri
uberon_la.save() # save the cleaned URI
if uberon_uri not in checked_uris:
# only check on a given URI once
checked_uris.append(uberon_uri)
try:
le = LinkEntity.objects.get(uri=uberon_uri)
except LinkEntity.DoesNotExist:
le = False
if le is False:
print('Getting missing data for: ' + uberon_uri)
u_api = uberonAPI()
label = u_api.get_uri_label_from_graph(uberon_uri)
if label is False:
print('Failed to read data for : ' + str(uberon_uri))
else:
print('Saving data for: ' + str(label) + ' (' +
uberon_uri + ')')
le = LinkEntity()
le.uri = uberon_uri
le.label = label
le.alt_label = label
le.ent_type = 'class'
le.vocab_uri = self.UBERON_VOCAB_URI
le.save()
def validate_fix_uberon_objects(self):
""" Searches for UBERON links in the
LinkAnnotations table, then fixes
badly URIs with cruft. Also
calls the UBERON API to get labels
for URIs with no record in the LinkEntity
table.
"""
checked_uris = []
uberon_las = LinkAnnotation.objects\
.filter(object_uri__icontains=self.UBERON_URI_PREFIX)
for uberon_la in uberon_las:
uberon_uri = uberon_la.object_uri
le_gen = LinkEntityGeneration()
uberon_uri = le_gen.make_clean_uri(uberon_uri) # strip off any cruft in the URI
if uberon_uri != uberon_la.object_uri:
print('Has cruft: ' + str(uberon_la.object_uri))
LinkAnnotation.objects\
.filter(hash_id=uberon_la.hash_id)\
.delete() # delete the old
uberon_la.object_uri = uberon_uri
uberon_la.save() # save the cleaned URI
if uberon_uri not in checked_uris:
# only check on a given URI once
checked_uris.append(uberon_uri)
try:
le = LinkEntity.objects.get(uri=uberon_uri)
except LinkEntity.DoesNotExist:
le = False
if le is False:
print('Getting missing data for: ' + uberon_uri)
u_api = uberonAPI()
label = u_api.get_uri_label_from_graph(uberon_uri)
if label is False:
print('Failed to read data for : ' + str(uberon_uri))
else:
print('Saving data for: ' + str(label) + ' (' + uberon_uri + ')')
le = LinkEntity()
le.uri = uberon_uri
le.label = label
le.alt_label = label
le.ent_type = 'class'
le.vocab_uri = self.UBERON_VOCAB_URI
le.save()
def validate_make_eol_hierarchy(self, child_uri, parent_uri):
""" Validated hierarchy relations for EOL entities.
If a child already has a parent, this will not do anything
otherwise it will create a hierachy relation
"""
ok_create = False
le_gen = LinkEntityGeneration()
child_uri = le_gen.make_clean_uri(
child_uri) # strip off any cruft in the URI
parent_uri = le_gen.make_clean_uri(parent_uri)
ent = Entity()
found = ent.dereference(child_uri)
if found:
lr = LinkRecursion()
parents = lr.get_jsonldish_entity_parents(child_uri, False)
if parents is False:
# no parents, so OK to make an assertion
ok_create = True
else:
if len(parents) == 0:
# no parents, so OK to make an assertion
ok_create = True
else:
ok_create = True # the child does not yet exist, so OK to make the relation
print('Getting missing data for: ' + child_uri)
self.get_save_entity_label(child_uri)
if ok_create:
print('OK, make rel for: ' + child_uri + ' in ' + parent_uri)
la = LinkAnnotation()
la.subject = child_uri
la.subject_type = 'uri'
la.project_uuid = '0'
la.source_id = 'manual-eol-manage'
la.predicate_uri = self.CHILD_PARENT_REL
la.object_uri = parent_uri
la.sort = 1
la.save()
else:
print('Already in hierarchy: ' + child_uri)
def get_jsonld_for_uri(self, uri):
"""
gets json-ld daa from the OCHRE URI
"""
le_gen = LinkEntityGeneration()
uri = le_gen.make_clean_uri(uri) # strip off any cruft in the URI
url = uri + '.jsonld'
self.request_url = url
if self.delay_before_request > 0:
# default to sleep BEFORE a request is sent, to
# give the remote service a break.
sleep(self.delay_before_request)
try:
gapi = GeneralAPI()
r = requests.get(url, timeout=240, headers=gapi.client_headers)
r.raise_for_status()
self.request_url = r.url
json_r = r.json()
except:
json_r = False
self.json_data = json_r
return self.json_data
def validate_make_eol_hierarchy(self, child_uri, parent_uri):
""" Validated hierarchy relations for EOL entities.
If a child already has a parent, this will not do anything
otherwise it will create a hierachy relation
"""
ok_create = False
le_gen = LinkEntityGeneration()
child_uri = le_gen.make_clean_uri(child_uri) # strip off any cruft in the URI
parent_uri = le_gen.make_clean_uri(parent_uri)
ent = Entity()
found = ent.dereference(child_uri)
if found:
lr = LinkRecursion()
parents = lr.get_jsonldish_entity_parents(child_uri, False)
if parents is False:
# no parents, so OK to make an assertion
ok_create = True
else:
if len(parents) == 0:
# no parents, so OK to make an assertion
ok_create = True
else:
ok_create = True # the child does not yet exist, so OK to make the relation
print('Getting missing data for: ' + child_uri)
self.get_save_entity_label(child_uri)
if ok_create:
print('OK, make rel for: ' + child_uri + ' in ' + parent_uri)
la = LinkAnnotation()
la.subject = child_uri
la.subject_type = 'uri'
la.project_uuid = '0'
la.source_id = 'manual-eol-manage'
la.predicate_uri = self.CHILD_PARENT_REL
la.object_uri = parent_uri
la.sort = 1
la.save()
else:
print('Already in hierarchy: ' + child_uri)
def suggest_vocabulary(self, uri):
""" suggests a vocabulary based on
the content of the URI
"""
vocab_uri = False
le_gen = LinkEntityGeneration()
uri = le_gen.make_clean_uri(uri)
if '/' in uri:
uri_ex = uri.split('/')
last_part = '/' + uri_ex[-1]
uri_prefix = uri.replace(last_part, '')
print('Checking uri prefix: ' + uri_prefix)
le_examps = LinkEntity.objects\
.filter(uri__contains=uri_prefix)
if len(le_examps) > 0:
vocab_uris = []
for le_ex in le_examps:
if le_ex.vocab_uri not in vocab_uris:
# doing this to make sure we have an unambiguous URI
vocab_uris.append(le_ex.vocab_uri)
if len(vocab_uris) == 1:
# the uri prefix was not ambiguous, so we can use it
vocab_uri = vocab_uris[0]
return vocab_uri
def get_jsonld_for_uri(self, uri):
"""
gets json-ld daa from the OCHRE URI
"""
le_gen = LinkEntityGeneration()
uri = le_gen.make_clean_uri(uri) # strip off any cruft in the URI
url = uri + '.jsonld'
self.request_url = url
if self.delay_before_request > 0:
# default to sleep BEFORE a request is sent, to
# give the remote service a break.
sleep(self.delay_before_request)
try:
gapi = GeneralAPI()
r = requests.get(url,
timeout=240,
headers=gapi.client_headers)
r.raise_for_status()
self.request_url = r.url
json_r = r.json()
except:
json_r = False
self.json_data = json_r
return self.json_data
def create_note_entity(self):
""" creates a note predicate entity if it does not yet
exist
"""
leg = LinkEntityGeneration()
leg.check_add_note_pred()
def add_update(self, post_data):
""" Creates or updates a linked data entity """
ok = True
uri = False
label = False
vocab_uri = False
alt_label = False
ent_type = 'class'
note = ''
action = 'attempted creation or update'
sent_uri = uri
sent_label = label
if 'uri' in post_data:
uri = post_data['uri']
sent_uri = uri
if not self.validate_web_uri(uri):
# must be a full web uri to use
note += '"' + uri + '" needs to be valid Web URI. '
uri = False
if 'label' in post_data:
label = post_data['label']
sent_label = label
alt_label = label # default for alt-label is label
if len(label) < 1:
note += 'The entity label cannot be blank. '
label = False
if 'alt_label' in post_data:
if len(post_data['alt_label']) > 0:
alt_label = post_data['alt_label']
if 'ent_type' in post_data:
ent_type = post_data['ent_type']
if 'vocab_uri' in post_data:
vocab_uri = post_data['vocab_uri']
if not self.validate_web_uri(vocab_uri)\
and ent_type != 'vocabulary':
# vocab_uri is not a full uri, so suggest one
# based on the URI for the request
vocab_uri = self.suggest_vocabulary(uri)
elif not self.validate_web_uri(vocab_uri)\
and ent_type == 'vocabulary':
vocab_uri = uri
else:
pass
if uri is not False \
and label is not False \
and vocab_uri is not False:
le_gen = LinkEntityGeneration()
uri = le_gen.make_clean_uri(uri)
if uri != vocab_uri:
# get the varient of the vocab_uri that's actually in use
# returns false if a varient can't be found
vocab_uri = self.check_vocab_uri(vocab_uri)
if vocab_uri is False:
# cannot find a varient for this vocabulary uri
vocab_ok = False
else:
vocab_ok = True
elif ent_type == 'vocabulary':
vocab_ok = True
else:
vocab_ok = False
if vocab_ok:
ok = True
try:
action = 'edit-update'
le = LinkEntity.objects.get(uri=uri)
except LinkEntity.DoesNotExist:
action = 'add-create'
le = LinkEntity()
le.uri = uri
# now add information to save
le.label = label
le.alt_label = alt_label
le.ent_type = ent_type
le.vocab_uri = vocab_uri
le.save()
uri = le.uri # in case the URI changed because of validation changes
else:
ok = False
note += 'Must first create a record for the vocabulary. '
else:
ok = False
note += 'Missing data required for this action. '
self.response = {
'action': action,
'uri': sent_uri,
'label': sent_label,
'ok': ok,
'change': {
'note': note
}
}
return self.response
def create_note_entity(self):
""" creates a note predicate entity if it does not yet
exist
"""
leg = LinkEntityGeneration()
leg.check_add_note_pred()