def add_description_note(self, note):
""" adds a description note about a new item
"""
# first make sure the has-note predicate exists
self.create_note_entity()
note = str(note)
if len(note) > 1:
# save the note as a string
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_obj = str_man.get_make_string(str(note))
# now make the assertion
new_ass = Assertion()
new_ass.uuid = self.uuid
new_ass.subject_type = self.item_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
if self.obs_node is False:
new_ass.obs_node = '#obs-' + str(self.obs_num)
else:
new_ass.obs_node = self.obs_node
new_ass.obs_num = self.obs_num
new_ass.sort = self.note_sort
new_ass.visibility = 1
new_ass.predicate_uuid = Assertion.PREDICATES_NOTE
new_ass.object_type = 'xsd:string'
new_ass.object_uuid = str_obj.uuid
new_ass.save()
# now clear the cache a change was made
self.clear_caches()
def add_description_note(self, note):
""" adds a description note about a new item
"""
# first make sure the has-note predicate exists
self.create_note_entity()
note = str(note)
if len(note) > 1:
# save the note as a string
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_obj = str_man.get_make_string(str(note))
# now make the assertion
new_ass = Assertion()
new_ass.uuid = self.uuid
new_ass.subject_type = self.item_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
if self.obs_node is False:
new_ass.obs_node = '#obs-' + str(self.obs_num)
else:
new_ass.obs_node = self.obs_node
new_ass.obs_num = self.obs_num
new_ass.sort = self.note_sort
new_ass.visibility = 1
new_ass.predicate_uuid = Assertion.PREDICATES_NOTE
new_ass.object_type = 'xsd:string'
new_ass.object_uuid = str_obj.uuid
new_ass.save()
# now clear the cache a change was made
self.clear_caches()
def get_make_string_w_suggested_uuid(self, content, string_uuid=False):
""" makes a string with a suggested uuid if it does not exist """
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_man.suggested_uuid = string_uuid
str_obj = str_man.get_make_string(content)
return str_obj
def get_make_string_w_suggested_uuid(self, content, string_uuid=False):
""" makes a string with a suggested uuid if it does not exist """
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_man.suggested_uuid = string_uuid
str_obj = str_man.get_make_string(content)
return str_obj
def get_make_type_within_pred_uuid(self, predicate_uuid, content):
"""
gets a type, filtered by a given predicate_uuid and content string
if a type does not exist, this function creates it, then returns the type
"""
str_manage = StringManagement()
str_manage.project_uuid = self.project_uuid
str_manage.source_id = self.source_id
str_manage.suggested_uuid = self.suggested_content_uuid
# get an existing or create a new string object
oc_string = str_manage.get_make_string(content)
self.content = content
self.get_make_type_pred_uuid_content_uuid(predicate_uuid, oc_string.uuid)
return self.oc_type
def get_make_type_within_pred_uuid(self, predicate_uuid, content):
"""
gets a type, filtered by a given predicate_uuid and content string
if a type does not exist, this function creates it, then returns the type
"""
str_manage = StringManagement()
str_manage.project_uuid = self.project_uuid
str_manage.source_id = self.source_id
str_manage.suggested_uuid = self.suggested_content_uuid
# get an existing or create a new string object
oc_string = str_manage.get_make_string(content)
self.content = content
self.get_make_type_pred_uuid_content_uuid(predicate_uuid,
oc_string.uuid)
return self.oc_type
def add_trinomial_to_item(self,
predicate,
sort,
manifest,
trinomial,
allow_multiple=False):
""" Adds trinomials to an item """
output = False
ok_to_create = False
if allow_multiple is False:
# first check to make sure the item doesn't have
# a predicate for a trinomial, if multiple are not allowed
t_ass = Assertion.objects\
.filter(uuid=manifest.uuid,
predicate_uuid=predicate.uuid)[:1]
if len(t_ass) < 1:
ok_to_create = True
else:
ok_to_create = True
if manifest.project_uuid in self.source_ids:
source_id = self.source_ids[manifest.project_uuid]
if ok_to_create:
# first make a string object
sm = StringManagement()
sm.project_uuid = self.dinaa_proj_uuid
sm.source_id = self.source_id
oc_string = sm.get_make_string(trinomial)
# now make the assertion for the trinomial
ass = Assertion()
ass.uuid = manifest.uuid
ass.subject_type = manifest.item_type
ass.project_uuid = manifest.project_uuid
ass.source_id = source_id
ass.obs_node = '#obs-' + str(self.obs_num)
ass.obs_num = self.obs_num
ass.sort = sort
ass.visibility = 1
ass.predicate_uuid = predicate.uuid
ass.object_uuid = oc_string.uuid
ass.object_type = 'xsd:string'
ass.save()
output = True
return output
def get_make_type_within_pred_uuid(self,
predicate_uuid,
content,
content_suffix_num=1):
"""
gets a type, filtered by a given predicate_uuid and content string
if a type does not exist, this function creates it, then returns the type
"""
original_content = content
str_manage = StringManagement()
str_manage.project_uuid = self.project_uuid
str_manage.source_id = self.source_id
str_manage.suggested_uuid = self.suggested_content_uuid
# get an existing or create a new string object
if content_suffix_num > 1:
# add a numeric suffix for content, in case we need to
# increment up to in rare cases when different types have the same string content
content += ' [' + str(content_suffix_num) + ']'
oc_string = str_manage.get_make_string(content)
self.content = content
self.get_make_type_pred_uuid_content_uuid(predicate_uuid,
oc_string.uuid)
if self.sup_reconcile_key is not None and\
self.sup_reconcile_value is not None:
# checks to see if the manifest supplemental JSON has a matching key, value pair
# this lets us further constrain reconciliation, say with FAIMS attributes (types)
man_obj = False
try:
man_obj = Manifest.objects.get(uuid=self.oc_type.uuid)
except Manifest.DoesNotExist:
man_obj = False
if man_obj is not False:
match_ok = man_obj.check_sup_json_key_value(
self.sup_reconcile_key, self.sup_reconcile_value)
if match_ok is False:
self.oc_type = False
content_suffix_num += 1
if content_suffix_num <= 10:
# so we don't get into endless loops
self.oc_type = self.get_make_type_within_pred_uuid(
predicate_uuid, original_content,
content_suffix_num)
return self.oc_type
def add_trinomial_to_item(self,
predicate,
sort,
manifest,
trinomial,
allow_multiple=False):
""" Adds trinomials to an item """
output = False
ok_to_create = False
if allow_multiple is False:
# first check to make sure the item doesn't have
# a predicate for a trinomial, if multiple are not allowed
t_ass = Assertion.objects\
.filter(uuid=manifest.uuid,
predicate_uuid=predicate.uuid)[:1]
if len(t_ass) < 1:
ok_to_create = True
else:
ok_to_create = True
if manifest.project_uuid in self.source_ids:
source_id = self.source_ids[manifest.project_uuid]
if ok_to_create:
# first make a string object
sm = StringManagement()
sm.project_uuid = self.dinaa_proj_uuid
sm.source_id = self.source_id
oc_string = sm.get_make_string(trinomial)
# now make the assertion for the trinomial
ass = Assertion()
ass.uuid = manifest.uuid
ass.subject_type = manifest.item_type
ass.project_uuid = manifest.project_uuid
ass.source_id = source_id
ass.obs_node = '#obs-' + str(self.obs_num)
ass.obs_num = self.obs_num
ass.sort = sort
ass.visibility = 1
ass.predicate_uuid = predicate.uuid
ass.object_uuid = oc_string.uuid
ass.object_type = 'xsd:string'
ass.save()
output = True
return output
def get_make_type_within_pred_uuid(self,
predicate_uuid,
content,
content_suffix_num=1):
"""
gets a type, filtered by a given predicate_uuid and content string
if a type does not exist, this function creates it, then returns the type
"""
original_content = content
str_manage = StringManagement()
str_manage.project_uuid = self.project_uuid
str_manage.source_id = self.source_id
str_manage.suggested_uuid = self.suggested_content_uuid
# get an existing or create a new string object
if content_suffix_num > 1:
# add a numeric suffix for content, in case we need to
# increment up to in rare cases when different types have the same string content
content += ' [' + str(content_suffix_num) + ']'
oc_string = str_manage.get_make_string(content)
self.content = content
self.get_make_type_pred_uuid_content_uuid(predicate_uuid, oc_string.uuid)
if self.sup_reconcile_key is not None and\
self.sup_reconcile_value is not None:
# checks to see if the manifest supplemental JSON has a matching key, value pair
# this lets us further constrain reconciliation, say with FAIMS attributes (types)
man_obj = False
try:
man_obj = Manifest.objects.get(uuid=self.oc_type.uuid)
except Manifest.DoesNotExist:
man_obj = False
if man_obj is not False:
match_ok = man_obj.check_sup_json_key_value(self.sup_reconcile_key,
self.sup_reconcile_value)
if match_ok is False:
self.oc_type = False
content_suffix_num += 1
if content_suffix_num <= 10:
# so we don't get into endless loops
self.oc_type = self.get_make_type_within_pred_uuid(predicate_uuid,
original_content,
content_suffix_num)
return self.oc_type
def make_datatype_wrong_assertion(self, predicate_uuid, content):
""" Makes an assertion for records that don't fit the
expected data_type / object_type
"""
pm = PredicateManagement()
pm.project_uuid = self.project_uuid
pm.source_id = self.source_id
pm.sort = self.sort
new_note_predicate = pm.get_make_related_note_predicate(predicate_uuid, ' (Note)')
if new_note_predicate is not False:
note_pred_uuid = new_note_predicate.uuid
# save a skos:related assertion linking the old and new predicates
ptm = PredicateTypeAssertions()
ptm.skos_relate_old_new_predicates(self.project_uuid,
self.source_id,
predicate_uuid,
note_pred_uuid)
sm = StringManagement()
sm.project_uuid = self.project_uuid
sm.source_id = self.source_id
oc_string = sm.get_make_string(content)
object_uuid = oc_string.uuid
if self.check_description_new(self.subject_uuid,
self.obs_num,
note_pred_uuid,
object_uuid,
None,
None):
# this asertion does not exist yet, OK to make it
new_ass = Assertion()
new_ass.uuid = self.subject_uuid
new_ass.subject_type = self.subject_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = self.obs_node
new_ass.obs_num = self.obs_num
new_ass.sort = self.sort + .1
new_ass.visibility = 1
new_ass.predicate_uuid = note_pred_uuid
new_ass.object_uuid = object_uuid
new_ass.object_type = 'xsd:string'
new_ass.save()
def reconcile_string_cell(self, imp_cell):
""" reconciles a predicate variable from the Import Field
"""
output = False
if len(imp_cell.record) > 0:
output = True
sm = StringManagement()
sm.project_uuid = imp_cell.project_uuid
sm.source_id = imp_cell.source_id
self.oc_string = sm.get_make_string(imp_cell.record)
self.uuid = self.oc_string.uuid
self.content = self.oc_string.content
# self.source_id = self.oc_string.source_id
if self.uuid != imp_cell.l_uuid:
# update the reconcilted UUID for import cells with same rec_hash
up_cells = ImportCell.objects\
.filter(source_id=self.source_id,
field_num=imp_cell.field_num,
rec_hash=imp_cell.rec_hash)
for up_cell in up_cells:
# save each cell with the correct UUID
up_cell.l_uuid = str(self.uuid)
up_cell.uuids_save()
return output
def update_string_content(self, content, content_type='content', post_data={}):
""" Updates the main string content of an item
(project, document, or table abstract)
"""
content = content.strip()
html_ok = self.valid_as_html(content) # check if valid, but allow invalid
if html_ok:
note = ''
else:
note = self.errors['html']
if self.manifest is not False:
# check for translation!
if 'language' in post_data:
language = post_data['language']
else:
language = Languages.DEFAULT_LANGUAGE
if 'script' in post_data:
script = post_data['script']
else:
script = None
if language != Languages.DEFAULT_LANGUAGE:
# editing another language, not the default
lan_obj = Languages()
localize_key = lan_obj.get_language_script_key(language, script)
else:
localize_key = False
if self.manifest.item_type == 'projects':
try:
cobj = Project.objects.get(uuid=self.manifest.uuid)
if localize_key is not False:
if content_type == 'short_des':
cobj.sm_localized_json = lan_obj.modify_localization_json(cobj.sm_localized_json,
localize_key,
content)
else:
cobj.lg_localized_json = lan_obj.modify_localization_json(cobj.lg_localized_json,
localize_key,
content)
else:
if content_type == 'short_des':
cobj.short_des = content
else:
cobj.content = content
cobj.save()
ok = True
except Project.DoesNotExist:
self.errors['uuid'] = self.manifest.uuid + ' not in projects'
ok = False
elif self.manifest.item_type == 'tables':
ex_id = ExpTableIdentifiers()
ex_id.make_all_identifiers(self.manifest.uuid)
try:
cobj = ExpTable.objects.get(table_id=ex_id.table_id)
if localize_key is not False:
if content_type == 'short_des':
cobj.sm_localized_json = lan_obj.modify_localization_json(cobj.sm_localized_json,
localize_key,
content)
else:
cobj.lg_localized_json = lan_obj.modify_localization_json(cobj.lg_localized_json,
localize_key,
content)
else:
if content_type == 'short_des':
cobj.short_des = content
else:
cobj.abstract = content
cobj.save()
ok = True
except ExpTable.DoesNotExist:
self.errors['uuid'] = ex_id.table_id + ' not in tables'
ok = False
elif self.manifest.item_type == 'documents' and content_type == 'content':
try:
cobj = OCdocument.objects.get(uuid=self.manifest.uuid)
if localize_key is not False:
cobj.localized_json = lan_obj.modify_localization_json(cobj.localized_json,
localize_key,
content)
else:
cobj.content = content
cobj.save()
ok = True
except OCdocument.DoesNotExist:
self.errors['uuid'] = self.manifest.uuid + ' not in documents'
ok = False
elif self.manifest.item_type == 'predicates' or self.manifest.item_type == 'types':
# make a skos not to document a predicate or type
ok = True
string_uuid = None
old_notes = Assertion.objects\
.filter(uuid=self.manifest.uuid,
predicate_uuid='skos:note')
for old_note in old_notes:
string_uuid = old_note.object_uuid
if localize_key is False:
# only delete if this is not a translation!
old_note.delete()
if localize_key is not False and string_uuid is not None:
# OK, we're just adding a translation
act_string = False
try:
act_string = OCstring.objects.get(uuid=string_uuid)
except OCstring.DoesNotExist:
act_string = False
string_uuid = None
if act_string is not False:
# update the localization JSON with the content
act_string.localized_json = lan_obj.modify_localization_json(act_string.localized_json,
localize_key,
content)
act_string.save()
if localize_key is False:
# this is for changing SKOS notes in cases where we're not
# adding a translation
if string_uuid is not None:
string_used = Assertion.objects\
.filter(project_uuid=self.manifest.project_uuid,
object_uuid=string_uuid)[:1]
if len(string_used) > 0:
# the string is used elsewhere, so we can't just use that
# string uuid
string_uuid = None
else:
# put the new content int the string that is not in use
# for other items
act_string = False
try:
act_string = OCstring.objects.get(uuid=string_uuid)
except OCstring.DoesNotExist:
act_string = False
string_uuid = None
if act_string is not False:
# save the content in the string to overwrite it
act_string.content = content
act_string.save()
if string_uuid is None:
# we don't have a string_uuid to overwrite
str_man = StringManagement()
str_man.project_uuid = self.manifest.project_uuid
str_man.source_id = 'web-form'
str_obj = str_man.get_make_string(str(content))
string_uuid = str_obj.uuid
# now make the assertion
new_ass = Assertion()
new_ass.uuid = uuid = self.manifest.uuid
new_ass.subject_type = self.manifest.item_type
new_ass.project_uuid = self.manifest.project_uuid
new_ass.source_id = 'web-form'
new_ass.obs_node = '#obs-1'
new_ass.obs_num = 1
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = 'skos:note'
new_ass.object_type = 'xsd:string'
new_ass.object_uuid = string_uuid
new_ass.save()
else:
ok = False
if ok:
# now clear the cache a change was made
self.clear_caches()
self.response = {'action': 'update-string-content',
'ok': ok,
'change': {'prop': content_type,
'new': content,
'old': '[Old content]',
'note': note}}
return self.response
def add_locus_coordinates(self):
""" adds coordinate html to the loci """
source_id = 'pc-locus-coord-html'
Assertion.objects.filter(source_id=source_id,
predicate_uuid=self.pred_uuid_open)\
.delete()
Assertion.objects.filter(source_id=source_id,
predicate_uuid=self.pred_uuid_close)\
.delete()
OCstring.objects.filter(source_id=source_id).delete()
open_fields = [3, 4, 5, 6]
close_fields = [7, 8, 9, 10]
l_id_objs = self.get_locus_objs_for_coordinates()
for l_id_obj in l_id_objs:
act_locus = l_id_obj['locus_obj']
rows = l_id_obj['rows']
open_html = self.make_coordinate_html(rows, open_fields)
close_html = self.make_coordinate_html(rows, close_fields)
str_m = StringManagement()
str_m.source_id = source_id
str_m.project_uuid = act_locus.project_uuid
open_str_obj = str_m.get_make_string(open_html)
str_m = StringManagement()
str_m.source_id = source_id
str_m.project_uuid = act_locus.project_uuid
close_str_obj = str_m.get_make_string(close_html)
# now make the assertions
print('Add coordinates to: ' + act_locus.uuid)
try:
new_ass = Assertion()
new_ass.uuid = act_locus.uuid
new_ass.subject_type = act_locus.item_type
new_ass.project_uuid = act_locus.project_uuid
new_ass.source_id = source_id
new_ass.obs_node = '#obs-2'
new_ass.obs_num = 2
new_ass.sort = 100
new_ass.visibility = 1
new_ass.predicate_uuid = self.pred_uuid_open
new_ass.object_uuid = open_str_obj.uuid
new_ass.object_type = 'xsd:string'
new_ass.save()
new_add = True
except:
new_add = False
try:
new_ass = Assertion()
new_ass.uuid = act_locus.uuid
new_ass.subject_type = act_locus.item_type
new_ass.project_uuid = act_locus.project_uuid
new_ass.source_id = source_id
new_ass.obs_node = '#obs-2'
new_ass.obs_num = 2
new_ass.sort = 101
new_ass.visibility = 1
new_ass.predicate_uuid = self.pred_uuid_close
new_ass.object_uuid = close_str_obj.uuid
new_ass.object_type = 'xsd:string'
new_ass.save()
new_add = True
except:
new_add = False
def process_complex_batch(self):
""" processes fields for documents
entities starting with a given row number.
This iterates over all containment fields, starting
with the root subjhect field
"""
self.clear_source() # clear prior import for this source
self.end_row = self.start_row + self.batch_size
self.get_complex_description_fields()
label_str_uuids = {}
if len(self.complex_des_fields) > 0:
print('Number of Complex Description Fields: ' +
str(len(self.complex_des_fields)))
cp_id_number = 0
for cp_field in self.complex_des_fields:
cp_id_number += 1
pc = ProcessCells(self.source_id, self.start_row)
distinct_records = pc.get_field_records_by_fl_uuid(
cp_field.describes_field.field_num, False)
if distinct_records is not False:
# sort the list in row_order from the import table
pg = ProcessGeneral(self.source_id)
distinct_records = pg.order_distinct_records(
distinct_records)
for row_key, dist_rec in distinct_records.items():
if cp_field.obs_num < 1:
obs_num = 1
else:
obs_num = cp_field.obs_num
obs_node = '#obs-' + str(obs_num)
subject_uuid = dist_rec['imp_cell_obj'].fl_uuid
subject_type = cp_field.describes_field.field_type
subject_ok = dist_rec['imp_cell_obj'].cell_ok
subject_record = dist_rec['imp_cell_obj'].record
if subject_uuid is False or\
len(subject_record) < 1:
subject_ok = False
if subject_uuid == 'False':
subject_ok = False
sort = 0
in_rows = dist_rec['rows']
print('Look for complex description labels in rows: ' +
str(in_rows))
if subject_ok is not False:
# OK! we have the subjects of complex descriptions
# with uuids, so now we can make an fl_uuid for each
# of the complex description fields.
complex_uuid = subject_uuid + self.FRAG_ID_PREFIX + str(
cp_id_number)
complex_recs = ImportCell.objects\
.filter(source_id=self.source_id,
field_num=cp_field.field_num,
row_num__in=in_rows)\
.exclude(record='')
if len(complex_recs) > 0:
# we have records in the complex description field that are not blank
# and are associated with the subject of the complex description.
# so now, let's record this association.
save_ok = False
new_ass = Assertion()
new_ass.uuid = subject_uuid
new_ass.subject_type = subject_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id + ProcessGeneral.COMPLEX_DESCRIPTION_SOURCE_SUFFIX
new_ass.obs_node = obs_node
new_ass.obs_num = obs_num
new_ass.sort = 100 + cp_id_number
new_ass.visibility = 1
new_ass.predicate_uuid = ComplexDescription.PREDICATE_COMPLEX_DES
new_ass.object_type = 'complex-description'
new_ass.object_uuid = complex_uuid
new_ass.save()
try:
print('Saved complex-description: ' +
complex_uuid)
new_ass.save()
save_ok = True
except:
save_ok = False
if save_ok:
self.count_new_assertions += 1
# now look through the complex description records and make labels
for comp_rec in complex_recs:
# first save the fl_uuid for the complex description
comp_rec.fl_uuid = complex_uuid
comp_rec.save()
if isinstance(cp_field.value_prefix, str):
cp_label = cp_field.value_prefix + comp_rec.record
else:
cp_label = comp_rec.record
if cp_label not in label_str_uuids:
# make a uuid for the record value
# adding a source_id suffix keeps this from being deleted as descriptions get processed
sm = StringManagement()
sm.project_uuid = self.project_uuid
sm.source_id = self.source_id + ProcessGeneral.COMPLEX_DESCRIPTION_SOURCE_SUFFIX
oc_string = sm.get_make_string(
cp_label)
content_uuid = oc_string.uuid
label_str_uuids[
cp_label] = content_uuid
content_uuid = label_str_uuids[cp_label]
save_ok = False
new_ass = Assertion()
new_ass.uuid = complex_uuid
new_ass.subject_type = 'complex-description'
new_ass.project_uuid = self.project_uuid
# adding a source_id suffix keeps this from being deleted as descriptions get processed
new_ass.source_id = self.source_id + ProcessGeneral.COMPLEX_DESCRIPTION_SOURCE_SUFFIX
new_ass.obs_node = '#obs-' + str(
self.
obs_num_complex_description_assertions)
new_ass.obs_num = self.obs_num_complex_description_assertions
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = ComplexDescription.PREDICATE_COMPLEX_DES_LABEL
new_ass.object_type = 'xsd:string'
new_ass.object_uuid = content_uuid
try:
new_ass.save()
save_ok = True
except:
save_ok = False
if save_ok:
self.count_new_assertions += 1
def validate_save_field_value(self,
item_man, # subject manifest object
field,
field_value_dict,
value_num):
""" Valididates a field value for a field and
then saves it to the database if valid
"""
error_key = str(field['id']) + '-' + str(value_num)
valid = True
id_val = None
literal_val = None
if isinstance(field_value_dict, dict):
if 'id' in field_value_dict:
id_val = field_value_dict['id']
if 'literal' in field_value_dict:
literal_val = field_value_dict['literal']
else:
valid = False
self.errors[error_key] = 'Value submitted to ' + field['label'] + ' has the wrong format'
data_type = self.get_predicate_data_type(field['predicate_uuid'])
object_uuid = None
object_type = data_type
data_num = None
data_date = None
if field['predicate_uuid'] in self.global_predicates:
# a global predicate, so get from dictionary set above
data_type = self.global_predicates[field['predicate_uuid']]['data_type']
else:
pred_man = self.get_manifest_item(field['predicate_uuid'])
if pred_man is False or data_type is False:
# could not find a predicate_uuid!!
valid = False
self.errors[error_key] = 'Problem with: ' + field['label'] + ' '
self.errors[error_key] += '(' + field['predicate_uuid'] + ') is not found.'
if valid:
# predicate_uuid exists
str_obj = None
if data_type == 'xsd:string':
str_man = StringManagement()
str_man.project_uuid = item_man.project_uuid
str_man.source_id = item_man.source_id
str_obj = str_man.get_make_string(str(literal_val))
object_uuid = str_obj.uuid
object_type = data_type
elif data_type == 'id':
# first check is to see if the field_value exists in the manifest
val_man = self.get_manifest_item(id_val)
if val_man is False:
valid = False
self.errors[error_key] = 'Problem with: ' + field['label'] + '; cannot find: ' + str(id_val)
if valid:
# OK, the field_value is an ID in the manifest
# so let's check to make sure it is OK to associate
object_type = val_man.item_type
object_uuid = val_man.uuid
if val_man.class_uri == 'variable' \
and val_man.item_type != 'types':
# we've got a variable that does not link to a type!
self.errors[error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += val_man.label + ' (' + val_man.uuid + ') is not a type/category.'
valid = False
elif data_type == 'xsd:boolean':
data_num = self.validate_convert_boolean(literal_val)
if t_f_data is None:
self.errors[error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += '"' + str(literal_val) + '" is not a recognized boolean (T/F) value.'
valid = False
elif data_type == 'xsd:integer':
try:
data_num = int(float(literal_val))
except:
self.errors[error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += '"' + str(literal_val) + '" is not an integer.'
valid = False
elif data_type == 'xsd:double':
try:
data_num = float(literal_val)
except:
self.errors[error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += '"' + str(literal_val) + '" is not a number.'
valid = False
elif data_type == 'xsd:date':
try:
data_date = parse(literal_val)
except Exception as e:
self.errors[error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += '"' + str(literal_val) + '" is not a yyyy-mm-dd date'
valid = False
if valid:
# we've validated the field_value data. Now to save the assertions!
new_ass = Assertion()
new_ass.uuid = item_man.uuid
new_ass.subject_type = item_man.item_type
new_ass.project_uuid = item_man.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-' + str(field['obs_num'])
new_ass.obs_num = field['obs_num']
new_ass.sort = field['sort'] + (value_num / 1000)
new_ass.visibility = 1
new_ass.predicate_uuid = field['predicate_uuid']
new_ass.object_type = object_type
if object_uuid is not None:
new_ass.object_uuid = object_uuid
if data_num is not None:
new_ass.data_num = data_num
if data_date is not None:
new_ass.data_date = data_date
try:
new_ass.save()
except:
valid = False
self.errors[error_key] = 'Same assertion for: ' + field['label'] + ' '
self.errors[error_key] += 'already exists.'
return valid
def add_description(self):
""" validates the value's data type
if the data type differs from the
predicate data type,
add it as a note
"""
# validate the faims record against its expected data type
# the validated_record will be None if not valid
if isinstance(self.subject_uuid, str) \
and isinstance(self.subject_type, str):
# we have subject information
ok_to_add = False
data_type = self.attrib_dict['data_type']
if isinstance(self.predicate_uuid, str):
predicate_uuid = self.predicate_uuid
elif 'predicate_uuid' in self.attrib_dict:
predicate_uuid = self.attrib_dict['predicate_uuid']
object_date = None
object_uuid = None
object_num = None
object_type = data_type
if data_type == 'id':
# we're processing a controlled vocab (oc-type) object
if isinstance(self.faims_record_id, str):
if self.faims_record_id in self.attrib_dict['objects']:
# we found the object for this item
obj_dict = self.attrib_dict['objects'][
self.faims_record_id]
object_uuid = obj_dict['type_uuid']
object_type = 'types'
ok_to_add = True
elif self.faims_record is not None:
# we're processing a literal value
ddt = DescriptionDataType()
validated_record = ddt.validate_literal_by_data_type(
data_type, self.faims_record)
if validated_record is not None:
# the record is valid for the data_type
ok_to_add = True
if data_type == 'xsd:date':
object_date = validated_record
elif data_type == 'xsd:string':
# we have a string, so add it to the database and get its uuid
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_obj = str_man.get_make_string(validated_record)
object_uuid = str(str_obj.uuid)
else:
# we have numeric data, so add to numeric data
object_num = validated_record
else:
# the record is not valid for the data_type, so treat it as a string
# so first we need to make a predicate for it, that is for
# string values of this associated with this attribute
object_type = 'xsd:string'
sup_dict = LastUpdatedOrderedDict()
sup_dict[self.reconcile_key] = self.attrib_dict[
'id'] # the faims attribute id
pm = PredicateManagement()
pm.project_uuid = self.project_uuid
pm.source_id = self.source_id
pm.sup_dict = sup_dict
pm.sup_reconcile_key = self.reconcile_key
pm.sup_reconcile_value = self.attrib_dict[
'id'] # the faims attribute id
pred_obj = pm.get_make_predicate(
self.attrib_dict['label'] + ' [Note]',
self.attrib_dict['predicate_type'], object_type)
if pred_obj is not False:
# we have a new predicate
predicate_uuid = pred_obj.uuid
# now make an object_uuid for the original faims record
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_obj = str_man.get_make_string(self.faims_record)
object_uuid = str(str_obj.uuid)
ok_to_add = True
else:
# not valid data, don't add
ok_to_add = False
if ok_to_add:
# now we're OK to add the descrpitive assertion
new_ass = Assertion()
new_ass.uuid = self.subject_uuid
new_ass.subject_type = self.subject_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-' + str(self.obs_num)
new_ass.obs_num = self.obs_num
new_ass.sort = self.sort_num
new_ass.visibility = 1
new_ass.predicate_uuid = predicate_uuid
new_ass.object_type = object_type
if object_num is not None:
new_ass.data_num = object_num
if object_date is not None:
new_ass.data_date = object_date
if object_uuid is not None:
new_ass.object_uuid = object_uuid
try:
new_ass.save()
except:
# probably already imported
pass
def add_description(self):
""" validates the value's data type
if the data type differs from the
predicate data type,
add it as a note
"""
# validate the faims record against its expected data type
# the validated_record will be None if not valid
if isinstance(self.subject_uuid, str) \
and isinstance(self.subject_type, str):
# we have subject information
ok_to_add = False
data_type = self.attrib_dict['data_type']
if isinstance(self.predicate_uuid, str):
predicate_uuid = self.predicate_uuid
elif 'predicate_uuid' in self.attrib_dict:
predicate_uuid = self.attrib_dict['predicate_uuid']
object_date = None
object_uuid = None
object_num = None
object_type = data_type
if data_type == 'id':
# we're processing a controlled vocab (oc-type) object
if isinstance(self.faims_record_id, str):
if self.faims_record_id in self.attrib_dict['objects']:
# we found the object for this item
obj_dict = self.attrib_dict['objects'][self.faims_record_id]
object_uuid = obj_dict['type_uuid']
object_type = 'types'
ok_to_add = True
elif self.faims_record is not None:
# we're processing a literal value
ddt = DescriptionDataType()
validated_record = ddt.validate_literal_by_data_type(data_type,
self.faims_record)
if validated_record is not None:
# the record is valid for the data_type
ok_to_add = True
if data_type == 'xsd:date':
object_date = validated_record
elif data_type == 'xsd:string':
# we have a string, so add it to the database and get its uuid
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_obj = str_man.get_make_string(validated_record)
object_uuid = str(str_obj.uuid)
else:
# we have numeric data, so add to numeric data
object_num = validated_record
else:
# the record is not valid for the data_type, so treat it as a string
# so first we need to make a predicate for it, that is for
# string values of this associated with this attribute
object_type = 'xsd:string'
sup_dict = LastUpdatedOrderedDict()
sup_dict[self.reconcile_key] = self.attrib_dict['id'] # the faims attribute id
pm = PredicateManagement()
pm.project_uuid = self.project_uuid
pm.source_id = self.source_id
pm.sup_dict = sup_dict
pm.sup_reconcile_key = self.reconcile_key
pm.sup_reconcile_value = self.attrib_dict['id'] # the faims attribute id
pred_obj = pm.get_make_predicate(self.attrib_dict['label'] + ' [Note]',
self.attrib_dict['predicate_type'],
object_type)
if pred_obj is not False:
# we have a new predicate
predicate_uuid = pred_obj.uuid
# now make an object_uuid for the original faims record
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_obj = str_man.get_make_string(self.faims_record)
object_uuid = str(str_obj.uuid)
ok_to_add = True
else:
# not valid data, don't add
ok_to_add = False
if ok_to_add:
# now we're OK to add the descrpitive assertion
new_ass = Assertion()
new_ass.uuid = self.subject_uuid
new_ass.subject_type = self.subject_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-' + str(self.obs_num)
new_ass.obs_num = self.obs_num
new_ass.sort = self.sort_num
new_ass.visibility = 1
new_ass.predicate_uuid = predicate_uuid
new_ass.object_type = object_type
if object_num is not None:
new_ass.data_num = object_num
if object_date is not None:
new_ass.data_date = object_date
if object_uuid is not None:
new_ass.object_uuid = object_uuid
try:
new_ass.save()
except:
# probably already imported
pass
def process_complex_batch(self):
""" processes fields for documents
entities starting with a given row number.
This iterates over all containment fields, starting
with the root subjhect field
"""
self.clear_source() # clear prior import for this source
self.end_row = self.start_row + self.batch_size
self.get_complex_description_fields()
label_str_uuids = {}
if len(self.complex_des_fields) > 0:
print('Number of Complex Description Fields: ' + str(len(self.complex_des_fields)))
cp_id_number = 0
for cp_field in self.complex_des_fields:
cp_id_number += 1
pc = ProcessCells(self.source_id,
self.start_row)
distinct_records = pc.get_field_records_by_fl_uuid(cp_field.describes_field.field_num,
False)
if distinct_records is not False:
# sort the list in row_order from the import table
pg = ProcessGeneral(self.source_id)
distinct_records = pg.order_distinct_records(distinct_records)
for row_key, dist_rec in distinct_records.items():
if cp_field.obs_num < 1:
obs_num = 1
else:
obs_num = cp_field.obs_num
obs_node = '#obs-' + str(obs_num)
subject_uuid = dist_rec['imp_cell_obj'].fl_uuid
subject_type = cp_field.describes_field.field_type
subject_ok = dist_rec['imp_cell_obj'].cell_ok
subject_record = dist_rec['imp_cell_obj'].record
if subject_uuid is False or\
len(subject_record) < 1:
subject_ok = False
if subject_uuid == 'False':
subject_ok = False
sort = 0
in_rows = dist_rec['rows']
print('Look for complex description labels in rows: ' + str(in_rows))
if subject_ok is not False:
# OK! we have the subjects of complex descriptions
# with uuids, so now we can make an fl_uuid for each
# of the complex description fields.
complex_uuid = subject_uuid + self.FRAG_ID_PREFIX + str(cp_id_number)
complex_recs = ImportCell.objects\
.filter(source_id=self.source_id,
field_num=cp_field.field_num,
row_num__in=in_rows)\
.exclude(record='')
if len(complex_recs) > 0:
# we have records in the complex description field that are not blank
# and are associated with the subject of the complex description.
# so now, let's record this association.
save_ok = False
new_ass = Assertion()
new_ass.uuid = subject_uuid
new_ass.subject_type = subject_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id + ProcessGeneral.COMPLEX_DESCRIPTION_SOURCE_SUFFIX
new_ass.obs_node = obs_node
new_ass.obs_num = obs_num
new_ass.sort = 100 + cp_id_number
new_ass.visibility = 1
new_ass.predicate_uuid = ComplexDescription.PREDICATE_COMPLEX_DES
new_ass.object_type = 'complex-description'
new_ass.object_uuid = complex_uuid
new_ass.save()
try:
print('Saved complex-description: ' + complex_uuid)
new_ass.save()
save_ok = True
except:
save_ok = False
if save_ok:
self.count_new_assertions += 1
# now look through the complex description records and make labels
for comp_rec in complex_recs:
# first save the fl_uuid for the complex description
comp_rec.fl_uuid = complex_uuid
comp_rec.save()
if isinstance(cp_field.value_prefix, str):
cp_label = cp_field.value_prefix + comp_rec.record
else:
cp_label = comp_rec.record
if cp_label not in label_str_uuids:
# make a uuid for the record value
# adding a source_id suffix keeps this from being deleted as descriptions get processed
sm = StringManagement()
sm.project_uuid = self.project_uuid
sm.source_id = self.source_id + ProcessGeneral.COMPLEX_DESCRIPTION_SOURCE_SUFFIX
oc_string = sm.get_make_string(cp_label)
content_uuid = oc_string.uuid
label_str_uuids[cp_label] = content_uuid
content_uuid = label_str_uuids[cp_label]
save_ok = False
new_ass = Assertion()
new_ass.uuid = complex_uuid
new_ass.subject_type = 'complex-description'
new_ass.project_uuid = self.project_uuid
# adding a source_id suffix keeps this from being deleted as descriptions get processed
new_ass.source_id = self.source_id + ProcessGeneral.COMPLEX_DESCRIPTION_SOURCE_SUFFIX
new_ass.obs_node = '#obs-' + str(self.obs_num_complex_description_assertions)
new_ass.obs_num = self.obs_num_complex_description_assertions
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = ComplexDescription.PREDICATE_COMPLEX_DES_LABEL
new_ass.object_type = 'xsd:string'
new_ass.object_uuid = content_uuid
try:
new_ass.save()
save_ok = True
except:
save_ok = False
if save_ok:
self.count_new_assertions += 1
def validate_save_field_value(
self,
item_man, # subject manifest object
field,
field_value_dict,
value_num):
""" Valididates a field value for a field and
then saves it to the database if valid
"""
error_key = str(field['id']) + '-' + str(value_num)
valid = True
id_val = None
literal_val = None
if isinstance(field_value_dict, dict):
if 'id' in field_value_dict:
id_val = field_value_dict['id']
if 'literal' in field_value_dict:
literal_val = field_value_dict['literal']
else:
valid = False
self.errors[error_key] = 'Value submitted to ' + field[
'label'] + ' has the wrong format'
data_type = self.get_predicate_data_type(field['predicate_uuid'])
object_uuid = None
object_type = data_type
data_num = None
data_date = None
if field['predicate_uuid'] in self.global_predicates:
# a global predicate, so get from dictionary set above
data_type = self.global_predicates[
field['predicate_uuid']]['data_type']
else:
pred_man = self.get_manifest_item(field['predicate_uuid'])
if pred_man is False or data_type is False:
# could not find a predicate_uuid!!
valid = False
self.errors[
error_key] = 'Problem with: ' + field['label'] + ' '
self.errors[error_key] += '(' + field[
'predicate_uuid'] + ') is not found.'
if valid:
# predicate_uuid exists
str_obj = None
if data_type == 'xsd:string':
str_man = StringManagement()
str_man.project_uuid = item_man.project_uuid
str_man.source_id = item_man.source_id
str_obj = str_man.get_make_string(str(literal_val))
object_uuid = str_obj.uuid
object_type = data_type
elif data_type == 'id':
# first check is to see if the field_value exists in the manifest
val_man = self.get_manifest_item(id_val)
if val_man is False:
valid = False
self.errors[error_key] = 'Problem with: ' + field[
'label'] + '; cannot find: ' + str(id_val)
if valid:
# OK, the field_value is an ID in the manifest
# so let's check to make sure it is OK to associate
object_type = val_man.item_type
object_uuid = val_man.uuid
if val_man.class_uri == 'variable' \
and val_man.item_type != 'types':
# we've got a variable that does not link to a type!
self.errors[error_key] = 'Problem with: ' + field[
'label'] + '; '
self.errors[
error_key] += val_man.label + ' (' + val_man.uuid + ') is not a type/category.'
valid = False
elif data_type == 'xsd:boolean':
data_num = self.validate_convert_boolean(literal_val)
if t_f_data is None:
self.errors[
error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += '"' + str(
literal_val
) + '" is not a recognized boolean (T/F) value.'
valid = False
elif data_type == 'xsd:integer':
try:
data_num = int(float(literal_val))
except:
self.errors[
error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += '"' + str(
literal_val) + '" is not an integer.'
valid = False
elif data_type == 'xsd:double':
try:
data_num = float(literal_val)
except:
self.errors[
error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += '"' + str(
literal_val) + '" is not a number.'
valid = False
elif data_type == 'xsd:date':
try:
data_date = parse(literal_val)
except Exception as e:
self.errors[
error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += '"' + str(
literal_val) + '" is not a yyyy-mm-dd date'
valid = False
if valid:
# we've validated the field_value data. Now to save the assertions!
new_ass = Assertion()
new_ass.uuid = item_man.uuid
new_ass.subject_type = item_man.item_type
new_ass.project_uuid = item_man.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-' + str(field['obs_num'])
new_ass.obs_num = field['obs_num']
new_ass.sort = field['sort'] + (value_num / 1000)
new_ass.visibility = 1
new_ass.predicate_uuid = field['predicate_uuid']
new_ass.object_type = object_type
if object_uuid is not None:
new_ass.object_uuid = object_uuid
if data_num is not None:
new_ass.data_num = data_num
if data_date is not None:
new_ass.data_date = data_date
try:
new_ass.save()
except:
valid = False
self.errors[error_key] = 'Same assertion for: ' + field[
'label'] + ' '
self.errors[error_key] += 'already exists.'
return valid