def add_description_note(self, note):
""" adds a description note about a new item
"""
# first make sure the has-note predicate exists
self.create_note_entity()
note = str(note)
if len(note) > 1:
# save the note as a string
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_obj = str_man.get_make_string(str(note))
# now make the assertion
new_ass = Assertion()
new_ass.uuid = self.uuid
new_ass.subject_type = self.item_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
if self.obs_node is False:
new_ass.obs_node = '#obs-' + str(self.obs_num)
else:
new_ass.obs_node = self.obs_node
new_ass.obs_num = self.obs_num
new_ass.sort = self.note_sort
new_ass.visibility = 1
new_ass.predicate_uuid = Assertion.PREDICATES_NOTE
new_ass.object_type = 'xsd:string'
new_ass.object_uuid = str_obj.uuid
new_ass.save()
# now clear the cache a change was made
self.clear_caches()
def get_make_string_w_suggested_uuid(self, content, string_uuid=False):
""" makes a string with a suggested uuid if it does not exist """
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_man.suggested_uuid = string_uuid
str_obj = str_man.get_make_string(content)
return str_obj
def validate_content_uuid(self, tm, post_data):
""" Checks to see if the content_uuid
is OK,
returns False if not valid
"""
ok = True
label = post_data['label'].strip()
predicate_uuid = post_data['predicate_uuid'].strip()
content_uuid = post_data['content_uuid'].strip()
if self.check_uuid_format(content_uuid):
str_manage = StringManagement()
str_manage.project_uuid = self.project_uuid
# get an existing or create a new string object
string_uuid = str_manage.check_string_exists(label, True)
# now check to make sure the content_uuid is not
# already used, skipping strings
uuid_exists = self.check_uuid_exists(content_uuid, False)
if uuid_exists:
# the uuid is used for something that is not a string
# this messes stuff up, so note the error
ok = False
self.errors[
'content_uuid'] = 'Cannot use the UUID: ' + content_uuid
self.errors['content_uuid'] += ', because it is already used.'
elif string_uuid is not False and string_uuid != content_uuid:
# conflict beteween the user submitted content_uuid
# and the string_uuid for the same label
ok = False
self.errors[
'content_uuid'] = 'Cannot create a category called "' + label + '" '
self.errors[
'content_uuid'] += ', becuase the submitted content UUID: ' + content_uuid
self.errors[
'content_uuid'] += ' conflicts with the existing UUID: ' + string_uuid
else:
# now, one last check is to make sure the
# current combinaiton of predicate_uuid and content_uuid does not exist
type_exists = tm.check_exists_pred_uuid_content(
predicate_uuid, content_uuid)
if type_exists:
ok = False
self.errors[
'content_uuid'] = 'Cannot create a category called "' + label + '" '
self.errors[
'content_uuid'] += ', becuase the submitted content UUID: ' + content_uuid
self.errors[
'content_uuid'] += ' is already used with the Predicate UUID: ' + predicate_uuid
else:
ok = False
self.errors[
'content_uuid'] = 'Cannot create a category called "' + label + '"'
self.errors[
'content_uuid'] += ', becuase the submitted content UUID: ' + content_uuid
self.errors['content_uuid'] += ' is badly formed.'
if ok:
return content_uuid
else:
return False
def get_make_type_within_pred_uuid(self, predicate_uuid, content):
"""
gets a type, filtered by a given predicate_uuid and content string
if a type does not exist, this function creates it, then returns the type
"""
str_manage = StringManagement()
str_manage.project_uuid = self.project_uuid
str_manage.source_id = self.source_id
str_manage.suggested_uuid = self.suggested_content_uuid
# get an existing or create a new string object
oc_string = str_manage.get_make_string(content)
self.content = content
self.get_make_type_pred_uuid_content_uuid(predicate_uuid,
oc_string.uuid)
return self.oc_type
def add_trinomial_to_item(self,
predicate,
sort,
manifest,
trinomial,
allow_multiple=False):
""" Adds trinomials to an item """
output = False
ok_to_create = False
if allow_multiple is False:
# first check to make sure the item doesn't have
# a predicate for a trinomial, if multiple are not allowed
t_ass = Assertion.objects\
.filter(uuid=manifest.uuid,
predicate_uuid=predicate.uuid)[:1]
if len(t_ass) < 1:
ok_to_create = True
else:
ok_to_create = True
if manifest.project_uuid in self.source_ids:
source_id = self.source_ids[manifest.project_uuid]
if ok_to_create:
# first make a string object
sm = StringManagement()
sm.project_uuid = self.dinaa_proj_uuid
sm.source_id = self.source_id
oc_string = sm.get_make_string(trinomial)
# now make the assertion for the trinomial
ass = Assertion()
ass.uuid = manifest.uuid
ass.subject_type = manifest.item_type
ass.project_uuid = manifest.project_uuid
ass.source_id = source_id
ass.obs_node = '#obs-' + str(self.obs_num)
ass.obs_num = self.obs_num
ass.sort = sort
ass.visibility = 1
ass.predicate_uuid = predicate.uuid
ass.object_uuid = oc_string.uuid
ass.object_type = 'xsd:string'
ass.save()
output = True
return output
def get_make_type_within_pred_uuid(self,
predicate_uuid,
content,
content_suffix_num=1):
"""
gets a type, filtered by a given predicate_uuid and content string
if a type does not exist, this function creates it, then returns the type
"""
original_content = content
str_manage = StringManagement()
str_manage.project_uuid = self.project_uuid
str_manage.source_id = self.source_id
str_manage.suggested_uuid = self.suggested_content_uuid
# get an existing or create a new string object
if content_suffix_num > 1:
# add a numeric suffix for content, in case we need to
# increment up to in rare cases when different types have the same string content
content += ' [' + str(content_suffix_num) + ']'
oc_string = str_manage.get_make_string(content)
self.content = content
self.get_make_type_pred_uuid_content_uuid(predicate_uuid,
oc_string.uuid)
if self.sup_reconcile_key is not None and\
self.sup_reconcile_value is not None:
# checks to see if the manifest supplemental JSON has a matching key, value pair
# this lets us further constrain reconciliation, say with FAIMS attributes (types)
man_obj = False
try:
man_obj = Manifest.objects.get(uuid=self.oc_type.uuid)
except Manifest.DoesNotExist:
man_obj = False
if man_obj is not False:
match_ok = man_obj.check_sup_json_key_value(
self.sup_reconcile_key, self.sup_reconcile_value)
if match_ok is False:
self.oc_type = False
content_suffix_num += 1
if content_suffix_num <= 10:
# so we don't get into endless loops
self.oc_type = self.get_make_type_within_pred_uuid(
predicate_uuid, original_content,
content_suffix_num)
return self.oc_type
def check_exists_pred_uuid_content(self,
predicate_uuid,
content,
show_uuid=False):
"""
Checks if a predicate_uuid and a content string already
exists
This is useful, since it does not change anything in the database,
it just checks to see if a content string is used with a predicate
"""
found = False
str_manage = StringManagement()
str_manage.project_uuid = self.project_uuid
# get an existing or create a new string object
content_uuid = str_manage.check_string_exists(content, True)
if content_uuid is not False:
# we've seen the string before, so
# now check if
found = self.check_exists_pred_uuid_content_uuid(
predicate_uuid, content_uuid, show_uuid)
return found
def add_locus_coordinates(self):
""" adds coordinate html to the loci """
source_id = 'pc-locus-coord-html'
Assertion.objects.filter(source_id=source_id,
predicate_uuid=self.pred_uuid_open)\
.delete()
Assertion.objects.filter(source_id=source_id,
predicate_uuid=self.pred_uuid_close)\
.delete()
OCstring.objects.filter(source_id=source_id).delete()
open_fields = [3, 4, 5, 6]
close_fields = [7, 8, 9, 10]
l_id_objs = self.get_locus_objs_for_coordinates()
for l_id_obj in l_id_objs:
act_locus = l_id_obj['locus_obj']
rows = l_id_obj['rows']
open_html = self.make_coordinate_html(rows, open_fields)
close_html = self.make_coordinate_html(rows, close_fields)
str_m = StringManagement()
str_m.source_id = source_id
str_m.project_uuid = act_locus.project_uuid
open_str_obj = str_m.get_make_string(open_html)
str_m = StringManagement()
str_m.source_id = source_id
str_m.project_uuid = act_locus.project_uuid
close_str_obj = str_m.get_make_string(close_html)
# now make the assertions
print('Add coordinates to: ' + act_locus.uuid)
try:
new_ass = Assertion()
new_ass.uuid = act_locus.uuid
new_ass.subject_type = act_locus.item_type
new_ass.project_uuid = act_locus.project_uuid
new_ass.source_id = source_id
new_ass.obs_node = '#obs-2'
new_ass.obs_num = 2
new_ass.sort = 100
new_ass.visibility = 1
new_ass.predicate_uuid = self.pred_uuid_open
new_ass.object_uuid = open_str_obj.uuid
new_ass.object_type = 'xsd:string'
new_ass.save()
new_add = True
except:
new_add = False
try:
new_ass = Assertion()
new_ass.uuid = act_locus.uuid
new_ass.subject_type = act_locus.item_type
new_ass.project_uuid = act_locus.project_uuid
new_ass.source_id = source_id
new_ass.obs_node = '#obs-2'
new_ass.obs_num = 2
new_ass.sort = 101
new_ass.visibility = 1
new_ass.predicate_uuid = self.pred_uuid_close
new_ass.object_uuid = close_str_obj.uuid
new_ass.object_type = 'xsd:string'
new_ass.save()
new_add = True
except:
new_add = False
def add_description(self):
""" validates the value's data type
if the data type differs from the
predicate data type,
add it as a note
"""
# validate the faims record against its expected data type
# the validated_record will be None if not valid
if isinstance(self.subject_uuid, str) \
and isinstance(self.subject_type, str):
# we have subject information
ok_to_add = False
data_type = self.attrib_dict['data_type']
if isinstance(self.predicate_uuid, str):
predicate_uuid = self.predicate_uuid
elif 'predicate_uuid' in self.attrib_dict:
predicate_uuid = self.attrib_dict['predicate_uuid']
object_date = None
object_uuid = None
object_num = None
object_type = data_type
if data_type == 'id':
# we're processing a controlled vocab (oc-type) object
if isinstance(self.faims_record_id, str):
if self.faims_record_id in self.attrib_dict['objects']:
# we found the object for this item
obj_dict = self.attrib_dict['objects'][
self.faims_record_id]
object_uuid = obj_dict['type_uuid']
object_type = 'types'
ok_to_add = True
elif self.faims_record is not None:
# we're processing a literal value
ddt = DescriptionDataType()
validated_record = ddt.validate_literal_by_data_type(
data_type, self.faims_record)
if validated_record is not None:
# the record is valid for the data_type
ok_to_add = True
if data_type == 'xsd:date':
object_date = validated_record
elif data_type == 'xsd:string':
# we have a string, so add it to the database and get its uuid
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_obj = str_man.get_make_string(validated_record)
object_uuid = str(str_obj.uuid)
else:
# we have numeric data, so add to numeric data
object_num = validated_record
else:
# the record is not valid for the data_type, so treat it as a string
# so first we need to make a predicate for it, that is for
# string values of this associated with this attribute
object_type = 'xsd:string'
sup_dict = LastUpdatedOrderedDict()
sup_dict[self.reconcile_key] = self.attrib_dict[
'id'] # the faims attribute id
pm = PredicateManagement()
pm.project_uuid = self.project_uuid
pm.source_id = self.source_id
pm.sup_dict = sup_dict
pm.sup_reconcile_key = self.reconcile_key
pm.sup_reconcile_value = self.attrib_dict[
'id'] # the faims attribute id
pred_obj = pm.get_make_predicate(
self.attrib_dict['label'] + ' [Note]',
self.attrib_dict['predicate_type'], object_type)
if pred_obj is not False:
# we have a new predicate
predicate_uuid = pred_obj.uuid
# now make an object_uuid for the original faims record
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_obj = str_man.get_make_string(self.faims_record)
object_uuid = str(str_obj.uuid)
ok_to_add = True
else:
# not valid data, don't add
ok_to_add = False
if ok_to_add:
# now we're OK to add the descrpitive assertion
new_ass = Assertion()
new_ass.uuid = self.subject_uuid
new_ass.subject_type = self.subject_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-' + str(self.obs_num)
new_ass.obs_num = self.obs_num
new_ass.sort = self.sort_num
new_ass.visibility = 1
new_ass.predicate_uuid = predicate_uuid
new_ass.object_type = object_type
if object_num is not None:
new_ass.data_num = object_num
if object_date is not None:
new_ass.data_date = object_date
if object_uuid is not None:
new_ass.object_uuid = object_uuid
try:
new_ass.save()
except:
# probably already imported
pass
def validate_save_field_value(
self,
item_man, # subject manifest object
field,
field_value_dict,
value_num):
""" Valididates a field value for a field and
then saves it to the database if valid
"""
error_key = str(field['id']) + '-' + str(value_num)
valid = True
id_val = None
literal_val = None
if isinstance(field_value_dict, dict):
if 'id' in field_value_dict:
id_val = field_value_dict['id']
if 'literal' in field_value_dict:
literal_val = field_value_dict['literal']
else:
valid = False
self.errors[error_key] = 'Value submitted to ' + field[
'label'] + ' has the wrong format'
data_type = self.get_predicate_data_type(field['predicate_uuid'])
object_uuid = None
object_type = data_type
data_num = None
data_date = None
if field['predicate_uuid'] in self.global_predicates:
# a global predicate, so get from dictionary set above
data_type = self.global_predicates[
field['predicate_uuid']]['data_type']
else:
pred_man = self.get_manifest_item(field['predicate_uuid'])
if pred_man is False or data_type is False:
# could not find a predicate_uuid!!
valid = False
self.errors[
error_key] = 'Problem with: ' + field['label'] + ' '
self.errors[error_key] += '(' + field[
'predicate_uuid'] + ') is not found.'
if valid:
# predicate_uuid exists
str_obj = None
if data_type == 'xsd:string':
str_man = StringManagement()
str_man.project_uuid = item_man.project_uuid
str_man.source_id = item_man.source_id
str_obj = str_man.get_make_string(str(literal_val))
object_uuid = str_obj.uuid
object_type = data_type
elif data_type == 'id':
# first check is to see if the field_value exists in the manifest
val_man = self.get_manifest_item(id_val)
if val_man is False:
valid = False
self.errors[error_key] = 'Problem with: ' + field[
'label'] + '; cannot find: ' + str(id_val)
if valid:
# OK, the field_value is an ID in the manifest
# so let's check to make sure it is OK to associate
object_type = val_man.item_type
object_uuid = val_man.uuid
if val_man.class_uri == 'variable' \
and val_man.item_type != 'types':
# we've got a variable that does not link to a type!
self.errors[error_key] = 'Problem with: ' + field[
'label'] + '; '
self.errors[
error_key] += val_man.label + ' (' + val_man.uuid + ') is not a type/category.'
valid = False
elif data_type == 'xsd:boolean':
data_num = self.validate_convert_boolean(literal_val)
if t_f_data is None:
self.errors[
error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += '"' + str(
literal_val
) + '" is not a recognized boolean (T/F) value.'
valid = False
elif data_type == 'xsd:integer':
try:
data_num = int(float(literal_val))
except:
self.errors[
error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += '"' + str(
literal_val) + '" is not an integer.'
valid = False
elif data_type == 'xsd:double':
try:
data_num = float(literal_val)
except:
self.errors[
error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += '"' + str(
literal_val) + '" is not a number.'
valid = False
elif data_type == 'xsd:date':
try:
data_date = parse(literal_val)
except Exception as e:
self.errors[
error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += '"' + str(
literal_val) + '" is not a yyyy-mm-dd date'
valid = False
if valid:
# we've validated the field_value data. Now to save the assertions!
new_ass = Assertion()
new_ass.uuid = item_man.uuid
new_ass.subject_type = item_man.item_type
new_ass.project_uuid = item_man.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-' + str(field['obs_num'])
new_ass.obs_num = field['obs_num']
new_ass.sort = field['sort'] + (value_num / 1000)
new_ass.visibility = 1
new_ass.predicate_uuid = field['predicate_uuid']
new_ass.object_type = object_type
if object_uuid is not None:
new_ass.object_uuid = object_uuid
if data_num is not None:
new_ass.data_num = data_num
if data_date is not None:
new_ass.data_date = data_date
try:
new_ass.save()
except:
valid = False
self.errors[error_key] = 'Same assertion for: ' + field[
'label'] + ' '
self.errors[error_key] += 'already exists.'
return valid
def process_complex_batch(self):
""" processes fields for documents
entities starting with a given row number.
This iterates over all containment fields, starting
with the root subjhect field
"""
self.clear_source() # clear prior import for this source
self.end_row = self.start_row + self.batch_size
self.get_complex_description_fields()
label_str_uuids = {}
if len(self.complex_des_fields) > 0:
print('Number of Complex Description Fields: ' +
str(len(self.complex_des_fields)))
cp_id_number = 0
for cp_field in self.complex_des_fields:
cp_id_number += 1
pc = ProcessCells(self.source_id, self.start_row)
distinct_records = pc.get_field_records_by_fl_uuid(
cp_field.describes_field.field_num, False)
if distinct_records is not False:
# sort the list in row_order from the import table
pg = ProcessGeneral(self.source_id)
distinct_records = pg.order_distinct_records(
distinct_records)
for row_key, dist_rec in distinct_records.items():
if cp_field.obs_num < 1:
obs_num = 1
else:
obs_num = cp_field.obs_num
obs_node = '#obs-' + str(obs_num)
subject_uuid = dist_rec['imp_cell_obj'].fl_uuid
subject_type = cp_field.describes_field.field_type
subject_ok = dist_rec['imp_cell_obj'].cell_ok
subject_record = dist_rec['imp_cell_obj'].record
if subject_uuid is False or\
len(subject_record) < 1:
subject_ok = False
if subject_uuid == 'False':
subject_ok = False
sort = 0
in_rows = dist_rec['rows']
print('Look for complex description labels in rows: ' +
str(in_rows))
if subject_ok is not False:
# OK! we have the subjects of complex descriptions
# with uuids, so now we can make an fl_uuid for each
# of the complex description fields.
complex_uuid = subject_uuid + self.FRAG_ID_PREFIX + str(
cp_id_number)
complex_recs = ImportCell.objects\
.filter(source_id=self.source_id,
field_num=cp_field.field_num,
row_num__in=in_rows)\
.exclude(record='')
if len(complex_recs) > 0:
# we have records in the complex description field that are not blank
# and are associated with the subject of the complex description.
# so now, let's record this association.
save_ok = False
new_ass = Assertion()
new_ass.uuid = subject_uuid
new_ass.subject_type = subject_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id + ProcessGeneral.COMPLEX_DESCRIPTION_SOURCE_SUFFIX
new_ass.obs_node = obs_node
new_ass.obs_num = obs_num
new_ass.sort = 100 + cp_id_number
new_ass.visibility = 1
new_ass.predicate_uuid = ComplexDescription.PREDICATE_COMPLEX_DES
new_ass.object_type = 'complex-description'
new_ass.object_uuid = complex_uuid
new_ass.save()
try:
print('Saved complex-description: ' +
complex_uuid)
new_ass.save()
save_ok = True
except:
save_ok = False
if save_ok:
self.count_new_assertions += 1
# now look through the complex description records and make labels
for comp_rec in complex_recs:
# first save the fl_uuid for the complex description
comp_rec.fl_uuid = complex_uuid
comp_rec.save()
if isinstance(cp_field.value_prefix, str):
cp_label = cp_field.value_prefix + comp_rec.record
else:
cp_label = comp_rec.record
if cp_label not in label_str_uuids:
# make a uuid for the record value
# adding a source_id suffix keeps this from being deleted as descriptions get processed
sm = StringManagement()
sm.project_uuid = self.project_uuid
sm.source_id = self.source_id + ProcessGeneral.COMPLEX_DESCRIPTION_SOURCE_SUFFIX
oc_string = sm.get_make_string(
cp_label)
content_uuid = oc_string.uuid
label_str_uuids[
cp_label] = content_uuid
content_uuid = label_str_uuids[cp_label]
save_ok = False
new_ass = Assertion()
new_ass.uuid = complex_uuid
new_ass.subject_type = 'complex-description'
new_ass.project_uuid = self.project_uuid
# adding a source_id suffix keeps this from being deleted as descriptions get processed
new_ass.source_id = self.source_id + ProcessGeneral.COMPLEX_DESCRIPTION_SOURCE_SUFFIX
new_ass.obs_node = '#obs-' + str(
self.
obs_num_complex_description_assertions)
new_ass.obs_num = self.obs_num_complex_description_assertions
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = ComplexDescription.PREDICATE_COMPLEX_DES_LABEL
new_ass.object_type = 'xsd:string'
new_ass.object_uuid = content_uuid
try:
new_ass.save()
save_ok = True
except:
save_ok = False
if save_ok:
self.count_new_assertions += 1
