def prepare_new_assertion(self, obs_num, assert_sort):
""" sets up a new assertion object """
new_ass = Assertion()
new_ass.uuid = self.item_man.uuid
new_ass.subject_type = self.item_man.item_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-' + str(obs_num)
new_ass.obs_num = obs_num
new_ass.sort = assert_sort
new_ass.visibility = 1
new_ass.certainty = 0
return new_ass
def store_records(self, act_table, recs):
"""
stores records retrieved for a given table
"""
i = 0
for record in recs:
i += 1
allow_write = self.check_allow_write(act_table, record)
record = self.prep_update_keep_old(act_table, record)
if (allow_write is False and self.update_keep_old is False):
print('\n Not allowed to overwite record.' + str(i))
else:
# print('\n Adding record:' + str(record))
newr = False
if (act_table == 'link_annotations'):
newr = LinkAnnotation(**record)
elif (act_table == 'link_entities'):
newr = LinkEntity(**record)
elif (act_table == 'oc_assertions'):
newr = Assertion(**record)
elif (act_table == 'oc_manifest'):
newr = Manifest(**record)
elif (act_table == 'oc_subjects'):
newr = Subject(**record)
elif (act_table == 'oc_mediafiles'):
newr = Mediafile(**record)
elif (act_table == 'oc_documents'):
newr = OCdocument(**record)
elif (act_table == 'oc_persons'):
newr = Person(**record)
elif (act_table == 'oc_projects'):
newr = Project(**record)
elif (act_table == 'oc_strings'):
newr = OCstring(**record)
elif (act_table == 'oc_types'):
newr = OCtype(**record)
elif (act_table == 'oc_geospace'):
newr = Geospace(**record)
elif (act_table == 'oc_events'):
newr = Event(**record)
elif (act_table == 'oc_predicates'):
newr = Predicate(**record)
elif (act_table == 'oc_identifiers'):
newr = StableIdentifer(**record)
elif (act_table == 'oc_obsmetadata'):
newr = ObsMetadata(**record)
if (newr is not False):
try:
newr.save(force_insert=self.force_insert,
force_update=self.update_keep_old)
except Exception as error:
print('Something slipped past in ' + act_table +
'...' + str(error))
def add_description(self):
""" validates the value's data type
if the data type differs from the
predicate data type,
add it as a note
"""
# validate the faims record against its expected data type
# the validated_record will be None if not valid
if isinstance(self.subject_uuid, str) \
and isinstance(self.subject_type, str):
# we have subject information
ok_to_add = False
data_type = self.attrib_dict['data_type']
if isinstance(self.predicate_uuid, str):
predicate_uuid = self.predicate_uuid
elif 'predicate_uuid' in self.attrib_dict:
predicate_uuid = self.attrib_dict['predicate_uuid']
object_date = None
object_uuid = None
object_num = None
object_type = data_type
if data_type == 'id':
# we're processing a controlled vocab (oc-type) object
if isinstance(self.faims_record_id, str):
if self.faims_record_id in self.attrib_dict['objects']:
# we found the object for this item
obj_dict = self.attrib_dict['objects'][
self.faims_record_id]
object_uuid = obj_dict['type_uuid']
object_type = 'types'
ok_to_add = True
elif self.faims_record is not None:
# we're processing a literal value
ddt = DescriptionDataType()
validated_record = ddt.validate_literal_by_data_type(
data_type, self.faims_record)
if validated_record is not None:
# the record is valid for the data_type
ok_to_add = True
if data_type == 'xsd:date':
object_date = validated_record
elif data_type == 'xsd:string':
# we have a string, so add it to the database and get its uuid
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_obj = str_man.get_make_string(validated_record)
object_uuid = str(str_obj.uuid)
else:
# we have numeric data, so add to numeric data
object_num = validated_record
else:
# the record is not valid for the data_type, so treat it as a string
# so first we need to make a predicate for it, that is for
# string values of this associated with this attribute
object_type = 'xsd:string'
sup_dict = LastUpdatedOrderedDict()
sup_dict[self.reconcile_key] = self.attrib_dict[
'id'] # the faims attribute id
pm = PredicateManagement()
pm.project_uuid = self.project_uuid
pm.source_id = self.source_id
pm.sup_dict = sup_dict
pm.sup_reconcile_key = self.reconcile_key
pm.sup_reconcile_value = self.attrib_dict[
'id'] # the faims attribute id
pred_obj = pm.get_make_predicate(
self.attrib_dict['label'] + ' [Note]',
self.attrib_dict['predicate_type'], object_type)
if pred_obj is not False:
# we have a new predicate
predicate_uuid = pred_obj.uuid
# now make an object_uuid for the original faims record
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_obj = str_man.get_make_string(self.faims_record)
object_uuid = str(str_obj.uuid)
ok_to_add = True
else:
# not valid data, don't add
ok_to_add = False
if ok_to_add:
# now we're OK to add the descrpitive assertion
new_ass = Assertion()
new_ass.uuid = self.subject_uuid
new_ass.subject_type = self.subject_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-' + str(self.obs_num)
new_ass.obs_num = self.obs_num
new_ass.sort = self.sort_num
new_ass.visibility = 1
new_ass.predicate_uuid = predicate_uuid
new_ass.object_type = object_type
if object_num is not None:
new_ass.data_num = object_num
if object_date is not None:
new_ass.data_date = object_date
if object_uuid is not None:
new_ass.object_uuid = object_uuid
try:
new_ass.save()
except:
# probably already imported
pass
def validate_save_field_value(self,
item_man, # subject manifest object
field,
field_value_dict,
value_num):
""" Valididates a field value for a field and
then saves it to the database if valid
"""
error_key = str(field['id']) + '-' + str(value_num)
valid = True
id_val = None
literal_val = None
if isinstance(field_value_dict, dict):
if 'id' in field_value_dict:
id_val = field_value_dict['id']
if 'literal' in field_value_dict:
literal_val = field_value_dict['literal']
else:
valid = False
self.errors[error_key] = 'Value submitted to ' + field['label'] + ' has the wrong format'
data_type = self.get_predicate_data_type(field['predicate_uuid'])
object_uuid = None
object_type = data_type
data_num = None
data_date = None
if field['predicate_uuid'] in self.global_predicates:
# a global predicate, so get from dictionary set above
data_type = self.global_predicates[field['predicate_uuid']]['data_type']
else:
pred_man = self.get_manifest_item(field['predicate_uuid'])
if pred_man is False or data_type is False:
# could not find a predicate_uuid!!
valid = False
self.errors[error_key] = 'Problem with: ' + field['label'] + ' '
self.errors[error_key] += '(' + field['predicate_uuid'] + ') is not found.'
if valid:
# predicate_uuid exists
str_obj = None
if data_type == 'xsd:string':
str_man = StringManagement()
str_man.project_uuid = item_man.project_uuid
str_man.source_id = item_man.source_id
str_obj = str_man.get_make_string(str(literal_val))
object_uuid = str_obj.uuid
object_type = data_type
elif data_type == 'id':
# first check is to see if the field_value exists in the manifest
val_man = self.get_manifest_item(id_val)
if val_man is False:
valid = False
self.errors[error_key] = 'Problem with: ' + field['label'] + '; cannot find: ' + str(id_val)
if valid:
# OK, the field_value is an ID in the manifest
# so let's check to make sure it is OK to associate
object_type = val_man.item_type
object_uuid = val_man.uuid
if val_man.class_uri == 'variable' \
and val_man.item_type != 'types':
# we've got a variable that does not link to a type!
self.errors[error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += val_man.label + ' (' + val_man.uuid + ') is not a type/category.'
valid = False
elif data_type == 'xsd:boolean':
data_num = self.validate_convert_boolean(literal_val)
if t_f_data is None:
self.errors[error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += '"' + str(literal_val) + '" is not a recognized boolean (T/F) value.'
valid = False
elif data_type == 'xsd:integer':
try:
data_num = int(float(literal_val))
except:
self.errors[error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += '"' + str(literal_val) + '" is not an integer.'
valid = False
elif data_type == 'xsd:double':
try:
data_num = float(literal_val)
except:
self.errors[error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += '"' + str(literal_val) + '" is not a number.'
valid = False
elif data_type == 'xsd:date':
try:
data_date = parse(literal_val)
except Exception as e:
self.errors[error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += '"' + str(literal_val) + '" is not a yyyy-mm-dd date'
valid = False
if valid:
# we've validated the field_value data. Now to save the assertions!
new_ass = Assertion()
new_ass.uuid = item_man.uuid
new_ass.subject_type = item_man.item_type
new_ass.project_uuid = item_man.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-' + str(field['obs_num'])
new_ass.obs_num = field['obs_num']
new_ass.sort = field['sort'] + (value_num / 1000)
new_ass.visibility = 1
new_ass.predicate_uuid = field['predicate_uuid']
new_ass.object_type = object_type
if object_uuid is not None:
new_ass.object_uuid = object_uuid
if data_num is not None:
new_ass.data_num = data_num
if data_date is not None:
new_ass.data_date = data_date
try:
new_ass.save()
except:
valid = False
self.errors[error_key] = 'Same assertion for: ' + field['label'] + ' '
self.errors[error_key] += 'already exists.'
return valid
def recontextualize_import(
project_uuid,
child_class_uri,
source_id,
child_prefix,
child_field,
parent_prefix,
parent_field,
over_write_existing_context=True):
"""Adds context assertions to existing items in the manifest """
mans = Manifest.objects\
.filter(item_type='subjects',
class_uri=child_class_uri,
source_id=source_id,
project_uuid=project_uuid)\
.order_by('sort')
changed_uuids = []
p_subs = {}
for man_obj in mans:
if over_write_existing_context:
Assertion.objects.filter(predicate_uuid=Assertion.PREDICATES_CONTAINS,
object_uuid=man_obj.uuid).delete()
cont_asses = []
else:
cont_asses = Assertion.objects.filter(predicate_uuid=Assertion.PREDICATES_CONTAINS,
object_uuid=man_obj.uuid)[:1]
if len(cont_asses):
continue
# need to fix missing context association
changed_uuids.append(man_obj.uuid)
act_id = man_obj.label.replace(child_prefix, '')
ch_cell = ImportCell.objects.filter(source_id=source_id, record=act_id, field_num=child_field)[:1][0]
par_cell = ImportCell.objects.get(source_id=source_id, field_num=parent_field, row_num=ch_cell.row_num)
p_context = parent_prefix + par_cell.record
print('Find Context: {} for {} import row: {}'.format(p_context, man_obj.label, ch_cell.row_num))
if p_context not in p_subs:
parent_sub = Subject.objects.get(context__endswith=p_context, project_uuid=project_uuid)
p_subs[p_context] = parent_sub
else:
parent_sub = p_subs[p_context]
print('Adding Context: {} : {}'.format(parent_sub.uuid, parent_sub.context))
new_ass = Assertion()
new_ass.uuid = parent_sub.uuid
new_ass.subject_type = 'subjects'
new_ass.project_uuid = man_obj.project_uuid
new_ass.source_id = source_id + '-fix'
new_ass.obs_node = '#contents-' + str(1)
new_ass.obs_num = 1
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = Assertion.PREDICATES_CONTAINS
new_ass.object_type = man_obj.item_type
new_ass.object_uuid = man_obj.uuid
new_ass.save()
sg = SubjectGeneration()
sg.generate_save_context_path_from_uuid(man_obj.uuid)
return changed_uuids
def make_media_links(self, scan_uuid, root_tb_uuid, tb_part, im_p_range):
""" makes linking relationships for the media item """
sub_asses = Assertion.objects\
.filter(subject_type='subjects',
object_uuid=root_tb_uuid)[:1]
if len(sub_asses) > 0:
subject_uuid = sub_asses[0].uuid
try:
new_ass = Assertion()
new_ass.uuid = subject_uuid
new_ass.subject_type = 'subjects'
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-1'
new_ass.obs_num = 1
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = 'oc-3'
new_ass.object_uuid = scan_uuid
new_ass.object_type = 'media'
new_ass.save()
new_add = True
except:
new_add = False
try:
new_ass = Assertion()
new_ass.uuid = tb_part.uuid
new_ass.subject_type = tb_part.item_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-1'
new_ass.obs_num = 1
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = 'oc-3'
new_ass.object_uuid = scan_uuid
new_ass.object_type = 'media'
new_ass.save()
new_add = True
except:
new_add = False
try:
new_ass = Assertion()
new_ass.uuid = scan_uuid
new_ass.subject_type = 'media'
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-1'
new_ass.obs_num = 1
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = 'oc-3'
new_ass.object_uuid = tb_part.item_type
new_ass.object_type = 'media'
new_ass.save()
new_add = True
except:
new_add = False
try:
# trench book scan image type
new_ass = Assertion()
new_ass.uuid = scan_uuid
new_ass.subject_type = 'media'
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-1'
new_ass.obs_num = 1
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = 'B8556EAA-CF52-446B-39FA-AE4798C13A6B'
new_ass.object_uuid = '6623B3D2-4A74-4B0B-6DDE-54802FCBF732'
new_ass.object_type = 'types'
new_ass.save()
new_add = True
except:
new_add = False
def add_trinomial_to_item(self,
predicate,
sort,
manifest,
trinomial,
allow_multiple=False):
""" Adds trinomials to an item """
output = False
ok_to_create = False
if allow_multiple is False:
# first check to make sure the item doesn't have
# a predicate for a trinomial, if multiple are not allowed
t_ass = Assertion.objects\
.filter(uuid=manifest.uuid,
predicate_uuid=predicate.uuid)[:1]
if len(t_ass) < 1:
ok_to_create = True
else:
ok_to_create = True
if manifest.project_uuid in self.source_ids:
source_id = self.source_ids[manifest.project_uuid]
if ok_to_create:
# first make a string object
sm = StringManagement()
sm.project_uuid = self.dinaa_proj_uuid
sm.source_id = self.source_id
oc_string = sm.get_make_string(trinomial)
# now make the assertion for the trinomial
ass = Assertion()
ass.uuid = manifest.uuid
ass.subject_type = manifest.item_type
ass.project_uuid = manifest.project_uuid
ass.source_id = source_id
ass.obs_node = '#obs-' + str(self.obs_num)
ass.obs_num = self.obs_num
ass.sort = sort
ass.visibility = 1
ass.predicate_uuid = predicate.uuid
ass.object_uuid = oc_string.uuid
ass.object_type = 'xsd:string'
ass.save()
output = True
return output
def create_description(self):
""" Creates a new descriptive assertion if data is valid """
self.is_valid = self.validate_creation()
if self.is_valid:
new_ass = Assertion()
new_ass.uuid = self.subject_uuid
new_ass.subject_type = self.subject_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = self.obs_node
new_ass.obs_num = self.obs_num
new_ass.sort = self.sort
new_ass.visibility = 1
new_ass.predicate_uuid = self.predicate_uuid
new_ass.object_type = self.object_type
if self.object_uuid is not False:
new_ass.object_uuid = self.object_uuid
if self.data_num is not None:
new_ass.data_num = self.data_num
if self.data_date is not None:
new_ass.data_date = self.data_date
new_ass.save()
def add_link_assertion(
project_uuid,
source_id,
subj_man_obj,
predicate_uuid,
obj_man_obj,
obs_num=1,
sort=0,
):
if not subj_man_obj or not obj_man_obj or not predicate_uuid:
# Skip out, we have some None objects, so no assertion
return None
ass = Assertion()
ass.uuid = subj_man_obj.uuid
ass.subject_type = subj_man_obj.item_type
ass.project_uuid = project_uuid
ass.source_id = source_id
ass.obs_node = '#obs-' + str(obs_num)
ass.obs_num = obs_num
ass.sort = sort
ass.visibility = 1
ass.predicate_uuid = predicate_uuid
ass.object_uuid = obj_man_obj.uuid
ass.object_type = obj_man_obj.item_type
try:
ass.save()
return True
except:
return False
def add_change_containment_assertion(self, parent_uuid, child_uuid):
""" adds or changes a containment assertion """
contain_pred = Assertion.PREDICATES_CONTAINS
del_old = Assertion.objects\
.filter(predicate_uuid=contain_pred,
object_uuid=child_uuid)\
.delete()
new_ass = Assertion()
new_ass.uuid = parent_uuid
new_ass.subject_type = 'subjects'
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#contents-' + str(1)
new_ass.obs_num = 1
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = contain_pred
new_ass.object_type = 'subjects'
new_ass.object_uuid = child_uuid
new_ass.save()
# now alter the subject item, regenerating its path
# for new containment
sg = SubjectGeneration()
sg.generate_save_context_path_from_uuid(child_uuid, True)
def create_missing_parent(self,
orphan_uuid,
prev_parent,
subjects_path,
parent):
""" Makes records for a missing parent in
manifest, oc_assertions, and oc_subjects
"""
orphan_man = False
try:
orphan_man = Manifest.objects.get(uuid=orphan_uuid)
except Manifest.DoesNotExist:
orphan_man = False
if orphan_man is not False:
# first check to see we didn't create this yet
if parent['uuid'] not in self.done_parents:
# now save the subject
label_path_dict = {'label': parent['label'],
'context': subjects_path}
label_path_dict = self.context_path_validate(label_path_dict)
parent['label'] = label_path_dict['label']
subjects_path = label_path_dict['context']
par_sub = Subject()
par_sub.uuid = parent['uuid']
par_sub.project_uuid = orphan_man.project_uuid
par_sub.source_id = 'child-xml-ref'
par_sub.context = subjects_path
par_sub.save()
# first make a manifest object for this item
parent_man = Manifest()
parent_man.uuid = parent['uuid']
parent_man.project_uuid = orphan_man.project_uuid
parent_man.source_id = 'child-xml-ref'
parent_man.item_type = 'subjects'
parent_man.class_uri = parent['class_uri']
parent_man.label = parent['label']
parent_man.save()
# now save an assertion of containment
if prev_parent is not False:
par_ass = Assertion()
par_ass.uuid = prev_parent
par_ass.subject_type = 'subjects'
par_ass.project_uuid = orphan_man.project_uuid
par_ass.source_id = 'child-xml-ref'
par_ass.obs_node = '#contents-1'
par_ass.obs_num = 1
par_ass.sort = 1
par_ass.visibility = 1
par_ass.predicate_uuid = Assertion.PREDICATES_CONTAINS
par_ass.object_uuid = parent['uuid']
par_ass.object_type = 'subjects'
par_ass.save()
# note we've made this parent, so we don't have to make it
self.done_parents.append(parent['uuid'])
# now check the orphan has a subject record
orphan_sub = False
try:
orphan_sub = Subject.objects.get(uuid=orphan_uuid)
except Subject.DoesNotExist:
orphan_sub = False
if orphan_sub is False:
orphan_sub = Subject()
orphan_sub.uuid = orphan_uuid
orphan_sub.project_uuid = orphan_man.project_uuid
orphan_sub.source_id = 'child-xml-ref'
orphan_sub.context = subjects_path + '/' + orphan_man.label
orphan_sub.save()
def load_context_row(project_uuid, source_id, row):
"""Loads a context record row into the database"""
parent_man_obj = Manifest.objects.filter(uuid=row['parent_uuid']).first()
if parent_man_obj is None:
print('Cannot find parent_uuid {} for uuid {}'.format(
row['parent_uuid'], row['context_uuid']))
# Skip the rest.
return False
# OK to continue
man_obj = Manifest.objects.filter(uuid=row['context_uuid']).first()
if man_obj is None:
man_obj = Manifest()
# Set up the new item in the Manifest
man_obj.uuid = row['context_uuid']
man_obj.source_id = source_id
man_obj.label = row['label']
man_obj.project_uuid = project_uuid
man_obj.item_type = 'subjects'
man_obj.class_uri = row['class_uri']
man_obj.save()
# Just to be sure, make sure this item does not
# have any existing parent relations.
Assertion.objects.filter(
predicate_uuid=Assertion.PREDICATES_CONTAINS,
object_uuid=man_obj.uuid,
).delete()
# Now add a context relation to it.
ass = Assertion()
ass.uuid = parent_man_obj.uuid
ass.subject_type = parent_man_obj.item_type
ass.project_uuid = parent_man_obj.project_uuid
ass.source_id = source_id
ass.obs_node = '#contents-{}'.format(DEFAULT_OBS_NUM)
ass.obs_num = DEFAULT_OBS_NUM
ass.sort = 1
ass.visibility = 1
ass.predicate_uuid = Assertion.PREDICATES_CONTAINS
ass.object_uuid = man_obj.uuid
ass.object_type = man_obj.item_type
ass.save()
sg = SubjectGeneration()
sg.generate_save_context_path_from_uuid(man_obj.uuid)
return True
def process_complex_batch(self):
""" processes fields for documents
entities starting with a given row number.
This iterates over all containment fields, starting
with the root subjhect field
"""
self.clear_source() # clear prior import for this source
self.end_row = self.start_row + self.batch_size
self.get_complex_description_fields()
label_str_uuids = {}
if len(self.complex_des_fields) > 0:
print('Number of Complex Description Fields: ' +
str(len(self.complex_des_fields)))
cp_id_number = 0
for cp_field in self.complex_des_fields:
cp_id_number += 1
pc = ProcessCells(self.source_id, self.start_row)
distinct_records = pc.get_field_records_by_fl_uuid(
cp_field.describes_field.field_num, False)
if distinct_records is not False:
# sort the list in row_order from the import table
pg = ProcessGeneral(self.source_id)
distinct_records = pg.order_distinct_records(
distinct_records)
for row_key, dist_rec in distinct_records.items():
if cp_field.obs_num < 1:
obs_num = 1
else:
obs_num = cp_field.obs_num
obs_node = '#obs-' + str(obs_num)
subject_uuid = dist_rec['imp_cell_obj'].fl_uuid
subject_type = cp_field.describes_field.field_type
subject_ok = dist_rec['imp_cell_obj'].cell_ok
subject_record = dist_rec['imp_cell_obj'].record
if subject_uuid is False or\
len(subject_record) < 1:
subject_ok = False
if subject_uuid == 'False':
subject_ok = False
sort = 0
in_rows = dist_rec['rows']
print('Look for complex description labels in rows: ' +
str(in_rows))
if subject_ok is not False:
# OK! we have the subjects of complex descriptions
# with uuids, so now we can make an fl_uuid for each
# of the complex description fields.
complex_uuid = subject_uuid + self.FRAG_ID_PREFIX + str(
cp_id_number)
complex_recs = ImportCell.objects\
.filter(source_id=self.source_id,
field_num=cp_field.field_num,
row_num__in=in_rows)\
.exclude(record='')
if len(complex_recs) > 0:
# we have records in the complex description field that are not blank
# and are associated with the subject of the complex description.
# so now, let's record this association.
save_ok = False
new_ass = Assertion()
new_ass.uuid = subject_uuid
new_ass.subject_type = subject_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id + ProcessGeneral.COMPLEX_DESCRIPTION_SOURCE_SUFFIX
new_ass.obs_node = obs_node
new_ass.obs_num = obs_num
new_ass.sort = 100 + cp_id_number
new_ass.visibility = 1
new_ass.predicate_uuid = ComplexDescription.PREDICATE_COMPLEX_DES
new_ass.object_type = 'complex-description'
new_ass.object_uuid = complex_uuid
new_ass.save()
try:
print('Saved complex-description: ' +
complex_uuid)
new_ass.save()
save_ok = True
except:
save_ok = False
if save_ok:
self.count_new_assertions += 1
# now look through the complex description records and make labels
for comp_rec in complex_recs:
# first save the fl_uuid for the complex description
comp_rec.fl_uuid = complex_uuid
comp_rec.save()
if isinstance(cp_field.value_prefix, str):
cp_label = cp_field.value_prefix + comp_rec.record
else:
cp_label = comp_rec.record
if cp_label not in label_str_uuids:
# make a uuid for the record value
# adding a source_id suffix keeps this from being deleted as descriptions get processed
sm = StringManagement()
sm.project_uuid = self.project_uuid
sm.source_id = self.source_id + ProcessGeneral.COMPLEX_DESCRIPTION_SOURCE_SUFFIX
oc_string = sm.get_make_string(
cp_label)
content_uuid = oc_string.uuid
label_str_uuids[
cp_label] = content_uuid
content_uuid = label_str_uuids[cp_label]
save_ok = False
new_ass = Assertion()
new_ass.uuid = complex_uuid
new_ass.subject_type = 'complex-description'
new_ass.project_uuid = self.project_uuid
# adding a source_id suffix keeps this from being deleted as descriptions get processed
new_ass.source_id = self.source_id + ProcessGeneral.COMPLEX_DESCRIPTION_SOURCE_SUFFIX
new_ass.obs_node = '#obs-' + str(
self.
obs_num_complex_description_assertions)
new_ass.obs_num = self.obs_num_complex_description_assertions
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = ComplexDescription.PREDICATE_COMPLEX_DES_LABEL
new_ass.object_type = 'xsd:string'
new_ass.object_uuid = content_uuid
try:
new_ass.save()
save_ok = True
except:
save_ok = False
if save_ok:
self.count_new_assertions += 1
def recontextualize_import(project_uuid,
child_class_uri,
source_id,
child_prefix,
child_field,
parent_prefix,
parent_field,
over_write_existing_context=True):
"""Adds context assertions to existing items in the manifest """
mans = Manifest.objects\
.filter(item_type='subjects',
class_uri=child_class_uri,
source_id=source_id,
project_uuid=project_uuid)\
.order_by('sort')
changed_uuids = []
p_subs = {}
for man_obj in mans:
if over_write_existing_context:
Assertion.objects.filter(
predicate_uuid=Assertion.PREDICATES_CONTAINS,
object_uuid=man_obj.uuid).delete()
cont_asses = []
else:
cont_asses = Assertion.objects.filter(
predicate_uuid=Assertion.PREDICATES_CONTAINS,
object_uuid=man_obj.uuid)[:1]
if len(cont_asses):
continue
# need to fix missing context association
changed_uuids.append(man_obj.uuid)
act_id = man_obj.label.replace(child_prefix, '')
ch_cell = ImportCell.objects.filter(source_id=source_id,
record=act_id,
field_num=child_field)[:1][0]
par_cell = ImportCell.objects.get(source_id=source_id,
field_num=parent_field,
row_num=ch_cell.row_num)
p_context = parent_prefix + par_cell.record
print('Find Context: {} for {} import row: {}'.format(
p_context, man_obj.label, ch_cell.row_num))
if p_context not in p_subs:
parent_sub = Subject.objects.get(context__endswith=p_context,
project_uuid=project_uuid)
p_subs[p_context] = parent_sub
else:
parent_sub = p_subs[p_context]
print('Adding Context: {} : {}'.format(parent_sub.uuid,
parent_sub.context))
new_ass = Assertion()
new_ass.uuid = parent_sub.uuid
new_ass.subject_type = 'subjects'
new_ass.project_uuid = man_obj.project_uuid
new_ass.source_id = source_id + '-fix'
new_ass.obs_node = '#contents-' + str(1)
new_ass.obs_num = 1
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = Assertion.PREDICATES_CONTAINS
new_ass.object_type = man_obj.item_type
new_ass.object_uuid = man_obj.uuid
new_ass.save()
sg = SubjectGeneration()
sg.generate_save_context_path_from_uuid(man_obj.uuid)
return changed_uuids
def prepare_new_assertion(self, obs_num, assert_sort):
""" sets up a new assertion object """
new_ass = Assertion()
new_ass.uuid = self.item_man.uuid
new_ass.subject_type = self.item_man.item_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-' + str(obs_num)
new_ass.obs_num = obs_num
new_ass.sort = assert_sort
new_ass.visibility = 1
new_ass.certainty = 0
return new_ass
def add_locus_coordinates(self):
""" adds coordinate html to the loci """
source_id = 'pc-locus-coord-html'
Assertion.objects.filter(source_id=source_id,
predicate_uuid=self.pred_uuid_open)\
.delete()
Assertion.objects.filter(source_id=source_id,
predicate_uuid=self.pred_uuid_close)\
.delete()
OCstring.objects.filter(source_id=source_id).delete()
open_fields = [3, 4, 5, 6]
close_fields = [7, 8, 9, 10]
l_id_objs = self.get_locus_objs_for_coordinates()
for l_id_obj in l_id_objs:
act_locus = l_id_obj['locus_obj']
rows = l_id_obj['rows']
open_html = self.make_coordinate_html(rows, open_fields)
close_html = self.make_coordinate_html(rows, close_fields)
str_m = StringManagement()
str_m.source_id = source_id
str_m.project_uuid = act_locus.project_uuid
open_str_obj = str_m.get_make_string(open_html)
str_m = StringManagement()
str_m.source_id = source_id
str_m.project_uuid = act_locus.project_uuid
close_str_obj = str_m.get_make_string(close_html)
# now make the assertions
print('Add coordinates to: ' + act_locus.uuid)
try:
new_ass = Assertion()
new_ass.uuid = act_locus.uuid
new_ass.subject_type = act_locus.item_type
new_ass.project_uuid = act_locus.project_uuid
new_ass.source_id = source_id
new_ass.obs_node = '#obs-2'
new_ass.obs_num = 2
new_ass.sort = 100
new_ass.visibility = 1
new_ass.predicate_uuid = self.pred_uuid_open
new_ass.object_uuid = open_str_obj.uuid
new_ass.object_type = 'xsd:string'
new_ass.save()
new_add = True
except:
new_add = False
try:
new_ass = Assertion()
new_ass.uuid = act_locus.uuid
new_ass.subject_type = act_locus.item_type
new_ass.project_uuid = act_locus.project_uuid
new_ass.source_id = source_id
new_ass.obs_node = '#obs-2'
new_ass.obs_num = 2
new_ass.sort = 101
new_ass.visibility = 1
new_ass.predicate_uuid = self.pred_uuid_close
new_ass.object_uuid = close_str_obj.uuid
new_ass.object_type = 'xsd:string'
new_ass.save()
new_add = True
except:
new_add = False
def create_link(self):
""" Creates a new link assertion if data is valid """
is_valid = self.validate_creation()
if is_valid:
# print('Subject added: ' + self.subject_uuid)
new_ass = Assertion()
new_ass.uuid = self.subject_uuid
new_ass.subject_type = self.subject_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = self.obs_node
new_ass.obs_num = self.obs_num
new_ass.sort = self.sort
new_ass.visibility = 1
new_ass.predicate_uuid = self.predicate_uuid
new_ass.object_type = self.object_type
new_ass.object_uuid = self.object_uuid
new_ass.save()
self.is_valid = True
def add_contain_assertion(self):
""" Adds a containment assertion for the new subject item """
if self.allow_new\
and self.parent_uuid is not False\
and self.uuid is not False:
old_ass = Assertion.objects\
.filter(uuid=self.parent_uuid,
obs_num=self.obs_num,
predicate_uuid=Assertion.PREDICATES_CONTAINS,
object_uuid=self.uuid)[:1]
if len(old_ass) < 1:
new_ass = Assertion()
new_ass.uuid = self.parent_uuid
new_ass.subject_type = 'subjects'
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#contents-' + str(self.obs_num)
new_ass.obs_num = self.obs_num
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = Assertion.PREDICATES_CONTAINS
new_ass.object_uuid = self.uuid
new_ass.object_type = 'subjects'
try:
# in case the relationship already exists
new_ass.save()
except:
print('Containment failed: ' + str(new_ass.uuid) + ' ' + str(new_ass.object_uuid))
else:
print('Containment already exists')
else:
print('No attempt at Containment: ' + str(self.parent_uuid) + ' ' + str(self.uuid))
def save_subject_link(self, subject_uuid, item_man):
""" create a containment relationship and a subject item """
try:
l_subject = Manifest.objects.get(uuid=subject_uuid)
except Manifest.DoesNotExist:
l_subject = False
self.ok = False
self.errors['subject_uuid'] = 'The linked subject (location or object): '
self.errors['subject_uuid'] += str(subject_uuid) + ' does not exist.'
if l_subject is not False:
if l_subject.item_type == 'subjects':
# the parent context exists, so we can create a containment relationship with it.
# first delete any existing containment relations
Assertion.objects\
.filter(predicate_uuid=Assertion.PREDICATES_LINK,
object_uuid=item_man.uuid)\
.delete()
# now create the new containment assertion
subl_ass = Assertion()
subl_ass.uuid = l_subject.uuid
subl_ass.subject_type = l_subject.item_type
subl_ass.project_uuid = self.project_uuid
subl_ass.source_id = self.make_source_id()
subl_ass.obs_node = '#obs-1'
subl_ass.obs_num = 1
subl_ass.sort = 1
subl_ass.visibility = self.visibility
subl_ass.predicate_uuid = Assertion.PREDICATES_LINK
subl_ass.object_uuid = item_man.uuid
subl_ass.object_type = item_man.item_type
subl_ass.save()
else:
self.ok = False
self.errors['context'] = 'Linked Subject ' + l_subject.label
self.errors['context'] += ' (' + l_subject.uuid + ') is: '
self.errors['context'] += l_subject.item_type
self.errors['context'] += '. It needs to be a "subjects" item.'
def save_contained_subject(self, context_uuid, item_man):
""" create a containment relationship and a subject item """
try:
parent = Manifest.objects.get(uuid=context_uuid)
except Manifest.DoesNotExist:
parent = False
self.ok = False
self.errors['context_uuid'] = 'The parent context: ' + str(
context_uuid) + ' does not exist.'
if parent is not False:
if parent.item_type == 'subjects' \
and item_man.item_type == 'subjects':
# the parent context exists, so we can create a containment relationship with it.
# first delete any existing containment relations
Assertion.objects\
.filter(predicate_uuid=Assertion.PREDICATES_CONTAINS,
object_uuid=item_man.uuid)\
.delete()
# now create the new containment assertion
con_ass = Assertion()
con_ass.uuid = parent.uuid
con_ass.subject_type = parent.item_type
con_ass.project_uuid = self.project_uuid
con_ass.source_id = self.make_source_id()
con_ass.obs_node = self.contain_obs_node
con_ass.obs_num = self.contain_obs_num
con_ass.sort = self.contain_sort
con_ass.visibility = self.visibility
con_ass.predicate_uuid = Assertion.PREDICATES_CONTAINS
con_ass.object_uuid = item_man.uuid
con_ass.object_type = item_man.item_type
con_ass.save()
# now make or update the subject record with context path for this item
subgen = SubjectGeneration()
subgen.generate_save_context_path_from_uuid(item_man.uuid)
else:
self.ok = False
self.errors[
'context'] = 'Parent context ' + parent.label + ' (' + parent.uuid + ') is: '
self.errors['context'] += parent.item_type + ', and '
self.errors[
'context'] += 'child item ' + item_man.label + ' (' + item_man.uuid + ') is: '
self.errors[
'context'] += item_man.item_type + '. Both need to be "subjects" items.'
def add_media_assertions(self, subj_uuid, media_uuid):
""" add assertions relating the media resource to the subject,
adding a little metadata
"""
#update old assertions for media
if self.sub_counts[subj_uuid] < 2:
# only update it once, not it we've seen it already
m_links = Assertion.objects\
.filter(uuid=subj_uuid,
object_type='media')\
.exclude(source_id__in=self.source_ids)
for m_link in m_links:
m_link.sort = m_link.sort + 10
m_link.save()
# make a new linking assertion
new_ass = Assertion()
new_ass.uuid = subj_uuid
new_ass.subject_type = 'subjects'
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = self.obs_node
new_ass.obs_num = self.obs_num
new_ass.sort = self.sub_counts[subj_uuid]
new_ass.visibility = 1
new_ass.predicate_uuid = 'oc-3'
new_ass.object_type = 'media'
new_ass.object_uuid = media_uuid
new_ass.save()
# make a new description assertion
new_ass = Assertion()
new_ass.uuid = media_uuid
new_ass.subject_type = 'media'
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = self.obs_node
new_ass.obs_num = self.obs_num
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = self.pred_image_type
new_ass.object_type = 'types'
new_ass.object_uuid = self.type_photo
new_ass.save()
def db_faims_relations_import(self, act_dir, filename='relns.xml'):
""" adds relationships to FAIMS entities that map to
Open Context containment relationships
"""
if self.tree is None:
# we have not imported the XML yet
self.tree = self.fm.load_xml_file(act_dir, filename)
if len(self.entities) < 1:
self.entities = self.fm.get_dict_from_file(self.oc_config_entities,
act_dir)
if len(self.relation_types) < 1:
self.relation_types = self.fm.get_dict_from_file(self.oc_config_relation_types,
act_dir)
if self.tree is not False and self.entities is not None and self.relation_types is not None:
# we've loaded the data we need!
print('Have all data needed to make relations')
rel_types = self.tree.xpath('/relationships/relationshipType')
sort_num = 1
for rel_type in rel_types:
sort_num += 1 # so non containment observations have different sorting
faims_id = rel_type.get('relntypeid')
if faims_id in self.relation_types:
# we found a configuration record for the current relationship type
rel_dict = self.relation_types[faims_id]
if isinstance(rel_dict['oc-equiv'], str)\
or isinstance(rel_dict['predicate_uuid'], str):
# we've got a reconciled predicate, so it's worth going through
# the relations XML
relations = rel_type.xpath('relationship')
for xml_rel in relations:
from_id = xml_rel.xpath('fromuuid')[0].text
to_id = xml_rel.xpath('touuid')[0].text
if from_id in self.entities and to_id in self.entities:
# we have a record for both the from and the to FAIMS-item-ids
subject_ent = self.entities[from_id]
object_ent = self.entities[to_id]
contain_pred = Assertion.PREDICATES_CONTAINS
if rel_dict['oc-equiv'] == contain_pred \
or rel_dict['predicate_uuid'] == contain_pred:
# we are making a new containment relationship
# the subject is the parent, the object is the child
self.add_change_containment_assertion(subject_ent['uuid'],
object_ent['uuid'])
else:
# we have a different kind of relationship to add, defined by
# the previously reconciled predicate_uuid
new_ass = Assertion()
new_ass.uuid = subject_ent['uuid']
new_ass.subject_type = subject_ent['item_type']
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-' + str(1)
new_ass.obs_num = 1
new_ass.sort = sort_num
new_ass.visibility = 1
new_ass.predicate_uuid = rel_dict['predicate_uuid']
new_ass.object_type = object_ent['item_type']
new_ass.object_uuid = object_ent['uuid']
try:
new_ass.save()
except:
# we probably already have this assertion
pass
def add_type_description(self, predicate_uuid, object_uuid, object_type='types'):
""" adds a description with a controlled vocabulary, where
both predicate_uuid and object_uuid are known
"""
if isinstance(self.subject_uuid, str) \
and isinstance(self.subject_type, str):
new_ass = Assertion()
new_ass.uuid = self.subject_uuid
new_ass.subject_type = self.subject_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-' + str(self.obs_num)
new_ass.obs_num = self.obs_num
new_ass.sort = self.sort_num
new_ass.visibility = 1
new_ass.predicate_uuid = predicate_uuid
new_ass.object_type = object_type
new_ass.object_uuid = object_uuid
try:
new_ass.save()
except:
# probably already imported
pass
def make_datatype_wrong_assertion(self, predicate_uuid, content):
""" Makes an assertion for records that don't fit the
expected data_type / object_type
"""
pm = PredicateManagement()
pm.project_uuid = self.project_uuid
pm.source_id = self.source_id
pm.sort = self.sort
new_note_predicate = pm.get_make_related_note_predicate(predicate_uuid, ' (Note)')
if new_note_predicate is not False:
note_pred_uuid = new_note_predicate.uuid
# save a skos:related assertion linking the old and new predicates
ptm = PredicateTypeAssertions()
ptm.skos_relate_old_new_predicates(self.project_uuid,
self.source_id,
predicate_uuid,
note_pred_uuid)
sm = StringManagement()
sm.project_uuid = self.project_uuid
sm.source_id = self.source_id
oc_string = sm.get_make_string(content)
object_uuid = oc_string.uuid
if self.check_description_new(self.subject_uuid,
self.obs_num,
note_pred_uuid,
object_uuid,
None,
None):
# this asertion does not exist yet, OK to make it
new_ass = Assertion()
new_ass.uuid = self.subject_uuid
new_ass.subject_type = self.subject_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = self.obs_node
new_ass.obs_num = self.obs_num
new_ass.sort = self.sort + .1
new_ass.visibility = 1
new_ass.predicate_uuid = note_pred_uuid
new_ass.object_uuid = object_uuid
new_ass.object_type = 'xsd:string'
new_ass.save()
def add_change_containment_assertion(self, parent_uuid, child_uuid):
""" adds or changes a containment assertion """
contain_pred = Assertion.PREDICATES_CONTAINS
del_old = Assertion.objects\
.filter(predicate_uuid=contain_pred,
object_uuid=child_uuid)\
.delete()
new_ass = Assertion()
new_ass.uuid = parent_uuid
new_ass.subject_type = 'subjects'
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#contents-' + str(1)
new_ass.obs_num = 1
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = contain_pred
new_ass.object_type = 'subjects'
new_ass.object_uuid = child_uuid
new_ass.save()
def add_containment_assertion(self, parent_uuid, child_uuid):
""" adds a new spatial containment assertion """
con_exists = Assertion.objects\
.filter(predicate_uuid=Assertion.PREDICATES_CONTAINS,
object_uuid=child_uuid)[:1]
if len(con_exists) < 1:
# child is not contained by something else, so make a containment rel
try:
parent = Manifest.objects.get(uuid=parent_uuid)
except Manifest.DoesNotExist:
parent = False
try:
child = Manifest.objects.get(uuid=child_uuid)
except Manifest.DoesNotExist:
child = False
if parent is not False and child is not False:
if parent.item_type == 'subjects'\
and child.item_type == 'subjects':
con_ass = Assertion()
con_ass.uuid = parent.uuid
con_ass.subject_type = parent.item_type
con_ass.project_uuid = child.project_uuid
con_ass.source_id = self.source_id
con_ass.obs_node = self.contain_obs_node
con_ass.obs_num = self.contain_obs_num
con_ass.sort = self.contain_sort
con_ass.visibility = self.visibility
con_ass.predicate_uuid = Assertion.PREDICATES_CONTAINS
con_ass.object_uuid = child.uuid
con_ass.object_type = child.item_type
con_ass.save()
cont = Containment()
child_children = cont.get_children_by_parent_uuid(
child.uuid, True)
dm = DeleteMerge()
dm.update_children_subjects(child_children)
else:
print('Item already containted in: ' + con_exists[0].uuid)
def add_change_containment_assertion(self, parent_uuid, child_uuid):
""" adds or changes a containment assertion """
contain_pred = Assertion.PREDICATES_CONTAINS
del_old = Assertion.objects\
.filter(predicate_uuid=contain_pred,
object_uuid=child_uuid)\
.delete()
new_ass = Assertion()
new_ass.uuid = parent_uuid
new_ass.subject_type = 'subjects'
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#contents-' + str(1)
new_ass.obs_num = 1
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = contain_pred
new_ass.object_type = 'subjects'
new_ass.object_uuid = child_uuid
new_ass.save()
# now alter the subject item, regenerating its path
# for new containment
sg = SubjectGeneration()
sg.generate_save_context_path_from_uuid(child_uuid,
True)
def add_description(self):
""" validates the value's data type
if the data type differs from the
predicate data type,
add it as a note
"""
# validate the faims record against its expected data type
# the validated_record will be None if not valid
if isinstance(self.subject_uuid, str) \
and isinstance(self.subject_type, str):
# we have subject information
ok_to_add = False
data_type = self.attrib_dict['data_type']
if isinstance(self.predicate_uuid, str):
predicate_uuid = self.predicate_uuid
elif 'predicate_uuid' in self.attrib_dict:
predicate_uuid = self.attrib_dict['predicate_uuid']
object_date = None
object_uuid = None
object_num = None
object_type = data_type
if data_type == 'id':
# we're processing a controlled vocab (oc-type) object
if isinstance(self.faims_record_id, str):
if self.faims_record_id in self.attrib_dict['objects']:
# we found the object for this item
obj_dict = self.attrib_dict['objects'][self.faims_record_id]
object_uuid = obj_dict['type_uuid']
object_type = 'types'
ok_to_add = True
elif self.faims_record is not None:
# we're processing a literal value
ddt = DescriptionDataType()
validated_record = ddt.validate_literal_by_data_type(data_type,
self.faims_record)
if validated_record is not None:
# the record is valid for the data_type
ok_to_add = True
if data_type == 'xsd:date':
object_date = validated_record
elif data_type == 'xsd:string':
# we have a string, so add it to the database and get its uuid
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_obj = str_man.get_make_string(validated_record)
object_uuid = str(str_obj.uuid)
else:
# we have numeric data, so add to numeric data
object_num = validated_record
else:
# the record is not valid for the data_type, so treat it as a string
# so first we need to make a predicate for it, that is for
# string values of this associated with this attribute
object_type = 'xsd:string'
sup_dict = LastUpdatedOrderedDict()
sup_dict[self.reconcile_key] = self.attrib_dict['id'] # the faims attribute id
pm = PredicateManagement()
pm.project_uuid = self.project_uuid
pm.source_id = self.source_id
pm.sup_dict = sup_dict
pm.sup_reconcile_key = self.reconcile_key
pm.sup_reconcile_value = self.attrib_dict['id'] # the faims attribute id
pred_obj = pm.get_make_predicate(self.attrib_dict['label'] + ' [Note]',
self.attrib_dict['predicate_type'],
object_type)
if pred_obj is not False:
# we have a new predicate
predicate_uuid = pred_obj.uuid
# now make an object_uuid for the original faims record
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_obj = str_man.get_make_string(self.faims_record)
object_uuid = str(str_obj.uuid)
ok_to_add = True
else:
# not valid data, don't add
ok_to_add = False
if ok_to_add:
# now we're OK to add the descrpitive assertion
new_ass = Assertion()
new_ass.uuid = self.subject_uuid
new_ass.subject_type = self.subject_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-' + str(self.obs_num)
new_ass.obs_num = self.obs_num
new_ass.sort = self.sort_num
new_ass.visibility = 1
new_ass.predicate_uuid = predicate_uuid
new_ass.object_type = object_type
if object_num is not None:
new_ass.data_num = object_num
if object_date is not None:
new_ass.data_date = object_date
if object_uuid is not None:
new_ass.object_uuid = object_uuid
try:
new_ass.save()
except:
# probably already imported
pass
def save_contained_subject(self, context_uuid, item_man):
""" create a containment relationship and a subject item """
try:
parent = Manifest.objects.get(uuid=context_uuid)
except Manifest.DoesNotExist:
parent = False
self.ok = False
self.errors['context_uuid'] = 'The parent context: ' + str(context_uuid) + ' does not exist.'
if parent is not False:
if parent.item_type == 'subjects' \
and item_man.item_type == 'subjects':
# the parent context exists, so we can create a containment relationship with it.
# first delete any existing containment relations
Assertion.objects\
.filter(predicate_uuid=Assertion.PREDICATES_CONTAINS,
object_uuid=item_man.uuid)\
.delete()
# now create the new containment assertion
con_ass = Assertion()
con_ass.uuid = parent.uuid
con_ass.subject_type = parent.item_type
con_ass.project_uuid = self.project_uuid
con_ass.source_id = self.make_source_id()
con_ass.obs_node = self.contain_obs_node
con_ass.obs_num = self.contain_obs_num
con_ass.sort = self.contain_sort
con_ass.visibility = self.visibility
con_ass.predicate_uuid = Assertion.PREDICATES_CONTAINS
con_ass.object_uuid = item_man.uuid
con_ass.object_type = item_man.item_type
con_ass.save()
# now make or update the subject record with context path for this item
subgen = SubjectGeneration()
subgen.generate_save_context_path_from_uuid(item_man.uuid)
else:
self.ok = False
self.errors['context'] = 'Parent context ' + parent.label + ' (' + parent.uuid + ') is: '
self.errors['context'] += parent.item_type + ', and '
self.errors['context'] += 'child item ' + item_man.label + ' (' + item_man.uuid + ') is: '
self.errors['context'] += item_man.item_type + '. Both need to be "subjects" items.'
def add_type_description(self,
predicate_uuid,
object_uuid,
object_type='types'):
""" adds a description with a controlled vocabulary, where
both predicate_uuid and object_uuid are known
"""
if isinstance(self.subject_uuid, str) \
and isinstance(self.subject_type, str):
new_ass = Assertion()
new_ass.uuid = self.subject_uuid
new_ass.subject_type = self.subject_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-' + str(self.obs_num)
new_ass.obs_num = self.obs_num
new_ass.sort = self.sort_num
new_ass.visibility = 1
new_ass.predicate_uuid = predicate_uuid
new_ass.object_type = object_type
new_ass.object_uuid = object_uuid
try:
new_ass.save()
except:
# probably already imported
pass
def add_edit_containment(self, field_data, item_man=False):
""" adds or edits containment data """
pred_uuid = Assertion.PREDICATES_CONTAINS
self.ok = True
note = ''
if item_man is False:
item_man = self.get_manifest_item(self.uuid)
if item_man is False:
self.ok = False
label = 'Item not found'
self.errors[
'uuid'] = 'Cannot find the item for editing assertions: ' + str(
self.uuid)
note = self.errors['uuid']
else:
label = item_man.label
self.uuid = item_man.uuid
if item_man.item_type != 'subjects':
self.ok = False
self.errors[
'uuid'] = 'Can only add containment to "subjects" items: ' + str(
self.uuid)
else:
for field_key, field in field_data.items():
new_field_data = []
if 'id' in field:
field_id = field['id']
else:
field_id = field_key
if 'predicate_uuid' not in field:
self.ok = False
self.errors[
field_key] = 'Missing a predicate_uuid for the field'
elif field['predicate_uuid'] != 'oc-gen:contained-in':
self.ok = False
self.errors[
field_key] = 'Must have predicate_uuid with "oc-gen:contained-in".'
else:
parent_uuid = False
if 'label' not in field:
field['label'] = 'Containment'
if 'sort' not in field:
field['sort'] = 1
if 'values' in field:
for field_value_dict in field['values']:
if 'id' in field_value_dict:
parent_uuid = field_value_dict['id']
if parent_uuid != 'ROOT':
parent = self.get_manifest_item(
parent_uuid)
if parent is not False:
if parent.item_type != 'subjects':
parent_uuid = False
self.ok = False
self.errors[
field_key] = 'Parent must be a "subjects" item.'
else:
self.ok = True
else:
parent_uuid = False
if parent_uuid is False:
self.ok = False
note = 'Could not find parent'
self.errors[
field_key] = 'Missing the parent_uuid in manifest'
else:
self.errors[
field_key] = 'Missing a "id" for parent_uuid'
if self.ok and parent_uuid is not False:
# first delete the old containment relationship
drev = DeletionRevision()
drev.project_uuid = item_man.project_uuid
drev.uuid = item_man.uuid
drev.item_type = item_man.item_type
drev.user_id = self.user_id
rev_label = 'Updated ' + item_man.label
rev_label += ', field: ' + field['label']
# delete prior containment relationship
del_objs = Assertion.objects\
.filter(object_uuid=item_man.uuid,
predicate_uuid=pred_uuid)
for del_obj in del_objs:
drev.assertion_keys.append(del_obj.hash_id)
del_obj.delete()
drev.save_delete_revision(rev_label, '')
if parent_uuid != 'ROOT':
# now add the new containment relationship
new_ass = Assertion()
new_ass.uuid = parent_uuid
new_ass.subject_type = 'subjects'
new_ass.project_uuid = item_man.project_uuid
new_ass.source_id = 'web-form'
new_ass.obs_node = '#contents-' + str(1)
new_ass.obs_num = 1
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = pred_uuid
new_ass.object_type = item_man.item_type
new_ass.object_uuid = item_man.uuid
try:
new_ass.save()
except:
self.ok = False
self.errors[
error_key] = 'Same containment assertion '
self.errors[error_key] += 'already exists.'
if self.ok:
# now change the path information for the Subjects
sg = SubjectGeneration()
sg.generate_save_context_path_from_uuid(
item_man.uuid, True)
if self.ok:
# now clear the cache a change was made
self.clear_caches()
self.response = {
'action': 'add-edit-item-containment',
'ok': self.ok,
'data': {
'parent_uuid': parent_uuid
},
'change': {
'uuid': self.uuid,
'label': label,
'note': note
}
}
return self.response
def add_edit_containment(self, field_data, item_man=False):
""" adds or edits containment data """
pred_uuid = Assertion.PREDICATES_CONTAINS
self.ok = True
note = ''
if item_man is False:
item_man = self.get_manifest_item(self.uuid)
if item_man is False:
self.ok = False
label = 'Item not found'
self.errors['uuid'] = 'Cannot find the item for editing assertions: ' + str(self.uuid)
note = self.errors['uuid']
else:
label = item_man.label
self.uuid = item_man.uuid
if item_man.item_type != 'subjects':
self.ok = False
self.errors['uuid'] = 'Can only add containment to "subjects" items: ' + str(self.uuid)
else:
for field_key, field in field_data.items():
new_field_data = []
if 'id' in field:
field_id = field['id']
else:
field_id = field_key
if 'predicate_uuid' not in field:
self.ok = False
self.errors[field_key] = 'Missing a predicate_uuid for the field'
elif field['predicate_uuid'] != 'oc-gen:contained-in':
self.ok = False
self.errors[field_key] = 'Must have predicate_uuid with "oc-gen:contained-in".'
else:
parent_uuid = False
if 'label' not in field:
field['label'] = 'Containment'
if 'sort' not in field:
field['sort'] = 1
if 'values' in field:
for field_value_dict in field['values']:
if 'id' in field_value_dict:
parent_uuid = field_value_dict['id']
if parent_uuid != 'ROOT':
parent = self.get_manifest_item(parent_uuid)
if parent is not False:
if parent.item_type != 'subjects':
parent_uuid = False
self.ok = False
self.errors[field_key] = 'Parent must be a "subjects" item.'
else:
self.ok = True
else:
parent_uuid = False
if parent_uuid is False:
self.ok = False
note = 'Could not find parent'
self.errors[field_key] = 'Missing the parent_uuid in manifest'
else:
self.errors[field_key] = 'Missing a "id" for parent_uuid'
if self.ok and parent_uuid is not False:
# first delete the old containment relationship
drev = DeletionRevision()
drev.project_uuid = item_man.project_uuid
drev.uuid = item_man.uuid
drev.item_type = item_man.item_type
drev.user_id = self.user_id
rev_label = 'Updated ' + item_man.label
rev_label += ', field: ' + field['label']
# delete prior containment relationship
del_objs = Assertion.objects\
.filter(object_uuid=item_man.uuid,
predicate_uuid=pred_uuid)
for del_obj in del_objs:
drev.assertion_keys.append(del_obj.hash_id)
del_obj.delete()
drev.save_delete_revision(rev_label, '')
if parent_uuid != 'ROOT':
# now add the new containment relationship
new_ass = Assertion()
new_ass.uuid = parent_uuid
new_ass.subject_type = 'subjects'
new_ass.project_uuid = item_man.project_uuid
new_ass.source_id = 'web-form'
new_ass.obs_node = '#contents-' + str(1)
new_ass.obs_num = 1
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = pred_uuid
new_ass.object_type = item_man.item_type
new_ass.object_uuid = item_man.uuid
try:
new_ass.save()
except:
self.ok = False
self.errors[error_key] = 'Same containment assertion '
self.errors[error_key] += 'already exists.'
if self.ok:
# now change the path information for the Subjects
sg = SubjectGeneration()
sg.generate_save_context_path_from_uuid(item_man.uuid,
True)
if self.ok:
# now clear the cache a change was made
self.clear_caches()
self.response = {'action': 'add-edit-item-containment',
'ok': self.ok,
'data': {'parent_uuid': parent_uuid},
'change': {'uuid': self.uuid,
'label': label,
'note': note}}
return self.response
def validate_save_field_value(
self,
item_man, # subject manifest object
field,
field_value_dict,
value_num):
""" Valididates a field value for a field and
then saves it to the database if valid
"""
error_key = str(field['id']) + '-' + str(value_num)
valid = True
id_val = None
literal_val = None
if isinstance(field_value_dict, dict):
if 'id' in field_value_dict:
id_val = field_value_dict['id']
if 'literal' in field_value_dict:
literal_val = field_value_dict['literal']
else:
valid = False
self.errors[error_key] = 'Value submitted to ' + field[
'label'] + ' has the wrong format'
data_type = self.get_predicate_data_type(field['predicate_uuid'])
object_uuid = None
object_type = data_type
data_num = None
data_date = None
if field['predicate_uuid'] in self.global_predicates:
# a global predicate, so get from dictionary set above
data_type = self.global_predicates[
field['predicate_uuid']]['data_type']
else:
pred_man = self.get_manifest_item(field['predicate_uuid'])
if pred_man is False or data_type is False:
# could not find a predicate_uuid!!
valid = False
self.errors[
error_key] = 'Problem with: ' + field['label'] + ' '
self.errors[error_key] += '(' + field[
'predicate_uuid'] + ') is not found.'
if valid:
# predicate_uuid exists
str_obj = None
if data_type == 'xsd:string':
str_man = StringManagement()
str_man.project_uuid = item_man.project_uuid
str_man.source_id = item_man.source_id
str_obj = str_man.get_make_string(str(literal_val))
object_uuid = str_obj.uuid
object_type = data_type
elif data_type == 'id':
# first check is to see if the field_value exists in the manifest
val_man = self.get_manifest_item(id_val)
if val_man is False:
valid = False
self.errors[error_key] = 'Problem with: ' + field[
'label'] + '; cannot find: ' + str(id_val)
if valid:
# OK, the field_value is an ID in the manifest
# so let's check to make sure it is OK to associate
object_type = val_man.item_type
object_uuid = val_man.uuid
if val_man.class_uri == 'variable' \
and val_man.item_type != 'types':
# we've got a variable that does not link to a type!
self.errors[error_key] = 'Problem with: ' + field[
'label'] + '; '
self.errors[
error_key] += val_man.label + ' (' + val_man.uuid + ') is not a type/category.'
valid = False
elif data_type == 'xsd:boolean':
data_num = self.validate_convert_boolean(literal_val)
if t_f_data is None:
self.errors[
error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += '"' + str(
literal_val
) + '" is not a recognized boolean (T/F) value.'
valid = False
elif data_type == 'xsd:integer':
try:
data_num = int(float(literal_val))
except:
self.errors[
error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += '"' + str(
literal_val) + '" is not an integer.'
valid = False
elif data_type == 'xsd:double':
try:
data_num = float(literal_val)
except:
self.errors[
error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += '"' + str(
literal_val) + '" is not a number.'
valid = False
elif data_type == 'xsd:date':
try:
data_date = parse(literal_val)
except Exception as e:
self.errors[
error_key] = 'Problem with: ' + field['label'] + '; '
self.errors[error_key] += '"' + str(
literal_val) + '" is not a yyyy-mm-dd date'
valid = False
if valid:
# we've validated the field_value data. Now to save the assertions!
new_ass = Assertion()
new_ass.uuid = item_man.uuid
new_ass.subject_type = item_man.item_type
new_ass.project_uuid = item_man.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-' + str(field['obs_num'])
new_ass.obs_num = field['obs_num']
new_ass.sort = field['sort'] + (value_num / 1000)
new_ass.visibility = 1
new_ass.predicate_uuid = field['predicate_uuid']
new_ass.object_type = object_type
if object_uuid is not None:
new_ass.object_uuid = object_uuid
if data_num is not None:
new_ass.data_num = data_num
if data_date is not None:
new_ass.data_date = data_date
try:
new_ass.save()
except:
valid = False
self.errors[error_key] = 'Same assertion for: ' + field[
'label'] + ' '
self.errors[error_key] += 'already exists.'
return valid
def add_description_note(self, note):
""" adds a description note about a new item
"""
# first make sure the has-note predicate exists
self.create_note_entity()
note = str(note)
if len(note) > 1:
# save the note as a string
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_obj = str_man.get_make_string(str(note))
# now make the assertion
new_ass = Assertion()
new_ass.uuid = self.uuid
new_ass.subject_type = self.item_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
if self.obs_node is False:
new_ass.obs_node = '#obs-' + str(self.obs_num)
else:
new_ass.obs_node = self.obs_node
new_ass.obs_num = self.obs_num
new_ass.sort = self.note_sort
new_ass.visibility = 1
new_ass.predicate_uuid = Assertion.PREDICATES_NOTE
new_ass.object_type = 'xsd:string'
new_ass.object_uuid = str_obj.uuid
new_ass.save()
# now clear the cache a change was made
self.clear_caches()
def add_description_note(self, note):
""" adds a description note about a new item
"""
# first make sure the has-note predicate exists
self.create_note_entity()
note = str(note)
if len(note) > 1:
# save the note as a string
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_obj = str_man.get_make_string(str(note))
# now make the assertion
new_ass = Assertion()
new_ass.uuid = self.uuid
new_ass.subject_type = self.item_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
if self.obs_node is False:
new_ass.obs_node = '#obs-' + str(self.obs_num)
else:
new_ass.obs_node = self.obs_node
new_ass.obs_num = self.obs_num
new_ass.sort = self.note_sort
new_ass.visibility = 1
new_ass.predicate_uuid = Assertion.PREDICATES_NOTE
new_ass.object_type = 'xsd:string'
new_ass.object_uuid = str_obj.uuid
new_ass.save()
# now clear the cache a change was made
self.clear_caches()
def add_trinomial_to_item(self,
predicate,
sort,
manifest,
trinomial,
allow_multiple=False):
""" Adds trinomials to an item """
output = False
ok_to_create = False
if allow_multiple is False:
# first check to make sure the item doesn't have
# a predicate for a trinomial, if multiple are not allowed
t_ass = Assertion.objects\
.filter(uuid=manifest.uuid,
predicate_uuid=predicate.uuid)[:1]
if len(t_ass) < 1:
ok_to_create = True
else:
ok_to_create = True
if manifest.project_uuid in self.source_ids:
source_id = self.source_ids[manifest.project_uuid]
if ok_to_create:
# first make a string object
sm = StringManagement()
sm.project_uuid = self.dinaa_proj_uuid
sm.source_id = self.source_id
oc_string = sm.get_make_string(trinomial)
# now make the assertion for the trinomial
ass = Assertion()
ass.uuid = manifest.uuid
ass.subject_type = manifest.item_type
ass.project_uuid = manifest.project_uuid
ass.source_id = source_id
ass.obs_node = '#obs-' + str(self.obs_num)
ass.obs_num = self.obs_num
ass.sort = sort
ass.visibility = 1
ass.predicate_uuid = predicate.uuid
ass.object_uuid = oc_string.uuid
ass.object_type = 'xsd:string'
ass.save()
output = True
return output
def create_link(self):
""" Creates a new link assertion if data is valid """
is_valid = self.validate_creation()
if is_valid:
# print('Subject added: ' + self.subject_uuid)
new_ass = Assertion()
new_ass.uuid = self.subject_uuid
new_ass.subject_type = self.subject_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = self.obs_node
new_ass.obs_num = self.obs_num
new_ass.sort = self.sort
new_ass.visibility = 1
new_ass.predicate_uuid = self.predicate_uuid
new_ass.object_type = self.object_type
new_ass.object_uuid = self.object_uuid
new_ass.save()
self.is_valid = True
def db_faims_relations_import(self, act_dir, filename='relns.xml'):
""" adds relationships to FAIMS entities that map to
Open Context containment relationships
"""
if self.tree is None:
# we have not imported the XML yet
self.tree = self.fm.load_xml_file(act_dir, filename)
if len(self.entities) < 1:
self.entities = self.fm.get_dict_from_file(self.oc_config_entities,
act_dir)
if len(self.relation_types) < 1:
self.relation_types = self.fm.get_dict_from_file(
self.oc_config_relation_types, act_dir)
if self.tree is not False and self.entities is not None and self.relation_types is not None:
# we've loaded the data we need!
print('Have all data needed to make relations')
rel_types = self.tree.xpath('/relationships/relationshipType')
sort_num = 1
for rel_type in rel_types:
sort_num += 1 # so non containment observations have different sorting
faims_id = rel_type.get('relntypeid')
if faims_id in self.relation_types:
# we found a configuration record for the current relationship type
rel_dict = self.relation_types[faims_id]
if isinstance(rel_dict['oc-equiv'], str)\
or isinstance(rel_dict['predicate_uuid'], str):
# we've got a reconciled predicate, so it's worth going through
# the relations XML
relations = rel_type.xpath('relationship')
for xml_rel in relations:
from_id = xml_rel.xpath('fromuuid')[0].text
to_id = xml_rel.xpath('touuid')[0].text
if from_id in self.entities and to_id in self.entities:
# we have a record for both the from and the to FAIMS-item-ids
subject_ent = self.entities[from_id]
object_ent = self.entities[to_id]
contain_pred = Assertion.PREDICATES_CONTAINS
if rel_dict['oc-equiv'] == contain_pred \
or rel_dict['predicate_uuid'] == contain_pred:
# we are making a new containment relationship
# the subject is the parent, the object is the child
self.add_change_containment_assertion(
subject_ent['uuid'],
object_ent['uuid'])
else:
# we have a different kind of relationship to add, defined by
# the previously reconciled predicate_uuid
new_ass = Assertion()
new_ass.uuid = subject_ent['uuid']
new_ass.subject_type = subject_ent[
'item_type']
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-' + str(1)
new_ass.obs_num = 1
new_ass.sort = sort_num
new_ass.visibility = 1
new_ass.predicate_uuid = rel_dict[
'predicate_uuid']
new_ass.object_type = object_ent[
'item_type']
new_ass.object_uuid = object_ent['uuid']
try:
new_ass.save()
except:
# we probably already have this assertion
pass
def add_containment_assertion(self, parent_uuid, child_uuid):
""" adds a new spatial containment assertion """
con_exists = Assertion.objects\
.filter(predicate_uuid=Assertion.PREDICATES_CONTAINS,
object_uuid=child_uuid)[:1]
if len(con_exists) < 1:
# child is not contained by something else, so make a containment rel
try:
parent = Manifest.objects.get(uuid=parent_uuid)
except Manifest.DoesNotExist:
parent = False
try:
child = Manifest.objects.get(uuid=child_uuid)
except Manifest.DoesNotExist:
child = False
if parent is not False and child is not False:
if parent.item_type == 'subjects'\
and child.item_type == 'subjects':
con_ass = Assertion()
con_ass.uuid = parent.uuid
con_ass.subject_type = parent.item_type
con_ass.project_uuid = child.project_uuid
con_ass.source_id = self.source_id
con_ass.obs_node = self.contain_obs_node
con_ass.obs_num = self.contain_obs_num
con_ass.sort = self.contain_sort
con_ass.visibility = self.visibility
con_ass.predicate_uuid = Assertion.PREDICATES_CONTAINS
con_ass.object_uuid = child.uuid
con_ass.object_type = child.item_type
con_ass.save()
cont = Containment()
child_children = cont.get_children_by_parent_uuid(child.uuid,
True)
dm = DeleteMerge()
dm.update_children_subjects(child_children)
else:
print('Item already containted in: ' + con_exists[0].uuid)
def update_string_content(self, content, content_type='content', post_data={}):
""" Updates the main string content of an item
(project, document, or table abstract)
"""
content = content.strip()
html_ok = self.valid_as_html(content) # check if valid, but allow invalid
if html_ok:
note = ''
else:
note = self.errors['html']
if self.manifest is not False:
# check for translation!
if 'language' in post_data:
language = post_data['language']
else:
language = Languages.DEFAULT_LANGUAGE
if 'script' in post_data:
script = post_data['script']
else:
script = None
if language != Languages.DEFAULT_LANGUAGE:
# editing another language, not the default
lan_obj = Languages()
localize_key = lan_obj.get_language_script_key(language, script)
else:
localize_key = False
if self.manifest.item_type == 'projects':
try:
cobj = Project.objects.get(uuid=self.manifest.uuid)
if localize_key is not False:
if content_type == 'short_des':
cobj.sm_localized_json = lan_obj.modify_localization_json(cobj.sm_localized_json,
localize_key,
content)
else:
cobj.lg_localized_json = lan_obj.modify_localization_json(cobj.lg_localized_json,
localize_key,
content)
else:
if content_type == 'short_des':
cobj.short_des = content
else:
cobj.content = content
cobj.save()
ok = True
except Project.DoesNotExist:
self.errors['uuid'] = self.manifest.uuid + ' not in projects'
ok = False
elif self.manifest.item_type == 'tables':
ex_id = ExpTableIdentifiers()
ex_id.make_all_identifiers(self.manifest.uuid)
try:
cobj = ExpTable.objects.get(table_id=ex_id.table_id)
if localize_key is not False:
if content_type == 'short_des':
cobj.sm_localized_json = lan_obj.modify_localization_json(cobj.sm_localized_json,
localize_key,
content)
else:
cobj.lg_localized_json = lan_obj.modify_localization_json(cobj.lg_localized_json,
localize_key,
content)
else:
if content_type == 'short_des':
cobj.short_des = content
else:
cobj.abstract = content
cobj.save()
ok = True
except ExpTable.DoesNotExist:
self.errors['uuid'] = ex_id.table_id + ' not in tables'
ok = False
elif self.manifest.item_type == 'documents' and content_type == 'content':
try:
cobj = OCdocument.objects.get(uuid=self.manifest.uuid)
if localize_key is not False:
cobj.localized_json = lan_obj.modify_localization_json(cobj.localized_json,
localize_key,
content)
else:
cobj.content = content
cobj.save()
ok = True
except OCdocument.DoesNotExist:
self.errors['uuid'] = self.manifest.uuid + ' not in documents'
ok = False
elif self.manifest.item_type == 'predicates' or self.manifest.item_type == 'types':
# make a skos not to document a predicate or type
ok = True
string_uuid = None
old_notes = Assertion.objects\
.filter(uuid=self.manifest.uuid,
predicate_uuid='skos:note')
for old_note in old_notes:
string_uuid = old_note.object_uuid
if localize_key is False:
# only delete if this is not a translation!
old_note.delete()
if localize_key is not False and string_uuid is not None:
# OK, we're just adding a translation
act_string = False
try:
act_string = OCstring.objects.get(uuid=string_uuid)
except OCstring.DoesNotExist:
act_string = False
string_uuid = None
if act_string is not False:
# update the localization JSON with the content
act_string.localized_json = lan_obj.modify_localization_json(act_string.localized_json,
localize_key,
content)
act_string.save()
if localize_key is False:
# this is for changing SKOS notes in cases where we're not
# adding a translation
if string_uuid is not None:
string_used = Assertion.objects\
.filter(project_uuid=self.manifest.project_uuid,
object_uuid=string_uuid)[:1]
if len(string_used) > 0:
# the string is used elsewhere, so we can't just use that
# string uuid
string_uuid = None
else:
# put the new content int the string that is not in use
# for other items
act_string = False
try:
act_string = OCstring.objects.get(uuid=string_uuid)
except OCstring.DoesNotExist:
act_string = False
string_uuid = None
if act_string is not False:
# save the content in the string to overwrite it
act_string.content = content
act_string.save()
if string_uuid is None:
# we don't have a string_uuid to overwrite
str_man = StringManagement()
str_man.project_uuid = self.manifest.project_uuid
str_man.source_id = 'web-form'
str_obj = str_man.get_make_string(str(content))
string_uuid = str_obj.uuid
# now make the assertion
new_ass = Assertion()
new_ass.uuid = uuid = self.manifest.uuid
new_ass.subject_type = self.manifest.item_type
new_ass.project_uuid = self.manifest.project_uuid
new_ass.source_id = 'web-form'
new_ass.obs_node = '#obs-1'
new_ass.obs_num = 1
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = 'skos:note'
new_ass.object_type = 'xsd:string'
new_ass.object_uuid = string_uuid
new_ass.save()
else:
ok = False
if ok:
# now clear the cache a change was made
self.clear_caches()
self.response = {'action': 'update-string-content',
'ok': ok,
'change': {'prop': content_type,
'new': content,
'old': '[Old content]',
'note': note}}
return self.response
def add_related_unit_links(self, root_trench_book_uuid, unit_uuids,
rel_objects):
""" adds links to desired unit_uuids if they do not
yet exist
"""
new_asserts = 0
if len(unit_uuids) > 1:
do_reciprocal = True
else:
do_reciprocal = False
i = -1
for unit_uuid in unit_uuids:
# first check to see if the root_trench_book_uuid
# already links to the unit_uuid, if not, add the link
i += 1
ex_asserts = Assertion.objects\
.filter(uuid=unit_uuid,
object_uuid=root_trench_book_uuid)[:1]
if len(ex_asserts) < 1:
# since this is the root trench book, make the subject
# of the assertion the excavation unit
new_ass = Assertion()
new_ass.uuid = unit_uuid
new_ass.subject_type = 'subjects'
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-1'
new_ass.obs_num = 1
new_ass.sort = 100 + (i / 1000)
new_ass.visibility = 1
new_ass.predicate_uuid = 'oc-3'
new_ass.object_type = 'documents'
new_ass.object_uuid = root_trench_book_uuid
new_ass.save()
new_asserts += 1
if do_reciprocal:
ex_asserts = Assertion.objects\
.filter(uuid=root_trench_book_uuid,
object_uuid=unit_uuid)[:1]
if len(ex_asserts) < 1:
# since this is the root trench book, make the subject
# of the assertion the excavation unit
new_ass = Assertion()
new_ass.uuid = root_trench_book_uuid
new_ass.subject_type = 'documents'
new_ass.project_uuid = self.project_uuid
new_ass.source_id = 'reciprocal-' + self.source_id
new_ass.obs_node = '#obs-1'
new_ass.obs_num = 1
new_ass.sort = 100 + (i / 1000)
new_ass.visibility = 1
new_ass.predicate_uuid = 'oc-3'
new_ass.object_type = 'subjects'
new_ass.object_uuid = unit_uuid
new_ass.save()
new_asserts += 1
# now go through and make sure we have links for the other
# related media and documents
for item_type, rel_uuids in rel_objects.items():
for rel_uuid in rel_uuids:
ex_asserts = Assertion.objects\
.filter(uuid=rel_uuid,
object_uuid=unit_uuid)[:1]
if len(ex_asserts
) < 1 and rel_uuid != root_trench_book_uuid:
# since this is a related trench book page or media item
# make the OBJECT of the assertion the excavation unit
new_ass = Assertion()
new_ass.uuid = rel_uuid
new_ass.subject_type = item_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = 'inferred-' + self.source_id
new_ass.obs_node = '#obs-1'
new_ass.obs_num = 1
new_ass.sort = 100 + (i / 1000)
new_ass.visibility = 1
new_ass.predicate_uuid = 'oc-3'
new_ass.object_type = 'subjects'
new_ass.object_uuid = unit_uuid
new_ass.save()
new_asserts += 1
return new_asserts
def save_subject_link(self, subject_uuid, item_man):
""" create a containment relationship and a subject item """
try:
l_subject = Manifest.objects.get(uuid=subject_uuid)
except Manifest.DoesNotExist:
l_subject = False
self.ok = False
self.errors[
'subject_uuid'] = 'The linked subject (location or object): '
self.errors['subject_uuid'] += str(
subject_uuid) + ' does not exist.'
if l_subject is not False:
if l_subject.item_type == 'subjects':
# the parent context exists, so we can create a containment relationship with it.
# first delete any existing containment relations
Assertion.objects\
.filter(predicate_uuid=Assertion.PREDICATES_LINK,
object_uuid=item_man.uuid)\
.delete()
# now create the new containment assertion
subl_ass = Assertion()
subl_ass.uuid = l_subject.uuid
subl_ass.subject_type = l_subject.item_type
subl_ass.project_uuid = self.project_uuid
subl_ass.source_id = self.make_source_id()
subl_ass.obs_node = '#obs-1'
subl_ass.obs_num = 1
subl_ass.sort = 1
subl_ass.visibility = self.visibility
subl_ass.predicate_uuid = Assertion.PREDICATES_LINK
subl_ass.object_uuid = item_man.uuid
subl_ass.object_type = item_man.item_type
subl_ass.save()
else:
self.ok = False
self.errors['context'] = 'Linked Subject ' + l_subject.label
self.errors['context'] += ' (' + l_subject.uuid + ') is: '
self.errors['context'] += l_subject.item_type
self.errors['context'] += '. It needs to be a "subjects" item.'
def add_change_containment_assertion(self, parent_uuid, child_uuid):
""" adds or changes a containment assertion """
contain_pred = Assertion.PREDICATES_CONTAINS
del_old = Assertion.objects\
.filter(predicate_uuid=contain_pred,
object_uuid=child_uuid)\
.delete()
new_ass = Assertion()
new_ass.uuid = parent_uuid
new_ass.subject_type = 'subjects'
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#contents-' + str(1)
new_ass.obs_num = 1
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = contain_pred
new_ass.object_type = 'subjects'
new_ass.object_uuid = child_uuid
new_ass.save()
def process_complex_batch(self):
""" processes fields for documents
entities starting with a given row number.
This iterates over all containment fields, starting
with the root subjhect field
"""
self.clear_source() # clear prior import for this source
self.end_row = self.start_row + self.batch_size
self.get_complex_description_fields()
label_str_uuids = {}
if len(self.complex_des_fields) > 0:
print('Number of Complex Description Fields: ' + str(len(self.complex_des_fields)))
cp_id_number = 0
for cp_field in self.complex_des_fields:
cp_id_number += 1
pc = ProcessCells(self.source_id,
self.start_row)
distinct_records = pc.get_field_records_by_fl_uuid(cp_field.describes_field.field_num,
False)
if distinct_records is not False:
# sort the list in row_order from the import table
pg = ProcessGeneral(self.source_id)
distinct_records = pg.order_distinct_records(distinct_records)
for row_key, dist_rec in distinct_records.items():
if cp_field.obs_num < 1:
obs_num = 1
else:
obs_num = cp_field.obs_num
obs_node = '#obs-' + str(obs_num)
subject_uuid = dist_rec['imp_cell_obj'].fl_uuid
subject_type = cp_field.describes_field.field_type
subject_ok = dist_rec['imp_cell_obj'].cell_ok
subject_record = dist_rec['imp_cell_obj'].record
if subject_uuid is False or\
len(subject_record) < 1:
subject_ok = False
if subject_uuid == 'False':
subject_ok = False
sort = 0
in_rows = dist_rec['rows']
print('Look for complex description labels in rows: ' + str(in_rows))
if subject_ok is not False:
# OK! we have the subjects of complex descriptions
# with uuids, so now we can make an fl_uuid for each
# of the complex description fields.
complex_uuid = subject_uuid + self.FRAG_ID_PREFIX + str(cp_id_number)
complex_recs = ImportCell.objects\
.filter(source_id=self.source_id,
field_num=cp_field.field_num,
row_num__in=in_rows)\
.exclude(record='')
if len(complex_recs) > 0:
# we have records in the complex description field that are not blank
# and are associated with the subject of the complex description.
# so now, let's record this association.
save_ok = False
new_ass = Assertion()
new_ass.uuid = subject_uuid
new_ass.subject_type = subject_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id + ProcessGeneral.COMPLEX_DESCRIPTION_SOURCE_SUFFIX
new_ass.obs_node = obs_node
new_ass.obs_num = obs_num
new_ass.sort = 100 + cp_id_number
new_ass.visibility = 1
new_ass.predicate_uuid = ComplexDescription.PREDICATE_COMPLEX_DES
new_ass.object_type = 'complex-description'
new_ass.object_uuid = complex_uuid
new_ass.save()
try:
print('Saved complex-description: ' + complex_uuid)
new_ass.save()
save_ok = True
except:
save_ok = False
if save_ok:
self.count_new_assertions += 1
# now look through the complex description records and make labels
for comp_rec in complex_recs:
# first save the fl_uuid for the complex description
comp_rec.fl_uuid = complex_uuid
comp_rec.save()
if isinstance(cp_field.value_prefix, str):
cp_label = cp_field.value_prefix + comp_rec.record
else:
cp_label = comp_rec.record
if cp_label not in label_str_uuids:
# make a uuid for the record value
# adding a source_id suffix keeps this from being deleted as descriptions get processed
sm = StringManagement()
sm.project_uuid = self.project_uuid
sm.source_id = self.source_id + ProcessGeneral.COMPLEX_DESCRIPTION_SOURCE_SUFFIX
oc_string = sm.get_make_string(cp_label)
content_uuid = oc_string.uuid
label_str_uuids[cp_label] = content_uuid
content_uuid = label_str_uuids[cp_label]
save_ok = False
new_ass = Assertion()
new_ass.uuid = complex_uuid
new_ass.subject_type = 'complex-description'
new_ass.project_uuid = self.project_uuid
# adding a source_id suffix keeps this from being deleted as descriptions get processed
new_ass.source_id = self.source_id + ProcessGeneral.COMPLEX_DESCRIPTION_SOURCE_SUFFIX
new_ass.obs_node = '#obs-' + str(self.obs_num_complex_description_assertions)
new_ass.obs_num = self.obs_num_complex_description_assertions
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = ComplexDescription.PREDICATE_COMPLEX_DES_LABEL
new_ass.object_type = 'xsd:string'
new_ass.object_uuid = content_uuid
try:
new_ass.save()
save_ok = True
except:
save_ok = False
if save_ok:
self.count_new_assertions += 1
def make_media_links(self, scan_uuid, root_tb_uuid, tb_part, im_p_range):
""" makes linking relationships for the media item """
sub_asses = Assertion.objects\
.filter(subject_type='subjects',
object_uuid=root_tb_uuid)[:1]
if len(sub_asses) > 0:
subject_uuid = sub_asses[0].uuid
try:
new_ass = Assertion()
new_ass.uuid = subject_uuid
new_ass.subject_type = 'subjects'
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-1'
new_ass.obs_num = 1
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = 'oc-3'
new_ass.object_uuid = scan_uuid
new_ass.object_type = 'media'
new_ass.save()
new_add = True
except:
new_add = False
try:
new_ass = Assertion()
new_ass.uuid = tb_part.uuid
new_ass.subject_type = tb_part.item_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-1'
new_ass.obs_num = 1
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = 'oc-3'
new_ass.object_uuid = scan_uuid
new_ass.object_type = 'media'
new_ass.save()
new_add = True
except:
new_add = False
try:
new_ass = Assertion()
new_ass.uuid = scan_uuid
new_ass.subject_type = 'media'
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-1'
new_ass.obs_num = 1
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = 'oc-3'
new_ass.object_uuid = tb_part.item_type
new_ass.object_type = 'media'
new_ass.save()
new_add = True
except:
new_add = False
try:
# trench book scan image type
new_ass = Assertion()
new_ass.uuid = scan_uuid
new_ass.subject_type = 'media'
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-1'
new_ass.obs_num = 1
new_ass.sort = 1
new_ass.visibility = 1
new_ass.predicate_uuid = 'B8556EAA-CF52-446B-39FA-AE4798C13A6B'
new_ass.object_uuid = '6623B3D2-4A74-4B0B-6DDE-54802FCBF732'
new_ass.object_type = 'types'
new_ass.save()
new_add = True
except:
new_add = False