def process_manifest_for_subjects(self):
"""
adds or updates subjects for the oc_subjects table
"""
done_count = 0
manifest_subjects = self.get_revised_manifest_subjects()
if (manifest_subjects is not False):
for sub_item in manifest_subjects:
act_context = self.generate_context_path(sub_item.uuid)
if (act_context is not False):
sub = Subject(uuid=sub_item.uuid,
project_uuid=sub_item.project_uuid,
source_id=sub_item.source_id,
context=act_context)
try:
sub.save()
except IntegrityError as e:
self.error_uuids[sub_item.uuid] = {
'context': act_context,
'error': e
}
done_count += 1
else:
self.error_uuids[sub_item.uuid] = {
'context': act_context,
'error': 'bad path'
}
return done_count
def generate_save_context_path_from_manifest_obj(self, man_obj):
""" Generates a context path for a manifest object, then saves it to the Subjects """
output = False
new_context = True
act_context = self.generate_context_path(man_obj.uuid)
if act_context is not False:
try:
sub_obj = Subject.objects.get(uuid=man_obj.uuid)
if sub_obj.context == act_context:
new_context = False
except Subject.DoesNotExist:
sub_obj = False
if sub_obj is False:
sub_obj = Subject()
sub_obj.uuid = man_obj.uuid
sub_obj.project_uuid = man_obj.project_uuid
sub_obj.source_id = man_obj.source_id
sub_obj.context = act_context
if new_context:
try:
sub_obj.save()
output = sub_obj
self.changes += 1
except IntegrityError as e:
self.error_uuids[sub_obj.uuid] = {
'context': act_context,
'error': e
}
else:
self.error_uuids[man_obj.uuid] = {
'context': act_context,
'error': 'bad path'
}
return output
def generate_save_context_path_from_manifest_obj(self, man_obj):
""" Generates a context path for a manifest object, then saves it to the Subjects """
output = False
new_context = True
act_context = self.generate_context_path(man_obj.uuid)
if act_context is not False:
print('Saving Path (' + str(man_obj.uuid) + '): ' + str(unidecode(act_context)))
try:
sub_obj = Subject.objects.get(uuid=man_obj.uuid)
if sub_obj.context == act_context:
new_context = False
except Subject.DoesNotExist:
sub_obj = False
if sub_obj is False:
sub_obj = Subject()
sub_obj.uuid = man_obj.uuid
sub_obj.project_uuid = man_obj.project_uuid
sub_obj.source_id = man_obj.source_id
sub_obj.context = act_context
if new_context:
try:
sub_obj.save()
output = sub_obj
self.changes += 1
except IntegrityError as e:
self.error_uuids[sub_obj.uuid] = {'context': act_context,
'error': e}
else:
self.error_uuids[man_obj.uuid] = {'context': act_context,
'error': 'bad path'}
return output
def db_create_initial_subject_item(self,
act_dir,
ent_dict,
faims_item_id,
item_label):
""" reconciles or makes a new subject item (manifest, subject,
initial containment assertion)
"""
if faims_item_id not in self.entities:
# a new item, not seen before
man_obj = self.check_get_faims_manifest_object(faims_item_id,
item_label,
ent_dict['item_type'],
ent_dict['class_uri'])
if man_obj is False:
# we did not find it, so make a new one
# first, make the supplemental dict object to help associate the faims_item_id
# with the manifest object. This makes reconcilation precise.
sup_dict = {}
sup_dict[self.reconcile_key] = faims_item_id
sup_dict['faims_label'] = item_label
# now, make sure the item label is unique
item_label = self.check_make_manifest_label_unique(item_label,
ent_dict['item_type'],
ent_dict['class_uri'])
# make the intial context, based on the root context's path
context = self.root_subject_context + '/' + item_label
uuid = GenUUID.uuid4()
uuid = str(uuid)
new_sub = Subject()
new_sub.uuid = uuid
new_sub.project_uuid = self.project_uuid
new_sub.source_id = self.source_id
new_sub.context = context
new_sub.save()
man_obj = Manifest()
man_obj.uuid = uuid
man_obj.project_uuid = self.project_uuid
man_obj.source_id = self.source_id
man_obj.item_type = 'subjects'
man_obj.repo = ''
man_obj.class_uri = ent_dict['class_uri']
man_obj.label = item_label
man_obj.des_predicate_uuid = ''
man_obj.views = 0
man_obj.sup_json = sup_dict
man_obj.save()
# now add the initial containment relationship
self.add_change_containment_assertion(self.root_subject_uuid,
man_obj.uuid)
# now save the open context uuid for the entity in the entities dict
self.entities[faims_item_id] = LastUpdatedOrderedDict()
self.entities[faims_item_id]['uuid'] = man_obj.uuid
self.entities[faims_item_id]['item_type'] = man_obj.item_type
self.fm.save_serialized_json(self.oc_config_entities,
act_dir,
self.entities)
def db_create_initial_subject_item(self, act_dir, ent_dict, faims_item_id,
item_label):
""" reconciles or makes a new subject item (manifest, subject,
initial containment assertion)
"""
if faims_item_id not in self.entities:
# a new item, not seen before
man_obj = self.check_get_faims_manifest_object(
faims_item_id, item_label, ent_dict['item_type'],
ent_dict['class_uri'])
if man_obj is False:
# we did not find it, so make a new one
# first, make the supplemental dict object to help associate the faims_item_id
# with the manifest object. This makes reconcilation precise.
sup_dict = {}
sup_dict[self.reconcile_key] = faims_item_id
sup_dict['faims_label'] = item_label
# now, make sure the item label is unique
item_label = self.check_make_manifest_label_unique(
item_label, ent_dict['item_type'], ent_dict['class_uri'])
# make the intial context, based on the root context's path
context = self.root_subject_context + '/' + item_label
uuid = GenUUID.uuid4()
uuid = str(uuid)
new_sub = Subject()
new_sub.uuid = uuid
new_sub.project_uuid = self.project_uuid
new_sub.source_id = self.source_id
new_sub.context = context
new_sub.save()
man_obj = Manifest()
man_obj.uuid = uuid
man_obj.project_uuid = self.project_uuid
man_obj.source_id = self.source_id
man_obj.item_type = 'subjects'
man_obj.repo = ''
man_obj.class_uri = ent_dict['class_uri']
man_obj.label = item_label
man_obj.des_predicate_uuid = ''
man_obj.views = 0
man_obj.sup_json = sup_dict
man_obj.save()
# now add the initial containment relationship
self.add_change_containment_assertion(self.root_subject_uuid,
man_obj.uuid)
# now save the open context uuid for the entity in the entities dict
self.entities[faims_item_id] = LastUpdatedOrderedDict()
self.entities[faims_item_id]['uuid'] = man_obj.uuid
self.entities[faims_item_id]['item_type'] = man_obj.item_type
self.fm.save_serialized_json(self.oc_config_entities, act_dir,
self.entities)
def store_records(self, act_table, recs):
"""
stores records retrieved for a given table
"""
i = 0
for record in recs:
i += 1
allow_write = self.check_allow_write(act_table, record)
record = self.prep_update_keep_old(act_table, record)
if (allow_write is False and self.update_keep_old is False):
print('\n Not allowed to overwite record.' + str(i))
else:
# print('\n Adding record:' + str(record))
newr = False
if (act_table == 'link_annotations'):
newr = LinkAnnotation(**record)
elif (act_table == 'link_entities'):
newr = LinkEntity(**record)
elif (act_table == 'oc_assertions'):
newr = Assertion(**record)
elif (act_table == 'oc_manifest'):
newr = Manifest(**record)
elif (act_table == 'oc_subjects'):
newr = Subject(**record)
elif (act_table == 'oc_mediafiles'):
newr = Mediafile(**record)
elif (act_table == 'oc_documents'):
newr = OCdocument(**record)
elif (act_table == 'oc_persons'):
newr = Person(**record)
elif (act_table == 'oc_projects'):
newr = Project(**record)
elif (act_table == 'oc_strings'):
newr = OCstring(**record)
elif (act_table == 'oc_types'):
newr = OCtype(**record)
elif (act_table == 'oc_geospace'):
newr = Geospace(**record)
elif (act_table == 'oc_events'):
newr = Event(**record)
elif (act_table == 'oc_predicates'):
newr = Predicate(**record)
elif (act_table == 'oc_identifiers'):
newr = StableIdentifer(**record)
elif (act_table == 'oc_obsmetadata'):
newr = ObsMetadata(**record)
if (newr is not False):
try:
newr.save(force_insert=self.force_insert,
force_update=self.update_keep_old)
except Exception as error:
print('Something slipped past in ' + act_table +
'...' + str(error))
def process_manifest_for_subjects(self):
"""
adds or updates subjects for the oc_subjects table
"""
done_count = 0
manifest_subjects = self.get_revised_manifest_subjects()
if(manifest_subjects is not False):
for sub_item in manifest_subjects:
act_context = self.generate_context_path(sub_item.uuid)
if(act_context is not False):
sub = Subject(uuid=sub_item.uuid,
project_uuid=sub_item.project_uuid,
source_id=sub_item.source_id,
context=act_context)
try:
sub.save()
except IntegrityError as e:
self.error_uuids[sub_item.uuid] = {'context': act_context,
'error': e}
done_count += 1
else:
self.error_uuids[sub_item.uuid] = {'context': act_context,
'error': 'bad path'}
return done_count
def db_create_temporary_root_subject(self):
""" makes a temporary root subject for the whole import
makes it easier to move subjects into hiearchies later
"""
if not isinstance(self.root_subject_label, str):
self.root_subject_label = self.source_id + '-root'
if not isinstance(self.root_subject_context, str):
self.root_subject_context = self.root_subject_label
if not isinstance(self.root_subject_uuid, str):
man_objs = Manifest.objects\
.filter(label=self.root_subject_label,
class_uri=self.root_subject_class,
project_uuid=self.project_uuid)[:1]
if len(man_objs) > 0:
self.root_subject_uuid = man_objs[0].uuid
else:
# did not find a root subject, so make one
sup_dict = {}
sup_dict[self.reconcile_key] = self.root_subject_sup_id
root_uuid = GenUUID.uuid4()
root_uuid = str(root_uuid)
self.root_subject_uuid = root_uuid
new_sub = Subject()
new_sub.uuid = self.root_subject_uuid
new_sub.project_uuid = self.project_uuid
new_sub.source_id = self.source_id
new_sub.context = self.root_subject_context
new_sub.save()
new_man = Manifest()
new_man.uuid = self.root_subject_uuid
new_man.project_uuid = self.project_uuid
new_man.source_id = self.source_id
new_man.item_type = 'subjects'
new_man.repo = ''
new_man.class_uri = self.root_subject_class
new_man.label = self.root_subject_label
new_man.des_predicate_uuid = ''
new_man.views = 0
new_man.sup_json = sup_dict
new_man.save()
def create_subject_item(self):
""" Create and save a new subject object"""
new_sub = Subject()
new_sub.uuid = self.uuid # use the previously assigned temporary UUID
new_sub.project_uuid = self.project_uuid
new_sub.source_id = self.source_id
new_sub.context = self.context
new_sub.save()
new_man = Manifest()
new_man.uuid = self.uuid
new_man.project_uuid = self.project_uuid
new_man.source_id = self.source_id
new_man.item_type = 'subjects'
new_man.repo = ''
new_man.class_uri = self.class_uri
new_man.label = self.label
new_man.des_predicate_uuid = ''
new_man.views = 0
new_man.save()
def db_create_temporary_root_subject(self):
""" makes a temporary root subject for the whole import
makes it easier to move subjects into hiearchies later
"""
if not isinstance(self.root_subject_label, str):
self.root_subject_label = self.source_id + '-root'
if not isinstance(self.root_subject_context, str):
self.root_subject_context = self.root_subject_label
if not isinstance(self.root_subject_uuid, str):
man_objs = Manifest.objects\
.filter(label=self.root_subject_label,
class_uri=self.root_subject_class,
project_uuid=self.project_uuid)[:1]
if len(man_objs) > 0:
self.root_subject_uuid = man_objs[0].uuid
else:
# did not find a root subject, so make one
sup_dict = {}
sup_dict[self.reconcile_key] = self.root_subject_sup_id
root_uuid = GenUUID.uuid4()
root_uuid = str(root_uuid)
self.root_subject_uuid = root_uuid
new_sub = Subject()
new_sub.uuid = self.root_subject_uuid
new_sub.project_uuid = self.project_uuid
new_sub.source_id = self.source_id
new_sub.context = self.root_subject_context
new_sub.save()
new_man = Manifest()
new_man.uuid = self.root_subject_uuid
new_man.project_uuid = self.project_uuid
new_man.source_id = self.source_id
new_man.item_type = 'subjects'
new_man.repo = ''
new_man.class_uri = self.root_subject_class
new_man.label = self.root_subject_label
new_man.des_predicate_uuid = ''
new_man.views = 0
new_man.sup_json = sup_dict
new_man.save()
def create_missing_parent(self,
orphan_uuid,
prev_parent,
subjects_path,
parent):
""" Makes records for a missing parent in
manifest, oc_assertions, and oc_subjects
"""
orphan_man = False
try:
orphan_man = Manifest.objects.get(uuid=orphan_uuid)
except Manifest.DoesNotExist:
orphan_man = False
if orphan_man is not False:
# first check to see we didn't create this yet
if parent['uuid'] not in self.done_parents:
# now save the subject
label_path_dict = {'label': parent['label'],
'context': subjects_path}
label_path_dict = self.context_path_validate(label_path_dict)
parent['label'] = label_path_dict['label']
subjects_path = label_path_dict['context']
par_sub = Subject()
par_sub.uuid = parent['uuid']
par_sub.project_uuid = orphan_man.project_uuid
par_sub.source_id = 'child-xml-ref'
par_sub.context = subjects_path
par_sub.save()
# first make a manifest object for this item
parent_man = Manifest()
parent_man.uuid = parent['uuid']
parent_man.project_uuid = orphan_man.project_uuid
parent_man.source_id = 'child-xml-ref'
parent_man.item_type = 'subjects'
parent_man.class_uri = parent['class_uri']
parent_man.label = parent['label']
parent_man.save()
# now save an assertion of containment
if prev_parent is not False:
par_ass = Assertion()
par_ass.uuid = prev_parent
par_ass.subject_type = 'subjects'
par_ass.project_uuid = orphan_man.project_uuid
par_ass.source_id = 'child-xml-ref'
par_ass.obs_node = '#contents-1'
par_ass.obs_num = 1
par_ass.sort = 1
par_ass.visibility = 1
par_ass.predicate_uuid = Assertion.PREDICATES_CONTAINS
par_ass.object_uuid = parent['uuid']
par_ass.object_type = 'subjects'
par_ass.save()
# note we've made this parent, so we don't have to make it
self.done_parents.append(parent['uuid'])
# now check the orphan has a subject record
orphan_sub = False
try:
orphan_sub = Subject.objects.get(uuid=orphan_uuid)
except Subject.DoesNotExist:
orphan_sub = False
if orphan_sub is False:
orphan_sub = Subject()
orphan_sub.uuid = orphan_uuid
orphan_sub.project_uuid = orphan_man.project_uuid
orphan_sub.source_id = 'child-xml-ref'
orphan_sub.context = subjects_path + '/' + orphan_man.label
orphan_sub.save()
