def store_records(self, act_table, recs):
"""
stores records retrieved for a given table
"""
i = 0
for record in recs:
i += 1
allow_write = self.check_allow_write(act_table, record)
record = self.prep_update_keep_old(act_table, record)
if (allow_write is False and self.update_keep_old is False):
print('\n Not allowed to overwite record.' + str(i))
else:
# print('\n Adding record:' + str(record))
newr = False
if (act_table == 'link_annotations'):
newr = LinkAnnotation(**record)
elif (act_table == 'link_entities'):
newr = LinkEntity(**record)
elif (act_table == 'oc_assertions'):
newr = Assertion(**record)
elif (act_table == 'oc_manifest'):
newr = Manifest(**record)
elif (act_table == 'oc_subjects'):
newr = Subject(**record)
elif (act_table == 'oc_mediafiles'):
newr = Mediafile(**record)
elif (act_table == 'oc_documents'):
newr = OCdocument(**record)
elif (act_table == 'oc_persons'):
newr = Person(**record)
elif (act_table == 'oc_projects'):
newr = Project(**record)
elif (act_table == 'oc_strings'):
newr = OCstring(**record)
elif (act_table == 'oc_types'):
newr = OCtype(**record)
elif (act_table == 'oc_geospace'):
newr = Geospace(**record)
elif (act_table == 'oc_events'):
newr = Event(**record)
elif (act_table == 'oc_predicates'):
newr = Predicate(**record)
elif (act_table == 'oc_identifiers'):
newr = StableIdentifer(**record)
elif (act_table == 'oc_obsmetadata'):
newr = ObsMetadata(**record)
if (newr is not False):
try:
newr.save(force_insert=self.force_insert,
force_update=self.update_keep_old)
except Exception as error:
print('Something slipped past in ' + act_table +
'...' + str(error))
def create_item(self, required_make_data):
""" creates an item based on required data """
if self.create_ok:
# we've got the required data to make the item
label = False
context_uuid = False
content = False
class_uri = ''
subject_uuid = False
for req_field in required_make_data:
if req_field['predicate_uuid'] == 'oc-gen:label':
label = req_field['value']
elif req_field['predicate_uuid'] == 'oc-gen:content':
content = req_field['value']
elif req_field['predicate_uuid'] == 'oc-gen:contained-in':
context_uuid = req_field['value']
elif req_field['predicate_uuid'] == 'oc-gen:class_uri':
class_uri = req_field['value']
elif req_field['predicate_uuid'] == 'oc-gen:subjects-link':
subject_uuid = req_field['value']
if self.create_ok:
item_man = Manifest()
item_man.uuid = self.edit_uuid
item_man.project_uuid = self.project_uuid
item_man.source_id = self.make_source_id()
item_man.item_type = self.item_type
item_man.repo = ''
item_man.class_uri = class_uri
item_man.label = label
item_man.des_predicate_uuid = ''
item_man.views = 0
item_man.save()
if context_uuid is not False \
and self.item_type == 'subjects':
self.save_contained_subject(context_uuid, item_man)
if subject_uuid is not False:
self.save_subject_link(subject_uuid, item_man)
if content is not False \
and self.item_type == 'documents':
doc = OCdocument()
doc.uuid = item_man.uuid
doc.project_uuid = item_man.project_uuid
doc.source_id = item_man.source_id
doc.content = content
doc.save()
else:
item_man = False
return item_man
def store_records(self, act_table, recs):
"""
stores records retrieved for a given table
"""
for rkey, record in recs.items():
if (act_table == 'link_annotations'):
newr = LinkAnnotation(**record)
newr.save()
elif (act_table == 'link_entities'):
newr = LinkEntity(**record)
newr.save()
elif (act_table == 'link_hierarchies'):
newr = LinkHierarchy(**record)
newr.save()
elif (act_table == 'oc_chronology'):
newr = Chronology(**record)
newr.save()
elif (act_table == 'oc_geodata'):
newr = Geodata(**record)
newr.save()
elif (act_table == 'oc_mediafiles'):
newr = Mediafile(**record)
newr.save()
elif (act_table == 'oc_documents'):
newr = OCdocument(**record)
newr.save()
elif (act_table == 'oc_persons'):
newr = Person(**record)
newr.save()
elif (act_table == 'oc_projects'):
newr = Project(**record)
newr.save()
elif (act_table == 'oc_strings'):
newr = OCstring(**record)
newr.save()
elif (act_table == 'oc_types'):
newr = OCtype(**record)
newr.save()
elif (act_table == 'oc_events'):
newr = Event(**record)
newr.save()
elif (act_table == 'oc_predicates'):
newr = Predicate(**record)
newr.save()
elif (act_table == 'oc_identifiers'):
newr = StableIdentifer(**record)
newr.save()
elif (act_table == 'oc_obsmetadata'):
newr = ObsMetadata(**record)
newr.save()
def update_required_make_data(self, item_man, required_make_data):
""" updates items based on required make data
we don't need to have these data complete if
the item already exists.
Required fields get some special handling
We don't update with the other (not required) fields
NOTE: the required fields list can be empty without an error
since this is for update only, not creation
"""
for req_field in required_make_data:
if req_field['predicate_uuid'] == 'oc-gen:label':
item_man.label = req_field['value']
item_man.save()
if item_man.item_type == 'subjects':
subj_gen = SubjectGeneration()
subj_gen.generate_save_context_path_from_uuid(
item_man.uuid)
# now get uuids for solr reindexing, including child items impacted by the changes
self.collect_solr_reindex_uuids(item_man.uuid)
elif req_field['predicate_uuid'] == 'oc-gen:content':
# string content, as in for documents
content = req_field['value']
if item_man.item_type == 'documents':
# save the content to the document
try:
doc = OCdocument.objects.get(uuid=item_man.uuid)
except OCdocument.DoesNotExist:
doc = OCdocument()
doc.uuid = item_man.uuid
doc.project_uuid = item_man.project_uuid
doc.source_id = item_man.source_id
doc.content = content
doc.save()
elif req_field['predicate_uuid'] == 'oc-gen:contained-in':
context_uuid = req_field['value']
if item_man.item_type == 'subjects':
self.save_contained_subject(context_uuid, item_man)
# now get uuids for solr reindexing, including child items impacted by the changes
self.collect_solr_reindex_uuids(item_man.uuid)
elif req_field['predicate_uuid'] == 'oc-gen:subjects-link':
subject_uuid = req_field['value']
self.save_subject_link(subject_uuid, item_man)
# now get uuids for solr reindexing, including child items impacted by the changes
self.collect_solr_reindex_uuids(item_man.uuid)
elif req_field['predicate_uuid'] == 'oc-gen:class_uri':
item_man.class_uri = req_field['value']
item_man.save()
def create_item(self, required_make_data):
""" creates an item based on required data """
if self.create_ok:
# we've got the required data to make the item
label = False
context_uuid = False
content = False
class_uri = ''
subject_uuid = False
for req_field in required_make_data:
if req_field['predicate_uuid'] == 'oc-gen:label':
label = req_field['value']
elif req_field['predicate_uuid'] == 'oc-gen:content':
content = req_field['value']
elif req_field['predicate_uuid'] == 'oc-gen:contained-in':
context_uuid = req_field['value']
elif req_field['predicate_uuid'] == 'oc-gen:class_uri':
class_uri = req_field['value']
elif req_field['predicate_uuid'] == 'oc-gen:subjects-link':
subject_uuid = req_field['value']
if self.create_ok:
item_man = Manifest()
item_man.uuid = self.edit_uuid
item_man.project_uuid = self.project_uuid
item_man.source_id = self.make_source_id()
item_man.item_type = self.item_type
item_man.repo = ''
item_man.class_uri = class_uri
item_man.label = label
item_man.des_predicate_uuid = ''
item_man.views = 0
item_man.save()
if context_uuid is not False \
and self.item_type == 'subjects':
self.save_contained_subject(context_uuid, item_man)
if subject_uuid is not False:
self.save_subject_link(subject_uuid, item_man)
if content is not False \
and self.item_type == 'documents':
doc = OCdocument()
doc.uuid = item_man.uuid
doc.project_uuid = item_man.project_uuid
doc.source_id = item_man.source_id
doc.content = content
doc.save()
else:
item_man = False
return item_man
def update_required_make_data(self, item_man, required_make_data):
""" updates items based on required make data
we don't need to have these data complete if
the item already exists.
Required fields get some special handling
We don't update with the other (not required) fields
NOTE: the required fields list can be empty without an error
since this is for update only, not creation
"""
for req_field in required_make_data:
if req_field['predicate_uuid'] == 'oc-gen:label':
item_man.label = req_field['value']
item_man.save()
if item_man.item_type == 'subjects':
subj_gen = SubjectGeneration()
subj_gen.generate_save_context_path_from_uuid(item_man.uuid)
# now get uuids for solr reindexing, including child items impacted by the changes
self.collect_solr_reindex_uuids(item_man.uuid)
elif req_field['predicate_uuid'] == 'oc-gen:content':
# string content, as in for documents
content = req_field['value']
if item_man.item_type == 'documents':
# save the content to the document
try:
doc = OCdocument.objects.get(uuid=item_man.uuid)
except OCdocument.DoesNotExist:
doc = OCdocument()
doc.uuid = item_man.uuid
doc.project_uuid = item_man.project_uuid
doc.source_id = item_man.source_id
doc.content = content
doc.save()
elif req_field['predicate_uuid'] == 'oc-gen:contained-in':
context_uuid = req_field['value']
if item_man.item_type == 'subjects':
self.save_contained_subject(context_uuid, item_man)
# now get uuids for solr reindexing, including child items impacted by the changes
self.collect_solr_reindex_uuids(item_man.uuid)
elif req_field['predicate_uuid'] == 'oc-gen:subjects-link':
subject_uuid = req_field['value']
self.save_subject_link(subject_uuid, item_man)
# now get uuids for solr reindexing, including child items impacted by the changes
self.collect_solr_reindex_uuids(item_man.uuid)
elif req_field['predicate_uuid'] == 'oc-gen:class_uri':
item_man.class_uri = req_field['value']
item_man.save()
def create_document_item(self, sup_metadata=None):
""" Create and save a new subject object"""
new_doc = OCdocument()
new_doc.uuid = self.uuid # use the previously assigned temporary UUID
new_doc.project_uuid = self.project_uuid
new_doc.source_id = self.source_id
new_doc.content = self.content
new_doc.save()
new_man = Manifest()
new_man.uuid = self.uuid
new_man.project_uuid = self.project_uuid
new_man.source_id = self.source_id
new_man.item_type = 'documents'
new_man.repo = ''
new_man.class_uri = ''
new_man.label = self.label
new_man.des_predicate_uuid = ''
new_man.views = 0
if isinstance(sup_metadata, dict):
new_man.sup_json = sup_metadata
new_man.save()
def reconcile_item(self, imp_cell_obj):
""" Checks to see if the item exists """
self.imp_cell_obj = imp_cell_obj
if len(imp_cell_obj.record) > 0:
self.label = imp_cell_obj.record
else:
pg = ProcessGeneral(self.source_id)
if self.import_rows is not False:
check_list = self.import_rows
else:
check_list = [imp_cell_obj.row_num]
if self.label is not False:
match_found = self.match_against_documents(self.label)
if match_found is False:
# create new document, manifest objects.
self.new_entity = True
sup_metadata = None
self.uuid = GenUUID.uuid4()
if self.metadata_obj is not None:
sup_metadata = self.metadata_obj.get_metadata(imp_cell_obj.field_num,
imp_cell_obj.row_num)
meta_uuid = self.metadata_obj.get_uuid_from_metadata_dict(sup_metadata)
if isinstance(meta_uuid, str):
# use the uuid in the metadata!
self.uuid = meta_uuid
self.create_document_item(sup_metadata)
else:
act_doc = None
exist_docs = OCdocument.objects\
.filter(uuid=self.uuid)[:1]
if len(exist_docs) < 1:
# problem! We have a manifest record for the document, but no document record,
# so make one
act_doc = OCdocument()
act_doc.uuid = self.uuid # use the previously assigned temporary UUID
act_doc.project_uuid = self.project_uuid
act_doc.source_id = self.source_id
act_doc.content = self.content
act_doc.save()
else:
act_doc = exist_docs[0]
if act_doc is not None:
if act_doc.content != self.content and self.content != self.DEFAULT_NO_CONTENT:
# update the document content with the latest content
act_doc.content = self.content
act_doc.save()
self.update_import_cell_uuid()
