def reconcile_predicate_var(self, des_field_obj):
""" reconciles a predicate variable from the Import Field
"""
output = False
self.setup_field(des_field_obj)
if len(self.label) > 0:
output = True
pm = PredicateManagement()
pm.project_uuid = self.project_uuid
pm.source_id = self.source_id
pm.sort = self.sort
predicate = pm.get_make_predicate(self.label,
'variable',
self.data_type)
self.uuid = predicate.uuid
self.predicate = predicate
if predicate.uuid != self.candidate_uuid:
if self.des_import_cell is False:
# update the reconcilted UUID with for the import field object
des_field_obj.f_uuid = self.uuid
des_field_obj.save()
else:
# update the reconcilted UUID for import cells with same rec_hash
up_cells = ImportCell.objects\
.filter(source_id=self.source_id,
field_num=self.des_import_cell.field_num,
rec_hash=self.des_import_cell.rec_hash)
for up_cell in up_cells:
# save each cell with the correct UUID
up_cell.fl_uuid = self.uuid
up_cell.uuids_save()
return output
def db_save_reconcile_entity_predicates_types(self, act_dir):
""" saves predicates and type items to the
Open Context database, and / or reconciles these
items with previously saved items from the same project
"""
key = self.oc_config_entity_types
json_obj = self.fm.get_dict_from_file(key, act_dir)
if json_obj is None:
print('Need to 1st generate an attributes file from the ArchEnts!')
ok = False
else:
# we have JSON with dictionary for the entity_types
self.entity_types = json_obj
make_entity_types_assertions = False
for faims_ent_type_id, ent_dict in json_obj.items():
if isinstance(ent_dict['item_type'], str) \
and ent_dict['add_type_as_attribute']:
# OK we have some items that need entity types made as
# a descriptive attribute
make_entity_types_assertions = True
break
if make_entity_types_assertions:
# we have entity_types that need to have a descriptive
# predicate, so create a new predicate in Open Context
# to describe entity_types for this project
sup_dict = LastUpdatedOrderedDict()
sup_dict[self.reconcile_key] = self.ent_type_pred_sup_id
pm = PredicateManagement()
pm.project_uuid = self.project_uuid
pm.source_id = self.source_id
pm.sup_dict = sup_dict
pm.sup_reconcile_key = self.reconcile_key
pm.sup_reconcile_value = self.ent_type_pred_sup_id
pred_obj = pm.get_make_predicate(
self.FAIMS_ENTITY_TYPE_PREDICATE_LABEL, 'variable', 'id')
if pred_obj is not False:
# we reconciled or created the predicate!
# now we mint oc_types for all the entity_types
predicate_uuid = str(pred_obj.uuid)
for faims_ent_type_id, ent_dict in json_obj.items():
if isinstance(ent_dict['item_type'], str) \
and ent_dict['add_type_as_attribute']:
# OK, we have an item entity type to be used as a description
sup_dict = LastUpdatedOrderedDict()
sup_dict[self.reconcile_key] = faims_ent_type_id
tm = TypeManagement()
tm.project_uuid = self.project_uuid
tm.source_id = self.source_id
tm.sup_dict = sup_dict
tm.sup_reconcile_key = self.reconcile_key
tm.sup_reconcile_value = faims_ent_type_id
type_obj = tm.get_make_type_within_pred_uuid(
predicate_uuid, ent_dict['label'])
if type_obj is not False:
# we have reconciled the type!
ent_dict['type_uuid'] = str(type_obj.uuid)
ent_dict['predicate_uuid'] = predicate_uuid
self.entity_types[faims_ent_type_id] = ent_dict
# now save the results
self.fm.save_serialized_json(key, act_dir, self.entity_types)
def make_reconcile_link_pred(self, label):
""" Makes a linking predicate from a given predicate label """
self.label = label
pm = PredicateManagement()
pm.project_uuid = self.project_uuid
pm.source_id = self.source_id
pm.data_type = self.data_type
pm.sort = self.sort
pm.get_make_predicate(label, self.class_uri, self.data_type)
self.uuid = pm.predicate.uuid
def db_save_reconcile_predicates(self, act_dir):
""" saves predicates to the
Open Context database, and / or reconciles these
items with previously saved items from the same project
"""
key = self.oc_config_relation_types
json_obj = self.fm.get_dict_from_file(key, act_dir)
if json_obj is None:
print('Need to 1st generate a oc-relation-types.json from the relns.xml!')
ok = False
else:
# we have JSON with dictionary for the attributes
ok = True
self.relation_types = json_obj
for faims_id_pred, rel_dict in json_obj.items():
if not isinstance(rel_dict['oc-equiv'], str):
# No equivalence, so reconcile the relation as a new predicate
sup_dict = LastUpdatedOrderedDict()
sup_dict[self.reconcile_key] = faims_id_pred
pm = PredicateManagement()
pm.project_uuid = self.project_uuid
pm.source_id = self.source_id
pm.sup_dict = sup_dict
pm.sup_reconcile_key = self.reconcile_key
pm.sup_reconcile_value = faims_id_pred
pred_obj = pm.get_make_predicate(rel_dict['label'],
rel_dict['predicate_type'],
rel_dict['data_type'])
if pred_obj is not False:
# we reconciled the predicate!
self.relation_types[faims_id_pred]['predicate_uuid'] = str(pred_obj.uuid)
# now save the results
self.fm.save_serialized_json(key,
act_dir,
self.relation_types)
def db_save_reconcile_predicates_types(self, act_dir):
""" saves predicates and type items to the
Open Context database, and / or reconciles these
items with previously saved items from the same project
"""
key = self.oc_config_attributes
json_obj = self.fm.get_dict_from_file(key, act_dir)
if json_obj is None:
print('Need to 1st generate an attributes file from the ArchEnts!')
ok = False
else:
# we have JSON with dictionary for the attributes
ok = True
self.attributes = json_obj
for faims_id_pred, attrib_dict in json_obj.items():
# default to always making a predicate and a type for attributes
sup_dict = LastUpdatedOrderedDict()
sup_dict[self.reconcile_key] = faims_id_pred
pm = PredicateManagement()
pm.project_uuid = self.project_uuid
pm.source_id = self.source_id
pm.sup_dict = sup_dict
pm.sup_reconcile_key = self.reconcile_key
pm.sup_reconcile_value = faims_id_pred
pred_obj = pm.get_make_predicate(attrib_dict['label'],
attrib_dict['predicate_type'],
attrib_dict['data_type'])
if pred_obj is not False:
# we reconciled the predicate!
self.attributes[faims_id_pred]['predicate_uuid'] = str(
pred_obj.uuid)
if 'objects' in attrib_dict:
for faims_id_type, type_dict in attrib_dict[
'objects'].items():
sup_dict = LastUpdatedOrderedDict()
sup_dict[self.reconcile_key] = faims_id_type
tm = TypeManagement()
tm.project_uuid = self.project_uuid
tm.source_id = self.source_id
tm.sup_dict = sup_dict
tm.sup_reconcile_key = self.reconcile_key
tm.sup_reconcile_value = faims_id_type
type_obj = tm.get_make_type_within_pred_uuid(
pred_obj.uuid, type_dict['label'])
if type_obj is not False:
# we have reconciled the type!
type_dict['type_uuid'] = str(type_obj.uuid)
type_dict['predicate_uuid'] = str(
pred_obj.uuid)
self.attributes[faims_id_pred]['objects'][
faims_id_type] = type_dict
# now save the results
self.fm.save_serialized_json(key, act_dir, self.attributes)
def make_datatype_wrong_assertion(self, predicate_uuid, content):
""" Makes an assertion for records that don't fit the
expected data_type / object_type
"""
pm = PredicateManagement()
pm.project_uuid = self.project_uuid
pm.source_id = self.source_id
pm.sort = self.sort
new_note_predicate = pm.get_make_related_note_predicate(predicate_uuid, ' (Note)')
if new_note_predicate is not False:
note_pred_uuid = new_note_predicate.uuid
# save a skos:related assertion linking the old and new predicates
ptm = PredicateTypeAssertions()
ptm.skos_relate_old_new_predicates(self.project_uuid,
self.source_id,
predicate_uuid,
note_pred_uuid)
sm = StringManagement()
sm.project_uuid = self.project_uuid
sm.source_id = self.source_id
oc_string = sm.get_make_string(content)
object_uuid = oc_string.uuid
if self.check_description_new(self.subject_uuid,
self.obs_num,
note_pred_uuid,
object_uuid,
None,
None):
# this asertion does not exist yet, OK to make it
new_ass = Assertion()
new_ass.uuid = self.subject_uuid
new_ass.subject_type = self.subject_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = self.obs_node
new_ass.obs_num = self.obs_num
new_ass.sort = self.sort + .1
new_ass.visibility = 1
new_ass.predicate_uuid = note_pred_uuid
new_ass.object_uuid = object_uuid
new_ass.object_type = 'xsd:string'
new_ass.save()
def make_reconcile_link_pred(self, label):
""" Makes a linking predicate from a given predicate label """
self.label = label
pm = PredicateManagement()
pm.project_uuid = self.project_uuid
pm.source_id = self.source_id
pm.data_type = self.data_type
pm.sort = self.sort
pm.get_make_predicate(label,
self.class_uri,
self.data_type)
self.uuid = pm.predicate.uuid
def db_save_reconcile_predicates_types(self, act_dir):
""" saves predicates and type items to the
Open Context database, and / or reconciles these
items with previously saved items from the same project
"""
key = self.oc_config_attributes
json_obj = self.fm.get_dict_from_file(key, act_dir)
if json_obj is None:
print('Need to 1st generate an attributes file from the ArchEnts!')
ok = False
else:
# we have JSON with dictionary for the attributes
ok = True
self.attributes = json_obj
for faims_id_pred, attrib_dict in json_obj.items():
# default to always making a predicate and a type for attributes
sup_dict = LastUpdatedOrderedDict()
sup_dict[self.reconcile_key] = faims_id_pred
pm = PredicateManagement()
pm.project_uuid = self.project_uuid
pm.source_id = self.source_id
pm.sup_dict = sup_dict
pm.sup_reconcile_key = self.reconcile_key
pm.sup_reconcile_value = faims_id_pred
pred_obj = pm.get_make_predicate(attrib_dict['label'],
attrib_dict['predicate_type'],
attrib_dict['data_type'])
if pred_obj is not False:
# we reconciled the predicate!
self.attributes[faims_id_pred]['predicate_uuid'] = str(pred_obj.uuid)
if 'objects' in attrib_dict:
for faims_id_type, type_dict in attrib_dict['objects'].items():
sup_dict = LastUpdatedOrderedDict()
sup_dict[self.reconcile_key] = faims_id_type
tm = TypeManagement()
tm.project_uuid = self.project_uuid
tm.source_id = self.source_id
tm.sup_dict = sup_dict
tm.sup_reconcile_key = self.reconcile_key
tm.sup_reconcile_value = faims_id_type
type_obj = tm.get_make_type_within_pred_uuid(pred_obj.uuid,
type_dict['label'])
if type_obj is not False:
# we have reconciled the type!
type_dict['type_uuid'] = str(type_obj.uuid)
type_dict['predicate_uuid'] = str(pred_obj.uuid)
self.attributes[faims_id_pred]['objects'][faims_id_type] = type_dict
# now save the results
self.fm.save_serialized_json(key,
act_dir,
self.attributes)
def create_pred_parents(self, new_hierachic_list):
""" Creates new types for
superior (more general) types from a list
of types that have hiearchies implicit in their labels
once the superior types are created,
linked data annotations noting hierarchy are stored
"""
parent_children_pairs = []
for manifest in new_hierachic_list:
try:
oc_pred = Predicate.objects.get(uuid=manifest.uuid)
except Predicate.DoesNotExist:
oc_pred = False
if (oc_pred is not False):
child_parts = manifest.label.split(self.HIERARCHY_DELIM)
act_delim = ''
act_new_label = ''
current_parent = False
for label_part in child_parts:
act_new_label = act_new_label + act_delim + label_part
act_delim = self.HIERARCHY_DELIM
pred_manage = PredicateManagement()
pred_manage.project_uuid = manifest.project_uuid
pred_manage.source_id = self.source_id
pred_manage.sort = oc_pred.sort
pred_manage.data_type = oc_pred.data_type
ppred = pred_manage.get_make_predicate(
act_new_label, manifest.class_uri)
if (ppred is not False and current_parent is not False):
parent_child = {
'parent': current_parent,
'child': ppred.uuid
}
parent_children_pairs.append(parent_child)
current_parent = ppred.uuid
if (len(parent_children_pairs) > 0):
# now make some linked data annotations
for parent_child in parent_children_pairs:
if (parent_child['parent'] is not False):
new_la = LinkAnnotation()
new_la.subject = parent_child['child']
new_la.subject_type = 'predicates'
new_la.project_uuid = manifest.project_uuid
new_la.source_id = self.source_id
new_la.predicate_uri = self.p_for_superobjs
new_la.object_uri = URImanagement.make_oc_uri(
parent_child['parent'], 'predicates')
new_la.creator_uuid = ''
new_la.save()
return parent_children_pairs
def db_save_reconcile_predicates(self, act_dir):
""" saves predicates to the
Open Context database, and / or reconciles these
items with previously saved items from the same project
"""
key = self.oc_config_relation_types
json_obj = self.fm.get_dict_from_file(key, act_dir)
if json_obj is None:
print(
'Need to 1st generate a oc-relation-types.json from the relns.xml!'
)
ok = False
else:
# we have JSON with dictionary for the attributes
ok = True
self.relation_types = json_obj
for faims_id_pred, rel_dict in json_obj.items():
if not isinstance(rel_dict['oc-equiv'], str):
# No equivalence, so reconcile the relation as a new predicate
sup_dict = LastUpdatedOrderedDict()
sup_dict[self.reconcile_key] = faims_id_pred
pm = PredicateManagement()
pm.project_uuid = self.project_uuid
pm.source_id = self.source_id
pm.sup_dict = sup_dict
pm.sup_reconcile_key = self.reconcile_key
pm.sup_reconcile_value = faims_id_pred
pred_obj = pm.get_make_predicate(
rel_dict['label'], rel_dict['predicate_type'],
rel_dict['data_type'])
if pred_obj is not False:
# we reconciled the predicate!
self.relation_types[faims_id_pred][
'predicate_uuid'] = str(pred_obj.uuid)
# now save the results
self.fm.save_serialized_json(key, act_dir, self.relation_types)
def create_pred_parents(self, new_hierachic_list):
""" Creates new types for
superior (more general) types from a list
of types that have hiearchies implicit in their labels
once the superior types are created,
linked data annotations noting hierarchy are stored
"""
parent_children_pairs = []
for manifest in new_hierachic_list:
try:
oc_pred = Predicate.objects.get(uuid=manifest.uuid)
except Predicate.DoesNotExist:
oc_pred = False
if oc_pred is not False:
child_parts = manifest.label.split(self.HIERARCHY_DELIM)
act_delim = ""
act_new_label = ""
current_parent = False
for label_part in child_parts:
act_new_label = act_new_label + act_delim + label_part
act_delim = self.HIERARCHY_DELIM
pred_manage = PredicateManagement()
pred_manage.project_uuid = manifest.project_uuid
pred_manage.source_id = self.source_id
pred_manage.sort = oc_pred.sort
pred_manage.data_type = oc_pred.data_type
ppred = pred_manage.get_make_predicate(act_new_label, manifest.class_uri)
if ppred is not False and current_parent is not False:
parent_child = {"parent": current_parent, "child": ppred.uuid}
parent_children_pairs.append(parent_child)
current_parent = ppred.uuid
if len(parent_children_pairs) > 0:
# now make some linked data annotations
for parent_child in parent_children_pairs:
if parent_child["parent"] is not False:
new_la = LinkAnnotation()
new_la.subject = parent_child["child"]
new_la.subject_type = "predicates"
new_la.project_uuid = manifest.project_uuid
new_la.source_id = self.source_id
new_la.predicate_uri = self.p_for_superobjs
new_la.object_uri = URImanagement.make_oc_uri(parent_child["parent"], "predicates")
new_la.creator_uuid = ""
new_la.save()
return parent_children_pairs
def add_trinomials_to_items(self):
""" adds trinomial identifiers to every item
with a trinomial added in the temporary
trinomial table
"""
proj_labels = {}
proj_uuids = self.dinaa_projs
projects = Project.objects.filter(uuid__in=proj_uuids)
source_index = 0
for project in projects:
project_uuid = project.uuid
proj_labels[project_uuid] = project.label
source_index = project.short_id
# check to see if the project already has a source_id for
# trinomial observation assertions
proj_source_id = self.check_project_source_id(project_uuid)
if proj_source_id is False:
# make a new source_id for this project
proj_source_id = self.source_id + '-' + str(source_index)
self.source_ids[project_uuid] = proj_source_id
self.create_obsmetadata(project_uuid)
else:
# use the existing source_id for this project
self.source_ids[project_uuid] = proj_source_id
print('Project uuid: ' + project_uuid + ' source: ' +
proj_source_id)
pm = PredicateManagement()
pm.project_uuid = self.dinaa_proj_uuid
pm.source_id = self.source_id
main_pred = pm.get_make_predicate(self.primary_label, 'variable')
pm = PredicateManagement()
pm.project_uuid = self.dinaa_proj_uuid
pm.source_id = self.source_id
sort_pred = pm.get_make_predicate(self.sort_label, 'variable')
pm = PredicateManagement()
pm.project_uuid = self.dinaa_proj_uuid
pm.source_id = self.source_id
aux_pred = pm.get_make_predicate(self.aux_label, 'variable')
for project_uuid, proj_label in proj_labels.items():
print('Adding trinomials for: ' + proj_label)
tris = Trinomial.objects.filter(trinomial__isnull=False,
project_label=proj_label)
for tri in tris:
# first check a manifest item exists for this item
manifest = False
try:
manifest = Manifest.objects.get(uuid=tri.uuid)
except Manifest.DoesNotExist:
manifest = False
if manifest is not False:
# OK we know this exists, so now make the assertion
# to add a trinomial
sorting = self.initial_sort
# add the cannonical trinomial first
ok = self.add_trinomial_to_item(main_pred, sorting,
manifest, tri.trinomial,
False)
if ok:
print('Added: ' + tri.trinomial + ' to: ' + tri.uri)
sorting += 1
# add the sorting trinomial
sort_trinomial = self.make_sort_trinomial(
tri.trinomial)
ok = self.add_trinomial_to_item(
sort_pred, sorting, manifest, sort_trinomial,
False)
# add a list of alternate versions
aux_list = self.make_aux_trinomial_list(tri.trinomial)
allow_multiple = False
for aux_trinomial in aux_list:
sorting += 1
# if successful in creation, then allow_multiple is true
allow_multiple = self.add_trinomial_to_item(
aux_pred, sorting, manifest, aux_trinomial,
allow_multiple)
else:
print('Bad news! Missing Manifest obj for: ' + str(tri))
def seperate_inconsistent_data_type(self, predicate_uuid):
""" This looks for assertions that have different object types than the
required main data_type of a given predicate. If such different assertions
are found, the offending assertion is deleted and replaced by a new predicate
and that new predicate is related back to the old predicate via 'skos:related'
"""
plabel = False
ptype = False
pdata_type = False
new_rel_predicates = [] # list of new predicates made from old
try:
pman = Manifest.objects.get(uuid=predicate_uuid)
plabel = pman.label
ptype = pman.class_uri
except Manifest.DoesNotExist:
pman = False
try: # look for the predicate item
pred = Predicate.objects.get(uuid=predicate_uuid)
pdata_type = pred.data_type
except Predicate.DoesNotExist:
pred = False
if (plabel is not False and ptype is not False
and pdata_type is not False):
adistinct = Assertion.objects.filter(predicate_uuid=predicate_uuid)\
.exclude(object_type=pdata_type)\
.order_by('object_type')\
.distinct('object_type') # get distinct list of object_types
for aitem in adistinct: # list of assertions that are not using the good data type
pm = PredicateManagement()
pm.source_id = pman.source_id
pm.project_uuid = pman.project_uuid
pm.sort = pred.sort
cpred = pm.get_make_predicate(plabel, ptype, aitem.object_type)
if (cpred is not False):
if (cpred.uuid not in new_rel_predicates):
new_rel_predicates.append(cpred.uuid)
if (len(new_rel_predicates) > 0):
# add annotations storing the relationship between the new predicates and the old
print('---- New predicates: ' + str(len(new_rel_predicates)))
for new_pred_uuid in new_rel_predicates:
print('New predicate: ' + str(new_pred_uuid))
self.skos_relate_old_new_predicates(
pman.project_uuid, pman.source_id, predicate_uuid,
new_pred_uuid)
alist = Assertion.objects.filter(
predicate_uuid=predicate_uuid).exclude(
object_type=pdata_type)
for aitem in alist: # list of assertions that are not using the good data type
pm = PredicateManagement()
pm.source_id = pman.source_id
pm.project_uuid = pman.project_uuid
pm.sort = pred.sort
cpred = pm.get_make_predicate(plabel, ptype,
aitem.object_type)
if (cpred is not False):
if (cpred.uuid not in new_rel_predicates):
new_rel_predicates.append(cpred.uuid)
Assertion.objects.filter(
hash_id=aitem.hash_id).delete()
# delete the old instance of the assertion
aitem.predicate_uuid = cpred.uuid # change the predicate to the new predicate
aitem.save(
) # save the assertion with the new predicate
else:
print('--OK Predicate--: ' + str(predicate_uuid))
def add_trinomials_to_items(self):
""" adds trinomial identifiers to every item
with a trinomial added in the temporary
trinomial table
"""
proj_labels = {}
proj_uuids = self.dinaa_projs
projects = Project.objects.filter(uuid__in=proj_uuids)
source_index = 0
for project in projects:
project_uuid = project.uuid
proj_labels[project_uuid] = project.label
source_index = project.short_id
# check to see if the project already has a source_id for
# trinomial observation assertions
proj_source_id = self.check_project_source_id(project_uuid)
if proj_source_id is False:
# make a new source_id for this project
proj_source_id = self.source_id + '-' + str(source_index)
self.source_ids[project_uuid] = proj_source_id
self.create_obsmetadata(project_uuid)
else:
# use the existing source_id for this project
self.source_ids[project_uuid] = proj_source_id
print('Project uuid: ' + project_uuid + ' source: ' + proj_source_id)
pm = PredicateManagement()
pm.project_uuid = self.dinaa_proj_uuid
pm.source_id = self.source_id
main_pred = pm.get_make_predicate(self.primary_label,
'variable')
pm = PredicateManagement()
pm.project_uuid = self.dinaa_proj_uuid
pm.source_id = self.source_id
sort_pred = pm.get_make_predicate(self.sort_label,
'variable')
pm = PredicateManagement()
pm.project_uuid = self.dinaa_proj_uuid
pm.source_id = self.source_id
aux_pred = pm.get_make_predicate(self.aux_label,
'variable')
for project_uuid, proj_label in proj_labels.items():
print('Adding trinomials for: ' + proj_label)
tris = Trinomial.objects.filter(trinomial__isnull=False,
project_label=proj_label)
for tri in tris:
# first check a manifest item exists for this item
manifest = False
try:
manifest = Manifest.objects.get(uuid=tri.uuid)
except Manifest.DoesNotExist:
manifest = False
if manifest is not False:
# OK we know this exists, so now make the assertion
# to add a trinomial
sorting = self.initial_sort
# add the cannonical trinomial first
ok = self.add_trinomial_to_item(main_pred,
sorting,
manifest,
tri.trinomial,
False)
if ok:
print('Added: ' + tri.trinomial + ' to: ' + tri.uri)
sorting += 1
# add the sorting trinomial
sort_trinomial = self.make_sort_trinomial(tri.trinomial)
ok = self.add_trinomial_to_item(sort_pred,
sorting,
manifest,
sort_trinomial,
False)
# add a list of alternate versions
aux_list = self.make_aux_trinomial_list(tri.trinomial)
allow_multiple = False
for aux_trinomial in aux_list:
sorting += 1
# if successful in creation, then allow_multiple is true
allow_multiple = self.add_trinomial_to_item(aux_pred,
sorting,
manifest,
aux_trinomial,
allow_multiple)
else:
print('Bad news! Missing Manifest obj for: ' + str(tri))
def seperate_inconsistent_data_type(self, predicate_uuid):
""" This looks for assertions that have different object types than the
required main data_type of a given predicate. If such different assertions
are found, the offending assertion is deleted and replaced by a new predicate
and that new predicate is related back to the old predicate via 'skos:related'
"""
plabel = False
ptype = False
pdata_type = False
new_rel_predicates = [] # list of new predicates made from old
try:
pman = Manifest.objects.get(uuid=predicate_uuid)
plabel = pman.label
ptype = pman.class_uri
except Manifest.DoesNotExist:
pman = False
try: # look for the predicate item
pred = Predicate.objects.get(uuid=predicate_uuid)
pdata_type = pred.data_type
except Predicate.DoesNotExist:
pred = False
if(plabel is not False
and ptype is not False
and pdata_type is not False):
adistinct = Assertion.objects.filter(predicate_uuid=predicate_uuid)\
.exclude(object_type=pdata_type)\
.order_by('object_type')\
.distinct('object_type') # get distinct list of object_types
for aitem in adistinct: # list of assertions that are not using the good data type
pm = PredicateManagement()
pm.source_id = pman.source_id
pm.project_uuid = pman.project_uuid
pm.sort = pred.sort
cpred = pm.get_make_predicate(plabel, ptype, aitem.object_type)
if(cpred is not False):
if(cpred.uuid not in new_rel_predicates):
new_rel_predicates.append(cpred.uuid)
if(len(new_rel_predicates) > 0):
# add annotations storing the relationship between the new predicates and the old
print('---- New predicates: ' + str(len(new_rel_predicates)))
for new_pred_uuid in new_rel_predicates:
print('New predicate: ' + str(new_pred_uuid))
self.skos_relate_old_new_predicates(pman.project_uuid,
pman.source_id,
predicate_uuid,
new_pred_uuid)
alist = Assertion.objects.filter(predicate_uuid=predicate_uuid).exclude(object_type=pdata_type)
for aitem in alist: # list of assertions that are not using the good data type
pm = PredicateManagement()
pm.source_id = pman.source_id
pm.project_uuid = pman.project_uuid
pm.sort = pred.sort
cpred = pm.get_make_predicate(plabel, ptype, aitem.object_type)
if(cpred is not False):
if(cpred.uuid not in new_rel_predicates):
new_rel_predicates.append(cpred.uuid)
Assertion.objects.filter(hash_id=aitem.hash_id).delete()
# delete the old instance of the assertion
aitem.predicate_uuid = cpred.uuid # change the predicate to the new predicate
aitem.save() # save the assertion with the new predicate
else:
print('--OK Predicate--: ' + str(predicate_uuid))
def add_description(self):
""" validates the value's data type
if the data type differs from the
predicate data type,
add it as a note
"""
# validate the faims record against its expected data type
# the validated_record will be None if not valid
if isinstance(self.subject_uuid, str) \
and isinstance(self.subject_type, str):
# we have subject information
ok_to_add = False
data_type = self.attrib_dict['data_type']
if isinstance(self.predicate_uuid, str):
predicate_uuid = self.predicate_uuid
elif 'predicate_uuid' in self.attrib_dict:
predicate_uuid = self.attrib_dict['predicate_uuid']
object_date = None
object_uuid = None
object_num = None
object_type = data_type
if data_type == 'id':
# we're processing a controlled vocab (oc-type) object
if isinstance(self.faims_record_id, str):
if self.faims_record_id in self.attrib_dict['objects']:
# we found the object for this item
obj_dict = self.attrib_dict['objects'][
self.faims_record_id]
object_uuid = obj_dict['type_uuid']
object_type = 'types'
ok_to_add = True
elif self.faims_record is not None:
# we're processing a literal value
ddt = DescriptionDataType()
validated_record = ddt.validate_literal_by_data_type(
data_type, self.faims_record)
if validated_record is not None:
# the record is valid for the data_type
ok_to_add = True
if data_type == 'xsd:date':
object_date = validated_record
elif data_type == 'xsd:string':
# we have a string, so add it to the database and get its uuid
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_obj = str_man.get_make_string(validated_record)
object_uuid = str(str_obj.uuid)
else:
# we have numeric data, so add to numeric data
object_num = validated_record
else:
# the record is not valid for the data_type, so treat it as a string
# so first we need to make a predicate for it, that is for
# string values of this associated with this attribute
object_type = 'xsd:string'
sup_dict = LastUpdatedOrderedDict()
sup_dict[self.reconcile_key] = self.attrib_dict[
'id'] # the faims attribute id
pm = PredicateManagement()
pm.project_uuid = self.project_uuid
pm.source_id = self.source_id
pm.sup_dict = sup_dict
pm.sup_reconcile_key = self.reconcile_key
pm.sup_reconcile_value = self.attrib_dict[
'id'] # the faims attribute id
pred_obj = pm.get_make_predicate(
self.attrib_dict['label'] + ' [Note]',
self.attrib_dict['predicate_type'], object_type)
if pred_obj is not False:
# we have a new predicate
predicate_uuid = pred_obj.uuid
# now make an object_uuid for the original faims record
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_obj = str_man.get_make_string(self.faims_record)
object_uuid = str(str_obj.uuid)
ok_to_add = True
else:
# not valid data, don't add
ok_to_add = False
if ok_to_add:
# now we're OK to add the descrpitive assertion
new_ass = Assertion()
new_ass.uuid = self.subject_uuid
new_ass.subject_type = self.subject_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-' + str(self.obs_num)
new_ass.obs_num = self.obs_num
new_ass.sort = self.sort_num
new_ass.visibility = 1
new_ass.predicate_uuid = predicate_uuid
new_ass.object_type = object_type
if object_num is not None:
new_ass.data_num = object_num
if object_date is not None:
new_ass.data_date = object_date
if object_uuid is not None:
new_ass.object_uuid = object_uuid
try:
new_ass.save()
except:
# probably already imported
pass
def add_description(self):
""" validates the value's data type
if the data type differs from the
predicate data type,
add it as a note
"""
# validate the faims record against its expected data type
# the validated_record will be None if not valid
if isinstance(self.subject_uuid, str) \
and isinstance(self.subject_type, str):
# we have subject information
ok_to_add = False
data_type = self.attrib_dict['data_type']
if isinstance(self.predicate_uuid, str):
predicate_uuid = self.predicate_uuid
elif 'predicate_uuid' in self.attrib_dict:
predicate_uuid = self.attrib_dict['predicate_uuid']
object_date = None
object_uuid = None
object_num = None
object_type = data_type
if data_type == 'id':
# we're processing a controlled vocab (oc-type) object
if isinstance(self.faims_record_id, str):
if self.faims_record_id in self.attrib_dict['objects']:
# we found the object for this item
obj_dict = self.attrib_dict['objects'][self.faims_record_id]
object_uuid = obj_dict['type_uuid']
object_type = 'types'
ok_to_add = True
elif self.faims_record is not None:
# we're processing a literal value
ddt = DescriptionDataType()
validated_record = ddt.validate_literal_by_data_type(data_type,
self.faims_record)
if validated_record is not None:
# the record is valid for the data_type
ok_to_add = True
if data_type == 'xsd:date':
object_date = validated_record
elif data_type == 'xsd:string':
# we have a string, so add it to the database and get its uuid
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_obj = str_man.get_make_string(validated_record)
object_uuid = str(str_obj.uuid)
else:
# we have numeric data, so add to numeric data
object_num = validated_record
else:
# the record is not valid for the data_type, so treat it as a string
# so first we need to make a predicate for it, that is for
# string values of this associated with this attribute
object_type = 'xsd:string'
sup_dict = LastUpdatedOrderedDict()
sup_dict[self.reconcile_key] = self.attrib_dict['id'] # the faims attribute id
pm = PredicateManagement()
pm.project_uuid = self.project_uuid
pm.source_id = self.source_id
pm.sup_dict = sup_dict
pm.sup_reconcile_key = self.reconcile_key
pm.sup_reconcile_value = self.attrib_dict['id'] # the faims attribute id
pred_obj = pm.get_make_predicate(self.attrib_dict['label'] + ' [Note]',
self.attrib_dict['predicate_type'],
object_type)
if pred_obj is not False:
# we have a new predicate
predicate_uuid = pred_obj.uuid
# now make an object_uuid for the original faims record
str_man = StringManagement()
str_man.project_uuid = self.project_uuid
str_man.source_id = self.source_id
str_obj = str_man.get_make_string(self.faims_record)
object_uuid = str(str_obj.uuid)
ok_to_add = True
else:
# not valid data, don't add
ok_to_add = False
if ok_to_add:
# now we're OK to add the descrpitive assertion
new_ass = Assertion()
new_ass.uuid = self.subject_uuid
new_ass.subject_type = self.subject_type
new_ass.project_uuid = self.project_uuid
new_ass.source_id = self.source_id
new_ass.obs_node = '#obs-' + str(self.obs_num)
new_ass.obs_num = self.obs_num
new_ass.sort = self.sort_num
new_ass.visibility = 1
new_ass.predicate_uuid = predicate_uuid
new_ass.object_type = object_type
if object_num is not None:
new_ass.data_num = object_num
if object_date is not None:
new_ass.data_date = object_date
if object_uuid is not None:
new_ass.object_uuid = object_uuid
try:
new_ass.save()
except:
# probably already imported
pass
def db_save_reconcile_entity_predicates_types(self, act_dir):
""" saves predicates and type items to the
Open Context database, and / or reconciles these
items with previously saved items from the same project
"""
key = self.oc_config_entity_types
json_obj = self.fm.get_dict_from_file(key, act_dir)
if json_obj is None:
print('Need to 1st generate an attributes file from the ArchEnts!')
ok = False
else:
# we have JSON with dictionary for the entity_types
self.entity_types = json_obj
make_entity_types_assertions = False
for faims_ent_type_id, ent_dict in json_obj.items():
if isinstance(ent_dict['item_type'], str) \
and ent_dict['add_type_as_attribute']:
# OK we have some items that need entity types made as
# a descriptive attribute
make_entity_types_assertions = True
break
if make_entity_types_assertions:
# we have entity_types that need to have a descriptive
# predicate, so create a new predicate in Open Context
# to describe entity_types for this project
sup_dict = LastUpdatedOrderedDict()
sup_dict[self.reconcile_key] = self.ent_type_pred_sup_id
pm = PredicateManagement()
pm.project_uuid = self.project_uuid
pm.source_id = self.source_id
pm.sup_dict = sup_dict
pm.sup_reconcile_key = self.reconcile_key
pm.sup_reconcile_value = self.ent_type_pred_sup_id
pred_obj = pm.get_make_predicate(self.FAIMS_ENTITY_TYPE_PREDICATE_LABEL,
'variable',
'id')
if pred_obj is not False:
# we reconciled or created the predicate!
# now we mint oc_types for all the entity_types
predicate_uuid = str(pred_obj.uuid)
for faims_ent_type_id, ent_dict in json_obj.items():
if isinstance(ent_dict['item_type'], str) \
and ent_dict['add_type_as_attribute']:
# OK, we have an item entity type to be used as a description
sup_dict = LastUpdatedOrderedDict()
sup_dict[self.reconcile_key] = faims_ent_type_id
tm = TypeManagement()
tm.project_uuid = self.project_uuid
tm.source_id = self.source_id
tm.sup_dict = sup_dict
tm.sup_reconcile_key = self.reconcile_key
tm.sup_reconcile_value = faims_ent_type_id
type_obj = tm.get_make_type_within_pred_uuid(predicate_uuid,
ent_dict['label'])
if type_obj is not False:
# we have reconciled the type!
ent_dict['type_uuid'] = str(type_obj.uuid)
ent_dict['predicate_uuid'] = predicate_uuid
self.entity_types[faims_ent_type_id] = ent_dict
# now save the results
self.fm.save_serialized_json(key,
act_dir,
self.entity_types)
