def db_save_reconcile_entity_predicates_types(self, act_dir): """ saves predicates and type items to the Open Context database, and / or reconciles these items with previously saved items from the same project """ key = self.oc_config_entity_types json_obj = self.fm.get_dict_from_file(key, act_dir) if json_obj is None: print('Need to 1st generate an attributes file from the ArchEnts!') ok = False else: # we have JSON with dictionary for the entity_types self.entity_types = json_obj make_entity_types_assertions = False for faims_ent_type_id, ent_dict in json_obj.items(): if isinstance(ent_dict['item_type'], str) \ and ent_dict['add_type_as_attribute']: # OK we have some items that need entity types made as # a descriptive attribute make_entity_types_assertions = True break if make_entity_types_assertions: # we have entity_types that need to have a descriptive # predicate, so create a new predicate in Open Context # to describe entity_types for this project sup_dict = LastUpdatedOrderedDict() sup_dict[self.reconcile_key] = self.ent_type_pred_sup_id pm = PredicateManagement() pm.project_uuid = self.project_uuid pm.source_id = self.source_id pm.sup_dict = sup_dict pm.sup_reconcile_key = self.reconcile_key pm.sup_reconcile_value = self.ent_type_pred_sup_id pred_obj = pm.get_make_predicate( self.FAIMS_ENTITY_TYPE_PREDICATE_LABEL, 'variable', 'id') if pred_obj is not False: # we reconciled or created the predicate! # now we mint oc_types for all the entity_types predicate_uuid = str(pred_obj.uuid) for faims_ent_type_id, ent_dict in json_obj.items(): if isinstance(ent_dict['item_type'], str) \ and ent_dict['add_type_as_attribute']: # OK, we have an item entity type to be used as a description sup_dict = LastUpdatedOrderedDict() sup_dict[self.reconcile_key] = faims_ent_type_id tm = TypeManagement() tm.project_uuid = self.project_uuid tm.source_id = self.source_id tm.sup_dict = sup_dict tm.sup_reconcile_key = self.reconcile_key tm.sup_reconcile_value = faims_ent_type_id type_obj = tm.get_make_type_within_pred_uuid( predicate_uuid, ent_dict['label']) if type_obj is not False: # we have reconciled the type! ent_dict['type_uuid'] = str(type_obj.uuid) ent_dict['predicate_uuid'] = predicate_uuid self.entity_types[faims_ent_type_id] = ent_dict # now save the results self.fm.save_serialized_json(key, act_dir, self.entity_types)
def db_save_reconcile_predicates(self, act_dir): """ saves predicates to the Open Context database, and / or reconciles these items with previously saved items from the same project """ key = self.oc_config_relation_types json_obj = self.fm.get_dict_from_file(key, act_dir) if json_obj is None: print('Need to 1st generate a oc-relation-types.json from the relns.xml!') ok = False else: # we have JSON with dictionary for the attributes ok = True self.relation_types = json_obj for faims_id_pred, rel_dict in json_obj.items(): if not isinstance(rel_dict['oc-equiv'], str): # No equivalence, so reconcile the relation as a new predicate sup_dict = LastUpdatedOrderedDict() sup_dict[self.reconcile_key] = faims_id_pred pm = PredicateManagement() pm.project_uuid = self.project_uuid pm.source_id = self.source_id pm.sup_dict = sup_dict pm.sup_reconcile_key = self.reconcile_key pm.sup_reconcile_value = faims_id_pred pred_obj = pm.get_make_predicate(rel_dict['label'], rel_dict['predicate_type'], rel_dict['data_type']) if pred_obj is not False: # we reconciled the predicate! self.relation_types[faims_id_pred]['predicate_uuid'] = str(pred_obj.uuid) # now save the results self.fm.save_serialized_json(key, act_dir, self.relation_types)
def db_save_reconcile_predicates_types(self, act_dir): """ saves predicates and type items to the Open Context database, and / or reconciles these items with previously saved items from the same project """ key = self.oc_config_attributes json_obj = self.fm.get_dict_from_file(key, act_dir) if json_obj is None: print('Need to 1st generate an attributes file from the ArchEnts!') ok = False else: # we have JSON with dictionary for the attributes ok = True self.attributes = json_obj for faims_id_pred, attrib_dict in json_obj.items(): # default to always making a predicate and a type for attributes sup_dict = LastUpdatedOrderedDict() sup_dict[self.reconcile_key] = faims_id_pred pm = PredicateManagement() pm.project_uuid = self.project_uuid pm.source_id = self.source_id pm.sup_dict = sup_dict pm.sup_reconcile_key = self.reconcile_key pm.sup_reconcile_value = faims_id_pred pred_obj = pm.get_make_predicate(attrib_dict['label'], attrib_dict['predicate_type'], attrib_dict['data_type']) if pred_obj is not False: # we reconciled the predicate! self.attributes[faims_id_pred]['predicate_uuid'] = str( pred_obj.uuid) if 'objects' in attrib_dict: for faims_id_type, type_dict in attrib_dict[ 'objects'].items(): sup_dict = LastUpdatedOrderedDict() sup_dict[self.reconcile_key] = faims_id_type tm = TypeManagement() tm.project_uuid = self.project_uuid tm.source_id = self.source_id tm.sup_dict = sup_dict tm.sup_reconcile_key = self.reconcile_key tm.sup_reconcile_value = faims_id_type type_obj = tm.get_make_type_within_pred_uuid( pred_obj.uuid, type_dict['label']) if type_obj is not False: # we have reconciled the type! type_dict['type_uuid'] = str(type_obj.uuid) type_dict['predicate_uuid'] = str( pred_obj.uuid) self.attributes[faims_id_pred]['objects'][ faims_id_type] = type_dict # now save the results self.fm.save_serialized_json(key, act_dir, self.attributes)
def db_save_reconcile_predicates_types(self, act_dir): """ saves predicates and type items to the Open Context database, and / or reconciles these items with previously saved items from the same project """ key = self.oc_config_attributes json_obj = self.fm.get_dict_from_file(key, act_dir) if json_obj is None: print('Need to 1st generate an attributes file from the ArchEnts!') ok = False else: # we have JSON with dictionary for the attributes ok = True self.attributes = json_obj for faims_id_pred, attrib_dict in json_obj.items(): # default to always making a predicate and a type for attributes sup_dict = LastUpdatedOrderedDict() sup_dict[self.reconcile_key] = faims_id_pred pm = PredicateManagement() pm.project_uuid = self.project_uuid pm.source_id = self.source_id pm.sup_dict = sup_dict pm.sup_reconcile_key = self.reconcile_key pm.sup_reconcile_value = faims_id_pred pred_obj = pm.get_make_predicate(attrib_dict['label'], attrib_dict['predicate_type'], attrib_dict['data_type']) if pred_obj is not False: # we reconciled the predicate! self.attributes[faims_id_pred]['predicate_uuid'] = str(pred_obj.uuid) if 'objects' in attrib_dict: for faims_id_type, type_dict in attrib_dict['objects'].items(): sup_dict = LastUpdatedOrderedDict() sup_dict[self.reconcile_key] = faims_id_type tm = TypeManagement() tm.project_uuid = self.project_uuid tm.source_id = self.source_id tm.sup_dict = sup_dict tm.sup_reconcile_key = self.reconcile_key tm.sup_reconcile_value = faims_id_type type_obj = tm.get_make_type_within_pred_uuid(pred_obj.uuid, type_dict['label']) if type_obj is not False: # we have reconciled the type! type_dict['type_uuid'] = str(type_obj.uuid) type_dict['predicate_uuid'] = str(pred_obj.uuid) self.attributes[faims_id_pred]['objects'][faims_id_type] = type_dict # now save the results self.fm.save_serialized_json(key, act_dir, self.attributes)
def db_save_reconcile_predicates(self, act_dir): """ saves predicates to the Open Context database, and / or reconciles these items with previously saved items from the same project """ key = self.oc_config_relation_types json_obj = self.fm.get_dict_from_file(key, act_dir) if json_obj is None: print( 'Need to 1st generate a oc-relation-types.json from the relns.xml!' ) ok = False else: # we have JSON with dictionary for the attributes ok = True self.relation_types = json_obj for faims_id_pred, rel_dict in json_obj.items(): if not isinstance(rel_dict['oc-equiv'], str): # No equivalence, so reconcile the relation as a new predicate sup_dict = LastUpdatedOrderedDict() sup_dict[self.reconcile_key] = faims_id_pred pm = PredicateManagement() pm.project_uuid = self.project_uuid pm.source_id = self.source_id pm.sup_dict = sup_dict pm.sup_reconcile_key = self.reconcile_key pm.sup_reconcile_value = faims_id_pred pred_obj = pm.get_make_predicate( rel_dict['label'], rel_dict['predicate_type'], rel_dict['data_type']) if pred_obj is not False: # we reconciled the predicate! self.relation_types[faims_id_pred][ 'predicate_uuid'] = str(pred_obj.uuid) # now save the results self.fm.save_serialized_json(key, act_dir, self.relation_types)
def add_description(self): """ validates the value's data type if the data type differs from the predicate data type, add it as a note """ # validate the faims record against its expected data type # the validated_record will be None if not valid if isinstance(self.subject_uuid, str) \ and isinstance(self.subject_type, str): # we have subject information ok_to_add = False data_type = self.attrib_dict['data_type'] if isinstance(self.predicate_uuid, str): predicate_uuid = self.predicate_uuid elif 'predicate_uuid' in self.attrib_dict: predicate_uuid = self.attrib_dict['predicate_uuid'] object_date = None object_uuid = None object_num = None object_type = data_type if data_type == 'id': # we're processing a controlled vocab (oc-type) object if isinstance(self.faims_record_id, str): if self.faims_record_id in self.attrib_dict['objects']: # we found the object for this item obj_dict = self.attrib_dict['objects'][ self.faims_record_id] object_uuid = obj_dict['type_uuid'] object_type = 'types' ok_to_add = True elif self.faims_record is not None: # we're processing a literal value ddt = DescriptionDataType() validated_record = ddt.validate_literal_by_data_type( data_type, self.faims_record) if validated_record is not None: # the record is valid for the data_type ok_to_add = True if data_type == 'xsd:date': object_date = validated_record elif data_type == 'xsd:string': # we have a string, so add it to the database and get its uuid str_man = StringManagement() str_man.project_uuid = self.project_uuid str_man.source_id = self.source_id str_obj = str_man.get_make_string(validated_record) object_uuid = str(str_obj.uuid) else: # we have numeric data, so add to numeric data object_num = validated_record else: # the record is not valid for the data_type, so treat it as a string # so first we need to make a predicate for it, that is for # string values of this associated with this attribute object_type = 'xsd:string' sup_dict = LastUpdatedOrderedDict() sup_dict[self.reconcile_key] = self.attrib_dict[ 'id'] # the faims attribute id pm = PredicateManagement() pm.project_uuid = self.project_uuid pm.source_id = self.source_id pm.sup_dict = sup_dict pm.sup_reconcile_key = self.reconcile_key pm.sup_reconcile_value = self.attrib_dict[ 'id'] # the faims attribute id pred_obj = pm.get_make_predicate( self.attrib_dict['label'] + ' [Note]', self.attrib_dict['predicate_type'], object_type) if pred_obj is not False: # we have a new predicate predicate_uuid = pred_obj.uuid # now make an object_uuid for the original faims record str_man = StringManagement() str_man.project_uuid = self.project_uuid str_man.source_id = self.source_id str_obj = str_man.get_make_string(self.faims_record) object_uuid = str(str_obj.uuid) ok_to_add = True else: # not valid data, don't add ok_to_add = False if ok_to_add: # now we're OK to add the descrpitive assertion new_ass = Assertion() new_ass.uuid = self.subject_uuid new_ass.subject_type = self.subject_type new_ass.project_uuid = self.project_uuid new_ass.source_id = self.source_id new_ass.obs_node = '#obs-' + str(self.obs_num) new_ass.obs_num = self.obs_num new_ass.sort = self.sort_num new_ass.visibility = 1 new_ass.predicate_uuid = predicate_uuid new_ass.object_type = object_type if object_num is not None: new_ass.data_num = object_num if object_date is not None: new_ass.data_date = object_date if object_uuid is not None: new_ass.object_uuid = object_uuid try: new_ass.save() except: # probably already imported pass
def add_description(self): """ validates the value's data type if the data type differs from the predicate data type, add it as a note """ # validate the faims record against its expected data type # the validated_record will be None if not valid if isinstance(self.subject_uuid, str) \ and isinstance(self.subject_type, str): # we have subject information ok_to_add = False data_type = self.attrib_dict['data_type'] if isinstance(self.predicate_uuid, str): predicate_uuid = self.predicate_uuid elif 'predicate_uuid' in self.attrib_dict: predicate_uuid = self.attrib_dict['predicate_uuid'] object_date = None object_uuid = None object_num = None object_type = data_type if data_type == 'id': # we're processing a controlled vocab (oc-type) object if isinstance(self.faims_record_id, str): if self.faims_record_id in self.attrib_dict['objects']: # we found the object for this item obj_dict = self.attrib_dict['objects'][self.faims_record_id] object_uuid = obj_dict['type_uuid'] object_type = 'types' ok_to_add = True elif self.faims_record is not None: # we're processing a literal value ddt = DescriptionDataType() validated_record = ddt.validate_literal_by_data_type(data_type, self.faims_record) if validated_record is not None: # the record is valid for the data_type ok_to_add = True if data_type == 'xsd:date': object_date = validated_record elif data_type == 'xsd:string': # we have a string, so add it to the database and get its uuid str_man = StringManagement() str_man.project_uuid = self.project_uuid str_man.source_id = self.source_id str_obj = str_man.get_make_string(validated_record) object_uuid = str(str_obj.uuid) else: # we have numeric data, so add to numeric data object_num = validated_record else: # the record is not valid for the data_type, so treat it as a string # so first we need to make a predicate for it, that is for # string values of this associated with this attribute object_type = 'xsd:string' sup_dict = LastUpdatedOrderedDict() sup_dict[self.reconcile_key] = self.attrib_dict['id'] # the faims attribute id pm = PredicateManagement() pm.project_uuid = self.project_uuid pm.source_id = self.source_id pm.sup_dict = sup_dict pm.sup_reconcile_key = self.reconcile_key pm.sup_reconcile_value = self.attrib_dict['id'] # the faims attribute id pred_obj = pm.get_make_predicate(self.attrib_dict['label'] + ' [Note]', self.attrib_dict['predicate_type'], object_type) if pred_obj is not False: # we have a new predicate predicate_uuid = pred_obj.uuid # now make an object_uuid for the original faims record str_man = StringManagement() str_man.project_uuid = self.project_uuid str_man.source_id = self.source_id str_obj = str_man.get_make_string(self.faims_record) object_uuid = str(str_obj.uuid) ok_to_add = True else: # not valid data, don't add ok_to_add = False if ok_to_add: # now we're OK to add the descrpitive assertion new_ass = Assertion() new_ass.uuid = self.subject_uuid new_ass.subject_type = self.subject_type new_ass.project_uuid = self.project_uuid new_ass.source_id = self.source_id new_ass.obs_node = '#obs-' + str(self.obs_num) new_ass.obs_num = self.obs_num new_ass.sort = self.sort_num new_ass.visibility = 1 new_ass.predicate_uuid = predicate_uuid new_ass.object_type = object_type if object_num is not None: new_ass.data_num = object_num if object_date is not None: new_ass.data_date = object_date if object_uuid is not None: new_ass.object_uuid = object_uuid try: new_ass.save() except: # probably already imported pass
def db_save_reconcile_entity_predicates_types(self, act_dir): """ saves predicates and type items to the Open Context database, and / or reconciles these items with previously saved items from the same project """ key = self.oc_config_entity_types json_obj = self.fm.get_dict_from_file(key, act_dir) if json_obj is None: print('Need to 1st generate an attributes file from the ArchEnts!') ok = False else: # we have JSON with dictionary for the entity_types self.entity_types = json_obj make_entity_types_assertions = False for faims_ent_type_id, ent_dict in json_obj.items(): if isinstance(ent_dict['item_type'], str) \ and ent_dict['add_type_as_attribute']: # OK we have some items that need entity types made as # a descriptive attribute make_entity_types_assertions = True break if make_entity_types_assertions: # we have entity_types that need to have a descriptive # predicate, so create a new predicate in Open Context # to describe entity_types for this project sup_dict = LastUpdatedOrderedDict() sup_dict[self.reconcile_key] = self.ent_type_pred_sup_id pm = PredicateManagement() pm.project_uuid = self.project_uuid pm.source_id = self.source_id pm.sup_dict = sup_dict pm.sup_reconcile_key = self.reconcile_key pm.sup_reconcile_value = self.ent_type_pred_sup_id pred_obj = pm.get_make_predicate(self.FAIMS_ENTITY_TYPE_PREDICATE_LABEL, 'variable', 'id') if pred_obj is not False: # we reconciled or created the predicate! # now we mint oc_types for all the entity_types predicate_uuid = str(pred_obj.uuid) for faims_ent_type_id, ent_dict in json_obj.items(): if isinstance(ent_dict['item_type'], str) \ and ent_dict['add_type_as_attribute']: # OK, we have an item entity type to be used as a description sup_dict = LastUpdatedOrderedDict() sup_dict[self.reconcile_key] = faims_ent_type_id tm = TypeManagement() tm.project_uuid = self.project_uuid tm.source_id = self.source_id tm.sup_dict = sup_dict tm.sup_reconcile_key = self.reconcile_key tm.sup_reconcile_value = faims_ent_type_id type_obj = tm.get_make_type_within_pred_uuid(predicate_uuid, ent_dict['label']) if type_obj is not False: # we have reconciled the type! ent_dict['type_uuid'] = str(type_obj.uuid) ent_dict['predicate_uuid'] = predicate_uuid self.entity_types[faims_ent_type_id] = ent_dict # now save the results self.fm.save_serialized_json(key, act_dir, self.entity_types)