def save(self): if self.value.strip() != '': self.id = self.id if (self.id != '' and self.id != None) else str( uuid.uuid4()) value = models.Value() value.pk = self.id value.value = self.value value.concept_id = self.conceptid # models.Concept.objects.get(pk=self.conceptid) value.valuetype_id = self.type # models.DValueType.objects.get(pk=self.type) if self.language != '': value.language_id = self.language # models.DLanguage.objects.get(pk=self.language) else: value.language_id = settings.LANGUAGE_CODE try: if value.value != 'Resource To Resource Relationship Types': value.save() except IntegrityError as e: valuetype = models.DValueType() valuetype.valuetype = value.valuetype_id valuetype.category = 'undefined' valuetype.namespace = 'arches' valuetype.save() value.save() self.category = value.valuetype.category
def get_value(self, valueid): try: return self.value_lookup[valueid] except: try: self.value_lookup[valueid] = models.Value.objects.get( pk=valueid) return self.value_lookup[valueid] except ObjectDoesNotExist: return models.Value()
def post(self, request, resourceid=None): es = Elasticsearch() se = SearchEngineFactory().create() res = dict(request.POST) relationship_type = res['relationship_properties[relationship_type]'][0] datefrom = res['relationship_properties[datefrom]'][0] dateto = res['relationship_properties[dateto]'][0] dateto = None if dateto == '' else dateto datefrom = None if datefrom == '' else datefrom notes = res['relationship_properties[notes]'][0] root_resourceinstanceid = res['root_resourceinstanceid'] instances_to_relate = [] relationships_to_update = [] if 'instances_to_relate[]' in res: instances_to_relate = res['instances_to_relate[]'] if 'relationship_ids[]' in res: relationships_to_update = res['relationship_ids[]'] for instanceid in instances_to_relate: rr = models.ResourceXResource.objects.create( resourceinstanceidfrom = Resource(root_resourceinstanceid[0]), resourceinstanceidto = Resource(instanceid), notes = notes, relationshiptype = models.Value(relationship_type), datestarted = datefrom, dateended = dateto ) document = model_to_dict(rr) se.index_data(index='resource_relations', doc_type='all', body=document, idfield='resourcexid') for relationshipid in relationships_to_update: rr = models.ResourceXResource.objects.get(pk=relationshipid) rr.notes = notes rr.relationshiptype = models.Value(relationship_type) rr.datestarted = datefrom rr.dateended = dateto rr.save() document = model_to_dict(rr) se.index_data(index='resource_relations', doc_type='all', body=document, idfield='resourcexid') start = request.GET.get('start', 0) es.indices.refresh(index="resource_relations") return JSONResponse(self.get_related_resources(root_resourceinstanceid[0], lang="en-us", start=start, limit=15), indent=4)
def save(self): if self.value.strip() != '': self.id = self.id if (self.id != '' and self.id != None) else str( uuid.uuid4()) value = models.Value() value.pk = self.id value.value = self.value value.concept_id = self.conceptid # models.Concept.objects.get(pk=self.conceptid) value.valuetype_id = self.type # models.DValueType.objects.get(pk=self.type) if self.language != '': value.language_id = self.language # models.DLanguage.objects.get(pk=self.language) value.save() self.category = value.valuetype.category
def save_concepts_from_skos(self, graph, overwrite_options="overwrite", staging_options="keep", bulk_load=False, path=""): """ given an RDF graph, tries to save the concpets to the system Keyword arguments: overwrite_options -- 'overwrite', 'ignore' staging_options -- 'stage', 'keep' """ baseuuid = uuid.uuid4() allowed_languages = models.DLanguage.objects.values_list("pk", flat=True) default_lang = settings.LANGUAGE_CODE if bulk_load is True: self.logger.setLevel(logging.ERROR) value_types = models.DValueType.objects.all() skos_value_types = value_types.filter( Q(namespace="skos") | Q(namespace="arches")) skos_value_types_list = list( skos_value_types.values_list("valuetype", flat=True)) skos_value_types = { valuetype.valuetype: valuetype for valuetype in skos_value_types } dcterms_value_types = value_types.filter(namespace="dcterms") dcterms_identifier_type = dcterms_value_types.get( valuetype=str(DCTERMS.identifier).replace(str(DCTERMS), "")) # if the graph is of the type rdflib.graph.Graph if isinstance(graph, Graph): values = [] # Search for ConceptSchemes first for scheme, v, o in graph.triples( (None, RDF.type, SKOS.ConceptScheme)): identifier = self.unwrapJsonLiteral(str(scheme)) scheme_id = self.generate_uuid_from_subject(baseuuid, scheme) if bulk_load is True: concept_scheme = models.Concept( pk=scheme_id, legacyoid=str(scheme), nodetype_id="ConceptScheme") else: concept_scheme = Concept({ 'id': scheme_id, 'legacyoid': str(scheme), 'nodetype': 'ConceptScheme' }) for predicate, object in graph.predicate_objects( subject=scheme): if str(DCTERMS) in predicate and predicate.replace( DCTERMS, "") in dcterms_value_types.values_list( "valuetype", flat=True): if not self.language_exists(object, allowed_languages): allowed_languages = models.DLanguage.objects.values_list( "pk", flat=True) try: # first try and get any values associated with the concept_scheme # predicate.replace(SKOS, '') should yield something like 'prefLabel' or 'scopeNote', etc.. value_type = dcterms_value_types.get( valuetype=predicate.replace(DCTERMS, "")) val = self.unwrapJsonLiteral(object) if predicate == DCTERMS.title: if bulk_load is True: values.append( models.Value( pk=val["value_id"] if (val["value_id"] != "" and val["value_id"] is not None) else str(uuid.uuid4()), concept_id=concept_scheme.pk, value=val["value"], language_id=object.language or default_lang, valuetype_id="prefLabel", )) else: concept_scheme.addvalue({ 'id': val['value_id'], 'value': val['value'], 'language': object.language or default_lang, 'type': 'prefLabel', 'category': value_type.category }) # print('Casting dcterms:title to skos:prefLabel') elif predicate == DCTERMS.description: if bulk_load is True: values.append( models.Value( pk=val["value_id"] if (val["value_id"] != "" and val["value_id"] is not None) else str(uuid.uuid4()), concept_id=concept_scheme.pk, value=val["value"], language_id=object.language or default_lang, valuetype_id="scopeNote", )) else: concept_scheme.addvalue({ 'id': val['value_id'], 'value': val['value'], 'language': object.language or default_lang, 'type': 'scopeNote', 'category': value_type.category }) # print('Casting dcterms:description to skos:scopeNote') elif predicate == DCTERMS.identifier: identifier = self.unwrapJsonLiteral( str(object)) except: pass if str(SKOS) in predicate: # print predicate if predicate == SKOS.hasTopConcept: top_concept_id = self.generate_uuid_from_subject( baseuuid, object) self.relations.append({ "source": scheme_id, "type": "hasTopConcept", "target": top_concept_id, }) if bulk_load is True: values.append( models.Value( pk=identifier["value_id"] if (identifier["value_id"] != "" and identifier["value_id"] is not None) else str( uuid.uuid4()), concept_id=concept_scheme.pk, value=identifier["value"], language_id=default_lang, valuetype_id=dcterms_identifier_type.valuetype, )) else: concept_scheme.addvalue({ 'id': identifier['value_id'], 'value': identifier['value'], 'language': default_lang, 'type': dcterms_identifier_type.valuetype, 'category': dcterms_identifier_type.category }) self.nodes.append(concept_scheme) # Search for Concepts for s, v, o in graph.triples((None, SKOS.inScheme, scheme)): identifier = self.unwrapJsonLiteral(str(s)) if bulk_load is True: concept = models.Concept( pk=self.generate_uuid_from_subject(baseuuid, s), legacyoid=str(s), nodetype_id="Concept", ) else: concept = Concept({ 'id': self.generate_uuid_from_subject(baseuuid, s), 'legacyoid': str(s), 'nodetype': 'Concept' }) # loop through all the elements within a <skos:Concept> element for predicate, object in graph.predicate_objects( subject=s): if str(SKOS) in predicate or str(ARCHES) in predicate: if not self.language_exists( object, allowed_languages): allowed_languages = models.DLanguage.objects.values_list( "pk", flat=True) # this is essentially the skos element type within a <skos:Concept> # element (eg: prefLabel, broader, etc...) relation_or_value_type = predicate.replace( SKOS, "").replace(ARCHES, "") if relation_or_value_type in skos_value_types_list: value_type = skos_value_types[ relation_or_value_type] val = self.unwrapJsonLiteral(object) if bulk_load is True: values.append( models.Value( pk=val["value_id"] if (val["value_id"] != "" and val["value_id"] is not None) else str(uuid.uuid4()), concept_id=concept.pk, value=val["value"], language_id=object.language or default_lang, valuetype_id=value_type.valuetype, )) else: concept.addvalue({ 'id': val['value_id'], 'value': val['value'], 'language': object.language or default_lang, 'type': value_type.valuetype, 'category': value_type.category }) elif predicate == SKOS.broader: self.relations.append({ "source": self.generate_uuid_from_subject( baseuuid, object), "type": "narrower", "target": self.generate_uuid_from_subject( baseuuid, s), }) elif predicate == SKOS.narrower: self.relations.append({ "source": self.generate_uuid_from_subject( baseuuid, s), "type": relation_or_value_type, "target": self.generate_uuid_from_subject( baseuuid, object), }) elif predicate == SKOS.related: self.relations.append({ "source": self.generate_uuid_from_subject( baseuuid, s), "type": relation_or_value_type, "target": self.generate_uuid_from_subject( baseuuid, object), }) elif predicate == DCTERMS.identifier: identifier = self.unwrapJsonLiteral(str(object)) if bulk_load is True: values.append( models.Value( pk=identifier["value_id"] if (identifier["value_id"] != "" and identifier["value_id"] is not None) else str(uuid.uuid4()), concept_id=concept.pk, value=identifier["value"], language_id=default_lang, valuetype_id=dcterms_identifier_type.valuetype, )) else: concept.addvalue({ 'id': identifier['value_id'], 'value': identifier['value'], 'language': default_lang, 'type': dcterms_identifier_type.valuetype, 'category': dcterms_identifier_type.category }) self.nodes.append(concept) # Search for SKOS.Collections for s, v, o in graph.triples((None, RDF.type, SKOS.Collection)): # print "%s %s %s " % (s,v,o) if bulk_load is True: concept = models.Concept( pk=self.generate_uuid_from_subject(baseuuid, s), legacyoid=str(s), nodetype_id="Collection", ) else: concept = Concept({ 'id': self.generate_uuid_from_subject(baseuuid, s), 'legacyoid': str(s), 'nodetype': 'Collection' }) # loop through all the elements within a <skos:Concept> element for predicate, object in graph.predicate_objects(subject=s): if str(SKOS) in predicate or str(ARCHES) in predicate: if not self.language_exists(object, allowed_languages): allowed_languages = models.DLanguage.objects.values_list( "pk", flat=True) # this is essentially the skos element type within a <skos:Concept> # element (eg: prefLabel, broader, etc...) relation_or_value_type = predicate.replace( SKOS, "").replace(ARCHES, "") if relation_or_value_type in skos_value_types_list: value_type = skos_value_types[ relation_or_value_type] val = self.unwrapJsonLiteral(object) if bulk_load is True: values.append( models.Value( pk=val["value_id"], concept_id=concept.pk, value=val["value"], language_id=object.language or default_lang, valuetype_id=value_type.valuetype, )) else: concept.addvalue({ 'id': val['value_id'], 'value': val['value'], 'language': object.language or default_lang, 'type': value_type.valuetype, 'category': value_type.category }) self.nodes.append(concept) for s, v, o in graph.triples((None, SKOS.member, None)): # print "%s %s %s " % (s,v,o) self.member_relations.append({ "source": self.generate_uuid_from_subject(baseuuid, s), "type": "member", "target": self.generate_uuid_from_subject(baseuuid, o), }) # insert and index the concpets scheme_node = None orphaned_concepts = {} concepts = [] # bulk_create() does NOT call the object's save() method, nor pre_save/post_save # TODO: figure out how to ensure functions get called with bulk_create() with transaction.atomic(): if bulk_load is True: models.Concept.objects.bulk_create(self.nodes, ignore_conflicts=True) models.Value.objects.bulk_create(values, ignore_conflicts=True) self.logger.info( f"Bulk created: {len(self.nodes)} concepts and {len(values)} values from {path}" ) for node in self.nodes: if node.nodetype.nodetype == "ConceptScheme": scheme_node = Concept({ "id": node.conceptid, "legacyoid": str(scheme), "nodetype": "ConceptScheme", }) elif node.nodetype.nodetype == "Concept": orphaned_concepts[str(node.conceptid)] = node if staging_options == "stage": try: models.Concept.objects.get(pk=node.conceptid) except: # this is a new concept, so add a reference to it in the Candiates schema if node.nodetype.nodetype != "ConceptScheme": self.relations.append({ "source": "00000000-0000-0000-0000-000000000006", "type": "narrower", "target": node.conceptid, }) if overwrite_options == "overwrite": node.save() # concepts.append(node) elif overwrite_options == "ignore": try: # don't do anything if the concept already exists models.Concept.objects.get(pk=node.conceptid) except: # else save it node.save() # concepts.append(node) else: for node in self.nodes: if node.nodetype == 'ConceptScheme': scheme_node = node elif node.nodetype == 'Concept': orphaned_concepts[str(node.id)] = node if staging_options == 'stage': try: models.Concept.objects.get(pk=node.id) except: # this is a new concept, so add a reference to it in the Candiates schema if node.nodetype != 'ConceptScheme': self.relations.append({ 'source': '00000000-0000-0000-0000-000000000006', 'type': 'narrower', 'target': node.id }) if overwrite_options == 'overwrite': node.save() elif overwrite_options == 'ignore': try: # don't do anything if the concept already exists models.Concept.objects.get(pk=node.id) except: # else save it node.save() # Concept().bulk_save(concepts, None) # insert the concept relations # TODO: make sure this still works with code commented out, then remove # relation_objs = [] for relation in self.relations: newrelation, created = models.Relation.objects.get_or_create( conceptfrom_id=relation["source"], conceptto_id=relation["target"], relationtype_id=relation["type"], ) # models.Relation.objects.bulk_create(relation_objs) # check for orphaned concepts, every concept except the concept scheme should have an edge pointing to it if (relation["type"] == "narrower" or relation["type"] == "hasTopConcept") and orphaned_concepts.get( relation["target"]) is not None: orphaned_concepts.pop(str(relation["target"])) # relation_objs.append(newrelation) if len(orphaned_concepts.keys()) > 0: if scheme_node: orphaned_scheme = Concept({ "id": uuid.uuid4(), "legacyoid": uuid.uuid4(), "nodetype": "ConceptScheme", }) orphaned_scheme_value = None for value in scheme_node.values: if value.type == "prefLabel": orphaned_scheme.addvalue({ "id": uuid.uuid4(), "value": "ORPHANS - " + value.value, "language": value.language, "type": value.type, "category": value.category, }) orphaned_scheme.save() for ( orphaned_concept_id, orphaned_concept, ) in orphaned_concepts.items(): models.Relation.objects.create( conceptfrom_id=str(orphaned_scheme.id), conceptto_id=orphaned_concept_id, relationtype_id="narrower", ) self.logger.warning( "The SKOS file appears to have orphaned concepts.") # need to index after the concepts and relations have been entered into the db # so that the proper context gets indexed with the concept if scheme_node: scheme_node.bulk_index() # insert the concept collection relations # we do this outide a transaction so that we can load incomplete collections # relation_objs = [] # TODO: debug bulk_create to speed up this section of skos for relation in self.member_relations: try: # if bulk_load is True: # newrelation = models.Relation( # conceptfrom_id=relation['source'], # conceptto_id=relation['target'], # relationtype_id=relation['type'] # ) # relation_objs.append(newrelation) # else: newrelation, created = models.Relation.objects.get_or_create( conceptfrom_id=relation["source"], conceptto_id=relation["target"], relationtype_id=relation["type"], ) except IntegrityError as e: self.logger.warning(e) pass # if bulk_load is True: # models.Relation.objects.bulk_create(relation_objs, ignore_conflicts=True) return scheme_node else: raise Exception( "graph argument should be of type rdflib.graph.Graph")
def post(self, request, resourceid=None): es = Elasticsearch() se = SearchEngineFactory().create() res = dict(request.POST) relationship_type = res['relationship_properties[relationship_type]'][ 0] datefrom = res['relationship_properties[datefrom]'][0] dateto = res['relationship_properties[dateto]'][0] dateto = None if dateto == '' else dateto datefrom = None if datefrom == '' else datefrom notes = res['relationship_properties[notes]'][0] root_resourceinstanceid = res['root_resourceinstanceid'] instances_to_relate = [] relationships_to_update = [] if 'instances_to_relate[]' in res: instances_to_relate = res['instances_to_relate[]'] if 'relationship_ids[]' in res: relationships_to_update = res['relationship_ids[]'] def get_relatable_resources(graphid): """ Takes the graphid of a resource, finds the graphs root node, and returns the relatable graphids """ nodes = models.Node.objects.filter(graph_id=graphid) top_node = [node for node in nodes if node.istopnode == True][0] relatable_resources = [ str(node.graph.graphid) for node in top_node.get_relatable_resources() ] return relatable_resources def confirm_relationship_permitted(to_id, from_id): resource_instance_to = models.ResourceInstance.objects.filter( resourceinstanceid=to_id)[0] resource_instance_from = models.ResourceInstance.objects.filter( resourceinstanceid=from_id)[0] relatable_to = get_relatable_resources( resource_instance_to.graph.graphid) relatable_from = get_relatable_resources( resource_instance_from.graph.graphid) relatable_to_is_valid = str( resource_instance_to.graph.graphid) in relatable_from relatable_from_is_valid = str( resource_instance_from.graph.graphid) in relatable_to return (relatable_to_is_valid == True and relatable_from_is_valid == True) for instanceid in instances_to_relate: permitted = confirm_relationship_permitted( instanceid, root_resourceinstanceid[0]) if permitted == True: rr = models.ResourceXResource( resourceinstanceidfrom=Resource( root_resourceinstanceid[0]), resourceinstanceidto=Resource(instanceid), notes=notes, relationshiptype=models.Value(relationship_type), datestarted=datefrom, dateended=dateto) rr.save() else: print 'relationship not permitted' for relationshipid in relationships_to_update: rr = models.ResourceXResource.objects.get(pk=relationshipid) rr.notes = notes rr.relationshiptype = models.Value(relationship_type) rr.datestarted = datefrom rr.dateended = dateto rr.save() start = request.GET.get('start', 0) es.indices.refresh(index="resource_relations") resource = Resource.objects.get(pk=root_resourceinstanceid[0]) related_resources = resource.get_related_resources(lang="en-US", start=start, limit=15) return JSONResponse(related_resources, indent=4)