def get(self, request, conceptid): lang = request.GET.get("lang", request.LANGUAGE_CODE) languages = sort_languages(models.DLanguage.objects.all(), lang) concept_schemes = [] for concept in models.Concept.objects.filter(nodetype="ConceptScheme"): concept_schemes.append(Concept().get( id=concept.pk, include=["label"]).get_preflabel(lang=lang)) collections = [] for concept in models.Concept.objects.filter(nodetype="Collection"): collections.append(Concept().get( id=concept.pk, include=["label"]).get_preflabel(lang=lang)) context = self.get_context_data( main_script="rdm", active_page="RDM", languages=languages, conceptid=conceptid, concept_schemes=concept_schemes, collections=collections, CORE_CONCEPTS=CORE_CONCEPTS, ) context["nav"]["icon"] = "fa fa-align-left" context["nav"]["title"] = _("Reference Data Manager") context["nav"]["help"] = { "title": _("Using the RDM"), "template": "rdm-help" } return render(request, "rdm.htm", context)
def __list_nodes(self, options): data = [] try: rm = GraphModel.objects.get(graphid=options['graph']) except GraphModel.DoesNotExist: rm = None if rm is None: self.__error("", "Invalid or missing graph UUID. Use --graph") else: for node in Node.objects.filter(graph=rm): value = {"nodeid": str(node.nodeid), "name": node.name, "datatype": str(node.datatype), "key": (re.sub(r'[^A-Z_]+', '_', node.name.replace(' ', '_').upper().strip('_')))} if ((value['datatype'] == 'concept') or (value['datatype'] == 'concept-list')): if 'rdmCollection' in node.config: conceptid = node.config['rdmCollection'] if not(conceptid is None): values = Concept().get_e55_domain(conceptid) value['values'] = [] for item in values: valueobj = get_preflabel_from_valueid(item['id'], 'en') valueid = valueobj['id'] label = get_preflabel_from_valueid(valueid, 'en') value['values'].append({'valueid': valueid, 'conceptid': item['conceptid'], 'label': label['value']}) for child in Concept().get_child_concepts(item['conceptid']): value['values'].append({'valueid': child[2], 'conceptid': child[0], 'label': child[1]}) data.append(value) return data
def forwards_func(apps, schema_editor): # We get the model from the versioned app registry; # if we directly import it, it'll be the wrong version arches_concept = Concept().get(id='00000000-0000-0000-0000-000000000007', include=['label']) arches_concept.index()
def test_get_label_no_exact_match(self): """ Given no match on language code or region, return the first prefLabel found """ concept = Concept() concept.values = [ ConceptValue({ 'type': 'prefLabel', 'value': 'bier', 'language': 'nl' }), ConceptValue({ 'type': 'prefLabel', 'category': 'label', 'value': 'beer', 'language': 'es-SP' }), ConceptValue({ 'type': 'altLabel', 'category': 'label', 'value': 'test alt label en-US', 'language': 'en-US' }) ] pl = concept.get_preflabel('fr-BE') self.assertEquals(pl.type,'prefLabel') self.assertEquals(pl.value,'bier' or 'beer') self.assertEquals(pl.language,'nl' or 'es-SP')
def make_collection(request, conceptid): concept = Concept().get(id=conceptid, values=[]) try: collection_concept = concept.make_collection() return JSONResponse({'collection': collection_concept, 'message':{'title': _('Success'), 'text': _('Collection successfully created from the selected concept')}}) except: return JSONResponse({'message':{'title': _('Unable to Make Collection'), 'text': _('Unable to make a collection from the selected concept.')}}, status=500)
def manage_parents(request, conceptid): # need to check user credentials here if request.method == "POST": json = request.body if json != None: data = JSONDeserializer().deserialize(json) with transaction.atomic(): if len(data["deleted"]) > 0: concept = Concept({"id": conceptid}) for deleted in data["deleted"]: concept.addparent(deleted) concept.delete() if len(data["added"]) > 0: concept = Concept({"id": conceptid}) for added in data["added"]: concept.addparent(added) concept.save() return JSONResponse(data) else: return HttpResponseNotAllowed(["POST"]) return HttpResponseNotFound()
def test_get_altlabel_when_no_preflabel_exists_and_altlabel_only_specifies_lang_code( self): """ Given a language and region and the system only has values that specify a language code, the the system should pick the altlabel even if the altlabel doesn't specifiy a region """ concept = Concept() concept.values = [ ConceptValue({ 'type': 'prefLabel', 'category': 'label', 'value': 'test pref label es', 'language': 'es' }), ConceptValue({ 'type': 'altLabel', 'category': 'label', 'value': 'test alt label en', 'language': 'en' }) ] self.assertEqual( concept.get_preflabel(lang='en-US').value, 'test alt label en')
def get(self, request, conceptid): lang = request.GET.get('lang', settings.LANGUAGE_CODE) languages = sort_languages(models.DLanguage.objects.all(), lang) concept_schemes = [] for concept in models.Concept.objects.filter(nodetype='ConceptScheme'): concept_schemes.append(Concept().get(id=concept.pk, include=['label']).get_preflabel(lang=lang)) collections = [] for concept in models.Concept.objects.filter(nodetype='Collection'): collections.append(Concept().get(id=concept.pk, include=['label']).get_preflabel(lang=lang)) context = self.get_context_data( main_script='rdm', active_page='RDM', languages=languages, conceptid=conceptid, concept_schemes=concept_schemes, collections=collections, CORE_CONCEPTS=CORE_CONCEPTS, ) context['nav']['icon'] = 'fa fa-align-left' context['nav']['title'] = _('Reference Data Manager') context['nav']['help'] = { 'title': _('Using the RDM'), 'template': 'rdm-help', } return render(request, 'rdm.htm', context)
def test_prefer_preflabel_with_just_lang_code_match_over_exact_match_with_altlabel(self): """ Given a language and region, test should pick the preflabel even if that preflabel specifies a language only and even if an altlabel exists with the exact match """ concept = Concept() concept.values = [ ConceptValue({ 'type': 'prefLabel', 'category': 'label', 'value': 'test pref label en', 'language': 'en' }), ConceptValue({ 'type': 'prefLabel', 'category': 'label', 'value': 'test pref label es', 'language': 'es-SP' }), ConceptValue({ 'type': 'altLabel', 'category': 'label', 'value': 'test alt label en-US', 'language': 'en-US' }) ] self.assertEqual(concept.get_preflabel(lang='en-US').value, 'test pref label en')
def test_get_region_specific_preflabel_when_language_only_version_does_not_exist(self): """ Given a language only and the system only has values that with regions specified, then the system should pick the first preflabel with the same language code """ concept = Concept() concept.values = [ ConceptValue({ 'type': 'prefLabel', 'category': 'label', 'value': 'test pref label en-US', 'language': 'en-US' }), ConceptValue({ 'type': 'prefLabel', 'category': 'label', 'value': 'test pref label es', 'language': 'es-SP' }), ConceptValue({ 'type': 'altLabel', 'category': 'label', 'value': 'test alt label en-US', 'language': 'en-US' }) ] self.assertEqual(concept.get_preflabel(lang='en').value, 'test pref label en-US')
def confirm_delete(request, conceptid): lang = request.GET.get("lang", request.LANGUAGE_CODE) concept = Concept().get(id=conceptid) concepts_to_delete = [ concept.get_preflabel(lang=lang).value for key, concept in Concept.gather_concepts_to_delete(concept, lang=lang).items() ] # return HttpResponse('<div>Showing only 50 of # %s concepts</div><ul><li>%s</ul>' % (len(concepts_to_delete), '<li>'.join(concepts_to_delete[:50]) + '')) return HttpResponse("<ul><li>%s</ul>" % ("<li>".join(concepts_to_delete) + ""))
def confirm_delete(request, conceptid): lang = request.GET.get("lang", settings.LANGUAGE_CODE) concept = Concept().get(id=conceptid) concepts_to_delete = [ concept.get_preflabel(lang=lang).value for key, concept in Concept.gather_concepts_to_delete(concept, lang=lang).iteritems() ] # return HttpResponse('<div>Showing only 50 of %s concepts</div><ul><li>%s</ul>' % (len(concepts_to_delete), '<li>'.join(concepts_to_delete[:50]) + '')) return HttpResponse("<ul><li>%s</ul>" % ("<li>".join(concepts_to_delete) + ""))
def collect_card_widget_node_data(self, graph_obj, graph, parentcard, nodegroupids=[]): nodegroupids.append(str(parentcard.nodegroup_id)) for node in graph_obj["nodes"]: if node["nodegroup_id"] == str(parentcard.nodegroup_id): found = False for widget in graph_obj["widgets"]: if node["nodeid"] == str(widget.node_id): found = True try: collection_id = node["config"]["rdmCollection"] concept_collection = Concept().get_child_collections_hierarchically(collection_id, offset=None) widget.config["options"] = concept_collection except Exception as e: pass break if not found: for card in graph_obj["cards"]: if card["nodegroup_id"] == node["nodegroup_id"]: widget = models.DDataType.objects.get(pk=node["datatype"]).defaultwidget if widget: widget_model = models.CardXNodeXWidget() widget_model.node_id = node["nodeid"] widget_model.card_id = card["cardid"] widget_model.widget_id = widget.pk widget_model.config = widget.defaultconfig try: collection_id = node["config"]["rdmCollection"] if collection_id: concept_collection = Concept().get_child_collections_hierarchically(collection_id, offset=None) widget_model.config["options"] = concept_collection except Exception as e: pass widget_model.label = node["name"] graph_obj["widgets"].append(widget_model) break if node["datatype"] == "resource-instance" or node["datatype"] == "resource-instance-list": if node["config"]["graphs"] is not None: graph_ids = [] for graph in node["config"]["graphs"]: graphuuid = uuid.UUID(graph["graphid"]) graph_ids.append(str(graphuuid)) node["config"]["options"] = [] for resource_instance in Resource.objects.filter(graph_id__in=graph_ids): node["config"]["options"].append( { "id": str(resource_instance.pk), "name": resource_instance.displayname, "graphid": str(resource_instance.graph_id), } ) for subcard in parentcard.cards: self.collect_card_widget_node_data(graph_obj, graph, subcard, nodegroupids) return graph_obj
def get_evaluation_path(valueid): value = models.Values.objects.get(pk=valueid) concept_graph = Concept().get(id=value.conceptid_id, include_subconcepts=False, include_parentconcepts=True, include_relatedconcepts=False, up_depth_limit=None, lang=lang) paths = [] for path in concept_graph.get_paths(lang=lang)[0]: if path['label'] != 'Arches' and path['label'] != 'Evaluation Criteria Type': paths.append(path['label']) return '; '.join(paths)
def index_concepts(): """ Collects all concepts and indexes both concepts and concept_labels """ se = SearchEngineFactory().create() se.delete_index(index='concept_labels') se.delete( index='term', body= '{"query":{"bool":{"must_not":[{"constant_score":{"filter":{"missing":{"field":"value.options.conceptid"}}}}],"must":[],"should":[]}}}' ) Resource().prepare_term_index(create=True) print 'indexing concepts' start = datetime.now() cursor = connection.cursor() cursor.execute("""select conceptid from concepts.concepts""") conceptids = cursor.fetchall() for c in conceptids: if c[0] not in CORE_CONCEPTS: concept = Concept().get(id=c[0], include_subconcepts=True, include_parentconcepts=False, include=['label']) concept.index() end = datetime.now() duration = end - start print 'indexing concepts required', duration.seconds, 'seconds' cursor = connection.cursor() sql = """ select conceptid, conceptlabel from concepts.vw_concepts where conceptid not in ('%s') """ % ("','".join(CORE_CONCEPTS)) cursor.execute(sql) concepts = cursor.fetchall() concept_index_results = {'count': len(concepts), 'passed': 0, 'failed': 0} for conceptid, conceptvalue in concepts: result = get_indexed_concepts(se, conceptid, conceptvalue) if result != 'passed': concept_index_results['failed'] += 1 else: concept_index_results['passed'] += 1 status = 'Passed' if concept_index_results['failed'] == 0 else 'Failed' print '\nConcept Index Results:' print "Status: {0}, In Database: {1}, Indexed: {2}".format( status, concept_index_results['count'], concept_index_results['passed'])
def collect_card_widget_node_data(self, graph_obj, graph, parentcard, nodegroupids=[]): nodegroupids.append(str(parentcard.nodegroup_id)) for node in graph_obj['nodes']: if node['nodegroup_id'] == str(parentcard.nodegroup_id): found = False for widget in graph_obj['widgets']: if node['nodeid'] == str(widget.node_id): found = True try: collection_id = node['config']['rdmCollection'] concept_collection = Concept().get_child_collections_hierarchically(collection_id, offset=None) widget.config['options'] = concept_collection except Exception as e: pass break if not found: for card in graph_obj['cards']: if card['nodegroup_id'] == node['nodegroup_id']: widget = models.DDataType.objects.get(pk=node['datatype']).defaultwidget if widget: widget_model = models.CardXNodeXWidget() widget_model.node_id = node['nodeid'] widget_model.card_id = card['cardid'] widget_model.widget_id = widget.pk widget_model.config = widget.defaultconfig try: collection_id = node['config']['rdmCollection'] if collection_id: concept_collection = Concept().get_child_collections_hierarchically(collection_id, offset=None) widget_model.config['options'] = concept_collection except Exception as e: pass widget_model.label = node['name'] graph_obj['widgets'].append(widget_model) break if node['datatype'] == 'resource-instance' or node['datatype'] == 'resource-instance-list': if node['config']['graphid'] is not None: try: graphuuid = uuid.UUID(node['config']['graphid'][0]) graph_id = unicode(graphuuid) except ValueError as e: graphuuid = uuid.UUID(node['config']['graphid']) graph_id = unicode(graphuuid) node['config']['options'] = [] for resource_instance in Resource.objects.filter(graph_id=graph_id): node['config']['options'].append({'id': str(resource_instance.pk), 'name': resource_instance.displayname}) for subcard in parentcard.cards: self.collect_card_widget_node_data(graph_obj, graph, subcard, nodegroupids) return graph_obj
def confirm_delete(request, conceptid): lang = request.GET.get('lang', settings.LANGUAGE_CODE) try: concept = Concept().get(id=conceptid) except Exception as es1: return render_to_response('404.htm', { 'main_script': '404', 'active_page': '404' }, context_instance=RequestContext(request)) concepts_to_delete = [concept.get_preflabel(lang=lang).value for key, concept in Concept.gather_concepts_to_delete(concept, lang=lang).iteritems()] #return HttpResponse('<div>Showing only 50 of %s concepts</div><ul><li>%s</ul>' % (len(concepts_to_delete), '<li>'.join(concepts_to_delete[:50]) + '')) return HttpResponse('<ul><li>%s</ul>' % ('<li>'.join(concepts_to_delete) + ''))
def test_get_altlabel_when_no_preflabel_exists(self): """ Given a language and region, test should pick the altlabel when no preflabel exists """ concept = Concept() concept.values = [ ConceptValue({"type": "prefLabel", "category": "label", "value": "test pref label es", "language": "es-SP"}), ConceptValue({"type": "altLabel", "category": "label", "value": "test alt label en-US", "language": "en-US"}), ] self.assertEqual(concept.get_preflabel(lang="en-US").value, "test alt label en-US")
def test_get_altlabel_when_no_preflabel_exists_and_altlabel_only_specifies_lang_code(self): """ Given a language and region and the system only has values that specify a language code, the the system should pick the altlabel even if the altlabel doesn't specifiy a region """ concept = Concept() concept.values = [ ConceptValue({"type": "prefLabel", "category": "label", "value": "test pref label es", "language": "es"}), ConceptValue({"type": "altLabel", "category": "label", "value": "test alt label en", "language": "en"}), ] self.assertEqual(concept.get_preflabel(lang="en-US").value, "test alt label en")
def forwards_func(apps, schema_editor): # We get the model from the versioned app registry; # if we directly import it, it'll be the wrong version # index base Arches concept arches_concept = Concept().get(id="00000000-0000-0000-0000-000000000001", include=["label"]) arches_concept.index() DValueType = apps.get_model("models", "DValueType") DValueType.objects.create(valuetype="identifier", category="identifiers", namespace="dcterms", datatype="text")
def import_reference_data(reference_data): # with transaction.atomic(): # if reference_data != '': for data in reference_data: print '\nLOADING {0} CONCEPT SCHEME FROM ARCHES JSON'.format( data['legacyoid']) print '---------------------------------------------' reporter = ConceptImportReporter(data) def create_collections(concept): relations = { '': 'hasCollection', 'hasTopConcept': 'member', 'narrower': 'member', 'narrowerTransitive': 'member' } for subconcept in concept.subconcepts: if concept.relationshiptype in relations.keys(): if concept.id == '00000000-0000-0000-0000-000000000001': concept.id = '00000000-0000-0000-0000-000000000003' models.Relation.objects.get_or_create( conceptfrom_id=concept.id, conceptto_id=subconcept.id, relationtype_id=relations[concept.relationshiptype]) reporter.update_concepts_saved() concept = Concept(data) concept.save() concept.traverse(create_collections, 'down') concept.index(scheme=concept) reporter.report_results()
def test_get_label_no_exact_match(self): """ Given no match on language code or region, return the first prefLabel found """ values = [ {'type': 'prefLabel', 'value': 'bier', 'language': 'nl'}, {'type': 'prefLabel', 'value': 'beer', 'language': 'en'}, ] c = Concept({'values': values}) pl = c.get_preflabel('fr-BE') self.assertEquals(pl.type,'prefLabel') self.assertEquals(pl.value,'bier') self.assertEquals(pl.language,'nl')
def get_reference_data_for_export(conceptids=None): reference_data_dict = {} reference_data = [] if conceptids is None or conceptids[0] == "all" or conceptids == [""]: reference_data.append(Concept().get( "00000000-0000-0000-0000-000000000001", include_subconcepts=True, semantic=True)) else: for conceptid in conceptids: reference_data.append(Concept().get(uuid.UUID(str(conceptid)), include_subconcepts=True, semantic=True)) reference_data_dict["reference_data"] = reference_data return reference_data_dict
def test_get_region_specific_preflabel_when_language_only_version_does_not_exist(self): """ Given a language only and the system only has values that with regions specified, then the system should pick the first preflabel with the same language code """ concept = Concept() concept.values = [ ConceptValue({"type": "prefLabel", "category": "label", "value": "test pref label en-US", "language": "en-US"}), ConceptValue({"type": "prefLabel", "category": "label", "value": "test pref label es", "language": "es-SP"}), ConceptValue({"type": "altLabel", "category": "label", "value": "test alt label en-US", "language": "en-US"}), ] self.assertEqual(concept.get_preflabel(lang="en").value, "test pref label en-US")
def test_prefer_preflabel_with_just_lang_code_match_over_exact_match_with_altlabel(self): """ Given a language and region, test should pick the preflabel even if that preflabel specifies a language only and even if an altlabel exists with the exact match """ concept = Concept() concept.values = [ ConceptValue({"type": "prefLabel", "category": "label", "value": "test pref label en", "language": "en"}), ConceptValue({"type": "prefLabel", "category": "label", "value": "test pref label es", "language": "es-SP"}), ConceptValue({"type": "altLabel", "category": "label", "value": "test alt label en-US", "language": "en-US"}), ] self.assertEqual(concept.get_preflabel(lang="en-US").value, "test pref label en")
def paged_dropdown(request): conceptid = request.GET.get("conceptid") query = request.GET.get("query", None) query = None if query == "" else query page = int(request.GET.get("page", 1)) limit = 50 offset = (page - 1) * limit results = Concept().get_child_collections_hierarchically(conceptid, offset=offset, limit=limit, query=query) total_count = results[0][3] if len(results) > 0 else 0 data = [ dict(list(zip(["valueto", "depth", "collector"], d))) for d in results ] data = [ dict(list( zip(["id", "text", "conceptid", "language", "type"], d["valueto"].values())), depth=d["depth"], collector=d["collector"]) for d in data ] return JSONResponse({ "results": data, "more": offset + limit < total_count })
def test_hierarchical_relationships(self): result = JSONDeserializer().deserialize(JSONSerializer().serialize( Concept().get(id='708e87fd-35bc-3270-b322-b551cd63c589', include_subconcepts=True, depth_limit=1))) children = len(result['subconcepts']) self.assertEqual(children, 3)
def test_value_import(self): result = JSONDeserializer().deserialize(JSONSerializer().serialize( Concept().get(id='32d8ca56-2aa1-3890-b95b-3fbb119460ad', include_subconcepts=True, depth_limit=1))) values = len(result['values']) self.assertEqual(values, 5)
def get_concept_uuid_for_aux_map_entry(self, concept_name, concept_value): ''' Reads concept labels(prefLabels) given in AUXILIARY_MAP and returns a list of conceptids. ''' concept = None try: concept = Concepts.objects.get(legacyoid=concept_name) except: print "No Concept found with the name %s" % concept_name if concept is not None: concept_graph = Concept().get(id=concept.pk, include_subconcepts=True, include=['label']) if concept_graph.subconcepts: for subconcept in concept_graph.subconcepts[0].subconcepts: for value in subconcept.values: if value.type == "prefLabel" and value.value == concept_value: return subconcept.id print '[ERROR]: %s is not found in %s' % (concept_value, concept_name) return
def index_concepts_by_entitytypeid(self, entitytypeid): entitytype = archesmodels.EntityTypes.objects.get(pk = entitytypeid) conceptid = entitytype.conceptid_id concept_graph = Concept().get(id=conceptid, include_subconcepts=True, exclude=['note']) if len(concept_graph.subconcepts) > 0: data = JSONSerializer().serializeToPython(concept_graph, ensure_ascii=True, indent=4) self.index(data, 'concept', entitytypeid, 'id', processdoc=None, getid=None, bulk=False)
def resource_manager(request, resourcetypeid='', form_id='default', resourceid=''): if resourceid != '': resource = Resource(resourceid) elif resourcetypeid != '': resource = Resource({'entitytypeid': resourcetypeid}) if form_id == 'default': form_id = resource.form_groups[0]['forms'][0]['id'] form = resource.get_form(form_id) if request.method == 'DELETE': resource.delete_index() se = SearchEngineFactory().create() realtionships = resource.get_related_resources(return_entities=False) for realtionship in realtionships: se.delete(index='resource_relations', doc_type='all', id=realtionship.resourcexid) realtionship.delete() resource.delete() return JSONResponse({ 'success': True }) if request.method == 'POST': data = JSONDeserializer().deserialize(request.POST.get('formdata', {})) form.update(data, request.FILES) with transaction.atomic(): if resourceid != '': resource.delete_index() resource.save(user=request.user) resource.index() resourceid = resource.entityid return redirect('resource_manager', resourcetypeid=resourcetypeid, form_id=form_id, resourceid=resourceid) min_max_dates = models.Dates.objects.aggregate(Min('val'), Max('val')) if request.method == 'GET': if form != None: lang = request.GET.get('lang', settings.LANGUAGE_CODE) form.load(lang) return render(request, 'resource-manager.htm', { 'form': form, 'formdata': JSONSerializer().serialize(form.data), 'form_template': 'views/forms/' + form_id + '.htm', 'form_id': form_id, 'resourcetypeid': resourcetypeid, 'resourceid': resourceid, 'main_script': 'resource-manager', 'active_page': 'ResourceManger', 'resource': resource, 'resource_name': resource.get_primary_name(), 'resource_type_name': resource.get_type_name(), 'form_groups': resource.form_groups, 'min_date': min_max_dates['val__min'].year if min_max_dates['val__min'] != None else 0, 'max_date': min_max_dates['val__max'].year if min_max_dates['val__min'] != None else 1, 'timefilterdata': JSONSerializer().serialize(Concept.get_time_filter_data()), }) else: return HttpResponseNotFound('<h1>Arches form not found.</h1>')
def get(self, request, cardid): try: card = Card.objects.get(cardid=cardid) self.graph = Graph.objects.get(graphid=card.graph_id) except (Card.DoesNotExist): # assume the cardid is actually a graph id card = Card.objects.get(cardid=Graph.objects.get( graphid=cardid).get_root_card().cardid) self.graph = Graph.objects.get(graphid=card.graph_id) if self.graph.isresource == True: return redirect('card_manager', graphid=cardid) card.confirm_enabled_state(request.user, card.nodegroup) for c in card.cards: c.confirm_enabled_state(request.user, c.nodegroup) datatypes = models.DDataType.objects.all() widgets = models.Widget.objects.all() geocoding_providers = models.Geocoder.objects.all() map_layers = models.MapLayer.objects.all() map_sources = models.MapSource.objects.all() resource_graphs = Graph.objects.exclude(pk=card.graph_id).exclude( pk=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID).exclude( isresource=False).exclude(isactive=False) lang = request.GET.get('lang', settings.LANGUAGE_CODE) concept_collections = Concept().concept_tree(mode='collections', lang=lang) ontology_properties = [] card_root_node = models.Node.objects.get(nodeid=card.nodegroup_id) for item in self.graph.get_valid_ontology_classes( nodeid=card.nodegroup_id): if card_root_node.ontologyclass in item['ontology_classes']: ontology_properties.append(item['ontology_property']) ontology_properties = sorted(ontology_properties, key=lambda item: item) context = self.get_context_data( main_script='views/graph/card-configuration-manager', graph_id=self.graph.pk, card=JSONSerializer().serialize(card), datatypes_json=JSONSerializer().serialize(datatypes), geocoding_providers=geocoding_providers, datatypes=datatypes, widgets=widgets, widgets_json=JSONSerializer().serialize(widgets), map_layers=map_layers, map_sources=map_sources, resource_graphs=resource_graphs, concept_collections=concept_collections, ontology_properties=JSONSerializer().serialize( ontology_properties), ) context['nav']['title'] = self.graph.name context['nav']['menu'] = True context['nav']['help'] = (_('Configuring Cards and Widgets'), 'help/card-designer-help.htm') return render(request, 'views/graph/card-configuration-manager.htm', context)
def export(request, conceptid): concept_graphs = [Concept().get(id=conceptid, include_subconcepts=True, include_parentconcepts=False, include_relatedconcepts=True, depth_limit=None, up_depth_limit=None)] skos = SKOSWriter() return HttpResponse(skos.write(concept_graphs, format="pretty-xml"), content_type="application/xml")
def get(self, request, conceptid=None): if user_can_read_concepts(user=request.user): include_subconcepts = request.GET.get('includesubconcepts', 'true') == 'true' include_parentconcepts = request.GET.get('includeparentconcepts', 'true') == 'true' include_relatedconcepts = request.GET.get('includerelatedconcepts', 'true') == 'true' depth_limit = request.GET.get('depthlimit', None) lang = request.GET.get('lang', settings.LANGUAGE_CODE) try: indent = int(request.GET.get('indent', None)) except: indent = None if conceptid: try: ret = [] concept_graph = Concept().get(id=conceptid, include_subconcepts=include_subconcepts, include_parentconcepts=include_parentconcepts, include_relatedconcepts=include_relatedconcepts, depth_limit=depth_limit, up_depth_limit=None, lang=lang) ret.append(concept_graph) except models.Concept.DoesNotExist: return JSONResponse(status=404) except: return JSONResponse(status=500) else: return JSONResponse(status=500) else: return JSONResponse(status=500) return JSONResponse(ret, indent=indent)
def forwards_func(apps, schema_editor): # We get the model from the versioned app registry; # if we directly import it, it'll be the wrong version extensions = [os.path.join(settings.ONTOLOGY_PATH, x) for x in settings.ONTOLOGY_EXT] management.call_command('load_ontology', source=os.path.join(settings.ONTOLOGY_PATH, settings.ONTOLOGY_BASE), version=settings.ONTOLOGY_BASE_VERSION, ontology_name=settings.ONTOLOGY_BASE_NAME, id=settings.ONTOLOGY_BASE_ID, extensions=','.join(extensions), verbosity=0) Ontology = apps.get_model("models", "Ontology") Node = apps.get_model("models", "Node") Edge = apps.get_model("models", "Edge") for ontology in Ontology.objects.filter(parentontology=None): g = Graph() g.parse(ontology.path.path) for extension in Ontology.objects.filter(parentontology=ontology): g.parse(extension.path.path) ontology_classes = set() ontology_properties = set() for ontology_property,p,o in g.triples((None, None, RDF.Property)): ontology_properties.add(ontology_property) for s,p,domain_class in g.triples((ontology_property, RDFS.domain, None)): ontology_classes.add(domain_class) for s,p,range_class in g.triples((ontology_property, RDFS.range, None)): ontology_classes.add(range_class) for ontology_class,p,o in g.triples((None, None, RDFS.Class)): ontology_classes.add(ontology_class) for ontology_class in ontology_classes: for node in Node.objects.filter(ontologyclass=str(ontology_class).split('/')[-1], graph__in=ontology.graphs.all()): node.ontologyclass = ontology_class node.save() for ontology_property in ontology_properties: for edge in Edge.objects.filter(ontologyproperty=str(ontology_property).split('/')[-1], graph__in=ontology.graphs.all()): edge.ontologyproperty = ontology_property edge.save() # index base Arches concept arches_concept = Concept().get(id='00000000-0000-0000-0000-000000000001', include=['label']) arches_concept.index() DValueType = apps.get_model("models", "DValueType") DValueType.objects.create(valuetype='identifier', category='identifiers', namespace='dcterms', datatype='text')
def index_concepts(): """ Collects all concepts and indexes both concepts and concept_labels """ se = SearchEngineFactory().create() se.delete_index(index='concept_labels') se.delete(index='term', body='{"query":{"bool":{"must_not":[{"constant_score":{"filter":{"missing":{"field":"value.options.conceptid"}}}}],"must":[],"should":[]}}}') Resource().prepare_term_index(create=True) print 'indexing concepts' start = datetime.now() cursor = connection.cursor() cursor.execute("""select conceptid from concepts.concepts""") conceptids = cursor.fetchall() for c in conceptids: if c[0] not in CORE_CONCEPTS: concept = Concept().get(id=c[0], include_subconcepts=True, include_parentconcepts=False, include=['label']) concept.index() end = datetime.now() duration = end - start print 'indexing concepts required', duration.seconds, 'seconds' cursor = connection.cursor() sql = """ select conceptid, conceptlabel from concepts.vw_concepts where conceptid not in ('%s') """ % ("','".join(CORE_CONCEPTS)) cursor.execute(sql) concepts = cursor.fetchall() concept_index_results = {'count':len(concepts), 'passed':0, 'failed':0} for conceptid, conceptvalue in concepts: result = get_indexed_concepts(se, conceptid, conceptvalue) if result != 'passed': concept_index_results['failed'] += 1 else: concept_index_results['passed'] += 1 status = 'Passed' if concept_index_results['failed'] == 0 else 'Failed' print '\nConcept Index Results:' print "Status: {0}, In Database: {1}, Indexed: {2}".format(status, concept_index_results['count'], concept_index_results['passed'])
def home_page(request): lang = request.GET.get('lang', settings.LANGUAGE_CODE) min_max_dates = models.Dates.objects.aggregate(Min('val'), Max('val')) return render(request, 'search.htm', { 'main_script': 'search', 'active_page': 'Search', 'min_date': min_max_dates['val__min'].year if min_max_dates['val__min'] != None else 0, 'max_date': min_max_dates['val__max'].year if min_max_dates['val__min'] != None else 1, 'timefilterdata': JSONSerializer().serialize(Concept.get_time_filter_data()), })
def add_concepts_from_sparql_endpoint(request, conceptid): if request.method == 'POST': json = request.body if json != None: data = JSONDeserializer().deserialize(json) parentconcept = Concept({ 'id': conceptid, 'nodetype': data['model']['nodetype'] }) if parentconcept.nodetype == 'Concept': relationshiptype = 'narrower' elif parentconcept.nodetype == 'ConceptScheme': relationshiptype = 'hasTopConcept' provider = get_sparql_providers(data['endpoint']) try: parentconcept.subconcepts = provider.get_concepts(data['ids']) except Exception as e: return HttpResponseServerError(e.message) for subconcept in parentconcept.subconcepts: subconcept.relationshiptype = relationshiptype parentconcept.save() parentconcept.index() return JSONResponse(parentconcept, indent=4) else: return HttpResponseNotAllowed(['POST']) return HttpResponseNotFound()
def get_concepts(self, uris): """ Get a list of concepts given a list of AAT uris like http://vocab.getty.edu/aat/300380087 """ concepts = [] langs = [] for lang in self.allowed_languages: langs.append('\"%s\"' % (lang)) for uri in uris.split(','): query = """ SELECT ?value ?type WHERE { { <%s> skos:prefLabel ?value . BIND('prefLabel' AS ?type) } UNION { <%s> skos:scopeNote [rdf:value ?value] . BIND('scopeNote' AS ?type) } FILTER (lang(?value) in (%s)) }""" % (uri, uri, ','.join(langs)) results = self.perform_sparql_query(query) if len(results["results"]["bindings"]) > 0 : concept = Concept() concept.nodetype = 'Concept' for result in results["results"]["bindings"]: concept.addvalue({ 'type': result["type"]["value"], 'value': result["value"]["value"], 'language': result["value"]["xml:lang"] }) concepts.append(concept) else: raise Exception(_("<strong>Error in SPARQL query:</strong><br>Test this query directly by pasting the query below into the Getty's own SPARQL endpoint at <a href='http://vocab.getty.edu/sparql' target='_blank'>http://vocab.getty.edu/sparql</a><i><pre>%s</pre></i>Query returned 0 results, please check the query for errors. You may need to add the appropriate languages into the database for this query to work<br><br>") % (query.replace('<', '<').replace('>', '>'))) return concepts
def add_concepts_from_sparql_endpoint(request, conceptid): if request.method == "POST": json = request.body if json != None: data = JSONDeserializer().deserialize(json) parentconcept = Concept({"id": conceptid, "nodetype": data["model"]["nodetype"]}) if parentconcept.nodetype == "Concept": relationshiptype = "narrower" elif parentconcept.nodetype == "ConceptScheme": relationshiptype = "hasTopConcept" provider = get_sparql_providers(data["endpoint"]) try: parentconcept.subconcepts = provider.get_concepts(data["ids"]) except Exception as e: return HttpResponseServerError(e.message) for subconcept in parentconcept.subconcepts: subconcept.relationshiptype = relationshiptype parentconcept.save() parentconcept.index() return JSONResponse(parentconcept, indent=4) else: return HttpResponseNotAllowed(["POST"]) return HttpResponseNotFound()
def import_concepts(reference_data): concepts = reference_data[0]['concepts'] values = reference_data[1]['values'] relations = reference_data[2]['relations'] concept_objs = {} for concept in concepts: concept_obj = Concept() concept_obj.id = concept['conceptid'] concept_obj.nodetype = concept['nodetype'] concept_obj.legacyoid = concept['legacyoid'] concept_obj.save() concept_objs[concept_obj.id] = concept_obj existing_valuetypes = [o.valuetype for o in models.DValueType.objects.all()] for value in values: if value['valuetype'] not in existing_valuetypes: models.DValueType.objects.create(valuetype = value['valuetype'], category = 'undefined', namespace = 'arches') existing_valuetypes.append(value['valuetype']) conceptvalue_obj = ConceptValue() conceptvalue_obj.id = value['valueid'] conceptvalue_obj.conceptid = value['conceptid'] conceptvalue_obj.type = value['valuetype'] conceptvalue_obj.value = value['value'] conceptvalue_obj.language = value['languageid'] conceptvalue_obj.save() for relation in relations: if relation['conceptidfrom'] in concept_objs and relation['conceptidto'] in concept_objs: conceptfrom = concept_objs[relation['conceptidfrom']] conceptto = concept_objs[relation['conceptidto']] conceptfrom.add_relation(conceptto, relation['relationtype'])
def home_page(request): lang = request.GET.get('lang', settings.LANGUAGE_CODE) min_max_dates = models.Dates.objects.aggregate(Min('val'), Max('val')) return render_to_response('search.htm', { 'main_script': 'search', 'active_page': 'Search', 'min_date': min_max_dates['val__min'].year if min_max_dates['val__min'] != None else 0, 'max_date': min_max_dates['val__max'].year if min_max_dates['val__min'] != None else 1, 'timefilterdata': JSONSerializer().serialize(Concept.get_time_filter_data()), 'group_options': settings.SEARCH_GROUP_ROOTS }, context_instance=RequestContext(request))
def test_get_altlabel_when_no_preflabel_exists(self): """ Given a language and region, test should pick the altlabel when no preflabel exists """ concept = Concept() concept.values = [ ConceptValue({ 'type': 'prefLabel', 'category': 'label', 'value': 'test pref label es', 'language': 'es-SP' }), ConceptValue({ 'type': 'altLabel', 'category': 'label', 'value': 'test alt label en-US', 'language': 'en-US' }) ] self.assertEqual(concept.get_preflabel(lang='en-US').value, 'test alt label en-US')
def test_prefer_preflabel_over_altlabel(self): """ Test to confirm the proper retrieval of the prefLabel based on different language requirements """ concept = Concept() concept.values = [ ConceptValue({ 'type': 'prefLabel', 'category': 'label', 'value': 'test pref label en-US', 'language': 'en-US' }), ConceptValue({ 'type': 'prefLabel', 'category': 'label', 'value': 'test pref label en', 'language': 'en' }), ConceptValue({ 'type': 'prefLabel', 'category': 'label', 'value': 'test pref label es-SP', 'language': 'es-SP' }), ConceptValue({ 'type': 'altLabel', 'category': 'label', 'value': 'test alt label en-US', 'language': 'en-US' }) ] self.assertEqual(concept.get_preflabel(lang='en-US').value, 'test pref label en-US') self.assertEqual(concept.get_preflabel(lang='en').value, 'test pref label en') self.assertEqual(concept.get_preflabel().value, 'test pref label %s' % (test_settings.LANGUAGE_CODE))
def test_get_altlabel_when_no_preflabel_exists_and_altlabel_only_specifies_lang_code(self): """ Given a language and region and the system only has values that specify a language code, the the system should pick the altlabel even if the altlabel doesn't specifiy a region """ concept = Concept() concept.values = [ ConceptValue({ 'type': 'prefLabel', 'category': 'label', 'value': 'test pref label es', 'language': 'es' }), ConceptValue({ 'type': 'altLabel', 'category': 'label', 'value': 'test alt label en', 'language': 'en' }) ] self.assertEqual(concept.get_preflabel(lang='en-US').value, 'test alt label en')
def home_page(request): lang = request.GET.get("lang", settings.LANGUAGE_CODE) min_max_dates = models.Dates.objects.aggregate(Min("val"), Max("val")) return render_to_response( "search.htm", { "main_script": "search", "active_page": "Search", "min_date": min_max_dates["val__min"].year if min_max_dates["val__min"] != None else 0, "max_date": min_max_dates["val__max"].year if min_max_dates["val__min"] != None else 1, "timefilterdata": JSONSerializer().serialize(Concept.get_time_filter_data()), }, context_instance=RequestContext(request), )
def import_reference_data(reference_data): # with transaction.atomic(): # if reference_data != '': for data in reference_data: print '\nLOADING {0} CONCEPT SCHEME FROM ARCHES JSON'.format(data['legacyoid']) print '---------------------------------------------' def create_collections(concept): relations = {'':'hasCollection', 'hasTopConcept':'member', 'narrower':'member', 'narrowerTransitive':'member'} for subconcept in concept.subconcepts: if concept.relationshiptype in relations.keys(): if concept.id == '00000000-0000-0000-0000-000000000001': concept.id = '00000000-0000-0000-0000-000000000003' models.Relation.objects.get_or_create(conceptfrom_id=concept.id, conceptto_id=subconcept.id, relationtype_id=relations[concept.relationshiptype]) concept = Concept(data) concept.save() concept.traverse(create_collections, 'down') concept.index(scheme=concept)
def test_create_concept(self): """ Test of basic CRUD on a Concept model """ concept_in = Concept() concept_in.nodetype = 'Concept' concept_in.values = [ConceptValue({ #id: '', #conceptid: '', 'type': 'prefLabel', 'category': 'label', 'value': 'test pref label', 'language': 'en-US' })] concept_in.save() concept_out = Concept().get(id=concept_in.id) self.assertEqual(concept_out.id, concept_in.id) self.assertEqual(concept_out.values[0].value, 'test pref label') label = concept_in.values[0] label.value = 'updated pref label' concept_in.values[0] = label concept_in.save() concept_out = Concept().get(id=concept_in.id) self.assertEqual(concept_out.values[0].value, 'updated pref label') concept_out.delete(delete_self=True) with self.assertRaises(models.Concept.DoesNotExist): deleted_concept = Concept().get(id=concept_out.id)
def concept(request, conceptid): f = request.GET.get('f', 'json') mode = request.GET.get('mode', '') lang = request.GET.get('lang', settings.LANGUAGE_CODE) pretty = request.GET.get('pretty', False) if request.method == 'GET': include_subconcepts = request.GET.get('include_subconcepts', 'true') == 'true' include_parentconcepts = request.GET.get('include_parentconcepts', 'true') == 'true' include_relatedconcepts = request.GET.get('include_relatedconcepts', 'true') == 'true' emulate_elastic_search = request.GET.get('emulate_elastic_search', 'false') == 'true' depth_limit = request.GET.get('depth_limit', None) if f == 'html': depth_limit = 1 if not conceptid: return render(request, 'views/rdm/concept-report.htm', { 'lang': lang, 'concept_count': models.Concept.objects.filter(nodetype='Concept').count(), 'collection_count': models.Concept.objects.filter(nodetype='Collection').count(), 'scheme_count': models.Concept.objects.filter(nodetype='ConceptScheme').count(), 'entitytype_count': models.Concept.objects.filter(nodetype='EntityType').count(), 'default_report': True }) ret = [] labels = [] this_concept = Concept().get(id=conceptid) if f == 'html': if mode == '' and (this_concept.nodetype == 'Concept' or this_concept.nodetype == 'ConceptScheme' or this_concept.nodetype == 'EntityType'): concept_graph = Concept().get(id=conceptid, include_subconcepts=include_subconcepts, include_parentconcepts=include_parentconcepts, include_relatedconcepts=include_relatedconcepts, depth_limit=depth_limit, up_depth_limit=None, lang=lang) else: concept_graph = Concept().get(id=conceptid, include_subconcepts=include_subconcepts, include_parentconcepts=include_parentconcepts, include_relatedconcepts=include_relatedconcepts, depth_limit=depth_limit, up_depth_limit=None, lang=lang, semantic=False) languages = models.DLanguage.objects.all() valuetypes = models.DValueType.objects.all() relationtypes = models.DRelationType.objects.all() prefLabel = concept_graph.get_preflabel(lang=lang) for subconcept in concept_graph.subconcepts: subconcept.prefLabel = subconcept.get_preflabel(lang=lang) for relatedconcept in concept_graph.relatedconcepts: relatedconcept.prefLabel = relatedconcept.get_preflabel(lang=lang) for value in concept_graph.values: if value.category == 'label': labels.append(value) if mode == '' and (this_concept.nodetype == 'Concept' or this_concept.nodetype == 'ConceptScheme' or this_concept.nodetype == 'EntityType'): if concept_graph.nodetype == 'ConceptScheme': parent_relations = relationtypes.filter(category='Properties') else: parent_relations = relationtypes.filter(category='Semantic Relations').exclude(relationtype = 'related').exclude(relationtype='broader').exclude(relationtype='broaderTransitive') return render(request, 'views/rdm/concept-report.htm', { 'lang': lang, 'prefLabel': prefLabel, 'labels': labels, 'concept': concept_graph, 'languages': languages, 'sparql_providers': get_sparql_providers(), 'valuetype_labels': valuetypes.filter(category='label'), 'valuetype_notes': valuetypes.filter(category='note'), 'valuetype_related_values': valuetypes.filter(category='undefined'), 'parent_relations': parent_relations, 'related_relations': relationtypes.filter(Q(category='Mapping Properties') | Q(relationtype = 'related')), 'concept_paths': concept_graph.get_paths(lang=lang), 'graph_json': JSONSerializer().serialize(concept_graph.get_node_and_links(lang=lang)), 'direct_parents': [parent.get_preflabel(lang=lang) for parent in concept_graph.parentconcepts] }) else: return render(request, 'views/rdm/entitytype-report.htm', { 'lang': lang, 'prefLabel': prefLabel, 'labels': labels, 'concept': concept_graph, 'languages': languages, 'valuetype_labels': valuetypes.filter(category='label'), 'valuetype_notes': valuetypes.filter(category='note'), 'valuetype_related_values': valuetypes.filter(category='undefined'), 'related_relations': relationtypes.filter(relationtype = 'member'), 'concept_paths': concept_graph.get_paths(lang=lang) }) concept_graph = Concept().get(id=conceptid, include_subconcepts=include_subconcepts, include_parentconcepts=include_parentconcepts, include_relatedconcepts=include_relatedconcepts, depth_limit=depth_limit, up_depth_limit=None, lang=lang) if f == 'skos': include_parentconcepts = False include_subconcepts = True depth_limit = None skos = SKOSWriter() return HttpResponse(skos.write(concept_graph, format="pretty-xml"), content_type="application/xml") if emulate_elastic_search: ret.append({'_type': id, '_source': concept_graph}) else: ret.append(concept_graph) if emulate_elastic_search: ret = {'hits':{'hits':ret}} return JSONResponse(ret, indent=4 if pretty else None) if request.method == 'POST': if len(request.FILES) > 0: skosfile = request.FILES.get('skosfile', None) imagefile = request.FILES.get('file', None) if imagefile: value = models.FileValue(valueid = str(uuid.uuid4()), value = request.FILES.get('file', None), conceptid_id = conceptid, valuetype_id = 'image',languageid_id = settings.LANGUAGE_CODE) value.save() return JSONResponse(value) elif skosfile: skos = SKOSReader() rdf = skos.read_file(skosfile) ret = skos.save_concepts_from_skos(rdf) return JSONResponse(ret) else: data = JSONDeserializer().deserialize(request.body) if data: with transaction.atomic(): concept = Concept(data) concept.save() concept.index() return JSONResponse(concept) if request.method == 'DELETE': data = JSONDeserializer().deserialize(request.body) if data: with transaction.atomic(): concept = Concept(data) delete_self = data['delete_self'] if 'delete_self' in data else False if not (delete_self and concept.id in CORE_CONCEPTS): in_use = False if delete_self: check_concept = Concept().get(data['id'], include_subconcepts=True) in_use = check_concept.check_if_concept_in_use() if 'subconcepts' in data: for subconcept in data['subconcepts']: if in_use == False: check_concept = Concept().get(subconcept['id'], include_subconcepts=True) in_use = check_concept.check_if_concept_in_use() if in_use == False: concept.delete_index(delete_self=delete_self) concept.delete(delete_self=delete_self) else: return JSONResponse({"in_use": in_use}) return JSONResponse(concept) return HttpResponseNotFound
def load_authority_file(cursor, path_to_authority_files, filename, auth_file_to_entity_concept_mapping): print filename.upper() start = time() value_types = models.ValueTypes.objects.all() filepath = os.path.join(path_to_authority_files, filename) unicodecsv.field_size_limit(sys.maxint) errors = [] lookups = Lookups() #create nodes for each authority document file and relate them to the authority document node in the concept schema auth_doc_file_name = str(filename) display_file_name = string.capwords(auth_doc_file_name.replace('_',' ').replace('AUTHORITY DOCUMENT.csv', '').strip()) if auth_doc_file_name.upper() != 'ARCHES RESOURCE CROSS-REFERENCE RELATIONSHIP TYPES.E32.CSV': top_concept = Concept() top_concept.id = str(uuid.uuid4()) top_concept.nodetype = 'Concept' top_concept.legacyoid = auth_doc_file_name top_concept.addvalue({'value':display_file_name, 'language': settings.LANGUAGE_CODE, 'type': 'prefLabel', 'category': 'label'}) lookups.add_relationship(source='00000000-0000-0000-0000-000000000001', type='hasTopConcept', target=top_concept.id) else: top_concept = Concept().get(id = '00000000-0000-0000-0000-000000000005') top_concept.legacyoid = 'ARCHES RESOURCE CROSS-REFERENCE RELATIONSHIP TYPES.E32.csv' lookups.add_lookup(concept=top_concept, rownum=0) try: with open(filepath, 'rU') as f: rows = unicodecsv.DictReader(f, fieldnames=['CONCEPTID','PREFLABEL','ALTLABELS','PARENTCONCEPTID','CONCEPTTYPE','PROVIDER'], encoding='utf-8-sig', delimiter=',', restkey='ADDITIONAL', restval='MISSING') rows.next() # skip header row for row in rows: try: if 'MISSING' in row: raise Exception('The row wasn\'t parsed properly. Missing %s' % (row['MISSING'])) else: legacyoid = row[u'CONCEPTID'] concept = Concept() concept.id = legacyoid if is_uuid(legacyoid) == True else str(uuid.uuid4()) concept.nodetype = 'Concept'# if row[u'CONCEPTTYPE'].upper() == 'INDEX' else 'Collection' concept.legacyoid = row[u'CONCEPTID'] concept.addvalue({'value':row[u'PREFLABEL'], 'language': settings.LANGUAGE_CODE, 'type': 'prefLabel', 'category': 'label'}) if row['CONCEPTTYPE'].lower() == 'collector': concept.addvalue({'value':row[u'PREFLABEL'], 'language': settings.LANGUAGE_CODE, 'type': 'collector', 'category': 'label'}) if row[u'ALTLABELS'] != '': altlabel_list = row[u'ALTLABELS'].split(';') for altlabel in altlabel_list: concept.addvalue({'value':altlabel, 'language': settings.LANGUAGE_CODE, 'type': 'altLabel', 'category': 'label'}) parent_concept_id = lookups.get_lookup(legacyoid=row[u'PARENTCONCEPTID']).id lookups.add_relationship(source=parent_concept_id, type='narrower', target=concept.id, rownum=rows.line_num) # don't add a member relationship between a top concept and it's children if parent_concept_id != top_concept.id: lookups.add_relationship(source=parent_concept_id, type='member', target=concept.id, rownum=rows.line_num) # add the member relationship from the E55 type (typically) to their top members if auth_doc_file_name in auth_file_to_entity_concept_mapping and row[u'PARENTCONCEPTID'] == auth_doc_file_name: for entitytype_info in auth_file_to_entity_concept_mapping[auth_doc_file_name]: lookups.add_relationship(source=entitytype_info['ENTITYTYPE_CONCEPTID'], type='member', target=concept.id, rownum=rows.line_num) if row[u'PARENTCONCEPTID'] == '' or (row[u'CONCEPTTYPE'].upper() != 'INDEX' and row[u'CONCEPTTYPE'].upper() != 'COLLECTOR'): raise Exception('The row has invalid values.') lookups.add_lookup(concept=concept, rownum=rows.line_num) except Exception as e: errors.append('ERROR in row %s: %s' % (rows.line_num, str(e))) except UnicodeDecodeError as e: errors.append('ERROR: Make sure the file is saved with UTF-8 encoding\n%s\n%s' % (str(e), traceback.format_exc())) except Exception as e: errors.append('ERROR: %s\n%s' % (str(e), traceback.format_exc())) if len(errors) > 0: errors.insert(0, 'ERRORS IN FILE: %s\n' % (filename)) errors.append('\n\n\n\n') try: # try and open the values file if it exists if exists(filepath.replace('.csv', '.values.csv')): with open(filepath.replace('.csv', '.values.csv'), 'rU') as f: rows = unicodecsv.DictReader(f, fieldnames=['CONCEPTID','VALUE','VALUETYPE','PROVIDER'], encoding='utf-8-sig', delimiter=',', restkey='ADDITIONAL', restval='MISSING') rows.next() # skip header row for row in rows: try: if 'ADDITIONAL' in row: raise Exception('The row wasn\'t parsed properly. Additional fields found %s. Add quotes to values that have commas in them.' % (row['ADDITIONAL'])) else: row_valuetype = row[u'VALUETYPE'].strip() if row_valuetype not in value_types.values_list('valuetype', flat=True): valuetype = models.ValueTypes() valuetype.valuetype = row_valuetype valuetype.category = 'undefined' valuetype.namespace = 'arches' valuetype.save() value_types = models.ValueTypes.objects.all() concept = lookups.get_lookup(legacyoid=row[u'CONCEPTID']) category = value_types.get(valuetype=row_valuetype).category concept.addvalue({'value':row[u'VALUE'], 'type': row[u'VALUETYPE'], 'category': category}) except Exception as e: errors.append('ERROR in row %s (%s): %s' % (rows.line_num, str(e), row)) except UnicodeDecodeError as e: errors.append('ERROR: Make sure the file is saved with UTF-8 encoding\n%s\n%s' % (str(e), traceback.format_exc())) except Exception as e: errors.append('ERROR: %s\n%s' % (str(e), traceback.format_exc())) if len(errors) > 0: errors.insert(0, 'ERRORS IN FILE: %s\n' % (filename.replace('.csv', '.values.csv'))) errors.append('\n\n\n\n') # insert and index the concpets for key in lookups.lookup: try: lookups.lookup[key]['concept'].save() except Exception as e: errors.append('ERROR in row %s (%s):\n%s\n' % (lookups.lookup[key]['rownum'], str(e), traceback.format_exc())) lookups.lookup[key]['concept'].index(scheme=top_concept) # insert the concept relations for relation in lookups.concept_relationships: sql = """ INSERT INTO concepts.relations(conceptidfrom, conceptidto, relationtype) VALUES ('%s', '%s', '%s'); """%(relation['source'], relation['target'], relation['type']) #print sql try: cursor.execute(sql) except Exception as e: errors.append('ERROR in row %s (%s):\n%s\n' % (relation['rownum'], str(e), traceback.format_exc())) if len(errors) > 0: errors.insert(0, 'ERRORS IN FILE: %s\n' % (filename)) errors.append('\n\n\n\n') #print 'Time to parse = %s' % ("{0:.2f}".format(time() - start)) return errors
def concept(request, conceptid): f = request.GET.get("f", "json") mode = request.GET.get("mode", "") lang = request.GET.get("lang", settings.LANGUAGE_CODE) pretty = request.GET.get("pretty", False) if request.method == "GET": include_subconcepts = request.GET.get("include_subconcepts", "true") == "true" include_parentconcepts = request.GET.get("include_parentconcepts", "true") == "true" include_relatedconcepts = request.GET.get("include_relatedconcepts", "true") == "true" emulate_elastic_search = request.GET.get("emulate_elastic_search", "false") == "true" depth_limit = request.GET.get("depth_limit", None) if f == "html": depth_limit = 1 if not conceptid: return render_to_response( "views/rdm/concept-report.htm", { "lang": lang, "concept_count": models.Concepts.objects.filter(nodetype="Concept").count(), "collection_count": models.Concepts.objects.filter(nodetype="Collection").count(), "scheme_count": models.Concepts.objects.filter(nodetype="ConceptScheme").count(), "entitytype_count": models.Concepts.objects.filter(nodetype="EntityType").count(), "default_report": True, }, context_instance=RequestContext(request), ) ret = [] labels = [] this_concept = Concept().get(id=conceptid) if f == "html": if mode == "" and ( this_concept.nodetype == "Concept" or this_concept.nodetype == "ConceptScheme" or this_concept.nodetype == "EntityType" ): concept_graph = Concept().get( id=conceptid, include_subconcepts=include_subconcepts, include_parentconcepts=include_parentconcepts, include_relatedconcepts=include_relatedconcepts, depth_limit=depth_limit, up_depth_limit=None, lang=lang, ) else: concept_graph = Concept().get( id=conceptid, include_subconcepts=include_subconcepts, include_parentconcepts=include_parentconcepts, include_relatedconcepts=include_relatedconcepts, depth_limit=depth_limit, up_depth_limit=None, lang=lang, semantic=False, ) languages = models.DLanguages.objects.all() valuetypes = models.ValueTypes.objects.all() relationtypes = models.DRelationtypes.objects.all() prefLabel = concept_graph.get_preflabel(lang=lang) for subconcept in concept_graph.subconcepts: subconcept.prefLabel = subconcept.get_preflabel(lang=lang) for relatedconcept in concept_graph.relatedconcepts: relatedconcept.prefLabel = relatedconcept.get_preflabel(lang=lang) for value in concept_graph.values: if value.category == "label": labels.append(value) if mode == "" and ( this_concept.nodetype == "Concept" or this_concept.nodetype == "ConceptScheme" or this_concept.nodetype == "EntityType" ): if concept_graph.nodetype == "ConceptScheme": parent_relations = relationtypes.filter(category="Properties") else: parent_relations = ( relationtypes.filter(category="Semantic Relations") .exclude(relationtype="related") .exclude(relationtype="broader") .exclude(relationtype="broaderTransitive") ) return render_to_response( "views/rdm/concept-report.htm", { "lang": lang, "prefLabel": prefLabel, "labels": labels, "concept": concept_graph, "languages": languages, "sparql_providers": get_sparql_providers(), "valuetype_labels": valuetypes.filter(category="label"), "valuetype_notes": valuetypes.filter(category="note"), "valuetype_related_values": valuetypes.filter(category="undefined"), "parent_relations": parent_relations, "related_relations": relationtypes.filter( Q(category="Mapping Properties") | Q(relationtype="related") ), "concept_paths": concept_graph.get_paths(lang=lang), "graph_json": JSONSerializer().serialize(concept_graph.get_node_and_links(lang=lang)), "direct_parents": [parent.get_preflabel(lang=lang) for parent in concept_graph.parentconcepts], }, context_instance=RequestContext(request), ) else: return render_to_response( "views/rdm/entitytype-report.htm", { "lang": lang, "prefLabel": prefLabel, "labels": labels, "concept": concept_graph, "languages": languages, "valuetype_labels": valuetypes.filter(category="label"), "valuetype_notes": valuetypes.filter(category="note"), "valuetype_related_values": valuetypes.filter(category="undefined"), "related_relations": relationtypes.filter(relationtype="member"), "concept_paths": concept_graph.get_paths(lang=lang), }, context_instance=RequestContext(request), ) concept_graph = Concept().get( id=conceptid, include_subconcepts=include_subconcepts, include_parentconcepts=include_parentconcepts, include_relatedconcepts=include_relatedconcepts, depth_limit=depth_limit, up_depth_limit=None, lang=lang, ) if f == "skos": include_parentconcepts = False include_subconcepts = True depth_limit = None skos = SKOSWriter() return HttpResponse(skos.write(concept_graph, format="pretty-xml"), content_type="application/xml") if emulate_elastic_search: ret.append({"_type": id, "_source": concept_graph}) else: ret.append(concept_graph) if emulate_elastic_search: ret = {"hits": {"hits": ret}} return JSONResponse(ret, indent=4 if pretty else None) if request.method == "POST": if len(request.FILES) > 0: skosfile = request.FILES.get("skosfile", None) imagefile = request.FILES.get("file", None) if imagefile: value = models.FileValues( valueid=str(uuid.uuid4()), value=request.FILES.get("file", None), conceptid_id=conceptid, valuetype_id="image", languageid_id=settings.LANGUAGE_CODE, ) value.save() return JSONResponse(value) elif skosfile: skos = SKOSReader() rdf = skos.read_file(skosfile) ret = skos.save_concepts_from_skos(rdf) return JSONResponse(ret) else: data = JSONDeserializer().deserialize(request.body) if data: with transaction.atomic(): concept = Concept(data) concept.save() concept.index() return JSONResponse(concept) if request.method == "DELETE": data = JSONDeserializer().deserialize(request.body) if data: with transaction.atomic(): concept = Concept(data) delete_self = data["delete_self"] if "delete_self" in data else False if not (delete_self and concept.id in CORE_CONCEPTS): in_use = False if delete_self: check_concept = Concept().get(data["id"], include_subconcepts=True) in_use = check_concept.check_if_concept_in_use() if "subconcepts" in data: for subconcept in data["subconcepts"]: if in_use == False: check_concept = Concept().get(subconcept["id"], include_subconcepts=True) in_use = check_concept.check_if_concept_in_use() if in_use == False: concept.delete_index(delete_self=delete_self) concept.delete(delete_self=delete_self) else: return JSONResponse({"in_use": in_use}) return JSONResponse(concept) return HttpResponseNotFound
def search(request): se = SearchEngineFactory().create() searchString = request.GET["q"] removechildren = request.GET.get("removechildren", None) query = Query(se, start=0, limit=100) phrase = Match(field="value", query=searchString.lower(), type="phrase_prefix") query.add_query(phrase) results = query.search(index="concept_labels") ids = [] if removechildren != None: concepts = Concept().get(id=removechildren, include_subconcepts=True, include=None) def get_children(concept): ids.append(concept.id) concepts.traverse(get_children) newresults = [] cached_scheme_names = {} for result in results["hits"]["hits"]: if result["_source"]["conceptid"] not in ids: # first look to see if we've already retrieved the scheme name # else look up the scheme name with ES and cache the result if result["_type"] in cached_scheme_names: result["in_scheme_name"] = cached_scheme_names[result["_type"]] else: query = Query(se, start=0, limit=100) phrase = Match(field="conceptid", query=result["_type"], type="phrase") query.add_query(phrase) scheme = query.search(index="concept_labels") for label in scheme["hits"]["hits"]: if label["_source"]["type"] == "prefLabel": cached_scheme_names[result["_type"]] = label["_source"]["value"] result["in_scheme_name"] = label["_source"]["value"] newresults.append(result) # Use the db to get the concept context but this is SLOW # for result in results['hits']['hits']: # if result['_source']['conceptid'] not in ids: # concept = Concept().get(id=result['_source']['conceptid'], include_parentconcepts=True) # pathlist = concept.get_paths() # result['in_scheme_name'] = pathlist[0][0]['label'] # newresults.append(result) # def crawl(conceptid, path=[]): # query = Query(se, start=0, limit=100) # bool = Bool() # bool.must(Match(field='conceptidto', query=conceptid, type='phrase')) # bool.must(Match(field='relationtype', query='narrower', type='phrase')) # query.add_query(bool) # relations = query.search(index='concept_relations') # for relation in relations['hits']['hits']: # path.insert(0, relation) # crawl(relation['_source']['conceptidfrom'], path=path) # return path # for result in results['hits']['hits']: # if result['_source']['conceptid'] not in ids: # concept_relations = crawl(result['_source']['conceptid'], path=[]) # if len(concept_relations) > 0: # conceptid = concept_relations[0]['_source']['conceptidfrom'] # if conceptid in cached_scheme_names: # result['in_scheme_name'] = cached_scheme_names[conceptid] # else: # result['in_scheme_name'] = get_preflabel_from_conceptid(conceptid, lang=settings.LANGUAGE_CODE)['value'] # cached_scheme_names[conceptid] = result['in_scheme_name'] # newresults.append(result) results["hits"]["hits"] = newresults return JSONResponse(results)
def save_concepts_from_skos(self, graph): """ given an RDF graph, tries to save the concpets to the system """ baseuuid = uuid.uuid4() allowed_languages = models.DLanguage.objects.values_list('pk', flat=True) value_types = models.DValueType.objects.all() skos_value_types = value_types.filter(namespace = 'skos') skos_value_types_list = skos_value_types.values_list('valuetype', flat=True) dcterms_value_types = value_types.filter(namespace = 'dcterms') relation_types = models.DRelationType.objects.all() skos_relation_types = relation_types.filter(namespace = 'skos') # if the graph is of the type rdflib.graph.Graph if isinstance(graph, Graph): # Search for ConceptSchemes first for scheme, v, o in graph.triples((None, RDF.type , SKOS.ConceptScheme)): scheme_id = self.generate_uuid_from_subject(baseuuid, scheme) concept_scheme = Concept({ 'id': scheme_id, 'legacyoid': str(scheme), 'nodetype': 'ConceptScheme' }) for predicate, object in graph.predicate_objects(subject = scheme): if str(DCTERMS) in predicate and predicate.replace(DCTERMS, '') in dcterms_value_types.values_list('valuetype', flat=True): if hasattr(object, 'language') and object.language not in allowed_languages: newlang = models.DLanguage() newlang.pk = object.language newlang.languagename = object.language newlang.isdefault = False newlang.save() allowed_languages = models.DLanguage.objects.values_list('pk', flat=True) try: # first try and get any values associated with the concept_scheme value_type = dcterms_value_types.get(valuetype=predicate.replace(DCTERMS, '')) # predicate.replace(SKOS, '') should yield something like 'prefLabel' or 'scopeNote', etc.. if predicate == DCTERMS.title: concept_scheme.addvalue({'value':object, 'language': object.language, 'type': 'prefLabel', 'category': value_type.category}) print 'Casting dcterms:title to skos:prefLabel' if predicate == DCTERMS.description: concept_scheme.addvalue({'value':object, 'language': object.language, 'type': 'scopeNote', 'category': value_type.category}) print 'Casting dcterms:description to skos:scopeNote' except: pass if str(SKOS) in predicate: if predicate == SKOS.hasTopConcept: self.relations.append({'source': scheme_id, 'type': 'hasTopConcept', 'target': self.generate_uuid_from_subject(baseuuid, object)}) self.nodes.append(concept_scheme) if len(self.nodes) == 0: raise Exception('No ConceptScheme found in file.') # Search for Concepts for s, v, o in graph.triples((None, SKOS.inScheme , scheme)): concept = Concept({ 'id': self.generate_uuid_from_subject(baseuuid, s), 'legacyoid': str(s), 'nodetype': 'Concept' }) # loop through all the elements within a <skos:Concept> element for predicate, object in graph.predicate_objects(subject = s): if str(SKOS) in predicate: if hasattr(object, 'language') and object.language not in allowed_languages: newlang = models.DLanguage() newlang.pk = object.language newlang.languagename = object.language newlang.isdefault = False newlang.save() allowed_languages = models.DLanguage.objects.values_list('pk', flat=True) relation_or_value_type = predicate.replace(SKOS, '') # this is essentially the skos element type within a <skos:Concept> element (eg: prefLabel, broader, etc...) if relation_or_value_type in skos_value_types_list: value_type = skos_value_types.get(valuetype=relation_or_value_type) concept.addvalue({'value':object, 'language': object.language, 'type': value_type.valuetype, 'category': value_type.category}) elif predicate == SKOS.broader: self.relations.append({'source': self.generate_uuid_from_subject(baseuuid, object), 'type': 'narrower', 'target': self.generate_uuid_from_subject(baseuuid, s)}) elif predicate == SKOS.narrower: self.relations.append({'source': self.generate_uuid_from_subject(baseuuid, s), 'type': relation_or_value_type, 'target': self.generate_uuid_from_subject(baseuuid, object)}) elif predicate == SKOS.related: self.relations.append({'source': self.generate_uuid_from_subject(baseuuid, s), 'type': relation_or_value_type, 'target': self.generate_uuid_from_subject(baseuuid, object)}) self.nodes.append(concept) # insert and index the concpets with transaction.atomic(): for node in self.nodes: node.save() # insert the concept relations for relation in self.relations: newrelation = models.Relation() newrelation.relationid = str(uuid.uuid4()) newrelation.conceptfrom_id = relation['source'] newrelation.conceptto_id = relation['target'] newrelation.relationtype_id = relation['type'] newrelation.save() # need to index after the concepts and relations have been entered into the db # so that the proper context gets indexed with the concept for node in self.nodes: node.index() return self else: raise Exception('graph argument should be of type rdflib.graph.Graph')
def create_reference_data(new_concepts, create_collections): errors = [] candidates = Concept().get(id='00000000-0000-0000-0000-000000000006') for arches_nodeid, concepts in new_concepts.iteritems(): collectionid = str(uuid.uuid4()) topconceptid = str(uuid.uuid4()) node = Node.objects.get(nodeid=arches_nodeid) # if node.datatype is concept or concept-list create concepts and collections if node.datatype in ['concept', 'concept-list']: # create collection if create_collections = create, otherwise append to collection already assigned to node if create_collections == True: collection_legacyoid = node.name + '_' + str(node.graph_id) + '_import' # check to see that there is not already a collection for this node if node.config['rdmCollection'] != None: errors.append({'type': 'WARNING', 'message': 'A collection already exists for the {0} node. Use the add option to add concepts to this collection.'.format(node.name)}) if len(errors) > 0: self.errors += errors collection = None else: # if there is no collection assigned to this node, create one and assign it to the node try: # check to see that a collection with this legacyid does not already exist collection = Concept().get(legacyoid=collection_legacyoid) errors.append({'type': 'WARNING', 'message': 'A collection with the legacyid {0} already exists.'.format(node.name + '_' + str(node.graph_id) + '_import')}) if len(errors) > 0: self.errors += errors except: collection = Concept({ 'id': collectionid, 'legacyoid': collection_legacyoid, 'nodetype': 'Collection' }) collection.addvalue({'id': str(uuid.uuid4()), 'value': node.name + '_import', 'language': settings.LANGUAGE_CODE, 'type': 'prefLabel'}) node.config['rdmCollection'] = collectionid node.save() collection.save() else: # if create collection = add check that there is a collection associated with node, if no collection associated with node create a collection and associated with the node try: collection = Concept().get(id=node.config['rdmCollection']) except: collection = Concept({ 'id': collectionid, 'legacyoid': node.name + '_' + str(node.graph_id) + '_import', 'nodetype': 'Collection' }) collection.addvalue({'id': str(uuid.uuid4()), 'value': node.name + '_import', 'language': settings.LANGUAGE_CODE, 'type': 'prefLabel'}) node.config['rdmCollection'] = collectionid node.save() collection.save() if collection != None: topconcept_legacyoid = node.name + '_' + str(node.graph_id) # Check if top concept already exists, if not create it and add to candidates scheme try: topconcept = Concept().get(legacyoid=topconcept_legacyoid) except: topconcept = Concept({ 'id': topconceptid, 'legacyoid': topconcept_legacyoid, 'nodetype': 'Concept' }) topconcept.addvalue({'id': str(uuid.uuid4()), 'value': node.name + '_import', 'language': settings.LANGUAGE_CODE, 'type': 'prefLabel'}) topconcept.save() candidates.add_relation(topconcept, 'narrower') # create child concepts and relate to top concept and collection accordingly for conceptid, value in concepts.iteritems(): concept_legacyoid = value + '_' + node.name + '_' + str(node.graph_id) # check if concept already exists, if not create and add to topconcept and collection try: conceptid = [concept for concept in topconcept.get_child_concepts(topconcept.id) if concept[1] == value][0][0] concept = Concept().get(id=conceptid) except: concept = Concept({ 'id': conceptid, 'legacyoid': concept_legacyoid, 'nodetype': 'Concept' }) concept.addvalue({'id': str(uuid.uuid4()), 'value': value, 'language': settings.LANGUAGE_CODE, 'type': 'prefLabel'}) concept.save() collection.add_relation(concept, 'member') topconcept.add_relation(concept, 'narrower') #if node.datatype is domain or domain-list create options array in node.config elif node.datatype in ['domain-value', 'domain-value-list']: for domainid, value in new_concepts[arches_nodeid].iteritems(): # check if value already exists in domain if value not in [t['text'] for t in node.config['options']]: domainvalue = { "text": value, "selected": False, "id": domainid } node.config['options'].append(domainvalue) node.save()