def manage_parents(request, conceptid): if request.method == 'POST': json = request.body if json != None: data = JSONDeserializer().deserialize(json) with transaction.atomic(): if len(data['deleted']) > 0: concept = Concept().get(id=conceptid, include=None) for deleted in data['deleted']: concept.addparent(deleted) concept.delete() concept.bulk_index() if len(data['added']) > 0: concept = Concept().get(id=conceptid) for added in data['added']: concept.addparent(added) concept.save() concept.bulk_index() return JSONResponse(data) else: return HttpResponseNotAllowed(['POST']) return HttpResponseNotFound()
def get(self, request, conceptid): lang = request.GET.get("lang", request.LANGUAGE_CODE) languages = sort_languages(models.DLanguage.objects.all(), lang) concept_schemes = [] for concept in models.Concept.objects.filter(nodetype="ConceptScheme"): concept_schemes.append(Concept().get( id=concept.pk, include=["label"]).get_preflabel(lang=lang)) collections = [] for concept in models.Concept.objects.filter(nodetype="Collection"): collections.append(Concept().get( id=concept.pk, include=["label"]).get_preflabel(lang=lang)) context = self.get_context_data( main_script="rdm", active_page="RDM", languages=languages, conceptid=conceptid, concept_schemes=concept_schemes, collections=collections, CORE_CONCEPTS=CORE_CONCEPTS, ) context["nav"]["icon"] = "fa fa-align-left" context["nav"]["title"] = _("Reference Data Manager") context["nav"]["help"] = { "title": _("Using the RDM"), "template": "rdm-help" } return render(request, "rdm.htm", context)
def manage_parents(request, conceptid): # need to check user credentials here if request.method == 'POST': json = request.body if json != None: data = JSONDeserializer().deserialize(json) with transaction.atomic(): if len(data['deleted']) > 0: concept = Concept({'id': conceptid}) for deleted in data['deleted']: concept.addparent(deleted) concept.delete() if len(data['added']) > 0: concept = Concept({'id': conceptid}) for added in data['added']: concept.addparent(added) concept.save() return JSONResponse(data) else: return HttpResponseNotAllowed(['POST']) return HttpResponseNotFound()
def get(self, request, conceptid): lang = request.GET.get('lang', settings.LANGUAGE_CODE) languages = sort_languages(models.DLanguage.objects.all(), lang) concept_schemes = [] for concept in models.Concept.objects.filter(nodetype='ConceptScheme'): concept_schemes.append(Concept().get(id=concept.pk, include=['label']).get_preflabel(lang=lang)) collections = [] for concept in models.Concept.objects.filter(nodetype='Collection'): collections.append(Concept().get(id=concept.pk, include=['label']).get_preflabel(lang=lang)) context = self.get_context_data( main_script='rdm', active_page='RDM', languages=languages, conceptid=conceptid, concept_schemes=concept_schemes, collections=collections, CORE_CONCEPTS=CORE_CONCEPTS, ) context['nav']['icon'] = 'fa fa-align-left' context['nav']['title'] = _('Reference Data Manager') context['nav']['help'] = { 'title': _('Using the RDM'), 'template': 'rdm-help', } return render(request, 'rdm.htm', context)
def test_create_concept(self): """ Test of basic CRUD on a Concept model """ concept_in = Concept() concept_in.nodetype = 'Concept' concept_in.values = [ConceptValue({ #id: '', #conceptid: '', 'type': 'prefLabel', 'category': 'label', 'value': 'test pref label', 'language': 'en-US' })] concept_in.save() concept_out = Concept().get(id=concept_in.id) self.assertEqual(concept_out.id, concept_in.id) self.assertEqual(concept_out.values[0].value, 'test pref label') label = concept_in.values[0] label.value = 'updated pref label' concept_in.values[0] = label concept_in.save() concept_out = Concept().get(id=concept_in.id) self.assertEqual(concept_out.values[0].value, 'updated pref label') concept_out.delete(delete_self=True) with self.assertRaises(models.Concept.DoesNotExist): deleted_concept = Concept().get(id=concept_out.id)
def __list_nodes(self, options): data = [] try: rm = GraphModel.objects.get(graphid=options['graph']) except GraphModel.DoesNotExist: rm = None if rm is None: self.__error("", "Invalid or missing graph UUID. Use --graph") else: for node in Node.objects.filter(graph=rm): value = {"nodeid": str(node.nodeid), "name": node.name, "datatype": str(node.datatype), "key": (re.sub(r'[^A-Z_]+', '_', node.name.replace(' ', '_').upper().strip('_')))} if ((value['datatype'] == 'concept') or (value['datatype'] == 'concept-list')): if 'rdmCollection' in node.config: conceptid = node.config['rdmCollection'] if not(conceptid is None): values = Concept().get_e55_domain(conceptid) value['values'] = [] for item in values: valueobj = get_preflabel_from_valueid(item['id'], 'en') valueid = valueobj['id'] label = get_preflabel_from_valueid(valueid, 'en') value['values'].append({'valueid': valueid, 'conceptid': item['conceptid'], 'label': label['value']}) for child in Concept().get_child_concepts(item['conceptid']): value['values'].append({'valueid': child[2], 'conceptid': child[0], 'label': child[1]}) data.append(value) return data
def collect_card_widget_node_data(self, graph_obj, graph, parentcard, nodegroupids=[]): nodegroupids.append(str(parentcard.nodegroup_id)) for node in graph_obj["nodes"]: if node["nodegroup_id"] == str(parentcard.nodegroup_id): found = False for widget in graph_obj["widgets"]: if node["nodeid"] == str(widget.node_id): found = True try: collection_id = node["config"]["rdmCollection"] concept_collection = Concept().get_child_collections_hierarchically(collection_id, offset=None) widget.config["options"] = concept_collection except Exception as e: pass break if not found: for card in graph_obj["cards"]: if card["nodegroup_id"] == node["nodegroup_id"]: widget = models.DDataType.objects.get(pk=node["datatype"]).defaultwidget if widget: widget_model = models.CardXNodeXWidget() widget_model.node_id = node["nodeid"] widget_model.card_id = card["cardid"] widget_model.widget_id = widget.pk widget_model.config = widget.defaultconfig try: collection_id = node["config"]["rdmCollection"] if collection_id: concept_collection = Concept().get_child_collections_hierarchically(collection_id, offset=None) widget_model.config["options"] = concept_collection except Exception as e: pass widget_model.label = node["name"] graph_obj["widgets"].append(widget_model) break if node["datatype"] == "resource-instance" or node["datatype"] == "resource-instance-list": if node["config"]["graphs"] is not None: graph_ids = [] for graph in node["config"]["graphs"]: graphuuid = uuid.UUID(graph["graphid"]) graph_ids.append(str(graphuuid)) node["config"]["options"] = [] for resource_instance in Resource.objects.filter(graph_id__in=graph_ids): node["config"]["options"].append( { "id": str(resource_instance.pk), "name": resource_instance.displayname, "graphid": str(resource_instance.graph_id), } ) for subcard in parentcard.cards: self.collect_card_widget_node_data(graph_obj, graph, subcard, nodegroupids) return graph_obj
def collect_card_widget_node_data(self, graph_obj, graph, parentcard, nodegroupids=[]): nodegroupids.append(str(parentcard.nodegroup_id)) for node in graph_obj['nodes']: if node['nodegroup_id'] == str(parentcard.nodegroup_id): found = False for widget in graph_obj['widgets']: if node['nodeid'] == str(widget.node_id): found = True try: collection_id = node['config']['rdmCollection'] concept_collection = Concept().get_child_collections_hierarchically(collection_id, offset=None) widget.config['options'] = concept_collection except Exception as e: pass break if not found: for card in graph_obj['cards']: if card['nodegroup_id'] == node['nodegroup_id']: widget = models.DDataType.objects.get(pk=node['datatype']).defaultwidget if widget: widget_model = models.CardXNodeXWidget() widget_model.node_id = node['nodeid'] widget_model.card_id = card['cardid'] widget_model.widget_id = widget.pk widget_model.config = widget.defaultconfig try: collection_id = node['config']['rdmCollection'] if collection_id: concept_collection = Concept().get_child_collections_hierarchically(collection_id, offset=None) widget_model.config['options'] = concept_collection except Exception as e: pass widget_model.label = node['name'] graph_obj['widgets'].append(widget_model) break if node['datatype'] == 'resource-instance' or node['datatype'] == 'resource-instance-list': if node['config']['graphid'] is not None: try: graphuuid = uuid.UUID(node['config']['graphid'][0]) graph_id = unicode(graphuuid) except ValueError as e: graphuuid = uuid.UUID(node['config']['graphid']) graph_id = unicode(graphuuid) node['config']['options'] = [] for resource_instance in Resource.objects.filter(graph_id=graph_id): node['config']['options'].append({'id': str(resource_instance.pk), 'name': resource_instance.displayname}) for subcard in parentcard.cards: self.collect_card_widget_node_data(graph_obj, graph, subcard, nodegroupids) return graph_obj
def index_entity_concept_lables(): from arches.app.models import models from arches.app.models.concept import Concept domains_concept = Concept('00000000-0000-0000-0000-000000000003') entitynodes_concept = Concept('00000000-0000-0000-0000-000000000004') entitytypes = models.EntityTypes.objects.all() for entitytype in entitytypes: concept = Concept(entitytype.conceptid).get(include=['label']) if entitytype.businesstablename == 'domains': concept.index(scheme=domains_concept) else: concept.index(scheme=entitynodes_concept)
def get_reference_data_for_export(conceptids=None): reference_data_dict = {} reference_data = [] if conceptids is None or conceptids[0] == "all" or conceptids == [""]: reference_data.append(Concept().get( "00000000-0000-0000-0000-000000000001", include_subconcepts=True, semantic=True)) else: for conceptid in conceptids: reference_data.append(Concept().get(uuid.UUID(str(conceptid)), include_subconcepts=True, semantic=True)) reference_data_dict["reference_data"] = reference_data return reference_data_dict
def test_prefer_preflabel_with_just_lang_code_match_over_exact_match_with_altlabel(self): """ Given a language and region, test should pick the preflabel even if that preflabel specifies a language only and even if an altlabel exists with the exact match """ concept = Concept() concept.values = [ ConceptValue({ 'type': 'prefLabel', 'category': 'label', 'value': 'test pref label en', 'language': 'en' }), ConceptValue({ 'type': 'prefLabel', 'category': 'label', 'value': 'test pref label es', 'language': 'es-SP' }), ConceptValue({ 'type': 'altLabel', 'category': 'label', 'value': 'test alt label en-US', 'language': 'en-US' }) ] self.assertEqual(concept.get_preflabel(lang='en-US').value, 'test pref label en')
def _get_child_concepts(conceptid): ret = set([conceptid]) for row in Concept().get_child_concepts(conceptid, ['narrower'], ['prefLabel'], 'prefLabel'): ret.add(row[0]) ret.add(row[1]) return list(ret)
def add_concepts_from_sparql_endpoint(request, conceptid): if request.method == 'POST': json = request.body if json != None: data = JSONDeserializer().deserialize(json) parentconcept = Concept({ 'id': conceptid, 'nodetype': data['model']['nodetype'] }) if parentconcept.nodetype == 'Concept': relationshiptype = 'narrower' elif parentconcept.nodetype == 'ConceptScheme': relationshiptype = 'hasTopConcept' provider = sparql_providers[data['endpoint']] try: parentconcept.subconcepts = provider.get_concepts(data['ids']) except Exception as e: return HttpResponseServerError(e.message) for subconcept in parentconcept.subconcepts: subconcept.relationshiptype = relationshiptype parentconcept.save() parentconcept.index() return JSONResponse(parentconcept, indent=4) else: return HttpResponseNotAllowed(['POST']) return HttpResponseNotFound()
def test_value_import(self): result = JSONDeserializer().deserialize(JSONSerializer().serialize( Concept().get(id='32d8ca56-2aa1-3890-b95b-3fbb119460ad', include_subconcepts=True, depth_limit=1))) values = len(result['values']) self.assertEqual(values, 5)
def test_hierarchical_relationships(self): result = JSONDeserializer().deserialize(JSONSerializer().serialize( Concept().get(id='708e87fd-35bc-3270-b322-b551cd63c589', include_subconcepts=True, depth_limit=1))) children = len(result['subconcepts']) self.assertEqual(children, 3)
def paged_dropdown(request): conceptid = request.GET.get('conceptid') query = request.GET.get('query', None) query = None if query == '' else query page = int(request.GET.get('page', 1)) limit = 50 offset = (page - 1) * limit results = Concept().get_child_collections_hierarchically(conceptid, offset=offset, limit=limit, query=query) total_count = results[0][3] if len(results) > 0 else 0 data = [ dict(list(zip(['valueto', 'depth', 'collector'], d))) for d in results ] data = [ dict(list( zip(['id', 'text', 'conceptid', 'language', 'type'], d['valueto'].values())), depth=d['depth'], collector=d['collector']) for d in data ] return JSONResponse({ 'results': data, 'more': offset + limit < total_count })
def get(self, request, cardid): try: card = Card.objects.get(cardid=cardid) self.graph = Graph.objects.get(graphid=card.graph_id) except (Card.DoesNotExist): # assume the cardid is actually a graph id card = Card.objects.get(cardid=Graph.objects.get( graphid=cardid).get_root_card().cardid) self.graph = Graph.objects.get(graphid=card.graph_id) if self.graph.isresource == True: return redirect('card_manager', graphid=cardid) card.confirm_enabled_state(request.user, card.nodegroup) for c in card.cards: c.confirm_enabled_state(request.user, c.nodegroup) datatypes = models.DDataType.objects.all() widgets = models.Widget.objects.all() geocoding_providers = models.Geocoder.objects.all() map_layers = models.MapLayer.objects.all() map_sources = models.MapSource.objects.all() resource_graphs = Graph.objects.exclude(pk=card.graph_id).exclude( pk=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID).exclude( isresource=False).exclude(isactive=False) lang = request.GET.get('lang', settings.LANGUAGE_CODE) concept_collections = Concept().concept_tree(mode='collections', lang=lang) ontology_properties = [] card_root_node = models.Node.objects.get(nodeid=card.nodegroup_id) for item in self.graph.get_valid_ontology_classes( nodeid=card.nodegroup_id): if card_root_node.ontologyclass in item['ontology_classes']: ontology_properties.append(item['ontology_property']) ontology_properties = sorted(ontology_properties, key=lambda item: item) context = self.get_context_data( main_script='views/graph/card-configuration-manager', graph_id=self.graph.pk, card=JSONSerializer().serialize(card), datatypes_json=JSONSerializer().serialize(datatypes), geocoding_providers=geocoding_providers, datatypes=datatypes, widgets=widgets, widgets_json=JSONSerializer().serialize(widgets), map_layers=map_layers, map_sources=map_sources, resource_graphs=resource_graphs, concept_collections=concept_collections, ontology_properties=JSONSerializer().serialize( ontology_properties), ) context['nav']['title'] = self.graph.name context['nav']['menu'] = True context['nav']['help'] = (_('Configuring Cards and Widgets'), 'help/card-designer-help.htm') return render(request, 'views/graph/card-configuration-manager.htm', context)
def get_concept_uuid_for_aux_map_entry(self, concept_name, concept_value): ''' Reads concept labels(prefLabels) given in AUXILIARY_MAP and returns a list of conceptids. ''' concept = None try: concept = Concepts.objects.get(legacyoid=concept_name) except: print "No Concept found with the name %s" % concept_name if concept is not None: concept_graph = Concept().get(id=concept.pk, include_subconcepts=True, include=['label']) if concept_graph.subconcepts: for subconcept in concept_graph.subconcepts[0].subconcepts: for value in subconcept.values: if value.type == "prefLabel" and value.value == concept_value: return subconcept.id print '[ERROR]: %s is not found in %s' % (concept_value, concept_name) return
def index_concepts_by_entitytypeid(self, entitytypeid): entitytype = archesmodels.EntityTypes.objects.get(pk = entitytypeid) conceptid = entitytype.conceptid_id concept_graph = Concept().get(id=conceptid, include_subconcepts=True, exclude=['note']) if len(concept_graph.subconcepts) > 0: data = JSONSerializer().serializeToPython(concept_graph, ensure_ascii=True, indent=4) self.index(data, 'concept', entitytypeid, 'id', processdoc=None, getid=None, bulk=False)
def export_skos(self, scheme_name, overwrite=False): """ Exports the specified concept scheme to an .xml SKOS file """ print "Exporting concept scheme: {}".format(scheme_name) outfile = datetime.now().strftime( "v3scheme-{}-%Y-%m-%d.xml".format(scheme_name.lower())) outfile = self.make_file_name("v3scheme",scheme_name.lower(),"xml") print "Output file: {}".format(outfile) # check if file exists and prompt for whether to overwrite it if os.path.isfile(outfile) and overwrite is False: confirm = raw_input(" File already exists. Overwrite? Y/n > ") if not confirm.lower().startswith("y") and not confirm == "": exit() print "\ncollecting concepts...", c = Concepts.objects.get(legacyoid__iexact=scheme_name) concept_graph = Concept().get(id=c.conceptid, include_subconcepts=True, include_parentconcepts=True, include_relatedconcepts=True, depth_limit=None, up_depth_limit=None, lang="en-US") print "done" print "writing file...", output = SKOSWriter().write(concept_graph, format="pretty-xml") with open(outfile,"wb") as outf: outf.write(output) print "done"
def add_concepts_from_sparql_endpoint(request, conceptid): if request.method == "POST": json = request.body if json is not None: data = JSONDeserializer().deserialize(json) parentconcept = Concept({ "id": conceptid, "nodetype": data["model"]["nodetype"] }) if parentconcept.nodetype == "Concept": relationshiptype = "narrower" elif parentconcept.nodetype == "ConceptScheme": relationshiptype = "hasTopConcept" provider = get_sparql_providers(data["endpoint"]) try: parentconcept.subconcepts = provider.get_concepts(data["ids"]) except Exception as e: return HttpResponseServerError(e.message) for subconcept in parentconcept.subconcepts: subconcept.relationshiptype = relationshiptype parentconcept.save() parentconcept.index() return JSONResponse(parentconcept, indent=4) else: return HttpResponseNotAllowed(["POST"]) return HttpResponseNotFound()
def get(self, request, conceptid=None): if user_can_read_concepts(user=request.user): include_subconcepts = request.GET.get('includesubconcepts', 'true') == 'true' include_parentconcepts = request.GET.get('includeparentconcepts', 'true') == 'true' include_relatedconcepts = request.GET.get('includerelatedconcepts', 'true') == 'true' depth_limit = request.GET.get('depthlimit', None) lang = request.GET.get('lang', settings.LANGUAGE_CODE) try: indent = int(request.GET.get('indent', None)) except: indent = None if conceptid: try: ret = [] concept_graph = Concept().get(id=conceptid, include_subconcepts=include_subconcepts, include_parentconcepts=include_parentconcepts, include_relatedconcepts=include_relatedconcepts, depth_limit=depth_limit, up_depth_limit=None, lang=lang) ret.append(concept_graph) except models.Concept.DoesNotExist: return JSONResponse(status=404) except: return JSONResponse(status=500) else: return JSONResponse(status=500) else: return JSONResponse(status=500) return JSONResponse(ret, indent=indent)
def import_reference_data(reference_data): # with transaction.atomic(): # if reference_data != '': for data in reference_data: print '\nLOADING {0} CONCEPT SCHEME FROM ARCHES JSON'.format( data['legacyoid']) print '---------------------------------------------' reporter = ConceptImportReporter(data) def create_collections(concept): relations = { '': 'hasCollection', 'hasTopConcept': 'member', 'narrower': 'member', 'narrowerTransitive': 'member' } for subconcept in concept.subconcepts: if concept.relationshiptype in relations.keys(): if concept.id == '00000000-0000-0000-0000-000000000001': concept.id = '00000000-0000-0000-0000-000000000003' models.Relation.objects.get_or_create( conceptfrom_id=concept.id, conceptto_id=subconcept.id, relationtype_id=relations[concept.relationshiptype]) reporter.update_concepts_saved() concept = Concept(data) concept.save() concept.traverse(create_collections, 'down') concept.index(scheme=concept) reporter.report_results()
def forwards_func(apps, schema_editor): # We get the model from the versioned app registry; # if we directly import it, it'll be the wrong version arches_concept = Concept().get(id='00000000-0000-0000-0000-000000000007', include=['label']) arches_concept.index()
def test_get_region_specific_preflabel_when_language_only_version_does_not_exist(self): """ Given a language only and the system only has values that with regions specified, then the system should pick the first preflabel with the same language code """ concept = Concept() concept.values = [ ConceptValue({ 'type': 'prefLabel', 'category': 'label', 'value': 'test pref label en-US', 'language': 'en-US' }), ConceptValue({ 'type': 'prefLabel', 'category': 'label', 'value': 'test pref label es', 'language': 'es-SP' }), ConceptValue({ 'type': 'altLabel', 'category': 'label', 'value': 'test alt label en-US', 'language': 'en-US' }) ] self.assertEqual(concept.get_preflabel(lang='en').value, 'test pref label en-US')
def export(request, conceptid): concept_graphs = [Concept().get(id=conceptid, include_subconcepts=True, include_parentconcepts=False, include_relatedconcepts=True, depth_limit=None, up_depth_limit=None)] skos = SKOSWriter() return HttpResponse(skos.write(concept_graphs, format="pretty-xml"), content_type="application/xml")
def test_get_label_no_exact_match(self): """ Given no match on language code or region, return the first prefLabel found """ concept = Concept() concept.values = [ ConceptValue({ 'type': 'prefLabel', 'value': 'bier', 'language': 'nl' }), ConceptValue({ 'type': 'prefLabel', 'category': 'label', 'value': 'beer', 'language': 'es-SP' }), ConceptValue({ 'type': 'altLabel', 'category': 'label', 'value': 'test alt label en-US', 'language': 'en-US' }) ] pl = concept.get_preflabel('fr-BE') self.assertEquals(pl.type,'prefLabel') self.assertEquals(pl.value,'bier' or 'beer') self.assertEquals(pl.language,'nl' or 'es-SP')
def make_collection(request, conceptid): concept = Concept().get(id=conceptid, values=[]) try: collection_concept = concept.make_collection() return JSONResponse({'collection': collection_concept, 'message':{'title': _('Success'), 'text': _('Collection successfully created from the selected concept')}}) except: return JSONResponse({'message':{'title': _('Unable to Make Collection'), 'text': _('Unable to make a collection from the selected concept.')}}, status=500)
def paged_dropdown(request): conceptid = request.GET.get("conceptid") query = request.GET.get("query", None) query = None if query == "" else query page = int(request.GET.get("page", 1)) limit = 50 offset = (page - 1) * limit results = Concept().get_child_collections_hierarchically(conceptid, offset=offset, limit=limit, query=query) total_count = results[0][3] if len(results) > 0 else 0 data = [ dict(list(zip(["valueto", "depth", "collector"], d))) for d in results ] data = [ dict(list( zip(["id", "text", "conceptid", "language", "type"], d["valueto"].values())), depth=d["depth"], collector=d["collector"]) for d in data ] return JSONResponse({ "results": data, "more": offset + limit < total_count })
def test_get_altlabel_when_no_preflabel_exists_and_altlabel_only_specifies_lang_code( self): """ Given a language and region and the system only has values that specify a language code, the the system should pick the altlabel even if the altlabel doesn't specifiy a region """ concept = Concept() concept.values = [ ConceptValue({ 'type': 'prefLabel', 'category': 'label', 'value': 'test pref label es', 'language': 'es' }), ConceptValue({ 'type': 'altLabel', 'category': 'label', 'value': 'test alt label en', 'language': 'en' }) ] self.assertEqual( concept.get_preflabel(lang='en-US').value, 'test alt label en')