def get(self, request, conceptid):
lang = request.GET.get("lang", request.LANGUAGE_CODE)
languages = sort_languages(models.DLanguage.objects.all(), lang)
concept_schemes = []
for concept in models.Concept.objects.filter(nodetype="ConceptScheme"):
concept_schemes.append(Concept().get(
id=concept.pk, include=["label"]).get_preflabel(lang=lang))
collections = []
for concept in models.Concept.objects.filter(nodetype="Collection"):
collections.append(Concept().get(
id=concept.pk, include=["label"]).get_preflabel(lang=lang))
context = self.get_context_data(
main_script="rdm",
active_page="RDM",
languages=languages,
conceptid=conceptid,
concept_schemes=concept_schemes,
collections=collections,
CORE_CONCEPTS=CORE_CONCEPTS,
)
context["nav"]["icon"] = "fa fa-align-left"
context["nav"]["title"] = _("Reference Data Manager")
context["nav"]["help"] = {
"title": _("Using the RDM"),
"template": "rdm-help"
}
return render(request, "rdm.htm", context)
def __list_nodes(self, options):
data = []
try:
rm = GraphModel.objects.get(graphid=options['graph'])
except GraphModel.DoesNotExist:
rm = None
if rm is None:
self.__error("", "Invalid or missing graph UUID. Use --graph")
else:
for node in Node.objects.filter(graph=rm):
value = {"nodeid": str(node.nodeid), "name": node.name, "datatype": str(node.datatype), "key": (re.sub(r'[^A-Z_]+', '_', node.name.replace(' ', '_').upper().strip('_')))}
if ((value['datatype'] == 'concept') or (value['datatype'] == 'concept-list')):
if 'rdmCollection' in node.config:
conceptid = node.config['rdmCollection']
if not(conceptid is None):
values = Concept().get_e55_domain(conceptid)
value['values'] = []
for item in values:
valueobj = get_preflabel_from_valueid(item['id'], 'en')
valueid = valueobj['id']
label = get_preflabel_from_valueid(valueid, 'en')
value['values'].append({'valueid': valueid, 'conceptid': item['conceptid'], 'label': label['value']})
for child in Concept().get_child_concepts(item['conceptid']):
value['values'].append({'valueid': child[2], 'conceptid': child[0], 'label': child[1]})
data.append(value)
return data
def forwards_func(apps, schema_editor):
# We get the model from the versioned app registry;
# if we directly import it, it'll be the wrong version
arches_concept = Concept().get(id='00000000-0000-0000-0000-000000000007',
include=['label'])
arches_concept.index()
def test_get_label_no_exact_match(self):
"""
Given no match on language code or region, return the first prefLabel found
"""
concept = Concept()
concept.values = [
ConceptValue({
'type': 'prefLabel',
'value': 'bier',
'language': 'nl'
}),
ConceptValue({
'type': 'prefLabel',
'category': 'label',
'value': 'beer',
'language': 'es-SP'
}),
ConceptValue({
'type': 'altLabel',
'category': 'label',
'value': 'test alt label en-US',
'language': 'en-US'
})
]
pl = concept.get_preflabel('fr-BE')
self.assertEquals(pl.type,'prefLabel')
self.assertEquals(pl.value,'bier' or 'beer')
self.assertEquals(pl.language,'nl' or 'es-SP')
def make_collection(request, conceptid):
concept = Concept().get(id=conceptid, values=[])
try:
collection_concept = concept.make_collection()
return JSONResponse({'collection': collection_concept, 'message':{'title': _('Success'), 'text': _('Collection successfully created from the selected concept')}})
except:
return JSONResponse({'message':{'title': _('Unable to Make Collection'), 'text': _('Unable to make a collection from the selected concept.')}}, status=500)
def manage_parents(request, conceptid):
# need to check user credentials here
if request.method == "POST":
json = request.body
if json != None:
data = JSONDeserializer().deserialize(json)
with transaction.atomic():
if len(data["deleted"]) > 0:
concept = Concept({"id": conceptid})
for deleted in data["deleted"]:
concept.addparent(deleted)
concept.delete()
if len(data["added"]) > 0:
concept = Concept({"id": conceptid})
for added in data["added"]:
concept.addparent(added)
concept.save()
return JSONResponse(data)
else:
return HttpResponseNotAllowed(["POST"])
return HttpResponseNotFound()
def test_get_altlabel_when_no_preflabel_exists_and_altlabel_only_specifies_lang_code(
self):
"""
Given a language and region and the system only has values that specify a language code, the
the system should pick the altlabel even if the altlabel doesn't specifiy a region
"""
concept = Concept()
concept.values = [
ConceptValue({
'type': 'prefLabel',
'category': 'label',
'value': 'test pref label es',
'language': 'es'
}),
ConceptValue({
'type': 'altLabel',
'category': 'label',
'value': 'test alt label en',
'language': 'en'
})
]
self.assertEqual(
concept.get_preflabel(lang='en-US').value, 'test alt label en')
def get(self, request, conceptid):
lang = request.GET.get('lang', settings.LANGUAGE_CODE)
languages = sort_languages(models.DLanguage.objects.all(), lang)
concept_schemes = []
for concept in models.Concept.objects.filter(nodetype='ConceptScheme'):
concept_schemes.append(Concept().get(id=concept.pk, include=['label']).get_preflabel(lang=lang))
collections = []
for concept in models.Concept.objects.filter(nodetype='Collection'):
collections.append(Concept().get(id=concept.pk, include=['label']).get_preflabel(lang=lang))
context = self.get_context_data(
main_script='rdm',
active_page='RDM',
languages=languages,
conceptid=conceptid,
concept_schemes=concept_schemes,
collections=collections,
CORE_CONCEPTS=CORE_CONCEPTS,
)
context['nav']['icon'] = 'fa fa-align-left'
context['nav']['title'] = _('Reference Data Manager')
context['nav']['help'] = {
'title': _('Using the RDM'),
'template': 'rdm-help',
}
return render(request, 'rdm.htm', context)
def test_prefer_preflabel_with_just_lang_code_match_over_exact_match_with_altlabel(self):
"""
Given a language and region, test should pick the preflabel even if that preflabel specifies a language only
and even if an altlabel exists with the exact match
"""
concept = Concept()
concept.values = [
ConceptValue({
'type': 'prefLabel',
'category': 'label',
'value': 'test pref label en',
'language': 'en'
}),
ConceptValue({
'type': 'prefLabel',
'category': 'label',
'value': 'test pref label es',
'language': 'es-SP'
}),
ConceptValue({
'type': 'altLabel',
'category': 'label',
'value': 'test alt label en-US',
'language': 'en-US'
})
]
self.assertEqual(concept.get_preflabel(lang='en-US').value, 'test pref label en')
def test_get_region_specific_preflabel_when_language_only_version_does_not_exist(self):
"""
Given a language only and the system only has values that with regions specified, then
the system should pick the first preflabel with the same language code
"""
concept = Concept()
concept.values = [
ConceptValue({
'type': 'prefLabel',
'category': 'label',
'value': 'test pref label en-US',
'language': 'en-US'
}),
ConceptValue({
'type': 'prefLabel',
'category': 'label',
'value': 'test pref label es',
'language': 'es-SP'
}),
ConceptValue({
'type': 'altLabel',
'category': 'label',
'value': 'test alt label en-US',
'language': 'en-US'
})
]
self.assertEqual(concept.get_preflabel(lang='en').value, 'test pref label en-US')
def make_collection(request, conceptid):
concept = Concept().get(id=conceptid, values=[])
try:
collection_concept = concept.make_collection()
return JSONResponse({'collection': collection_concept, 'message':{'title': _('Success'), 'text': _('Collection successfully created from the selected concept')}})
except:
return JSONResponse({'message':{'title': _('Unable to Make Collection'), 'text': _('Unable to make a collection from the selected concept.')}}, status=500)
def confirm_delete(request, conceptid):
lang = request.GET.get("lang", request.LANGUAGE_CODE)
concept = Concept().get(id=conceptid)
concepts_to_delete = [
concept.get_preflabel(lang=lang).value for key, concept in Concept.gather_concepts_to_delete(concept, lang=lang).items()
]
# return HttpResponse('<div>Showing only 50 of
# %s concepts</div><ul><li>%s</ul>' % (len(concepts_to_delete), '<li>'.join(concepts_to_delete[:50]) + ''))
return HttpResponse("<ul><li>%s</ul>" % ("<li>".join(concepts_to_delete) + ""))
def confirm_delete(request, conceptid):
lang = request.GET.get("lang", settings.LANGUAGE_CODE)
concept = Concept().get(id=conceptid)
concepts_to_delete = [
concept.get_preflabel(lang=lang).value
for key, concept in Concept.gather_concepts_to_delete(concept, lang=lang).iteritems()
]
# return HttpResponse('<div>Showing only 50 of %s concepts</div><ul><li>%s</ul>' % (len(concepts_to_delete), '<li>'.join(concepts_to_delete[:50]) + ''))
return HttpResponse("<ul><li>%s</ul>" % ("<li>".join(concepts_to_delete) + ""))
def collect_card_widget_node_data(self, graph_obj, graph, parentcard, nodegroupids=[]):
nodegroupids.append(str(parentcard.nodegroup_id))
for node in graph_obj["nodes"]:
if node["nodegroup_id"] == str(parentcard.nodegroup_id):
found = False
for widget in graph_obj["widgets"]:
if node["nodeid"] == str(widget.node_id):
found = True
try:
collection_id = node["config"]["rdmCollection"]
concept_collection = Concept().get_child_collections_hierarchically(collection_id, offset=None)
widget.config["options"] = concept_collection
except Exception as e:
pass
break
if not found:
for card in graph_obj["cards"]:
if card["nodegroup_id"] == node["nodegroup_id"]:
widget = models.DDataType.objects.get(pk=node["datatype"]).defaultwidget
if widget:
widget_model = models.CardXNodeXWidget()
widget_model.node_id = node["nodeid"]
widget_model.card_id = card["cardid"]
widget_model.widget_id = widget.pk
widget_model.config = widget.defaultconfig
try:
collection_id = node["config"]["rdmCollection"]
if collection_id:
concept_collection = Concept().get_child_collections_hierarchically(collection_id, offset=None)
widget_model.config["options"] = concept_collection
except Exception as e:
pass
widget_model.label = node["name"]
graph_obj["widgets"].append(widget_model)
break
if node["datatype"] == "resource-instance" or node["datatype"] == "resource-instance-list":
if node["config"]["graphs"] is not None:
graph_ids = []
for graph in node["config"]["graphs"]:
graphuuid = uuid.UUID(graph["graphid"])
graph_ids.append(str(graphuuid))
node["config"]["options"] = []
for resource_instance in Resource.objects.filter(graph_id__in=graph_ids):
node["config"]["options"].append(
{
"id": str(resource_instance.pk),
"name": resource_instance.displayname,
"graphid": str(resource_instance.graph_id),
}
)
for subcard in parentcard.cards:
self.collect_card_widget_node_data(graph_obj, graph, subcard, nodegroupids)
return graph_obj
def get_evaluation_path(valueid):
value = models.Values.objects.get(pk=valueid)
concept_graph = Concept().get(id=value.conceptid_id, include_subconcepts=False,
include_parentconcepts=True, include_relatedconcepts=False, up_depth_limit=None, lang=lang)
paths = []
for path in concept_graph.get_paths(lang=lang)[0]:
if path['label'] != 'Arches' and path['label'] != 'Evaluation Criteria Type':
paths.append(path['label'])
return '; '.join(paths)
def get_evaluation_path(valueid):
value = models.Values.objects.get(pk=valueid)
concept_graph = Concept().get(id=value.conceptid_id, include_subconcepts=False,
include_parentconcepts=True, include_relatedconcepts=False, up_depth_limit=None, lang=lang)
paths = []
for path in concept_graph.get_paths(lang=lang)[0]:
if path['label'] != 'Arches' and path['label'] != 'Evaluation Criteria Type':
paths.append(path['label'])
return '; '.join(paths)
def index_concepts():
"""
Collects all concepts and indexes both concepts and concept_labels
"""
se = SearchEngineFactory().create()
se.delete_index(index='concept_labels')
se.delete(
index='term',
body=
'{"query":{"bool":{"must_not":[{"constant_score":{"filter":{"missing":{"field":"value.options.conceptid"}}}}],"must":[],"should":[]}}}'
)
Resource().prepare_term_index(create=True)
print 'indexing concepts'
start = datetime.now()
cursor = connection.cursor()
cursor.execute("""select conceptid from concepts.concepts""")
conceptids = cursor.fetchall()
for c in conceptids:
if c[0] not in CORE_CONCEPTS:
concept = Concept().get(id=c[0],
include_subconcepts=True,
include_parentconcepts=False,
include=['label'])
concept.index()
end = datetime.now()
duration = end - start
print 'indexing concepts required', duration.seconds, 'seconds'
cursor = connection.cursor()
sql = """
select conceptid, conceptlabel from concepts.vw_concepts where conceptid not in ('%s')
""" % ("','".join(CORE_CONCEPTS))
cursor.execute(sql)
concepts = cursor.fetchall()
concept_index_results = {'count': len(concepts), 'passed': 0, 'failed': 0}
for conceptid, conceptvalue in concepts:
result = get_indexed_concepts(se, conceptid, conceptvalue)
if result != 'passed':
concept_index_results['failed'] += 1
else:
concept_index_results['passed'] += 1
status = 'Passed' if concept_index_results['failed'] == 0 else 'Failed'
print '\nConcept Index Results:'
print "Status: {0}, In Database: {1}, Indexed: {2}".format(
status, concept_index_results['count'],
concept_index_results['passed'])
def collect_card_widget_node_data(self, graph_obj, graph, parentcard, nodegroupids=[]):
nodegroupids.append(str(parentcard.nodegroup_id))
for node in graph_obj['nodes']:
if node['nodegroup_id'] == str(parentcard.nodegroup_id):
found = False
for widget in graph_obj['widgets']:
if node['nodeid'] == str(widget.node_id):
found = True
try:
collection_id = node['config']['rdmCollection']
concept_collection = Concept().get_child_collections_hierarchically(collection_id, offset=None)
widget.config['options'] = concept_collection
except Exception as e:
pass
break
if not found:
for card in graph_obj['cards']:
if card['nodegroup_id'] == node['nodegroup_id']:
widget = models.DDataType.objects.get(pk=node['datatype']).defaultwidget
if widget:
widget_model = models.CardXNodeXWidget()
widget_model.node_id = node['nodeid']
widget_model.card_id = card['cardid']
widget_model.widget_id = widget.pk
widget_model.config = widget.defaultconfig
try:
collection_id = node['config']['rdmCollection']
if collection_id:
concept_collection = Concept().get_child_collections_hierarchically(collection_id, offset=None)
widget_model.config['options'] = concept_collection
except Exception as e:
pass
widget_model.label = node['name']
graph_obj['widgets'].append(widget_model)
break
if node['datatype'] == 'resource-instance' or node['datatype'] == 'resource-instance-list':
if node['config']['graphid'] is not None:
try:
graphuuid = uuid.UUID(node['config']['graphid'][0])
graph_id = unicode(graphuuid)
except ValueError as e:
graphuuid = uuid.UUID(node['config']['graphid'])
graph_id = unicode(graphuuid)
node['config']['options'] = []
for resource_instance in Resource.objects.filter(graph_id=graph_id):
node['config']['options'].append({'id': str(resource_instance.pk), 'name': resource_instance.displayname})
for subcard in parentcard.cards:
self.collect_card_widget_node_data(graph_obj, graph, subcard, nodegroupids)
return graph_obj
def confirm_delete(request, conceptid):
lang = request.GET.get('lang', settings.LANGUAGE_CODE)
try:
concept = Concept().get(id=conceptid)
except Exception as es1:
return render_to_response('404.htm', {
'main_script': '404',
'active_page': '404'
},
context_instance=RequestContext(request))
concepts_to_delete = [concept.get_preflabel(lang=lang).value for key, concept in Concept.gather_concepts_to_delete(concept, lang=lang).iteritems()]
#return HttpResponse('<div>Showing only 50 of %s concepts</div><ul><li>%s</ul>' % (len(concepts_to_delete), '<li>'.join(concepts_to_delete[:50]) + ''))
return HttpResponse('<ul><li>%s</ul>' % ('<li>'.join(concepts_to_delete) + ''))
def test_get_altlabel_when_no_preflabel_exists(self):
"""
Given a language and region, test should pick the altlabel when no preflabel exists
"""
concept = Concept()
concept.values = [
ConceptValue({"type": "prefLabel", "category": "label", "value": "test pref label es", "language": "es-SP"}),
ConceptValue({"type": "altLabel", "category": "label", "value": "test alt label en-US", "language": "en-US"}),
]
self.assertEqual(concept.get_preflabel(lang="en-US").value, "test alt label en-US")
def test_get_altlabel_when_no_preflabel_exists_and_altlabel_only_specifies_lang_code(self):
"""
Given a language and region and the system only has values that specify a language code, the
the system should pick the altlabel even if the altlabel doesn't specifiy a region
"""
concept = Concept()
concept.values = [
ConceptValue({"type": "prefLabel", "category": "label", "value": "test pref label es", "language": "es"}),
ConceptValue({"type": "altLabel", "category": "label", "value": "test alt label en", "language": "en"}),
]
self.assertEqual(concept.get_preflabel(lang="en-US").value, "test alt label en")
def forwards_func(apps, schema_editor):
# We get the model from the versioned app registry;
# if we directly import it, it'll be the wrong version
# index base Arches concept
arches_concept = Concept().get(id="00000000-0000-0000-0000-000000000001",
include=["label"])
arches_concept.index()
DValueType = apps.get_model("models", "DValueType")
DValueType.objects.create(valuetype="identifier",
category="identifiers",
namespace="dcterms",
datatype="text")
def import_reference_data(reference_data):
# with transaction.atomic():
# if reference_data != '':
for data in reference_data:
print '\nLOADING {0} CONCEPT SCHEME FROM ARCHES JSON'.format(
data['legacyoid'])
print '---------------------------------------------'
reporter = ConceptImportReporter(data)
def create_collections(concept):
relations = {
'': 'hasCollection',
'hasTopConcept': 'member',
'narrower': 'member',
'narrowerTransitive': 'member'
}
for subconcept in concept.subconcepts:
if concept.relationshiptype in relations.keys():
if concept.id == '00000000-0000-0000-0000-000000000001':
concept.id = '00000000-0000-0000-0000-000000000003'
models.Relation.objects.get_or_create(
conceptfrom_id=concept.id,
conceptto_id=subconcept.id,
relationtype_id=relations[concept.relationshiptype])
reporter.update_concepts_saved()
concept = Concept(data)
concept.save()
concept.traverse(create_collections, 'down')
concept.index(scheme=concept)
reporter.report_results()
def test_get_label_no_exact_match(self):
"""
Given no match on language code or region, return the first prefLabel found
"""
values = [
{'type': 'prefLabel', 'value': 'bier', 'language': 'nl'},
{'type': 'prefLabel', 'value': 'beer', 'language': 'en'},
]
c = Concept({'values': values})
pl = c.get_preflabel('fr-BE')
self.assertEquals(pl.type,'prefLabel')
self.assertEquals(pl.value,'bier')
self.assertEquals(pl.language,'nl')
def get_reference_data_for_export(conceptids=None):
reference_data_dict = {}
reference_data = []
if conceptids is None or conceptids[0] == "all" or conceptids == [""]:
reference_data.append(Concept().get(
"00000000-0000-0000-0000-000000000001",
include_subconcepts=True,
semantic=True))
else:
for conceptid in conceptids:
reference_data.append(Concept().get(uuid.UUID(str(conceptid)),
include_subconcepts=True,
semantic=True))
reference_data_dict["reference_data"] = reference_data
return reference_data_dict
def test_get_region_specific_preflabel_when_language_only_version_does_not_exist(self):
"""
Given a language only and the system only has values that with regions specified, then
the system should pick the first preflabel with the same language code
"""
concept = Concept()
concept.values = [
ConceptValue({"type": "prefLabel", "category": "label", "value": "test pref label en-US", "language": "en-US"}),
ConceptValue({"type": "prefLabel", "category": "label", "value": "test pref label es", "language": "es-SP"}),
ConceptValue({"type": "altLabel", "category": "label", "value": "test alt label en-US", "language": "en-US"}),
]
self.assertEqual(concept.get_preflabel(lang="en").value, "test pref label en-US")
def test_prefer_preflabel_with_just_lang_code_match_over_exact_match_with_altlabel(self):
"""
Given a language and region, test should pick the preflabel even if that preflabel specifies a language only
and even if an altlabel exists with the exact match
"""
concept = Concept()
concept.values = [
ConceptValue({"type": "prefLabel", "category": "label", "value": "test pref label en", "language": "en"}),
ConceptValue({"type": "prefLabel", "category": "label", "value": "test pref label es", "language": "es-SP"}),
ConceptValue({"type": "altLabel", "category": "label", "value": "test alt label en-US", "language": "en-US"}),
]
self.assertEqual(concept.get_preflabel(lang="en-US").value, "test pref label en")
def paged_dropdown(request):
conceptid = request.GET.get("conceptid")
query = request.GET.get("query", None)
query = None if query == "" else query
page = int(request.GET.get("page", 1))
limit = 50
offset = (page - 1) * limit
results = Concept().get_child_collections_hierarchically(conceptid,
offset=offset,
limit=limit,
query=query)
total_count = results[0][3] if len(results) > 0 else 0
data = [
dict(list(zip(["valueto", "depth", "collector"], d))) for d in results
]
data = [
dict(list(
zip(["id", "text", "conceptid", "language", "type"],
d["valueto"].values())),
depth=d["depth"],
collector=d["collector"]) for d in data
]
return JSONResponse({
"results": data,
"more": offset + limit < total_count
})
def test_hierarchical_relationships(self):
result = JSONDeserializer().deserialize(JSONSerializer().serialize(
Concept().get(id='708e87fd-35bc-3270-b322-b551cd63c589',
include_subconcepts=True,
depth_limit=1)))
children = len(result['subconcepts'])
self.assertEqual(children, 3)
def test_value_import(self):
result = JSONDeserializer().deserialize(JSONSerializer().serialize(
Concept().get(id='32d8ca56-2aa1-3890-b95b-3fbb119460ad',
include_subconcepts=True,
depth_limit=1)))
values = len(result['values'])
self.assertEqual(values, 5)
def get_concept_uuid_for_aux_map_entry(self, concept_name, concept_value):
'''
Reads concept labels(prefLabels) given in AUXILIARY_MAP and returns a list of conceptids.
'''
concept = None
try:
concept = Concepts.objects.get(legacyoid=concept_name)
except:
print "No Concept found with the name %s" % concept_name
if concept is not None:
concept_graph = Concept().get(id=concept.pk,
include_subconcepts=True,
include=['label'])
if concept_graph.subconcepts:
for subconcept in concept_graph.subconcepts[0].subconcepts:
for value in subconcept.values:
if value.type == "prefLabel" and value.value == concept_value:
return subconcept.id
print '[ERROR]: %s is not found in %s' % (concept_value,
concept_name)
return
def index_concepts_by_entitytypeid(self, entitytypeid):
entitytype = archesmodels.EntityTypes.objects.get(pk = entitytypeid)
conceptid = entitytype.conceptid_id
concept_graph = Concept().get(id=conceptid, include_subconcepts=True, exclude=['note'])
if len(concept_graph.subconcepts) > 0:
data = JSONSerializer().serializeToPython(concept_graph, ensure_ascii=True, indent=4)
self.index(data, 'concept', entitytypeid, 'id', processdoc=None, getid=None, bulk=False)
def resource_manager(request, resourcetypeid='', form_id='default', resourceid=''):
if resourceid != '':
resource = Resource(resourceid)
elif resourcetypeid != '':
resource = Resource({'entitytypeid': resourcetypeid})
if form_id == 'default':
form_id = resource.form_groups[0]['forms'][0]['id']
form = resource.get_form(form_id)
if request.method == 'DELETE':
resource.delete_index()
se = SearchEngineFactory().create()
realtionships = resource.get_related_resources(return_entities=False)
for realtionship in realtionships:
se.delete(index='resource_relations', doc_type='all', id=realtionship.resourcexid)
realtionship.delete()
resource.delete()
return JSONResponse({ 'success': True })
if request.method == 'POST':
data = JSONDeserializer().deserialize(request.POST.get('formdata', {}))
form.update(data, request.FILES)
with transaction.atomic():
if resourceid != '':
resource.delete_index()
resource.save(user=request.user)
resource.index()
resourceid = resource.entityid
return redirect('resource_manager', resourcetypeid=resourcetypeid, form_id=form_id, resourceid=resourceid)
min_max_dates = models.Dates.objects.aggregate(Min('val'), Max('val'))
if request.method == 'GET':
if form != None:
lang = request.GET.get('lang', settings.LANGUAGE_CODE)
form.load(lang)
return render(request, 'resource-manager.htm', {
'form': form,
'formdata': JSONSerializer().serialize(form.data),
'form_template': 'views/forms/' + form_id + '.htm',
'form_id': form_id,
'resourcetypeid': resourcetypeid,
'resourceid': resourceid,
'main_script': 'resource-manager',
'active_page': 'ResourceManger',
'resource': resource,
'resource_name': resource.get_primary_name(),
'resource_type_name': resource.get_type_name(),
'form_groups': resource.form_groups,
'min_date': min_max_dates['val__min'].year if min_max_dates['val__min'] != None else 0,
'max_date': min_max_dates['val__max'].year if min_max_dates['val__min'] != None else 1,
'timefilterdata': JSONSerializer().serialize(Concept.get_time_filter_data()),
})
else:
return HttpResponseNotFound('<h1>Arches form not found.</h1>')
def get(self, request, cardid):
try:
card = Card.objects.get(cardid=cardid)
self.graph = Graph.objects.get(graphid=card.graph_id)
except (Card.DoesNotExist):
# assume the cardid is actually a graph id
card = Card.objects.get(cardid=Graph.objects.get(
graphid=cardid).get_root_card().cardid)
self.graph = Graph.objects.get(graphid=card.graph_id)
if self.graph.isresource == True:
return redirect('card_manager', graphid=cardid)
card.confirm_enabled_state(request.user, card.nodegroup)
for c in card.cards:
c.confirm_enabled_state(request.user, c.nodegroup)
datatypes = models.DDataType.objects.all()
widgets = models.Widget.objects.all()
geocoding_providers = models.Geocoder.objects.all()
map_layers = models.MapLayer.objects.all()
map_sources = models.MapSource.objects.all()
resource_graphs = Graph.objects.exclude(pk=card.graph_id).exclude(
pk=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID).exclude(
isresource=False).exclude(isactive=False)
lang = request.GET.get('lang', settings.LANGUAGE_CODE)
concept_collections = Concept().concept_tree(mode='collections',
lang=lang)
ontology_properties = []
card_root_node = models.Node.objects.get(nodeid=card.nodegroup_id)
for item in self.graph.get_valid_ontology_classes(
nodeid=card.nodegroup_id):
if card_root_node.ontologyclass in item['ontology_classes']:
ontology_properties.append(item['ontology_property'])
ontology_properties = sorted(ontology_properties,
key=lambda item: item)
context = self.get_context_data(
main_script='views/graph/card-configuration-manager',
graph_id=self.graph.pk,
card=JSONSerializer().serialize(card),
datatypes_json=JSONSerializer().serialize(datatypes),
geocoding_providers=geocoding_providers,
datatypes=datatypes,
widgets=widgets,
widgets_json=JSONSerializer().serialize(widgets),
map_layers=map_layers,
map_sources=map_sources,
resource_graphs=resource_graphs,
concept_collections=concept_collections,
ontology_properties=JSONSerializer().serialize(
ontology_properties),
)
context['nav']['title'] = self.graph.name
context['nav']['menu'] = True
context['nav']['help'] = (_('Configuring Cards and Widgets'),
'help/card-designer-help.htm')
return render(request, 'views/graph/card-configuration-manager.htm',
context)
def export(request, conceptid):
concept_graphs = [Concept().get(id=conceptid, include_subconcepts=True,
include_parentconcepts=False, include_relatedconcepts=True,
depth_limit=None, up_depth_limit=None)]
skos = SKOSWriter()
return HttpResponse(skos.write(concept_graphs, format="pretty-xml"), content_type="application/xml")
def get(self, request, conceptid=None):
if user_can_read_concepts(user=request.user):
include_subconcepts = request.GET.get('includesubconcepts', 'true') == 'true'
include_parentconcepts = request.GET.get('includeparentconcepts', 'true') == 'true'
include_relatedconcepts = request.GET.get('includerelatedconcepts', 'true') == 'true'
depth_limit = request.GET.get('depthlimit', None)
lang = request.GET.get('lang', settings.LANGUAGE_CODE)
try:
indent = int(request.GET.get('indent', None))
except:
indent = None
if conceptid:
try:
ret = []
concept_graph = Concept().get(id=conceptid, include_subconcepts=include_subconcepts,
include_parentconcepts=include_parentconcepts, include_relatedconcepts=include_relatedconcepts,
depth_limit=depth_limit, up_depth_limit=None, lang=lang)
ret.append(concept_graph)
except models.Concept.DoesNotExist:
return JSONResponse(status=404)
except:
return JSONResponse(status=500)
else:
return JSONResponse(status=500)
else:
return JSONResponse(status=500)
return JSONResponse(ret, indent=indent)
def forwards_func(apps, schema_editor):
# We get the model from the versioned app registry;
# if we directly import it, it'll be the wrong version
extensions = [os.path.join(settings.ONTOLOGY_PATH, x) for x in settings.ONTOLOGY_EXT]
management.call_command('load_ontology', source=os.path.join(settings.ONTOLOGY_PATH, settings.ONTOLOGY_BASE),
version=settings.ONTOLOGY_BASE_VERSION, ontology_name=settings.ONTOLOGY_BASE_NAME, id=settings.ONTOLOGY_BASE_ID, extensions=','.join(extensions), verbosity=0)
Ontology = apps.get_model("models", "Ontology")
Node = apps.get_model("models", "Node")
Edge = apps.get_model("models", "Edge")
for ontology in Ontology.objects.filter(parentontology=None):
g = Graph()
g.parse(ontology.path.path)
for extension in Ontology.objects.filter(parentontology=ontology):
g.parse(extension.path.path)
ontology_classes = set()
ontology_properties = set()
for ontology_property,p,o in g.triples((None, None, RDF.Property)):
ontology_properties.add(ontology_property)
for s,p,domain_class in g.triples((ontology_property, RDFS.domain, None)):
ontology_classes.add(domain_class)
for s,p,range_class in g.triples((ontology_property, RDFS.range, None)):
ontology_classes.add(range_class)
for ontology_class,p,o in g.triples((None, None, RDFS.Class)):
ontology_classes.add(ontology_class)
for ontology_class in ontology_classes:
for node in Node.objects.filter(ontologyclass=str(ontology_class).split('/')[-1], graph__in=ontology.graphs.all()):
node.ontologyclass = ontology_class
node.save()
for ontology_property in ontology_properties:
for edge in Edge.objects.filter(ontologyproperty=str(ontology_property).split('/')[-1], graph__in=ontology.graphs.all()):
edge.ontologyproperty = ontology_property
edge.save()
# index base Arches concept
arches_concept = Concept().get(id='00000000-0000-0000-0000-000000000001', include=['label'])
arches_concept.index()
DValueType = apps.get_model("models", "DValueType")
DValueType.objects.create(valuetype='identifier', category='identifiers', namespace='dcterms', datatype='text')
def index_concepts():
"""
Collects all concepts and indexes both concepts and concept_labels
"""
se = SearchEngineFactory().create()
se.delete_index(index='concept_labels')
se.delete(index='term', body='{"query":{"bool":{"must_not":[{"constant_score":{"filter":{"missing":{"field":"value.options.conceptid"}}}}],"must":[],"should":[]}}}')
Resource().prepare_term_index(create=True)
print 'indexing concepts'
start = datetime.now()
cursor = connection.cursor()
cursor.execute("""select conceptid from concepts.concepts""")
conceptids = cursor.fetchall()
for c in conceptids:
if c[0] not in CORE_CONCEPTS:
concept = Concept().get(id=c[0], include_subconcepts=True, include_parentconcepts=False, include=['label'])
concept.index()
end = datetime.now()
duration = end - start
print 'indexing concepts required', duration.seconds, 'seconds'
cursor = connection.cursor()
sql = """
select conceptid, conceptlabel from concepts.vw_concepts where conceptid not in ('%s')
""" % ("','".join(CORE_CONCEPTS))
cursor.execute(sql)
concepts = cursor.fetchall()
concept_index_results = {'count':len(concepts), 'passed':0, 'failed':0}
for conceptid, conceptvalue in concepts:
result = get_indexed_concepts(se, conceptid, conceptvalue)
if result != 'passed':
concept_index_results['failed'] += 1
else:
concept_index_results['passed'] += 1
status = 'Passed' if concept_index_results['failed'] == 0 else 'Failed'
print '\nConcept Index Results:'
print "Status: {0}, In Database: {1}, Indexed: {2}".format(status, concept_index_results['count'], concept_index_results['passed'])
def home_page(request):
lang = request.GET.get('lang', settings.LANGUAGE_CODE)
min_max_dates = models.Dates.objects.aggregate(Min('val'), Max('val'))
return render(request, 'search.htm', {
'main_script': 'search',
'active_page': 'Search',
'min_date': min_max_dates['val__min'].year if min_max_dates['val__min'] != None else 0,
'max_date': min_max_dates['val__max'].year if min_max_dates['val__min'] != None else 1,
'timefilterdata': JSONSerializer().serialize(Concept.get_time_filter_data()),
})
def add_concepts_from_sparql_endpoint(request, conceptid):
if request.method == 'POST':
json = request.body
if json != None:
data = JSONDeserializer().deserialize(json)
parentconcept = Concept({
'id': conceptid,
'nodetype': data['model']['nodetype']
})
if parentconcept.nodetype == 'Concept':
relationshiptype = 'narrower'
elif parentconcept.nodetype == 'ConceptScheme':
relationshiptype = 'hasTopConcept'
provider = get_sparql_providers(data['endpoint'])
try:
parentconcept.subconcepts = provider.get_concepts(data['ids'])
except Exception as e:
return HttpResponseServerError(e.message)
for subconcept in parentconcept.subconcepts:
subconcept.relationshiptype = relationshiptype
parentconcept.save()
parentconcept.index()
return JSONResponse(parentconcept, indent=4)
else:
return HttpResponseNotAllowed(['POST'])
return HttpResponseNotFound()
def get_concepts(self, uris):
"""
Get a list of concepts given a list of AAT uris like http://vocab.getty.edu/aat/300380087
"""
concepts = []
langs = []
for lang in self.allowed_languages:
langs.append('\"%s\"' % (lang))
for uri in uris.split(','):
query = """
SELECT ?value ?type WHERE {
{
<%s> skos:prefLabel ?value .
BIND('prefLabel' AS ?type)
}
UNION
{
<%s> skos:scopeNote [rdf:value ?value] .
BIND('scopeNote' AS ?type)
}
FILTER (lang(?value) in (%s))
}""" % (uri, uri, ','.join(langs))
results = self.perform_sparql_query(query)
if len(results["results"]["bindings"]) > 0 :
concept = Concept()
concept.nodetype = 'Concept'
for result in results["results"]["bindings"]:
concept.addvalue({
'type': result["type"]["value"],
'value': result["value"]["value"],
'language': result["value"]["xml:lang"]
})
concepts.append(concept)
else:
raise Exception(_("<strong>Error in SPARQL query:</strong><br>Test this query directly by pasting the query below into the Getty's own SPARQL endpoint at <a href='http://vocab.getty.edu/sparql' target='_blank'>http://vocab.getty.edu/sparql</a><i><pre>%s</pre></i>Query returned 0 results, please check the query for errors. You may need to add the appropriate languages into the database for this query to work<br><br>") % (query.replace('<', '<').replace('>', '>')))
return concepts
def add_concepts_from_sparql_endpoint(request, conceptid):
if request.method == "POST":
json = request.body
if json != None:
data = JSONDeserializer().deserialize(json)
parentconcept = Concept({"id": conceptid, "nodetype": data["model"]["nodetype"]})
if parentconcept.nodetype == "Concept":
relationshiptype = "narrower"
elif parentconcept.nodetype == "ConceptScheme":
relationshiptype = "hasTopConcept"
provider = get_sparql_providers(data["endpoint"])
try:
parentconcept.subconcepts = provider.get_concepts(data["ids"])
except Exception as e:
return HttpResponseServerError(e.message)
for subconcept in parentconcept.subconcepts:
subconcept.relationshiptype = relationshiptype
parentconcept.save()
parentconcept.index()
return JSONResponse(parentconcept, indent=4)
else:
return HttpResponseNotAllowed(["POST"])
return HttpResponseNotFound()
def import_concepts(reference_data):
concepts = reference_data[0]['concepts']
values = reference_data[1]['values']
relations = reference_data[2]['relations']
concept_objs = {}
for concept in concepts:
concept_obj = Concept()
concept_obj.id = concept['conceptid']
concept_obj.nodetype = concept['nodetype']
concept_obj.legacyoid = concept['legacyoid']
concept_obj.save()
concept_objs[concept_obj.id] = concept_obj
existing_valuetypes = [o.valuetype for o in models.DValueType.objects.all()]
for value in values:
if value['valuetype'] not in existing_valuetypes:
models.DValueType.objects.create(valuetype = value['valuetype'], category = 'undefined', namespace = 'arches')
existing_valuetypes.append(value['valuetype'])
conceptvalue_obj = ConceptValue()
conceptvalue_obj.id = value['valueid']
conceptvalue_obj.conceptid = value['conceptid']
conceptvalue_obj.type = value['valuetype']
conceptvalue_obj.value = value['value']
conceptvalue_obj.language = value['languageid']
conceptvalue_obj.save()
for relation in relations:
if relation['conceptidfrom'] in concept_objs and relation['conceptidto'] in concept_objs:
conceptfrom = concept_objs[relation['conceptidfrom']]
conceptto = concept_objs[relation['conceptidto']]
conceptfrom.add_relation(conceptto, relation['relationtype'])
def home_page(request):
lang = request.GET.get('lang', settings.LANGUAGE_CODE)
min_max_dates = models.Dates.objects.aggregate(Min('val'), Max('val'))
return render_to_response('search.htm', {
'main_script': 'search',
'active_page': 'Search',
'min_date': min_max_dates['val__min'].year if min_max_dates['val__min'] != None else 0,
'max_date': min_max_dates['val__max'].year if min_max_dates['val__min'] != None else 1,
'timefilterdata': JSONSerializer().serialize(Concept.get_time_filter_data()),
'group_options': settings.SEARCH_GROUP_ROOTS
},
context_instance=RequestContext(request))
def test_get_altlabel_when_no_preflabel_exists(self):
"""
Given a language and region, test should pick the altlabel when no preflabel exists
"""
concept = Concept()
concept.values = [
ConceptValue({
'type': 'prefLabel',
'category': 'label',
'value': 'test pref label es',
'language': 'es-SP'
}),
ConceptValue({
'type': 'altLabel',
'category': 'label',
'value': 'test alt label en-US',
'language': 'en-US'
})
]
self.assertEqual(concept.get_preflabel(lang='en-US').value, 'test alt label en-US')
def test_prefer_preflabel_over_altlabel(self):
"""
Test to confirm the proper retrieval of the prefLabel based on different language requirements
"""
concept = Concept()
concept.values = [
ConceptValue({
'type': 'prefLabel',
'category': 'label',
'value': 'test pref label en-US',
'language': 'en-US'
}),
ConceptValue({
'type': 'prefLabel',
'category': 'label',
'value': 'test pref label en',
'language': 'en'
}),
ConceptValue({
'type': 'prefLabel',
'category': 'label',
'value': 'test pref label es-SP',
'language': 'es-SP'
}),
ConceptValue({
'type': 'altLabel',
'category': 'label',
'value': 'test alt label en-US',
'language': 'en-US'
})
]
self.assertEqual(concept.get_preflabel(lang='en-US').value, 'test pref label en-US')
self.assertEqual(concept.get_preflabel(lang='en').value, 'test pref label en')
self.assertEqual(concept.get_preflabel().value, 'test pref label %s' % (test_settings.LANGUAGE_CODE))
def test_get_altlabel_when_no_preflabel_exists_and_altlabel_only_specifies_lang_code(self):
"""
Given a language and region and the system only has values that specify a language code, the
the system should pick the altlabel even if the altlabel doesn't specifiy a region
"""
concept = Concept()
concept.values = [
ConceptValue({
'type': 'prefLabel',
'category': 'label',
'value': 'test pref label es',
'language': 'es'
}),
ConceptValue({
'type': 'altLabel',
'category': 'label',
'value': 'test alt label en',
'language': 'en'
})
]
self.assertEqual(concept.get_preflabel(lang='en-US').value, 'test alt label en')
def home_page(request):
lang = request.GET.get("lang", settings.LANGUAGE_CODE)
min_max_dates = models.Dates.objects.aggregate(Min("val"), Max("val"))
return render_to_response(
"search.htm",
{
"main_script": "search",
"active_page": "Search",
"min_date": min_max_dates["val__min"].year if min_max_dates["val__min"] != None else 0,
"max_date": min_max_dates["val__max"].year if min_max_dates["val__min"] != None else 1,
"timefilterdata": JSONSerializer().serialize(Concept.get_time_filter_data()),
},
context_instance=RequestContext(request),
)
def import_reference_data(reference_data):
# with transaction.atomic():
# if reference_data != '':
for data in reference_data:
print '\nLOADING {0} CONCEPT SCHEME FROM ARCHES JSON'.format(data['legacyoid'])
print '---------------------------------------------'
def create_collections(concept):
relations = {'':'hasCollection', 'hasTopConcept':'member', 'narrower':'member', 'narrowerTransitive':'member'}
for subconcept in concept.subconcepts:
if concept.relationshiptype in relations.keys():
if concept.id == '00000000-0000-0000-0000-000000000001':
concept.id = '00000000-0000-0000-0000-000000000003'
models.Relation.objects.get_or_create(conceptfrom_id=concept.id, conceptto_id=subconcept.id, relationtype_id=relations[concept.relationshiptype])
concept = Concept(data)
concept.save()
concept.traverse(create_collections, 'down')
concept.index(scheme=concept)
def test_create_concept(self):
"""
Test of basic CRUD on a Concept model
"""
concept_in = Concept()
concept_in.nodetype = 'Concept'
concept_in.values = [ConceptValue({
#id: '',
#conceptid: '',
'type': 'prefLabel',
'category': 'label',
'value': 'test pref label',
'language': 'en-US'
})]
concept_in.save()
concept_out = Concept().get(id=concept_in.id)
self.assertEqual(concept_out.id, concept_in.id)
self.assertEqual(concept_out.values[0].value, 'test pref label')
label = concept_in.values[0]
label.value = 'updated pref label'
concept_in.values[0] = label
concept_in.save()
concept_out = Concept().get(id=concept_in.id)
self.assertEqual(concept_out.values[0].value, 'updated pref label')
concept_out.delete(delete_self=True)
with self.assertRaises(models.Concept.DoesNotExist):
deleted_concept = Concept().get(id=concept_out.id)
def concept(request, conceptid):
f = request.GET.get('f', 'json')
mode = request.GET.get('mode', '')
lang = request.GET.get('lang', settings.LANGUAGE_CODE)
pretty = request.GET.get('pretty', False)
if request.method == 'GET':
include_subconcepts = request.GET.get('include_subconcepts', 'true') == 'true'
include_parentconcepts = request.GET.get('include_parentconcepts', 'true') == 'true'
include_relatedconcepts = request.GET.get('include_relatedconcepts', 'true') == 'true'
emulate_elastic_search = request.GET.get('emulate_elastic_search', 'false') == 'true'
depth_limit = request.GET.get('depth_limit', None)
if f == 'html':
depth_limit = 1
if not conceptid:
return render(request, 'views/rdm/concept-report.htm', {
'lang': lang,
'concept_count': models.Concept.objects.filter(nodetype='Concept').count(),
'collection_count': models.Concept.objects.filter(nodetype='Collection').count(),
'scheme_count': models.Concept.objects.filter(nodetype='ConceptScheme').count(),
'entitytype_count': models.Concept.objects.filter(nodetype='EntityType').count(),
'default_report': True
})
ret = []
labels = []
this_concept = Concept().get(id=conceptid)
if f == 'html':
if mode == '' and (this_concept.nodetype == 'Concept' or this_concept.nodetype == 'ConceptScheme' or this_concept.nodetype == 'EntityType'):
concept_graph = Concept().get(id=conceptid, include_subconcepts=include_subconcepts,
include_parentconcepts=include_parentconcepts, include_relatedconcepts=include_relatedconcepts,
depth_limit=depth_limit, up_depth_limit=None, lang=lang)
else:
concept_graph = Concept().get(id=conceptid, include_subconcepts=include_subconcepts,
include_parentconcepts=include_parentconcepts, include_relatedconcepts=include_relatedconcepts,
depth_limit=depth_limit, up_depth_limit=None, lang=lang, semantic=False)
languages = models.DLanguage.objects.all()
valuetypes = models.DValueType.objects.all()
relationtypes = models.DRelationType.objects.all()
prefLabel = concept_graph.get_preflabel(lang=lang)
for subconcept in concept_graph.subconcepts:
subconcept.prefLabel = subconcept.get_preflabel(lang=lang)
for relatedconcept in concept_graph.relatedconcepts:
relatedconcept.prefLabel = relatedconcept.get_preflabel(lang=lang)
for value in concept_graph.values:
if value.category == 'label':
labels.append(value)
if mode == '' and (this_concept.nodetype == 'Concept' or this_concept.nodetype == 'ConceptScheme' or this_concept.nodetype == 'EntityType'):
if concept_graph.nodetype == 'ConceptScheme':
parent_relations = relationtypes.filter(category='Properties')
else:
parent_relations = relationtypes.filter(category='Semantic Relations').exclude(relationtype = 'related').exclude(relationtype='broader').exclude(relationtype='broaderTransitive')
return render(request, 'views/rdm/concept-report.htm', {
'lang': lang,
'prefLabel': prefLabel,
'labels': labels,
'concept': concept_graph,
'languages': languages,
'sparql_providers': get_sparql_providers(),
'valuetype_labels': valuetypes.filter(category='label'),
'valuetype_notes': valuetypes.filter(category='note'),
'valuetype_related_values': valuetypes.filter(category='undefined'),
'parent_relations': parent_relations,
'related_relations': relationtypes.filter(Q(category='Mapping Properties') | Q(relationtype = 'related')),
'concept_paths': concept_graph.get_paths(lang=lang),
'graph_json': JSONSerializer().serialize(concept_graph.get_node_and_links(lang=lang)),
'direct_parents': [parent.get_preflabel(lang=lang) for parent in concept_graph.parentconcepts]
})
else:
return render(request, 'views/rdm/entitytype-report.htm', {
'lang': lang,
'prefLabel': prefLabel,
'labels': labels,
'concept': concept_graph,
'languages': languages,
'valuetype_labels': valuetypes.filter(category='label'),
'valuetype_notes': valuetypes.filter(category='note'),
'valuetype_related_values': valuetypes.filter(category='undefined'),
'related_relations': relationtypes.filter(relationtype = 'member'),
'concept_paths': concept_graph.get_paths(lang=lang)
})
concept_graph = Concept().get(id=conceptid, include_subconcepts=include_subconcepts,
include_parentconcepts=include_parentconcepts, include_relatedconcepts=include_relatedconcepts,
depth_limit=depth_limit, up_depth_limit=None, lang=lang)
if f == 'skos':
include_parentconcepts = False
include_subconcepts = True
depth_limit = None
skos = SKOSWriter()
return HttpResponse(skos.write(concept_graph, format="pretty-xml"), content_type="application/xml")
if emulate_elastic_search:
ret.append({'_type': id, '_source': concept_graph})
else:
ret.append(concept_graph)
if emulate_elastic_search:
ret = {'hits':{'hits':ret}}
return JSONResponse(ret, indent=4 if pretty else None)
if request.method == 'POST':
if len(request.FILES) > 0:
skosfile = request.FILES.get('skosfile', None)
imagefile = request.FILES.get('file', None)
if imagefile:
value = models.FileValue(valueid = str(uuid.uuid4()), value = request.FILES.get('file', None), conceptid_id = conceptid, valuetype_id = 'image',languageid_id = settings.LANGUAGE_CODE)
value.save()
return JSONResponse(value)
elif skosfile:
skos = SKOSReader()
rdf = skos.read_file(skosfile)
ret = skos.save_concepts_from_skos(rdf)
return JSONResponse(ret)
else:
data = JSONDeserializer().deserialize(request.body)
if data:
with transaction.atomic():
concept = Concept(data)
concept.save()
concept.index()
return JSONResponse(concept)
if request.method == 'DELETE':
data = JSONDeserializer().deserialize(request.body)
if data:
with transaction.atomic():
concept = Concept(data)
delete_self = data['delete_self'] if 'delete_self' in data else False
if not (delete_self and concept.id in CORE_CONCEPTS):
in_use = False
if delete_self:
check_concept = Concept().get(data['id'], include_subconcepts=True)
in_use = check_concept.check_if_concept_in_use()
if 'subconcepts' in data:
for subconcept in data['subconcepts']:
if in_use == False:
check_concept = Concept().get(subconcept['id'], include_subconcepts=True)
in_use = check_concept.check_if_concept_in_use()
if in_use == False:
concept.delete_index(delete_self=delete_self)
concept.delete(delete_self=delete_self)
else:
return JSONResponse({"in_use": in_use})
return JSONResponse(concept)
return HttpResponseNotFound
def load_authority_file(cursor, path_to_authority_files, filename, auth_file_to_entity_concept_mapping):
print filename.upper()
start = time()
value_types = models.ValueTypes.objects.all()
filepath = os.path.join(path_to_authority_files, filename)
unicodecsv.field_size_limit(sys.maxint)
errors = []
lookups = Lookups()
#create nodes for each authority document file and relate them to the authority document node in the concept schema
auth_doc_file_name = str(filename)
display_file_name = string.capwords(auth_doc_file_name.replace('_',' ').replace('AUTHORITY DOCUMENT.csv', '').strip())
if auth_doc_file_name.upper() != 'ARCHES RESOURCE CROSS-REFERENCE RELATIONSHIP TYPES.E32.CSV':
top_concept = Concept()
top_concept.id = str(uuid.uuid4())
top_concept.nodetype = 'Concept'
top_concept.legacyoid = auth_doc_file_name
top_concept.addvalue({'value':display_file_name, 'language': settings.LANGUAGE_CODE, 'type': 'prefLabel', 'category': 'label'})
lookups.add_relationship(source='00000000-0000-0000-0000-000000000001', type='hasTopConcept', target=top_concept.id)
else:
top_concept = Concept().get(id = '00000000-0000-0000-0000-000000000005')
top_concept.legacyoid = 'ARCHES RESOURCE CROSS-REFERENCE RELATIONSHIP TYPES.E32.csv'
lookups.add_lookup(concept=top_concept, rownum=0)
try:
with open(filepath, 'rU') as f:
rows = unicodecsv.DictReader(f, fieldnames=['CONCEPTID','PREFLABEL','ALTLABELS','PARENTCONCEPTID','CONCEPTTYPE','PROVIDER'],
encoding='utf-8-sig', delimiter=',', restkey='ADDITIONAL', restval='MISSING')
rows.next() # skip header row
for row in rows:
try:
if 'MISSING' in row:
raise Exception('The row wasn\'t parsed properly. Missing %s' % (row['MISSING']))
else:
legacyoid = row[u'CONCEPTID']
concept = Concept()
concept.id = legacyoid if is_uuid(legacyoid) == True else str(uuid.uuid4())
concept.nodetype = 'Concept'# if row[u'CONCEPTTYPE'].upper() == 'INDEX' else 'Collection'
concept.legacyoid = row[u'CONCEPTID']
concept.addvalue({'value':row[u'PREFLABEL'], 'language': settings.LANGUAGE_CODE, 'type': 'prefLabel', 'category': 'label'})
if row['CONCEPTTYPE'].lower() == 'collector':
concept.addvalue({'value':row[u'PREFLABEL'], 'language': settings.LANGUAGE_CODE, 'type': 'collector', 'category': 'label'})
if row[u'ALTLABELS'] != '':
altlabel_list = row[u'ALTLABELS'].split(';')
for altlabel in altlabel_list:
concept.addvalue({'value':altlabel, 'language': settings.LANGUAGE_CODE, 'type': 'altLabel', 'category': 'label'})
parent_concept_id = lookups.get_lookup(legacyoid=row[u'PARENTCONCEPTID']).id
lookups.add_relationship(source=parent_concept_id, type='narrower', target=concept.id, rownum=rows.line_num)
# don't add a member relationship between a top concept and it's children
if parent_concept_id != top_concept.id:
lookups.add_relationship(source=parent_concept_id, type='member', target=concept.id, rownum=rows.line_num)
# add the member relationship from the E55 type (typically) to their top members
if auth_doc_file_name in auth_file_to_entity_concept_mapping and row[u'PARENTCONCEPTID'] == auth_doc_file_name:
for entitytype_info in auth_file_to_entity_concept_mapping[auth_doc_file_name]:
lookups.add_relationship(source=entitytype_info['ENTITYTYPE_CONCEPTID'], type='member', target=concept.id, rownum=rows.line_num)
if row[u'PARENTCONCEPTID'] == '' or (row[u'CONCEPTTYPE'].upper() != 'INDEX' and row[u'CONCEPTTYPE'].upper() != 'COLLECTOR'):
raise Exception('The row has invalid values.')
lookups.add_lookup(concept=concept, rownum=rows.line_num)
except Exception as e:
errors.append('ERROR in row %s: %s' % (rows.line_num, str(e)))
except UnicodeDecodeError as e:
errors.append('ERROR: Make sure the file is saved with UTF-8 encoding\n%s\n%s' % (str(e), traceback.format_exc()))
except Exception as e:
errors.append('ERROR: %s\n%s' % (str(e), traceback.format_exc()))
if len(errors) > 0:
errors.insert(0, 'ERRORS IN FILE: %s\n' % (filename))
errors.append('\n\n\n\n')
try:
# try and open the values file if it exists
if exists(filepath.replace('.csv', '.values.csv')):
with open(filepath.replace('.csv', '.values.csv'), 'rU') as f:
rows = unicodecsv.DictReader(f, fieldnames=['CONCEPTID','VALUE','VALUETYPE','PROVIDER'],
encoding='utf-8-sig', delimiter=',', restkey='ADDITIONAL', restval='MISSING')
rows.next() # skip header row
for row in rows:
try:
if 'ADDITIONAL' in row:
raise Exception('The row wasn\'t parsed properly. Additional fields found %s. Add quotes to values that have commas in them.' % (row['ADDITIONAL']))
else:
row_valuetype = row[u'VALUETYPE'].strip()
if row_valuetype not in value_types.values_list('valuetype', flat=True):
valuetype = models.ValueTypes()
valuetype.valuetype = row_valuetype
valuetype.category = 'undefined'
valuetype.namespace = 'arches'
valuetype.save()
value_types = models.ValueTypes.objects.all()
concept = lookups.get_lookup(legacyoid=row[u'CONCEPTID'])
category = value_types.get(valuetype=row_valuetype).category
concept.addvalue({'value':row[u'VALUE'], 'type': row[u'VALUETYPE'], 'category': category})
except Exception as e:
errors.append('ERROR in row %s (%s): %s' % (rows.line_num, str(e), row))
except UnicodeDecodeError as e:
errors.append('ERROR: Make sure the file is saved with UTF-8 encoding\n%s\n%s' % (str(e), traceback.format_exc()))
except Exception as e:
errors.append('ERROR: %s\n%s' % (str(e), traceback.format_exc()))
if len(errors) > 0:
errors.insert(0, 'ERRORS IN FILE: %s\n' % (filename.replace('.csv', '.values.csv')))
errors.append('\n\n\n\n')
# insert and index the concpets
for key in lookups.lookup:
try:
lookups.lookup[key]['concept'].save()
except Exception as e:
errors.append('ERROR in row %s (%s):\n%s\n' % (lookups.lookup[key]['rownum'], str(e), traceback.format_exc()))
lookups.lookup[key]['concept'].index(scheme=top_concept)
# insert the concept relations
for relation in lookups.concept_relationships:
sql = """
INSERT INTO concepts.relations(conceptidfrom, conceptidto, relationtype)
VALUES ('%s', '%s', '%s');
"""%(relation['source'], relation['target'], relation['type'])
#print sql
try:
cursor.execute(sql)
except Exception as e:
errors.append('ERROR in row %s (%s):\n%s\n' % (relation['rownum'], str(e), traceback.format_exc()))
if len(errors) > 0:
errors.insert(0, 'ERRORS IN FILE: %s\n' % (filename))
errors.append('\n\n\n\n')
#print 'Time to parse = %s' % ("{0:.2f}".format(time() - start))
return errors
def concept(request, conceptid):
f = request.GET.get("f", "json")
mode = request.GET.get("mode", "")
lang = request.GET.get("lang", settings.LANGUAGE_CODE)
pretty = request.GET.get("pretty", False)
if request.method == "GET":
include_subconcepts = request.GET.get("include_subconcepts", "true") == "true"
include_parentconcepts = request.GET.get("include_parentconcepts", "true") == "true"
include_relatedconcepts = request.GET.get("include_relatedconcepts", "true") == "true"
emulate_elastic_search = request.GET.get("emulate_elastic_search", "false") == "true"
depth_limit = request.GET.get("depth_limit", None)
if f == "html":
depth_limit = 1
if not conceptid:
return render_to_response(
"views/rdm/concept-report.htm",
{
"lang": lang,
"concept_count": models.Concepts.objects.filter(nodetype="Concept").count(),
"collection_count": models.Concepts.objects.filter(nodetype="Collection").count(),
"scheme_count": models.Concepts.objects.filter(nodetype="ConceptScheme").count(),
"entitytype_count": models.Concepts.objects.filter(nodetype="EntityType").count(),
"default_report": True,
},
context_instance=RequestContext(request),
)
ret = []
labels = []
this_concept = Concept().get(id=conceptid)
if f == "html":
if mode == "" and (
this_concept.nodetype == "Concept"
or this_concept.nodetype == "ConceptScheme"
or this_concept.nodetype == "EntityType"
):
concept_graph = Concept().get(
id=conceptid,
include_subconcepts=include_subconcepts,
include_parentconcepts=include_parentconcepts,
include_relatedconcepts=include_relatedconcepts,
depth_limit=depth_limit,
up_depth_limit=None,
lang=lang,
)
else:
concept_graph = Concept().get(
id=conceptid,
include_subconcepts=include_subconcepts,
include_parentconcepts=include_parentconcepts,
include_relatedconcepts=include_relatedconcepts,
depth_limit=depth_limit,
up_depth_limit=None,
lang=lang,
semantic=False,
)
languages = models.DLanguages.objects.all()
valuetypes = models.ValueTypes.objects.all()
relationtypes = models.DRelationtypes.objects.all()
prefLabel = concept_graph.get_preflabel(lang=lang)
for subconcept in concept_graph.subconcepts:
subconcept.prefLabel = subconcept.get_preflabel(lang=lang)
for relatedconcept in concept_graph.relatedconcepts:
relatedconcept.prefLabel = relatedconcept.get_preflabel(lang=lang)
for value in concept_graph.values:
if value.category == "label":
labels.append(value)
if mode == "" and (
this_concept.nodetype == "Concept"
or this_concept.nodetype == "ConceptScheme"
or this_concept.nodetype == "EntityType"
):
if concept_graph.nodetype == "ConceptScheme":
parent_relations = relationtypes.filter(category="Properties")
else:
parent_relations = (
relationtypes.filter(category="Semantic Relations")
.exclude(relationtype="related")
.exclude(relationtype="broader")
.exclude(relationtype="broaderTransitive")
)
return render_to_response(
"views/rdm/concept-report.htm",
{
"lang": lang,
"prefLabel": prefLabel,
"labels": labels,
"concept": concept_graph,
"languages": languages,
"sparql_providers": get_sparql_providers(),
"valuetype_labels": valuetypes.filter(category="label"),
"valuetype_notes": valuetypes.filter(category="note"),
"valuetype_related_values": valuetypes.filter(category="undefined"),
"parent_relations": parent_relations,
"related_relations": relationtypes.filter(
Q(category="Mapping Properties") | Q(relationtype="related")
),
"concept_paths": concept_graph.get_paths(lang=lang),
"graph_json": JSONSerializer().serialize(concept_graph.get_node_and_links(lang=lang)),
"direct_parents": [parent.get_preflabel(lang=lang) for parent in concept_graph.parentconcepts],
},
context_instance=RequestContext(request),
)
else:
return render_to_response(
"views/rdm/entitytype-report.htm",
{
"lang": lang,
"prefLabel": prefLabel,
"labels": labels,
"concept": concept_graph,
"languages": languages,
"valuetype_labels": valuetypes.filter(category="label"),
"valuetype_notes": valuetypes.filter(category="note"),
"valuetype_related_values": valuetypes.filter(category="undefined"),
"related_relations": relationtypes.filter(relationtype="member"),
"concept_paths": concept_graph.get_paths(lang=lang),
},
context_instance=RequestContext(request),
)
concept_graph = Concept().get(
id=conceptid,
include_subconcepts=include_subconcepts,
include_parentconcepts=include_parentconcepts,
include_relatedconcepts=include_relatedconcepts,
depth_limit=depth_limit,
up_depth_limit=None,
lang=lang,
)
if f == "skos":
include_parentconcepts = False
include_subconcepts = True
depth_limit = None
skos = SKOSWriter()
return HttpResponse(skos.write(concept_graph, format="pretty-xml"), content_type="application/xml")
if emulate_elastic_search:
ret.append({"_type": id, "_source": concept_graph})
else:
ret.append(concept_graph)
if emulate_elastic_search:
ret = {"hits": {"hits": ret}}
return JSONResponse(ret, indent=4 if pretty else None)
if request.method == "POST":
if len(request.FILES) > 0:
skosfile = request.FILES.get("skosfile", None)
imagefile = request.FILES.get("file", None)
if imagefile:
value = models.FileValues(
valueid=str(uuid.uuid4()),
value=request.FILES.get("file", None),
conceptid_id=conceptid,
valuetype_id="image",
languageid_id=settings.LANGUAGE_CODE,
)
value.save()
return JSONResponse(value)
elif skosfile:
skos = SKOSReader()
rdf = skos.read_file(skosfile)
ret = skos.save_concepts_from_skos(rdf)
return JSONResponse(ret)
else:
data = JSONDeserializer().deserialize(request.body)
if data:
with transaction.atomic():
concept = Concept(data)
concept.save()
concept.index()
return JSONResponse(concept)
if request.method == "DELETE":
data = JSONDeserializer().deserialize(request.body)
if data:
with transaction.atomic():
concept = Concept(data)
delete_self = data["delete_self"] if "delete_self" in data else False
if not (delete_self and concept.id in CORE_CONCEPTS):
in_use = False
if delete_self:
check_concept = Concept().get(data["id"], include_subconcepts=True)
in_use = check_concept.check_if_concept_in_use()
if "subconcepts" in data:
for subconcept in data["subconcepts"]:
if in_use == False:
check_concept = Concept().get(subconcept["id"], include_subconcepts=True)
in_use = check_concept.check_if_concept_in_use()
if in_use == False:
concept.delete_index(delete_self=delete_self)
concept.delete(delete_self=delete_self)
else:
return JSONResponse({"in_use": in_use})
return JSONResponse(concept)
return HttpResponseNotFound
def search(request):
se = SearchEngineFactory().create()
searchString = request.GET["q"]
removechildren = request.GET.get("removechildren", None)
query = Query(se, start=0, limit=100)
phrase = Match(field="value", query=searchString.lower(), type="phrase_prefix")
query.add_query(phrase)
results = query.search(index="concept_labels")
ids = []
if removechildren != None:
concepts = Concept().get(id=removechildren, include_subconcepts=True, include=None)
def get_children(concept):
ids.append(concept.id)
concepts.traverse(get_children)
newresults = []
cached_scheme_names = {}
for result in results["hits"]["hits"]:
if result["_source"]["conceptid"] not in ids:
# first look to see if we've already retrieved the scheme name
# else look up the scheme name with ES and cache the result
if result["_type"] in cached_scheme_names:
result["in_scheme_name"] = cached_scheme_names[result["_type"]]
else:
query = Query(se, start=0, limit=100)
phrase = Match(field="conceptid", query=result["_type"], type="phrase")
query.add_query(phrase)
scheme = query.search(index="concept_labels")
for label in scheme["hits"]["hits"]:
if label["_source"]["type"] == "prefLabel":
cached_scheme_names[result["_type"]] = label["_source"]["value"]
result["in_scheme_name"] = label["_source"]["value"]
newresults.append(result)
# Use the db to get the concept context but this is SLOW
# for result in results['hits']['hits']:
# if result['_source']['conceptid'] not in ids:
# concept = Concept().get(id=result['_source']['conceptid'], include_parentconcepts=True)
# pathlist = concept.get_paths()
# result['in_scheme_name'] = pathlist[0][0]['label']
# newresults.append(result)
# def crawl(conceptid, path=[]):
# query = Query(se, start=0, limit=100)
# bool = Bool()
# bool.must(Match(field='conceptidto', query=conceptid, type='phrase'))
# bool.must(Match(field='relationtype', query='narrower', type='phrase'))
# query.add_query(bool)
# relations = query.search(index='concept_relations')
# for relation in relations['hits']['hits']:
# path.insert(0, relation)
# crawl(relation['_source']['conceptidfrom'], path=path)
# return path
# for result in results['hits']['hits']:
# if result['_source']['conceptid'] not in ids:
# concept_relations = crawl(result['_source']['conceptid'], path=[])
# if len(concept_relations) > 0:
# conceptid = concept_relations[0]['_source']['conceptidfrom']
# if conceptid in cached_scheme_names:
# result['in_scheme_name'] = cached_scheme_names[conceptid]
# else:
# result['in_scheme_name'] = get_preflabel_from_conceptid(conceptid, lang=settings.LANGUAGE_CODE)['value']
# cached_scheme_names[conceptid] = result['in_scheme_name']
# newresults.append(result)
results["hits"]["hits"] = newresults
return JSONResponse(results)
def save_concepts_from_skos(self, graph):
"""
given an RDF graph, tries to save the concpets to the system
"""
baseuuid = uuid.uuid4()
allowed_languages = models.DLanguage.objects.values_list('pk', flat=True)
value_types = models.DValueType.objects.all()
skos_value_types = value_types.filter(namespace = 'skos')
skos_value_types_list = skos_value_types.values_list('valuetype', flat=True)
dcterms_value_types = value_types.filter(namespace = 'dcterms')
relation_types = models.DRelationType.objects.all()
skos_relation_types = relation_types.filter(namespace = 'skos')
# if the graph is of the type rdflib.graph.Graph
if isinstance(graph, Graph):
# Search for ConceptSchemes first
for scheme, v, o in graph.triples((None, RDF.type , SKOS.ConceptScheme)):
scheme_id = self.generate_uuid_from_subject(baseuuid, scheme)
concept_scheme = Concept({
'id': scheme_id,
'legacyoid': str(scheme),
'nodetype': 'ConceptScheme'
})
for predicate, object in graph.predicate_objects(subject = scheme):
if str(DCTERMS) in predicate and predicate.replace(DCTERMS, '') in dcterms_value_types.values_list('valuetype', flat=True):
if hasattr(object, 'language') and object.language not in allowed_languages:
newlang = models.DLanguage()
newlang.pk = object.language
newlang.languagename = object.language
newlang.isdefault = False
newlang.save()
allowed_languages = models.DLanguage.objects.values_list('pk', flat=True)
try:
# first try and get any values associated with the concept_scheme
value_type = dcterms_value_types.get(valuetype=predicate.replace(DCTERMS, '')) # predicate.replace(SKOS, '') should yield something like 'prefLabel' or 'scopeNote', etc..
if predicate == DCTERMS.title:
concept_scheme.addvalue({'value':object, 'language': object.language, 'type': 'prefLabel', 'category': value_type.category})
print 'Casting dcterms:title to skos:prefLabel'
if predicate == DCTERMS.description:
concept_scheme.addvalue({'value':object, 'language': object.language, 'type': 'scopeNote', 'category': value_type.category})
print 'Casting dcterms:description to skos:scopeNote'
except:
pass
if str(SKOS) in predicate:
if predicate == SKOS.hasTopConcept:
self.relations.append({'source': scheme_id, 'type': 'hasTopConcept', 'target': self.generate_uuid_from_subject(baseuuid, object)})
self.nodes.append(concept_scheme)
if len(self.nodes) == 0:
raise Exception('No ConceptScheme found in file.')
# Search for Concepts
for s, v, o in graph.triples((None, SKOS.inScheme , scheme)):
concept = Concept({
'id': self.generate_uuid_from_subject(baseuuid, s),
'legacyoid': str(s),
'nodetype': 'Concept'
})
# loop through all the elements within a <skos:Concept> element
for predicate, object in graph.predicate_objects(subject = s):
if str(SKOS) in predicate:
if hasattr(object, 'language') and object.language not in allowed_languages:
newlang = models.DLanguage()
newlang.pk = object.language
newlang.languagename = object.language
newlang.isdefault = False
newlang.save()
allowed_languages = models.DLanguage.objects.values_list('pk', flat=True)
relation_or_value_type = predicate.replace(SKOS, '') # this is essentially the skos element type within a <skos:Concept> element (eg: prefLabel, broader, etc...)
if relation_or_value_type in skos_value_types_list:
value_type = skos_value_types.get(valuetype=relation_or_value_type)
concept.addvalue({'value':object, 'language': object.language, 'type': value_type.valuetype, 'category': value_type.category})
elif predicate == SKOS.broader:
self.relations.append({'source': self.generate_uuid_from_subject(baseuuid, object), 'type': 'narrower', 'target': self.generate_uuid_from_subject(baseuuid, s)})
elif predicate == SKOS.narrower:
self.relations.append({'source': self.generate_uuid_from_subject(baseuuid, s), 'type': relation_or_value_type, 'target': self.generate_uuid_from_subject(baseuuid, object)})
elif predicate == SKOS.related:
self.relations.append({'source': self.generate_uuid_from_subject(baseuuid, s), 'type': relation_or_value_type, 'target': self.generate_uuid_from_subject(baseuuid, object)})
self.nodes.append(concept)
# insert and index the concpets
with transaction.atomic():
for node in self.nodes:
node.save()
# insert the concept relations
for relation in self.relations:
newrelation = models.Relation()
newrelation.relationid = str(uuid.uuid4())
newrelation.conceptfrom_id = relation['source']
newrelation.conceptto_id = relation['target']
newrelation.relationtype_id = relation['type']
newrelation.save()
# need to index after the concepts and relations have been entered into the db
# so that the proper context gets indexed with the concept
for node in self.nodes:
node.index()
return self
else:
raise Exception('graph argument should be of type rdflib.graph.Graph')
def create_reference_data(new_concepts, create_collections):
errors = []
candidates = Concept().get(id='00000000-0000-0000-0000-000000000006')
for arches_nodeid, concepts in new_concepts.iteritems():
collectionid = str(uuid.uuid4())
topconceptid = str(uuid.uuid4())
node = Node.objects.get(nodeid=arches_nodeid)
# if node.datatype is concept or concept-list create concepts and collections
if node.datatype in ['concept', 'concept-list']:
# create collection if create_collections = create, otherwise append to collection already assigned to node
if create_collections == True:
collection_legacyoid = node.name + '_' + str(node.graph_id) + '_import'
# check to see that there is not already a collection for this node
if node.config['rdmCollection'] != None:
errors.append({'type': 'WARNING', 'message': 'A collection already exists for the {0} node. Use the add option to add concepts to this collection.'.format(node.name)})
if len(errors) > 0:
self.errors += errors
collection = None
else:
# if there is no collection assigned to this node, create one and assign it to the node
try:
# check to see that a collection with this legacyid does not already exist
collection = Concept().get(legacyoid=collection_legacyoid)
errors.append({'type': 'WARNING', 'message': 'A collection with the legacyid {0} already exists.'.format(node.name + '_' + str(node.graph_id) + '_import')})
if len(errors) > 0:
self.errors += errors
except:
collection = Concept({
'id': collectionid,
'legacyoid': collection_legacyoid,
'nodetype': 'Collection'
})
collection.addvalue({'id': str(uuid.uuid4()), 'value': node.name + '_import', 'language': settings.LANGUAGE_CODE, 'type': 'prefLabel'})
node.config['rdmCollection'] = collectionid
node.save()
collection.save()
else:
# if create collection = add check that there is a collection associated with node, if no collection associated with node create a collection and associated with the node
try:
collection = Concept().get(id=node.config['rdmCollection'])
except:
collection = Concept({
'id': collectionid,
'legacyoid': node.name + '_' + str(node.graph_id) + '_import',
'nodetype': 'Collection'
})
collection.addvalue({'id': str(uuid.uuid4()), 'value': node.name + '_import', 'language': settings.LANGUAGE_CODE, 'type': 'prefLabel'})
node.config['rdmCollection'] = collectionid
node.save()
collection.save()
if collection != None:
topconcept_legacyoid = node.name + '_' + str(node.graph_id)
# Check if top concept already exists, if not create it and add to candidates scheme
try:
topconcept = Concept().get(legacyoid=topconcept_legacyoid)
except:
topconcept = Concept({
'id': topconceptid,
'legacyoid': topconcept_legacyoid,
'nodetype': 'Concept'
})
topconcept.addvalue({'id': str(uuid.uuid4()), 'value': node.name + '_import', 'language': settings.LANGUAGE_CODE, 'type': 'prefLabel'})
topconcept.save()
candidates.add_relation(topconcept, 'narrower')
# create child concepts and relate to top concept and collection accordingly
for conceptid, value in concepts.iteritems():
concept_legacyoid = value + '_' + node.name + '_' + str(node.graph_id)
# check if concept already exists, if not create and add to topconcept and collection
try:
conceptid = [concept for concept in topconcept.get_child_concepts(topconcept.id) if concept[1] == value][0][0]
concept = Concept().get(id=conceptid)
except:
concept = Concept({
'id': conceptid,
'legacyoid': concept_legacyoid,
'nodetype': 'Concept'
})
concept.addvalue({'id': str(uuid.uuid4()), 'value': value, 'language': settings.LANGUAGE_CODE, 'type': 'prefLabel'})
concept.save()
collection.add_relation(concept, 'member')
topconcept.add_relation(concept, 'narrower')
#if node.datatype is domain or domain-list create options array in node.config
elif node.datatype in ['domain-value', 'domain-value-list']:
for domainid, value in new_concepts[arches_nodeid].iteritems():
# check if value already exists in domain
if value not in [t['text'] for t in node.config['options']]:
domainvalue = {
"text": value,
"selected": False,
"id": domainid
}
node.config['options'].append(domainvalue)
node.save()
