def forwards_func(apps, schema_editor):
# We get the model from the versioned app registry;
# if we directly import it, it'll be the wrong version
arches_concept = Concept().get(id='00000000-0000-0000-0000-000000000007',
include=['label'])
arches_concept.index()
def add_concepts_from_sparql_endpoint(request, conceptid):
if request.method == "POST":
json = request.body
if json is not None:
data = JSONDeserializer().deserialize(json)
parentconcept = Concept({
"id": conceptid,
"nodetype": data["model"]["nodetype"]
})
if parentconcept.nodetype == "Concept":
relationshiptype = "narrower"
elif parentconcept.nodetype == "ConceptScheme":
relationshiptype = "hasTopConcept"
provider = get_sparql_providers(data["endpoint"])
try:
parentconcept.subconcepts = provider.get_concepts(data["ids"])
except Exception as e:
return HttpResponseServerError(e.message)
for subconcept in parentconcept.subconcepts:
subconcept.relationshiptype = relationshiptype
parentconcept.save()
parentconcept.index()
return JSONResponse(parentconcept, indent=4)
else:
return HttpResponseNotAllowed(["POST"])
return HttpResponseNotFound()
def add_concepts_from_sparql_endpoint(request, conceptid):
if request.method == 'POST':
json = request.body
if json != None:
data = JSONDeserializer().deserialize(json)
parentconcept = Concept({
'id': conceptid,
'nodetype': data['model']['nodetype']
})
if parentconcept.nodetype == 'Concept':
relationshiptype = 'narrower'
elif parentconcept.nodetype == 'ConceptScheme':
relationshiptype = 'hasTopConcept'
provider = sparql_providers[data['endpoint']]
try:
parentconcept.subconcepts = provider.get_concepts(data['ids'])
except Exception as e:
return HttpResponseServerError(e.message)
for subconcept in parentconcept.subconcepts:
subconcept.relationshiptype = relationshiptype
parentconcept.save()
parentconcept.index()
return JSONResponse(parentconcept, indent=4)
else:
return HttpResponseNotAllowed(['POST'])
return HttpResponseNotFound()
def import_reference_data(reference_data):
# with transaction.atomic():
# if reference_data != '':
for data in reference_data:
print '\nLOADING {0} CONCEPT SCHEME FROM ARCHES JSON'.format(
data['legacyoid'])
print '---------------------------------------------'
reporter = ConceptImportReporter(data)
def create_collections(concept):
relations = {
'': 'hasCollection',
'hasTopConcept': 'member',
'narrower': 'member',
'narrowerTransitive': 'member'
}
for subconcept in concept.subconcepts:
if concept.relationshiptype in relations.keys():
if concept.id == '00000000-0000-0000-0000-000000000001':
concept.id = '00000000-0000-0000-0000-000000000003'
models.Relation.objects.get_or_create(
conceptfrom_id=concept.id,
conceptto_id=subconcept.id,
relationtype_id=relations[concept.relationshiptype])
reporter.update_concepts_saved()
concept = Concept(data)
concept.save()
concept.traverse(create_collections, 'down')
concept.index(scheme=concept)
reporter.report_results()
def add_concepts_from_sparql_endpoint(request, conceptid):
if request.method == "POST":
json = request.body
if json != None:
data = JSONDeserializer().deserialize(json)
parentconcept = Concept({"id": conceptid, "nodetype": data["model"]["nodetype"]})
if parentconcept.nodetype == "Concept":
relationshiptype = "narrower"
elif parentconcept.nodetype == "ConceptScheme":
relationshiptype = "hasTopConcept"
provider = get_sparql_providers(data["endpoint"])
try:
parentconcept.subconcepts = provider.get_concepts(data["ids"])
except Exception as e:
return HttpResponseServerError(e.message)
for subconcept in parentconcept.subconcepts:
subconcept.relationshiptype = relationshiptype
parentconcept.save()
parentconcept.index()
return JSONResponse(parentconcept, indent=4)
else:
return HttpResponseNotAllowed(["POST"])
return HttpResponseNotFound()
def add_concepts_from_sparql_endpoint(request, conceptid):
if request.method == 'POST':
json = request.body
if json != None:
data = JSONDeserializer().deserialize(json)
parentconcept = Concept({
'id': conceptid,
'nodetype': data['model']['nodetype']
})
if parentconcept.nodetype == 'Concept':
relationshiptype = 'narrower'
elif parentconcept.nodetype == 'ConceptScheme':
relationshiptype = 'hasTopConcept'
provider = get_sparql_providers(data['endpoint'])
try:
parentconcept.subconcepts = provider.get_concepts(data['ids'])
except Exception as e:
return HttpResponseServerError(e.message)
for subconcept in parentconcept.subconcepts:
subconcept.relationshiptype = relationshiptype
parentconcept.save()
parentconcept.index()
return JSONResponse(parentconcept, indent=4)
else:
return HttpResponseNotAllowed(['POST'])
return HttpResponseNotFound()
def index_concepts():
"""
Collects all concepts and indexes both concepts and concept_labels
"""
se = SearchEngineFactory().create()
se.delete_index(index='concept_labels')
se.delete(
index='term',
body=
'{"query":{"bool":{"must_not":[{"constant_score":{"filter":{"missing":{"field":"value.options.conceptid"}}}}],"must":[],"should":[]}}}'
)
Resource().prepare_term_index(create=True)
print 'indexing concepts'
start = datetime.now()
cursor = connection.cursor()
cursor.execute("""select conceptid from concepts.concepts""")
conceptids = cursor.fetchall()
for c in conceptids:
if c[0] not in CORE_CONCEPTS:
concept = Concept().get(id=c[0],
include_subconcepts=True,
include_parentconcepts=False,
include=['label'])
concept.index()
end = datetime.now()
duration = end - start
print 'indexing concepts required', duration.seconds, 'seconds'
cursor = connection.cursor()
sql = """
select conceptid, conceptlabel from concepts.vw_concepts where conceptid not in ('%s')
""" % ("','".join(CORE_CONCEPTS))
cursor.execute(sql)
concepts = cursor.fetchall()
concept_index_results = {'count': len(concepts), 'passed': 0, 'failed': 0}
for conceptid, conceptvalue in concepts:
result = get_indexed_concepts(se, conceptid, conceptvalue)
if result != 'passed':
concept_index_results['failed'] += 1
else:
concept_index_results['passed'] += 1
status = 'Passed' if concept_index_results['failed'] == 0 else 'Failed'
print '\nConcept Index Results:'
print "Status: {0}, In Database: {1}, Indexed: {2}".format(
status, concept_index_results['count'],
concept_index_results['passed'])
def forwards_func(apps, schema_editor):
# We get the model from the versioned app registry;
# if we directly import it, it'll be the wrong version
# index base Arches concept
arches_concept = Concept().get(id="00000000-0000-0000-0000-000000000001",
include=["label"])
arches_concept.index()
DValueType = apps.get_model("models", "DValueType")
DValueType.objects.create(valuetype="identifier",
category="identifiers",
namespace="dcterms",
datatype="text")
def index_entity_concept_lables():
from arches.app.models import models
from arches.app.models.concept import Concept
domains_concept = Concept('00000000-0000-0000-0000-000000000003')
entitynodes_concept = Concept('00000000-0000-0000-0000-000000000004')
entitytypes = models.EntityTypes.objects.all()
for entitytype in entitytypes:
concept = Concept(entitytype.conceptid).get(include=['label'])
if entitytype.businesstablename == 'domains':
concept.index(scheme=domains_concept)
else:
concept.index(scheme=entitynodes_concept)
def index_concepts():
"""
Collects all concepts and indexes both concepts and concept_labels
"""
se = SearchEngineFactory().create()
se.delete_index(index='concept_labels')
se.delete(index='term', body='{"query":{"bool":{"must_not":[{"constant_score":{"filter":{"missing":{"field":"value.options.conceptid"}}}}],"must":[],"should":[]}}}')
Resource().prepare_term_index(create=True)
print 'indexing concepts'
start = datetime.now()
cursor = connection.cursor()
cursor.execute("""select conceptid from concepts.concepts""")
conceptids = cursor.fetchall()
for c in conceptids:
if c[0] not in CORE_CONCEPTS:
concept = Concept().get(id=c[0], include_subconcepts=True, include_parentconcepts=False, include=['label'])
concept.index()
end = datetime.now()
duration = end - start
print 'indexing concepts required', duration.seconds, 'seconds'
cursor = connection.cursor()
sql = """
select conceptid, conceptlabel from concepts.vw_concepts where conceptid not in ('%s')
""" % ("','".join(CORE_CONCEPTS))
cursor.execute(sql)
concepts = cursor.fetchall()
concept_index_results = {'count':len(concepts), 'passed':0, 'failed':0}
for conceptid, conceptvalue in concepts:
result = get_indexed_concepts(se, conceptid, conceptvalue)
if result != 'passed':
concept_index_results['failed'] += 1
else:
concept_index_results['passed'] += 1
status = 'Passed' if concept_index_results['failed'] == 0 else 'Failed'
print '\nConcept Index Results:'
print "Status: {0}, In Database: {1}, Indexed: {2}".format(status, concept_index_results['count'], concept_index_results['passed'])
def forwards_func(apps, schema_editor):
# We get the model from the versioned app registry;
# if we directly import it, it'll be the wrong version
extensions = [os.path.join(settings.ONTOLOGY_PATH, x) for x in settings.ONTOLOGY_EXT]
management.call_command('load_ontology', source=os.path.join(settings.ONTOLOGY_PATH, settings.ONTOLOGY_BASE),
version=settings.ONTOLOGY_BASE_VERSION, ontology_name=settings.ONTOLOGY_BASE_NAME, id=settings.ONTOLOGY_BASE_ID, extensions=','.join(extensions), verbosity=0)
Ontology = apps.get_model("models", "Ontology")
Node = apps.get_model("models", "Node")
Edge = apps.get_model("models", "Edge")
for ontology in Ontology.objects.filter(parentontology=None):
g = Graph()
g.parse(ontology.path.path)
for extension in Ontology.objects.filter(parentontology=ontology):
g.parse(extension.path.path)
ontology_classes = set()
ontology_properties = set()
for ontology_property,p,o in g.triples((None, None, RDF.Property)):
ontology_properties.add(ontology_property)
for s,p,domain_class in g.triples((ontology_property, RDFS.domain, None)):
ontology_classes.add(domain_class)
for s,p,range_class in g.triples((ontology_property, RDFS.range, None)):
ontology_classes.add(range_class)
for ontology_class,p,o in g.triples((None, None, RDFS.Class)):
ontology_classes.add(ontology_class)
for ontology_class in ontology_classes:
for node in Node.objects.filter(ontologyclass=str(ontology_class).split('/')[-1], graph__in=ontology.graphs.all()):
node.ontologyclass = ontology_class
node.save()
for ontology_property in ontology_properties:
for edge in Edge.objects.filter(ontologyproperty=str(ontology_property).split('/')[-1], graph__in=ontology.graphs.all()):
edge.ontologyproperty = ontology_property
edge.save()
# index base Arches concept
arches_concept = Concept().get(id='00000000-0000-0000-0000-000000000001', include=['label'])
arches_concept.index()
DValueType = apps.get_model("models", "DValueType")
DValueType.objects.create(valuetype='identifier', category='identifiers', namespace='dcterms', datatype='text')
def forwards_func(apps, schema_editor):
# We get the model from the versioned app registry;
# if we directly import it, it'll be the wrong version
extensions = [os.path.join(settings.ONTOLOGY_PATH, x) for x in settings.ONTOLOGY_EXT]
management.call_command('load_ontology', source=os.path.join(settings.ONTOLOGY_PATH, settings.ONTOLOGY_BASE),
version=settings.ONTOLOGY_BASE_VERSION, ontology_name=settings.ONTOLOGY_BASE_NAME, id=settings.ONTOLOGY_BASE_ID, extensions=','.join(extensions), verbosity=0)
Ontology = apps.get_model("models", "Ontology")
Node = apps.get_model("models", "Node")
Edge = apps.get_model("models", "Edge")
for ontology in Ontology.objects.filter(parentontology=None):
g = Graph()
g.parse(ontology.path.path)
for extension in Ontology.objects.filter(parentontology=ontology):
g.parse(extension.path.path)
ontology_classes = set()
ontology_properties = set()
for ontology_property,p,o in g.triples((None, None, RDF.Property)):
ontology_properties.add(ontology_property)
for s,p,domain_class in g.triples((ontology_property, RDFS.domain, None)):
ontology_classes.add(domain_class)
for s,p,range_class in g.triples((ontology_property, RDFS.range, None)):
ontology_classes.add(range_class)
for ontology_class,p,o in g.triples((None, None, RDFS.Class)):
ontology_classes.add(ontology_class)
for ontology_class in ontology_classes:
for node in Node.objects.filter(ontologyclass=str(ontology_class).split('/')[-1], graph__in=ontology.graphs.all()):
node.ontologyclass = ontology_class
node.save()
for ontology_property in ontology_properties:
for edge in Edge.objects.filter(ontologyproperty=str(ontology_property).split('/')[-1], graph__in=ontology.graphs.all()):
edge.ontologyproperty = ontology_property
edge.save()
# index base Arches concept
arches_concept = Concept().get(id='00000000-0000-0000-0000-000000000001', include=['label'])
arches_concept.index()
DValueType = apps.get_model("models", "DValueType")
DValueType.objects.create(valuetype='identifier', category='identifiers', namespace='dcterms', datatype='text')
def import_reference_data(reference_data):
# with transaction.atomic():
# if reference_data != '':
for data in reference_data:
print '\nLOADING {0} CONCEPT SCHEME FROM ARCHES JSON'.format(data['legacyoid'])
print '---------------------------------------------'
def create_collections(concept):
relations = {'':'hasCollection', 'hasTopConcept':'member', 'narrower':'member', 'narrowerTransitive':'member'}
for subconcept in concept.subconcepts:
if concept.relationshiptype in relations.keys():
if concept.id == '00000000-0000-0000-0000-000000000001':
concept.id = '00000000-0000-0000-0000-000000000003'
models.Relation.objects.get_or_create(conceptfrom_id=concept.id, conceptto_id=subconcept.id, relationtype_id=relations[concept.relationshiptype])
concept = Concept(data)
concept.save()
concept.traverse(create_collections, 'down')
concept.index(scheme=concept)
def concept(request, conceptid):
f = request.GET.get("f", "json")
mode = request.GET.get("mode", "")
lang = request.GET.get("lang", settings.LANGUAGE_CODE)
pretty = request.GET.get("pretty", False)
if request.method == "GET":
include_subconcepts = request.GET.get("include_subconcepts", "true") == "true"
include_parentconcepts = request.GET.get("include_parentconcepts", "true") == "true"
include_relatedconcepts = request.GET.get("include_relatedconcepts", "true") == "true"
emulate_elastic_search = request.GET.get("emulate_elastic_search", "false") == "true"
depth_limit = request.GET.get("depth_limit", None)
if f == "html":
depth_limit = 1
if not conceptid:
return render_to_response(
"views/rdm/concept-report.htm",
{
"lang": lang,
"concept_count": models.Concepts.objects.filter(nodetype="Concept").count(),
"collection_count": models.Concepts.objects.filter(nodetype="Collection").count(),
"scheme_count": models.Concepts.objects.filter(nodetype="ConceptScheme").count(),
"entitytype_count": models.Concepts.objects.filter(nodetype="EntityType").count(),
"default_report": True,
},
context_instance=RequestContext(request),
)
ret = []
labels = []
this_concept = Concept().get(id=conceptid)
if f == "html":
if mode == "" and (
this_concept.nodetype == "Concept"
or this_concept.nodetype == "ConceptScheme"
or this_concept.nodetype == "EntityType"
):
concept_graph = Concept().get(
id=conceptid,
include_subconcepts=include_subconcepts,
include_parentconcepts=include_parentconcepts,
include_relatedconcepts=include_relatedconcepts,
depth_limit=depth_limit,
up_depth_limit=None,
lang=lang,
)
else:
concept_graph = Concept().get(
id=conceptid,
include_subconcepts=include_subconcepts,
include_parentconcepts=include_parentconcepts,
include_relatedconcepts=include_relatedconcepts,
depth_limit=depth_limit,
up_depth_limit=None,
lang=lang,
semantic=False,
)
languages = models.DLanguages.objects.all()
valuetypes = models.ValueTypes.objects.all()
relationtypes = models.DRelationtypes.objects.all()
prefLabel = concept_graph.get_preflabel(lang=lang)
for subconcept in concept_graph.subconcepts:
subconcept.prefLabel = subconcept.get_preflabel(lang=lang)
for relatedconcept in concept_graph.relatedconcepts:
relatedconcept.prefLabel = relatedconcept.get_preflabel(lang=lang)
for value in concept_graph.values:
if value.category == "label":
labels.append(value)
if mode == "" and (
this_concept.nodetype == "Concept"
or this_concept.nodetype == "ConceptScheme"
or this_concept.nodetype == "EntityType"
):
if concept_graph.nodetype == "ConceptScheme":
parent_relations = relationtypes.filter(category="Properties")
else:
parent_relations = (
relationtypes.filter(category="Semantic Relations")
.exclude(relationtype="related")
.exclude(relationtype="broader")
.exclude(relationtype="broaderTransitive")
)
return render_to_response(
"views/rdm/concept-report.htm",
{
"lang": lang,
"prefLabel": prefLabel,
"labels": labels,
"concept": concept_graph,
"languages": languages,
"sparql_providers": get_sparql_providers(),
"valuetype_labels": valuetypes.filter(category="label"),
"valuetype_notes": valuetypes.filter(category="note"),
"valuetype_related_values": valuetypes.filter(category="undefined"),
"parent_relations": parent_relations,
"related_relations": relationtypes.filter(
Q(category="Mapping Properties") | Q(relationtype="related")
),
"concept_paths": concept_graph.get_paths(lang=lang),
"graph_json": JSONSerializer().serialize(concept_graph.get_node_and_links(lang=lang)),
"direct_parents": [parent.get_preflabel(lang=lang) for parent in concept_graph.parentconcepts],
},
context_instance=RequestContext(request),
)
else:
return render_to_response(
"views/rdm/entitytype-report.htm",
{
"lang": lang,
"prefLabel": prefLabel,
"labels": labels,
"concept": concept_graph,
"languages": languages,
"valuetype_labels": valuetypes.filter(category="label"),
"valuetype_notes": valuetypes.filter(category="note"),
"valuetype_related_values": valuetypes.filter(category="undefined"),
"related_relations": relationtypes.filter(relationtype="member"),
"concept_paths": concept_graph.get_paths(lang=lang),
},
context_instance=RequestContext(request),
)
concept_graph = Concept().get(
id=conceptid,
include_subconcepts=include_subconcepts,
include_parentconcepts=include_parentconcepts,
include_relatedconcepts=include_relatedconcepts,
depth_limit=depth_limit,
up_depth_limit=None,
lang=lang,
)
if f == "skos":
include_parentconcepts = False
include_subconcepts = True
depth_limit = None
skos = SKOSWriter()
return HttpResponse(skos.write(concept_graph, format="pretty-xml"), content_type="application/xml")
if emulate_elastic_search:
ret.append({"_type": id, "_source": concept_graph})
else:
ret.append(concept_graph)
if emulate_elastic_search:
ret = {"hits": {"hits": ret}}
return JSONResponse(ret, indent=4 if pretty else None)
if request.method == "POST":
if len(request.FILES) > 0:
skosfile = request.FILES.get("skosfile", None)
imagefile = request.FILES.get("file", None)
if imagefile:
value = models.FileValues(
valueid=str(uuid.uuid4()),
value=request.FILES.get("file", None),
conceptid_id=conceptid,
valuetype_id="image",
languageid_id=settings.LANGUAGE_CODE,
)
value.save()
return JSONResponse(value)
elif skosfile:
skos = SKOSReader()
rdf = skos.read_file(skosfile)
ret = skos.save_concepts_from_skos(rdf)
return JSONResponse(ret)
else:
data = JSONDeserializer().deserialize(request.body)
if data:
with transaction.atomic():
concept = Concept(data)
concept.save()
concept.index()
return JSONResponse(concept)
if request.method == "DELETE":
data = JSONDeserializer().deserialize(request.body)
if data:
with transaction.atomic():
concept = Concept(data)
delete_self = data["delete_self"] if "delete_self" in data else False
if not (delete_self and concept.id in CORE_CONCEPTS):
in_use = False
if delete_self:
check_concept = Concept().get(data["id"], include_subconcepts=True)
in_use = check_concept.check_if_concept_in_use()
if "subconcepts" in data:
for subconcept in data["subconcepts"]:
if in_use == False:
check_concept = Concept().get(subconcept["id"], include_subconcepts=True)
in_use = check_concept.check_if_concept_in_use()
if in_use == False:
concept.delete_index(delete_self=delete_self)
concept.delete(delete_self=delete_self)
else:
return JSONResponse({"in_use": in_use})
return JSONResponse(concept)
return HttpResponseNotFound
def concept(request, conceptid):
f = request.GET.get('f', 'json')
mode = request.GET.get('mode', '')
lang = request.GET.get('lang', settings.LANGUAGE_CODE)
pretty = request.GET.get('pretty', False)
if request.method == 'GET':
include_subconcepts = request.GET.get('include_subconcepts', 'true') == 'true'
include_parentconcepts = request.GET.get('include_parentconcepts', 'true') == 'true'
include_relatedconcepts = request.GET.get('include_relatedconcepts', 'true') == 'true'
emulate_elastic_search = request.GET.get('emulate_elastic_search', 'false') == 'true'
depth_limit = request.GET.get('depth_limit', None)
depth_limit = 1
if not conceptid:
return render(request, 'views/rdm/concept-report.htm', {
'lang': lang,
'concept_count': models.Concept.objects.filter(nodetype='Concept').count(),
'collection_count': models.Concept.objects.filter(nodetype='Collection').count(),
'scheme_count': models.Concept.objects.filter(nodetype='ConceptScheme').count(),
'entitytype_count': models.Concept.objects.filter(nodetype='EntityType').count(),
'default_report': True
})
labels = []
concept_graph = Concept().get(id=conceptid, include_subconcepts=include_subconcepts,
include_parentconcepts=include_parentconcepts, include_relatedconcepts=include_relatedconcepts,
depth_limit=depth_limit, up_depth_limit=None, lang=lang, semantic=(mode == 'semantic' or mode == ''))
languages = sort_languages(models.DLanguage.objects.all(), lang)
valuetypes = models.DValueType.objects.all()
relationtypes = models.DRelationType.objects.all()
prefLabel = concept_graph.get_preflabel(lang=lang)
for subconcept in concept_graph.subconcepts:
subconcept.prefLabel = subconcept.get_preflabel(lang=lang)
for relatedconcept in concept_graph.relatedconcepts:
relatedconcept.prefLabel = relatedconcept.get_preflabel(lang=lang)
for value in concept_graph.values:
if value.category == 'label':
labels.append(value)
if (mode == 'semantic' or mode == '') and (concept_graph.nodetype == 'Concept' or concept_graph.nodetype == 'ConceptScheme' or concept_graph.nodetype == 'EntityType'):
if concept_graph.nodetype == 'ConceptScheme':
parent_relations = relationtypes.filter(category='Properties')
else:
parent_relations = relationtypes.filter(category='Semantic Relations').exclude(relationtype = 'related').exclude(relationtype='broader').exclude(relationtype='broaderTransitive')
return render(request, 'views/rdm/concept-report.htm', {
'lang': lang,
'prefLabel': prefLabel,
'labels': labels,
'concept': concept_graph,
'languages': languages,
'sparql_providers': get_sparql_providers(),
'valuetype_labels': valuetypes.filter(category='label'),
'valuetype_notes': valuetypes.filter(category='note'),
'valuetype_related_values': valuetypes.filter(category__in=['undefined','identifiers']),
'parent_relations': parent_relations,
'related_relations': relationtypes.filter(Q(category='Mapping Properties') | Q(relationtype = 'related')),
'concept_paths': concept_graph.get_paths(lang=lang),
'graph_json': JSONSerializer().serialize(concept_graph.get_node_and_links(lang=lang)),
'direct_parents': [parent.get_preflabel(lang=lang) for parent in concept_graph.parentconcepts]
})
elif mode == 'collections':
return render(request, 'views/rdm/entitytype-report.htm', {
'lang': lang,
'prefLabel': prefLabel,
'labels': labels,
'concept': concept_graph,
'languages': languages,
'valuetype_labels': valuetypes.filter(category='label'),
'valuetype_notes': valuetypes.filter(category='note'),
'valuetype_related_values': valuetypes.filter(category__in=['undefined','identifiers']),
'related_relations': relationtypes.filter(relationtype = 'member'),
'concept_paths': concept_graph.get_paths(lang=lang)
})
if request.method == 'POST':
if len(request.FILES) > 0:
skosfile = request.FILES.get('skosfile', None)
imagefile = request.FILES.get('file', None)
if imagefile:
value = models.FileValue(valueid = str(uuid.uuid4()), value = request.FILES.get('file', None), concept_id = conceptid, valuetype_id = 'image',language_id = settings.LANGUAGE_CODE)
value.save()
return JSONResponse(value)
elif skosfile:
overwrite_options = request.POST.get('overwrite_options', None)
staging_options = request.POST.get('staging_options', None)
skos = SKOSReader()
try:
rdf = skos.read_file(skosfile)
ret = skos.save_concepts_from_skos(rdf, overwrite_options, staging_options)
return JSONResponse(ret)
except:
return JSONResponse({'message':{'title': _('Unable to Load SKOS File'), 'text': _('There was an issue saving the contents of the file to Arches.')}}, status=500)
else:
data = JSONDeserializer().deserialize(request.body)
if data:
with transaction.atomic():
concept = Concept(data)
concept.save()
concept.index()
return JSONResponse(concept)
if request.method == 'DELETE':
data = JSONDeserializer().deserialize(request.body)
if data:
with transaction.atomic():
concept = Concept(data)
delete_self = data['delete_self'] if 'delete_self' in data else False
if not (delete_self and concept.id in CORE_CONCEPTS):
if concept.nodetype == 'Collection':
concept.delete(delete_self=delete_self)
else:
in_use = False
if delete_self:
check_concept = Concept().get(data['id'], include_subconcepts=True)
in_use = check_concept.check_if_concept_in_use()
if 'subconcepts' in data:
for subconcept in data['subconcepts']:
if in_use == False:
check_concept = Concept().get(subconcept['id'], include_subconcepts=True)
in_use = check_concept.check_if_concept_in_use()
if in_use == False:
concept.delete_index(delete_self=delete_self)
concept.delete(delete_self=delete_self)
else:
return JSONResponse({"in_use": in_use, 'message':{'title': _('Unable to Delete'), 'text': _('This concept or one of it\'s subconcepts is already in use by an existing resource.')}})
return JSONResponse(concept)
return HttpResponseNotFound
def forwards_func(apps, schema_editor):
# We get the model from the versioned app registry;
# if we directly import it, it'll be the wrong version
arches_concept = Concept().get(id='00000000-0000-0000-0000-000000000007', include=['label'])
arches_concept.index()
def concept(request, conceptid):
f = request.GET.get('f', 'json')
mode = request.GET.get('mode', '')
lang = request.GET.get('lang', settings.LANGUAGE_CODE)
pretty = request.GET.get('pretty', False)
if request.method == 'GET':
include_subconcepts = request.GET.get('include_subconcepts', 'true') == 'true'
include_parentconcepts = request.GET.get('include_parentconcepts', 'true') == 'true'
include_relatedconcepts = request.GET.get('include_relatedconcepts', 'true') == 'true'
emulate_elastic_search = request.GET.get('emulate_elastic_search', 'false') == 'true'
depth_limit = request.GET.get('depth_limit', None)
if f == 'html':
depth_limit = 1
if not conceptid:
return render(request, 'views/rdm/concept-report.htm', {
'lang': lang,
'concept_count': models.Concept.objects.filter(nodetype='Concept').count(),
'collection_count': models.Concept.objects.filter(nodetype='Collection').count(),
'scheme_count': models.Concept.objects.filter(nodetype='ConceptScheme').count(),
'entitytype_count': models.Concept.objects.filter(nodetype='EntityType').count(),
'default_report': True
})
ret = []
labels = []
this_concept = Concept().get(id=conceptid)
if f == 'html':
if mode == '' and (this_concept.nodetype == 'Concept' or this_concept.nodetype == 'ConceptScheme' or this_concept.nodetype == 'EntityType'):
concept_graph = Concept().get(id=conceptid, include_subconcepts=include_subconcepts,
include_parentconcepts=include_parentconcepts, include_relatedconcepts=include_relatedconcepts,
depth_limit=depth_limit, up_depth_limit=None, lang=lang)
else:
concept_graph = Concept().get(id=conceptid, include_subconcepts=include_subconcepts,
include_parentconcepts=include_parentconcepts, include_relatedconcepts=include_relatedconcepts,
depth_limit=depth_limit, up_depth_limit=None, lang=lang, semantic=False)
languages = models.DLanguage.objects.all()
valuetypes = models.DValueType.objects.all()
relationtypes = models.DRelationType.objects.all()
prefLabel = concept_graph.get_preflabel(lang=lang)
for subconcept in concept_graph.subconcepts:
subconcept.prefLabel = subconcept.get_preflabel(lang=lang)
for relatedconcept in concept_graph.relatedconcepts:
relatedconcept.prefLabel = relatedconcept.get_preflabel(lang=lang)
for value in concept_graph.values:
if value.category == 'label':
labels.append(value)
if mode == '' and (this_concept.nodetype == 'Concept' or this_concept.nodetype == 'ConceptScheme' or this_concept.nodetype == 'EntityType'):
if concept_graph.nodetype == 'ConceptScheme':
parent_relations = relationtypes.filter(category='Properties')
else:
parent_relations = relationtypes.filter(category='Semantic Relations').exclude(relationtype = 'related').exclude(relationtype='broader').exclude(relationtype='broaderTransitive')
return render(request, 'views/rdm/concept-report.htm', {
'lang': lang,
'prefLabel': prefLabel,
'labels': labels,
'concept': concept_graph,
'languages': languages,
'sparql_providers': get_sparql_providers(),
'valuetype_labels': valuetypes.filter(category='label'),
'valuetype_notes': valuetypes.filter(category='note'),
'valuetype_related_values': valuetypes.filter(category='undefined'),
'parent_relations': parent_relations,
'related_relations': relationtypes.filter(Q(category='Mapping Properties') | Q(relationtype = 'related')),
'concept_paths': concept_graph.get_paths(lang=lang),
'graph_json': JSONSerializer().serialize(concept_graph.get_node_and_links(lang=lang)),
'direct_parents': [parent.get_preflabel(lang=lang) for parent in concept_graph.parentconcepts]
})
else:
return render(request, 'views/rdm/entitytype-report.htm', {
'lang': lang,
'prefLabel': prefLabel,
'labels': labels,
'concept': concept_graph,
'languages': languages,
'valuetype_labels': valuetypes.filter(category='label'),
'valuetype_notes': valuetypes.filter(category='note'),
'valuetype_related_values': valuetypes.filter(category='undefined'),
'related_relations': relationtypes.filter(relationtype = 'member'),
'concept_paths': concept_graph.get_paths(lang=lang)
})
concept_graph = Concept().get(id=conceptid, include_subconcepts=include_subconcepts,
include_parentconcepts=include_parentconcepts, include_relatedconcepts=include_relatedconcepts,
depth_limit=depth_limit, up_depth_limit=None, lang=lang)
if f == 'skos':
include_parentconcepts = False
include_subconcepts = True
depth_limit = None
skos = SKOSWriter()
return HttpResponse(skos.write(concept_graph, format="pretty-xml"), content_type="application/xml")
if emulate_elastic_search:
ret.append({'_type': id, '_source': concept_graph})
else:
ret.append(concept_graph)
if emulate_elastic_search:
ret = {'hits':{'hits':ret}}
return JSONResponse(ret, indent=4 if pretty else None)
if request.method == 'POST':
if len(request.FILES) > 0:
skosfile = request.FILES.get('skosfile', None)
imagefile = request.FILES.get('file', None)
if imagefile:
value = models.FileValue(valueid = str(uuid.uuid4()), value = request.FILES.get('file', None), conceptid_id = conceptid, valuetype_id = 'image',languageid_id = settings.LANGUAGE_CODE)
value.save()
return JSONResponse(value)
elif skosfile:
skos = SKOSReader()
rdf = skos.read_file(skosfile)
ret = skos.save_concepts_from_skos(rdf)
return JSONResponse(ret)
else:
data = JSONDeserializer().deserialize(request.body)
if data:
with transaction.atomic():
concept = Concept(data)
concept.save()
concept.index()
return JSONResponse(concept)
if request.method == 'DELETE':
data = JSONDeserializer().deserialize(request.body)
if data:
with transaction.atomic():
concept = Concept(data)
delete_self = data['delete_self'] if 'delete_self' in data else False
if not (delete_self and concept.id in CORE_CONCEPTS):
in_use = False
if delete_self:
check_concept = Concept().get(data['id'], include_subconcepts=True)
in_use = check_concept.check_if_concept_in_use()
if 'subconcepts' in data:
for subconcept in data['subconcepts']:
if in_use == False:
check_concept = Concept().get(subconcept['id'], include_subconcepts=True)
in_use = check_concept.check_if_concept_in_use()
if in_use == False:
concept.delete_index(delete_self=delete_self)
concept.delete(delete_self=delete_self)
else:
return JSONResponse({"in_use": in_use})
return JSONResponse(concept)
return HttpResponseNotFound
def concept(request, conceptid):
f = request.GET.get('f', 'json')
mode = request.GET.get('mode', '')
lang = request.GET.get('lang', settings.LANGUAGE_CODE)
pretty = request.GET.get('pretty', False)
if request.method == 'GET':
include_subconcepts = request.GET.get('include_subconcepts',
'true') == 'true'
include_parentconcepts = request.GET.get('include_parentconcepts',
'true') == 'true'
include_relatedconcepts = request.GET.get('include_relatedconcepts',
'true') == 'true'
emulate_elastic_search = request.GET.get('emulate_elastic_search',
'false') == 'true'
depth_limit = request.GET.get('depth_limit', None)
if f == 'html':
depth_limit = 1
if not conceptid:
return render_to_response(
'views/rdm/concept-report.htm', {
'lang':
lang,
'concept_count':
models.Concepts.objects.filter(
nodetype='Concept').count(),
'collection_count':
models.Concepts.objects.filter(
nodetype='Collection').count(),
'scheme_count':
models.Concepts.objects.filter(
nodetype='ConceptScheme').count(),
'entitytype_count':
models.Concepts.objects.filter(
nodetype='EntityType').count(),
'default_report':
True
},
context_instance=RequestContext(request))
ret = []
labels = []
this_concept = Concept().get(id=conceptid)
if f == 'html':
if mode == '' and (this_concept.nodetype == 'Concept'
or this_concept.nodetype == 'ConceptScheme'
or this_concept.nodetype == 'EntityType'):
concept_graph = Concept().get(
id=conceptid,
include_subconcepts=include_subconcepts,
include_parentconcepts=include_parentconcepts,
include_relatedconcepts=include_relatedconcepts,
depth_limit=depth_limit,
up_depth_limit=None,
lang=lang)
else:
concept_graph = Concept().get(
id=conceptid,
include_subconcepts=include_subconcepts,
include_parentconcepts=include_parentconcepts,
include_relatedconcepts=include_relatedconcepts,
depth_limit=depth_limit,
up_depth_limit=None,
lang=lang,
semantic=False)
languages = models.DLanguages.objects.all()
valuetypes = models.ValueTypes.objects.all()
relationtypes = models.DRelationtypes.objects.all()
prefLabel = concept_graph.get_preflabel(lang=lang)
for subconcept in concept_graph.subconcepts:
subconcept.prefLabel = subconcept.get_preflabel(lang=lang)
for relatedconcept in concept_graph.relatedconcepts:
relatedconcept.prefLabel = relatedconcept.get_preflabel(
lang=lang)
for value in concept_graph.values:
if value.category == 'label':
labels.append(value)
if mode == '' and (this_concept.nodetype == 'Concept'
or this_concept.nodetype == 'ConceptScheme'
or this_concept.nodetype == 'EntityType'):
if concept_graph.nodetype == 'ConceptScheme':
parent_relations = relationtypes.filter(
category='Properties')
else:
parent_relations = relationtypes.filter(
category='Semantic Relations').exclude(
relationtype='related').exclude(
relationtype='broader').exclude(
relationtype='broaderTransitive')
return render_to_response(
'views/rdm/concept-report.htm', {
'lang':
lang,
'prefLabel':
prefLabel,
'labels':
labels,
'concept':
concept_graph,
'languages':
languages,
'sparql_providers':
sparql_providers,
'valuetype_labels':
valuetypes.filter(category='label'),
'valuetype_notes':
valuetypes.filter(category='note'),
'valuetype_related_values':
valuetypes.filter(category='undefined'),
'parent_relations':
parent_relations,
'related_relations':
relationtypes.filter(
Q(category='Mapping Properties')
| Q(relationtype='related')),
'concept_paths':
concept_graph.get_paths(lang=lang),
'graph_json':
JSONSerializer().serialize(
concept_graph.get_node_and_links(lang=lang)),
'direct_parents': [
parent.get_preflabel(lang=lang)
for parent in concept_graph.parentconcepts
]
},
context_instance=RequestContext(request))
else:
return render_to_response(
'views/rdm/entitytype-report.htm', {
'lang':
lang,
'prefLabel':
prefLabel,
'labels':
labels,
'concept':
concept_graph,
'languages':
languages,
'valuetype_labels':
valuetypes.filter(category='label'),
'valuetype_notes':
valuetypes.filter(category='note'),
'valuetype_related_values':
valuetypes.filter(category='undefined'),
'related_relations':
relationtypes.filter(relationtype='member'),
'concept_paths':
concept_graph.get_paths(lang=lang)
},
context_instance=RequestContext(request))
concept_graph = Concept().get(
id=conceptid,
include_subconcepts=include_subconcepts,
include_parentconcepts=include_parentconcepts,
include_relatedconcepts=include_relatedconcepts,
depth_limit=depth_limit,
up_depth_limit=None,
lang=lang)
if f == 'skos':
include_parentconcepts = False
include_subconcepts = True
depth_limit = None
skos = SKOSWriter()
return HttpResponse(skos.write(concept_graph, format="pretty-xml"),
content_type="application/xml")
if emulate_elastic_search:
ret.append({'_type': id, '_source': concept_graph})
else:
ret.append(concept_graph)
if emulate_elastic_search:
ret = {'hits': {'hits': ret}}
return JSONResponse(ret, indent=4 if pretty else None)
if request.method == 'POST':
if len(request.FILES) > 0:
skosfile = request.FILES.get('skosfile', None)
imagefile = request.FILES.get('file', None)
if imagefile:
value = models.FileValues(valueid=str(uuid.uuid4()),
value=request.FILES.get(
'file', None),
conceptid_id=conceptid,
valuetype_id='image',
languageid_id=settings.LANGUAGE_CODE)
value.save()
return JSONResponse(value)
elif skosfile:
skos = SKOSReader()
rdf = skos.read_file(skosfile)
ret = skos.save_concepts_from_skos(rdf)
return JSONResponse(ret)
else:
data = JSONDeserializer().deserialize(request.body)
if data:
with transaction.atomic():
concept = Concept(data)
concept.save()
concept.index()
return JSONResponse(concept)
if request.method == 'DELETE':
data = JSONDeserializer().deserialize(request.body)
if data:
with transaction.atomic():
concept = Concept(data)
delete_self = data[
'delete_self'] if 'delete_self' in data else False
if not (delete_self and concept.id in CORE_CONCEPTS):
in_use = False
if delete_self:
check_concept = Concept().get(data['id'],
include_subconcepts=True)
in_use = check_concept.check_if_concept_in_use()
if 'subconcepts' in data:
for subconcept in data['subconcepts']:
if in_use == False:
check_concept = Concept().get(
subconcept['id'], include_subconcepts=True)
in_use = check_concept.check_if_concept_in_use(
)
if in_use == False:
concept.delete_index(delete_self=delete_self)
concept.delete(delete_self=delete_self)
else:
return JSONResponse({"in_use": in_use})
return JSONResponse(concept)
return HttpResponseNotFound
def concept(request, conceptid):
f = request.GET.get("f", "json")
mode = request.GET.get("mode", "")
lang = request.GET.get("lang", request.LANGUAGE_CODE)
pretty = request.GET.get("pretty", False)
if request.method == "GET":
include_subconcepts = request.GET.get("include_subconcepts",
"true") == "true"
include_parentconcepts = request.GET.get("include_parentconcepts",
"true") == "true"
include_relatedconcepts = request.GET.get("include_relatedconcepts",
"true") == "true"
emulate_elastic_search = request.GET.get("emulate_elastic_search",
"false") == "true"
depth_limit = request.GET.get("depth_limit", None)
depth_limit = 1
if not conceptid:
return render(
request,
"views/rdm/concept-report.htm",
{
"lang":
lang,
"concept_count":
models.Concept.objects.filter(nodetype="Concept").count(),
"collection_count":
models.Concept.objects.filter(
nodetype="Collection").count(),
"scheme_count":
models.Concept.objects.filter(
nodetype="ConceptScheme").count(),
"entitytype_count":
models.Concept.objects.filter(
nodetype="EntityType").count(),
"default_report":
True,
},
)
labels = []
concept_graph = Concept().get(
id=conceptid,
include_subconcepts=include_subconcepts,
include_parentconcepts=include_parentconcepts,
include_relatedconcepts=include_relatedconcepts,
depth_limit=depth_limit,
up_depth_limit=None,
lang=lang,
semantic=(mode == "semantic" or mode == ""),
)
languages = sort_languages(models.DLanguage.objects.all(), lang)
valuetypes = models.DValueType.objects.all()
relationtypes = models.DRelationType.objects.all()
prefLabel = concept_graph.get_preflabel(lang=lang)
for subconcept in concept_graph.subconcepts:
subconcept.prefLabel = subconcept.get_preflabel(lang=lang)
for relatedconcept in concept_graph.relatedconcepts:
relatedconcept.prefLabel = relatedconcept.get_preflabel(lang=lang)
for value in concept_graph.values:
if value.category == "label":
labels.append(value)
if value.type == "image":
value.full_image_url = (
(settings.FORCE_SCRIPT_NAME
if settings.FORCE_SCRIPT_NAME is not None else "") +
settings.MEDIA_URL + value.value).replace("//", "/")
if (mode == "semantic"
or mode == "") and (concept_graph.nodetype == "Concept" or
concept_graph.nodetype == "ConceptScheme"
or concept_graph.nodetype == "EntityType"):
if concept_graph.nodetype == "ConceptScheme":
parent_relations = relationtypes.filter(category="Properties")
else:
parent_relations = (relationtypes.filter(
category="Semantic Relations").exclude(
relationtype="related").exclude(
relationtype="broader").exclude(
relationtype="broaderTransitive"))
return render(
request,
"views/rdm/concept-report.htm",
{
"FORCE_SCRIPT_NAME":
settings.FORCE_SCRIPT_NAME,
"lang":
lang,
"prefLabel":
prefLabel,
"labels":
labels,
"concept":
concept_graph,
"languages":
languages,
"sparql_providers":
get_sparql_providers(),
"valuetype_labels":
valuetypes.filter(category="label"),
"valuetype_notes":
valuetypes.filter(category="note"),
"valuetype_related_values":
valuetypes.filter(
category__in=["undefined", "identifiers"]),
"parent_relations":
parent_relations,
"related_relations":
relationtypes.filter(
Q(category="Mapping Properties")
| Q(relationtype="related")),
"concept_paths":
concept_graph.get_paths(lang=lang),
"graph_json":
JSONSerializer().serialize(
concept_graph.get_node_and_links(lang=lang)),
"direct_parents": [
parent.get_preflabel(lang=lang)
for parent in concept_graph.parentconcepts
],
},
)
elif mode == "collections":
return render(
request,
"views/rdm/entitytype-report.htm",
{
"lang":
lang,
"prefLabel":
prefLabel,
"labels":
labels,
"concept":
concept_graph,
"languages":
languages,
"valuetype_labels":
valuetypes.filter(category="label"),
"valuetype_notes":
valuetypes.filter(category="note"),
"valuetype_related_values":
valuetypes.filter(
category__in=["undefined", "identifiers"]),
"related_relations":
relationtypes.filter(relationtype="member"),
"concept_paths":
concept_graph.get_paths(lang=lang),
},
)
if request.method == "POST":
if len(request.FILES) > 0:
skosfile = request.FILES.get("skosfile", None)
imagefile = request.FILES.get("file", None)
if imagefile:
value = models.FileValue(
valueid=str(uuid.uuid4()),
value=request.FILES.get("file", None),
concept_id=conceptid,
valuetype_id="image",
language_id=lang,
)
value.save()
return JSONResponse(value)
elif skosfile:
overwrite_options = request.POST.get("overwrite_options", None)
staging_options = request.POST.get("staging_options", None)
skos = SKOSReader()
try:
rdf = skos.read_file(skosfile)
ret = skos.save_concepts_from_skos(rdf, overwrite_options,
staging_options)
return JSONResponse(ret)
except Exception as e:
return JSONErrorResponse(
_('Unable to Load SKOS File'),
_('There was an issue saving the contents of the file to Arches. '
) + str(e))
else:
data = JSONDeserializer().deserialize(request.body)
if data:
with transaction.atomic():
concept = Concept(data)
concept.save()
concept.index()
return JSONResponse(concept)
if request.method == "DELETE":
data = JSONDeserializer().deserialize(request.body)
if data:
with transaction.atomic():
concept = Concept(data)
delete_self = data[
"delete_self"] if "delete_self" in data else False
if not (delete_self and concept.id in CORE_CONCEPTS):
if concept.nodetype == "Collection":
concept.delete(delete_self=delete_self)
else:
in_use = False
if delete_self:
check_concept = Concept().get(
data["id"], include_subconcepts=True)
in_use = check_concept.check_if_concept_in_use()
if "subconcepts" in data:
for subconcept in data["subconcepts"]:
if in_use == False:
check_concept = Concept().get(
subconcept["id"],
include_subconcepts=True)
in_use = check_concept.check_if_concept_in_use(
)
if in_use == False:
concept.delete_index(delete_self=delete_self)
concept.delete(delete_self=delete_self)
else:
return JSONErrorResponse(
_('Unable to Delete'),
_('This concept or one of it\'s subconcepts is already in use by an existing resource.'
), {"in_use": in_use})
return JSONResponse(concept)
return HttpResponseNotFound
