def setUpClass(cls):
for path in test_settings.RESOURCE_GRAPH_LOCATIONS:
management.call_command('packages',
operation='import_graphs',
source=path)
sql = """
INSERT INTO public.resource_instances(resourceinstanceid, legacyid, graphid, createdtime)
VALUES ('40000000-0000-0000-0000-000000000000', '40000000-0000-0000-0000-000000000000', '2f7f8e40-adbc-11e6-ac7f-14109fd34195', '1/1/2000');
INSERT INTO node_groups(nodegroupid, legacygroupid, cardinality)
VALUES ('99999999-0000-0000-0000-000000000001', '', 'n');
INSERT INTO node_groups(nodegroupid, legacygroupid, cardinality)
VALUES ('32999999-0000-0000-0000-000000000000', '', 'n');
INSERT INTO node_groups(nodegroupid, legacygroupid, cardinality)
VALUES ('19999999-0000-0000-0000-000000000000', '', 'n');
INSERT INTO node_groups(nodegroupid, legacygroupid, cardinality)
VALUES ('21111111-0000-0000-0000-000000000000', '', 'n');
"""
cursor = connection.cursor()
cursor.execute(sql)
delete_terms_index()
delete_concepts_index()
delete_search_index()
prepare_terms_index(create=True)
prepare_concepts_index(create=True)
prepare_search_index(create=True)
def forwards_func(apps, schema_editor):
# We get the model from the versioned app registry;
# if we directly import it, it'll be the wrong version
delete_search_index()
for graphid in GraphModel.objects.filter(isresource=True).values_list(
'graphid', flat=True):
prepare_search_index(str(graphid), create=True)
settings_data_file = os.path.join(settings.ROOT_DIR, 'db',
'system_settings',
'Arches_System_Settings.json')
local_settings_available = os.path.isfile(
os.path.join(settings.SYSTEM_SETTINGS_LOCAL_PATH))
if local_settings_available == True:
settings_data_file = settings.SYSTEM_SETTINGS_LOCAL_PATH
management.call_command('es', operation='index_resources')
management.call_command('es', operation='index_resources')
management.call_command('packages',
operation='import_graphs',
source=os.path.join(
settings.ROOT_DIR, 'db', 'system_settings',
'Arches_System_Settings_Model.json'))
management.call_command('packages',
operation='import_business_data',
source=settings_data_file,
overwrite='overwrite')
def setUpTestPackage():
"""
see https://nose.readthedocs.io/en/latest/writing_tests.html#test-packages
this is called from __init__.py
"""
cursor = connection.cursor()
sql = """
INSERT INTO public.oauth2_provider_application(
id,client_id, redirect_uris, client_type, authorization_grant_type,
client_secret,
name, user_id, skip_authorization, created, updated)
VALUES (
44,'{oauth_client_id}', 'http://localhost:8000/test', 'public', 'client-credentials',
'{oauth_client_secret}',
'TEST APP', {user_id}, false, '1-1-2000', '1-1-2000');
"""
sql = sql.format(user_id=1,
oauth_client_id=OAUTH_CLIENT_ID,
oauth_client_secret=OAUTH_CLIENT_SECRET)
cursor.execute(sql)
prepare_terms_index(create=True)
prepare_concepts_index(create=True)
prepare_search_index(create=True)
prepare_resource_relations_index(create=True)
def save(self):
"""
Saves an a graph and it's nodes, edges, and nodegroups back to the db
creates associated card objects if any of the nodegroups don't already have a card
"""
self.validate()
with transaction.atomic():
super(Graph, self).save()
for nodegroup in self.get_nodegroups():
nodegroup.save()
for node in self.nodes.itervalues():
node.save()
for edge in self.edges.itervalues():
edge.save()
for card in self.cards.itervalues():
card.save()
for widget in self.widgets.itervalues():
widget.save()
for nodegroup in self._nodegroups_to_delete:
nodegroup.delete()
self._nodegroups_to_delete = []
if self.isresource:
prepare_search_index(self.graphid, create=True)
return self
def forwards_func(apps, schema_editor):
# We get the model from the versioned app registry;
# if we directly import it, it'll be the wrong version
GraphModel = apps.get_model("models", "GraphModel")
delete_search_index()
for graphid in GraphModel.objects.filter(isresource=True).values_list(
'graphid', flat=True):
prepare_search_index(str(graphid), create=True)
def setup_indexes(self):
prepare_terms_index(create=True)
prepare_concepts_index(create=True)
prepare_resource_relations_index(create=True)
prepare_search_index(create=True)
# add custom indexes
for index in settings.ELASTICSEARCH_CUSTOM_INDEXES:
self.register_index(index['name'])
def setup_indexes(self, name=None):
if name is None:
prepare_terms_index(create=True)
prepare_concepts_index(create=True)
prepare_resource_relations_index(create=True)
prepare_search_index(create=True)
# add custom indexes
for index in settings.ELASTICSEARCH_CUSTOM_INDEXES:
self.register_index(index["name"])
else:
self.register_index(name)
def forwards_func(apps, schema_editor):
# We get the model from the versioned app registry;
# if we directly import it, it'll be the wrong version
delete_search_index()
for graphid in GraphModel.objects.filter(isresource=True).values_list('graphid', flat=True):
prepare_search_index(str(graphid), create=True)
settings_data_file = os.path.join(settings.ROOT_DIR, 'db', 'system_settings', 'Arches_System_Settings.json')
local_settings_available = os.path.isfile(os.path.join(settings.SYSTEM_SETTINGS_LOCAL_PATH))
if local_settings_available == True:
settings_data_file = settings.SYSTEM_SETTINGS_LOCAL_PATH
management.call_command('es', operation='index_resources')
management.call_command('packages', operation='import_graphs', source=os.path.join(settings.ROOT_DIR, 'db', 'system_settings', 'Arches_System_Settings_Model.json'))
management.call_command('packages', operation='import_business_data', source=settings_data_file, overwrite='overwrite')
def forwards_func(apps, schema_editor):
se = SearchEngineFactory().create()
prefix = settings.ELASTICSEARCH_PREFIX
if (se.es.indices.exists(index="%s_strings" % prefix)):
prepare_terms_index(create=True)
doc = {
"source": {
"index": "%s_strings" % prefix,
"type": "term"
},
"dest": {
"index": "%s_terms" % prefix,
"type": "_doc"
}
}
se.es.reindex(body=doc)
prepare_concepts_index(create=True)
doc = {
"source": {
"index": "%s_strings" % prefix,
"type": "concept"
},
"dest": {
"index": "%s_concepts" % prefix,
"type": "_doc"
}
}
se.es.reindex(body=doc)
if (se.es.indices.exists(index="%s_resource" % prefix)):
prepare_search_index(create=True)
doc = {
"source": {
"index": "%s_resource" % prefix
},
"dest": {
"index": "%s_resources" % prefix,
"type": "_doc"
}
}
se.es.reindex(body=doc)
def save(self):
"""
Saves an a graph and it's nodes, edges, and nodegroups back to the db
creates associated card objects if any of the nodegroups don't already have a card
"""
self.validate()
with transaction.atomic():
super(Graph, self).save()
for nodegroup in self.get_nodegroups():
nodegroup.save()
for node in self.nodes.itervalues():
node.save()
for edge in self.edges.itervalues():
edge.save()
for card in self.cards.itervalues():
card.save()
for widget in self.widgets.itervalues():
widget.save()
for functionxgraph in self._functions:
# Right now this only saves a functionxgraph record if the function is present in the database. Otherwise it silently fails.
if functionxgraph.function_id in [
str(id) for id in models.Function.objects.values_list(
'functionid', flat=True)
]:
functionxgraph.save()
for nodegroup in self._nodegroups_to_delete:
nodegroup.delete()
self._nodegroups_to_delete = []
if self.isresource:
prepare_search_index(self.graphid, create=True)
return self
def save(self):
"""
Saves an a graph and it's nodes, edges, and nodegroups back to the db
creates associated card objects if any of the nodegroups don't already have a card
"""
self.validate()
with transaction.atomic():
super(Graph, self).save()
for nodegroup in self.get_nodegroups():
nodegroup.save()
for node in self.nodes.itervalues():
node.save()
for edge in self.edges.itervalues():
edge.save()
for card in self.cards.itervalues():
card.save()
for widget in self.widgets.itervalues():
widget.save()
for functionxgraph in self._functions:
# Right now this only saves a functionxgraph record if the function is present in the database. Otherwise it silently fails.
if functionxgraph.function_id in [str(id) for id in models.Function.objects.values_list('functionid', flat=True)]:
functionxgraph.save()
for nodegroup in self._nodegroups_to_delete:
nodegroup.delete()
self._nodegroups_to_delete = []
if self.isresource:
prepare_search_index(self.graphid, create=True)
return self
def forwards_func(apps, schema_editor):
if settings.SEARCH_BACKEND != 'tests.base_test.TestSearchEngine':
se = SearchEngineFactory().create()
se_old = SearchEngineFactory().create(
hosts=[settings.ELASTICSEARCH_TEMP_HTTP_ENDPOINT])
prefix = settings.ELASTICSEARCH_PREFIX
if (se.es.indices.exists(index="%s_terms" % prefix) is False
and se_old.es.indices.exists(index="%s_strings" % prefix)
is True):
prepare_terms_index(create=True)
doc = {
"source": {
"remote": {
"host": settings.ELASTICSEARCH_TEMP_HTTP_ENDPOINT
},
"index": "%s_strings" % prefix,
"type": "term"
},
"dest": {
"index": "%s_terms" % prefix,
"type": "_doc"
}
}
se.es.reindex(body=doc)
if (se.es.indices.exists(index="%s_concepts" % prefix) is False
and se_old.es.indices.exists(index="%s_strings" % prefix)
is True):
prepare_concepts_index(create=True)
doc = {
"source": {
"remote": {
"host": settings.ELASTICSEARCH_TEMP_HTTP_ENDPOINT
},
"index": "%s_strings" % prefix,
"type": "concept"
},
"dest": {
"index": "%s_concepts" % prefix,
"type": "_doc"
}
}
se.es.reindex(body=doc)
if (se.es.indices.exists(index="%s_resources" % prefix) is False
and se_old.es.indices.exists(index="%s_resource" % prefix)
is True):
prepare_search_index(create=True)
doc = {
"source": {
"remote": {
"host": settings.ELASTICSEARCH_TEMP_HTTP_ENDPOINT
},
"index": "%s_resource" % prefix
},
"dest": {
"index": "%s_resources" % prefix,
"type": "_doc"
}
}
se.es.reindex(body=doc)
if (se.es.indices.exists(index="%s_resource_relations" % prefix) is
False and se_old.es.indices.exists(
index="%s_resource_relations" % prefix) is True):
prepare_resource_relations_index(create=True)
doc = {
"source": {
"remote": {
"host": settings.ELASTICSEARCH_TEMP_HTTP_ENDPOINT
},
"index": "%s_resource_relations" % prefix
},
"dest": {
"index": "%s_resource_relations" % prefix,
"type": "_doc"
}
}
se.es.reindex(body=doc)
def setUpClass(cls):
delete_terms_index()
delete_concepts_index()
delete_search_index()
prepare_terms_index(create=True)
prepare_concepts_index(create=True)
prepare_search_index(create=True)
cls.client = Client()
cls.client.login(username="******", password="******")
models.ResourceInstance.objects.all().delete()
with open(
os.path.join(
"tests/fixtures/resource_graphs/Search Test Model.json"),
"rU") as f:
archesfile = JSONDeserializer().deserialize(f)
ResourceGraphImporter(archesfile["graph"])
cls.search_model_graphid = "d291a445-fa5f-11e6-afa8-14109fd34195"
cls.search_model_cultural_period_nodeid = "7a182580-fa60-11e6-96d1-14109fd34195"
cls.search_model_creation_date_nodeid = "1c1d05f5-fa60-11e6-887f-14109fd34195"
cls.search_model_destruction_date_nodeid = "e771b8a1-65fe-11e7-9163-14109fd34195"
cls.search_model_name_nodeid = "2fe14de3-fa61-11e6-897b-14109fd34195"
cls.search_model_sensitive_info_nodeid = "57446fae-65ff-11e7-b63a-14109fd34195"
cls.search_model_geom_nodeid = "3ebc6785-fa61-11e6-8c85-14109fd34195"
cls.user = User.objects.create_user(
"unpriviliged_user", "*****@*****.**", "test")
cls.user.groups.add(Group.objects.get(name="Guest"))
nodegroup = models.NodeGroup.objects.get(
pk=cls.search_model_destruction_date_nodeid)
assign_perm("no_access_to_nodegroup", cls.user, nodegroup)
# Add a concept that defines a min and max date
concept = {
"id":
"00000000-0000-0000-0000-000000000001",
"legacyoid":
"ARCHES",
"nodetype":
"ConceptScheme",
"values": [],
"subconcepts": [{
"values": [
{
"value": "Mock concept",
"language": "en-US",
"category": "label",
"type": "prefLabel",
"id": "",
"conceptid": ""
},
{
"value": "1950",
"language": "en-US",
"category": "note",
"type": "min_year",
"id": "",
"conceptid": ""
},
{
"value": "1980",
"language": "en-US",
"category": "note",
"type": "max_year",
"id": "",
"conceptid": ""
},
],
"relationshiptype":
"hasTopConcept",
"nodetype":
"Concept",
"id":
"",
"legacyoid":
"",
"subconcepts": [],
"parentconcepts": [],
"relatedconcepts": [],
}],
}
post_data = JSONSerializer().serialize(concept)
content_type = "application/x-www-form-urlencoded"
response = cls.client.post(
reverse(
"concept",
kwargs={"conceptid": "00000000-0000-0000-0000-000000000001"}),
post_data, content_type)
response_json = json.loads(response.content)
valueid = response_json["subconcepts"][0]["values"][0]["id"]
cls.conceptid = response_json["subconcepts"][0]["id"]
# add resource instance with only a cultural period defined
cls.cultural_period_resource = Resource(
graph_id=cls.search_model_graphid)
tile = Tile(data={cls.search_model_cultural_period_nodeid: [valueid]},
nodegroup_id=cls.search_model_cultural_period_nodeid)
cls.cultural_period_resource.tiles.append(tile)
cls.cultural_period_resource.save()
# add resource instance with a creation and destruction date defined
cls.date_resource = Resource(graph_id=cls.search_model_graphid)
tile = Tile(data={cls.search_model_creation_date_nodeid: "1941-01-01"},
nodegroup_id=cls.search_model_creation_date_nodeid)
cls.date_resource.tiles.append(tile)
tile = Tile(
data={cls.search_model_destruction_date_nodeid: "1948-01-01"},
nodegroup_id=cls.search_model_destruction_date_nodeid)
cls.date_resource.tiles.append(tile)
tile = Tile(data={cls.search_model_name_nodeid: "testing 123"},
nodegroup_id=cls.search_model_name_nodeid)
cls.date_resource.tiles.append(tile)
cls.date_resource.save()
# add resource instance with a creation date and a cultural period defined
cls.date_and_cultural_period_resource = Resource(
graph_id=cls.search_model_graphid)
tile = Tile(data={cls.search_model_creation_date_nodeid: "1942-01-01"},
nodegroup_id=cls.search_model_creation_date_nodeid)
cls.date_and_cultural_period_resource.tiles.append(tile)
tile = Tile(data={cls.search_model_cultural_period_nodeid: [valueid]},
nodegroup_id=cls.search_model_cultural_period_nodeid)
cls.date_and_cultural_period_resource.tiles.append(tile)
cls.date_and_cultural_period_resource.save()
# add resource instance with with no dates or periods defined
cls.name_resource = Resource(graph_id=cls.search_model_graphid)
tile = Tile(data={cls.search_model_name_nodeid: "some test name"},
nodegroup_id=cls.search_model_name_nodeid)
cls.name_resource.tiles.append(tile)
geom = {
"type":
"FeatureCollection",
"features": [{
"geometry": {
"type": "Point",
"coordinates": [0, 0]
},
"type": "Feature",
"properties": {}
}],
}
tile = Tile(data={cls.search_model_geom_nodeid: geom},
nodegroup_id=cls.search_model_geom_nodeid)
cls.name_resource.tiles.append(tile)
cls.name_resource.save()
# add delay to allow for indexes to be updated
time.sleep(1)
def setup_indexes(self):
prepare_terms_index(create=True)
prepare_concepts_index(create=True)
prepare_resource_relations_index(create=True)
prepare_search_index(create=True)
def setUpClass(cls):
delete_terms_index()
delete_concepts_index()
delete_search_index()
prepare_terms_index(create=True)
prepare_concepts_index(create=True)
prepare_search_index(create=True)
prepare_resource_relations_index(create=True)
cls.client = Client()
cls.client.login(username='******', password='******')
models.ResourceInstance.objects.all().delete()
with open(
os.path.join(
'tests/fixtures/resource_graphs/Resource Test Model.json'),
'rU') as f:
archesfile = JSONDeserializer().deserialize(f)
resource_graph_importer(archesfile['graph'])
cls.search_model_graphid = 'e503a445-fa5f-11e6-afa8-14109fd34195'
cls.search_model_cultural_period_nodeid = '7a182580-fa60-11e6-96d1-14109fd34195'
cls.search_model_creation_date_nodeid = '1c1d05f5-fa60-11e6-887f-14109fd34195'
cls.search_model_destruction_date_nodeid = 'e771b8a1-65fe-11e7-9163-14109fd34195'
cls.search_model_name_nodeid = '2fe14de3-fa61-11e6-897b-14109fd34195'
cls.search_model_sensitive_info_nodeid = '57446fae-65ff-11e7-b63a-14109fd34195'
cls.search_model_geom_nodeid = '3ebc6785-fa61-11e6-8c85-14109fd34195'
cls.user = User.objects.create_user('test', '*****@*****.**',
'test')
cls.user.save()
cls.user.groups.add(Group.objects.get(name='Guest'))
nodegroup = models.NodeGroup.objects.get(
pk=cls.search_model_destruction_date_nodeid)
assign_perm('no_access_to_nodegroup', cls.user, nodegroup)
# Add a concept that defines a min and max date
concept = {
"id":
"00000000-0000-0000-0000-000000000001",
"legacyoid":
"ARCHES",
"nodetype":
"ConceptScheme",
"values": [],
"subconcepts": [{
"values": [{
"value": "Mock concept",
"language": "en-US",
"category": "label",
"type": "prefLabel",
"id": "",
"conceptid": ""
}, {
"value": "1950",
"language": "en-US",
"category": "note",
"type": "min_year",
"id": "",
"conceptid": ""
}, {
"value": "1980",
"language": "en-US",
"category": "note",
"type": "max_year",
"id": "",
"conceptid": ""
}],
"relationshiptype":
"hasTopConcept",
"nodetype":
"Concept",
"id":
"",
"legacyoid":
"",
"subconcepts": [],
"parentconcepts": [],
"relatedconcepts": []
}]
}
post_data = JSONSerializer().serialize(concept)
content_type = 'application/x-www-form-urlencoded'
response = cls.client.post(
reverse(
'concept',
kwargs={'conceptid': '00000000-0000-0000-0000-000000000001'}),
post_data, content_type)
response_json = json.loads(response.content)
valueid = response_json['subconcepts'][0]['values'][0]['id']
cls.conceptid = response_json['subconcepts'][0]['id']
# Add resource with Name, Cultural Period, Creation Date and Geometry
cls.test_resource = Resource(graph_id=cls.search_model_graphid)
# Add Name
tile = Tile(data={cls.search_model_name_nodeid: 'Test Name 1'},
nodegroup_id=cls.search_model_name_nodeid)
cls.test_resource.tiles.append(tile)
# Add Cultural Period
tile = Tile(data={cls.search_model_cultural_period_nodeid: [valueid]},
nodegroup_id=cls.search_model_cultural_period_nodeid)
cls.test_resource.tiles.append(tile)
# Add Creation Date
tile = Tile(data={cls.search_model_creation_date_nodeid: '1941-01-01'},
nodegroup_id=cls.search_model_creation_date_nodeid)
cls.test_resource.tiles.append(tile)
# Add Gometry
cls.geom = {
"type":
"FeatureCollection",
"features": [{
"geometry": {
"type": "Point",
"coordinates": [0, 0]
},
"type": "Feature",
"properties": {}
}]
}
tile = Tile(data={cls.search_model_geom_nodeid: cls.geom},
nodegroup_id=cls.search_model_geom_nodeid)
cls.test_resource.tiles.append(tile)
cls.test_resource.save()
# add delay to allow for indexes to be updated
time.sleep(1)
def setup_indexes(self):
prepare_term_index(create=True)
prepare_resource_relations_index(create=True)
for graphid in models.GraphModel.objects.filter(isresource=True).values_list('graphid', flat=True):
prepare_search_index(str(graphid), create=True)
