def check_status(self):
urls = settings.ES_URLS
try:
from search import get_es_client
es = get_es_client()
return es.indices.exists(settings.SITE_NAME)
except Exception as e:
raise ServiceUnavailable("Connection Error")
def setUpClass(cls):
from search import get_es_client
cls.client = get_es_client()
cls.index = "nested_test"
if cls.client.indices.exists(cls.index):
cls.client.indices.delete(cls.index)
cls.client.indices.create(index=cls.index, body=mappings)
cls.s = Search(using=cls.client, index=cls.index)
def get_indexed_datasets(self):
from elasticsearch_dsl import Search, A
from search import get_es_client
client = get_es_client()
s = Search(using=client)
a = A('terms', field='delving_spec.raw', size=500)
s.aggs.bucket('delving_spec', a)
response = s.execute()
self.total_records = response.hits.total
return response.aggregations.delving_spec.buckets
def data_counts():
"""
return the total number of 'active' datasets
:return: dataset_count
"""
try:
dataset_count = DataSet.objects.count()
object_count = get_es_client().count(index=settings.SITE_NAME, doc_type="void_edmrecord")['count']
return {'datasets': dataset_count, 'objects': object_count}
except:
return {'datasets': 0, 'objects': 0}
def test_save_cached_resource(self):
from search import get_es_client
client = get_es_client()
s = Search(client).index("test")
del_response = client.delete_by_query(index='test', q="*:*")
es_response = s.execute()
self.assertEquals(
es_response.hits.total,
0
)
assert CacheResource.objects.count() == 0
store = rdfstore._rdfstore_test
store._clear_all()
graph = load_nquad_fixtures()
assert len(graph) > 0
resource = CacheResource()
resource.source_uri = "http://nl.dbpedia.org/resource/Ton_Smits"
resource.save()
response = tasks.store_cache_resource.delay(obj=resource, store=store)
assert response.status
assert response.result
response = tasks.update_rdf_in_index.delay(obj=resource, store=store, index='test')
time.sleep(1)
assert response.status
assert response.result
assert CacheResource.objects.count() == 1
es_response = s.execute()
self.assertEquals(
es_response.hits.total,
1
)
record = es_response.hits[0]
# assert len(list(record['_source'].keys())) > 2
resource = CacheResource.objects.first()
es_action = resource.create_es_action(index="test", store=store, record_type="cached")
assert es_action is not None
def get_es():
from search import get_es_client
return get_es_client()
def test_synchronise_dataset(self):
from search import get_es_client
client = get_es_client()
s = Search(client).index(self.index_name)
del_response = client.delete_by_query(index=self.index_name, q="*:*")
es_response = s.execute()
self.assertEquals(
es_response.hits.total,
0
)
self.assertEquals(
EDMRecord.objects.count(),
0
)
assert self.store.ask(query="""where {{
GRAPH <http://localhost:8000/resource/aggregation/ton-smits-huis/454/graph>
{{?s <http://schemas.delving.eu/narthex/terms/synced> false}}
}}"""
)
response = tasks.synchronise_dataset_records(
dataset_graph_uri=self.dataset_graph_uri,
store=self.store,
index=self.index_name
)
# self.assertTrue(response.successful)
# self.assertEquals(
# response.result,
# 1
# )
self.assertEquals(
EDMRecord.objects.count(),
1
)
time.sleep(2)
es_response = s.execute()
self.assertEquals(
es_response.hits.total,
1,
"there should be one record in the test index"
)
record = es_response.hits[0]
self.assertEquals(
record.meta.doc_type,
"void_edmrecord"
)
self.assertEquals(
"_".join(record.meta.id.split('_')[1:]),
"ton-smits-huis_454"
)
# test if switch is flipped
assert self.store.ask(query="""where {{
GRAPH <http://localhost:8000/resource/aggregation/ton-smits-huis/454/graph>
{{?s <http://schemas.delving.eu/narthex/terms/synced> true}}
}}"""
)
# test if dataset is deleted from index
ds = DataSet.get_dataset_from_graph(
dataset_graph_uri=self.dataset_graph_uri, store=self.store)
ds.delete_from_index(self.index_name)
es_response = s.execute()
self.assertEquals(
es_response.hits.total,
0,
"there should be no records in the test index after the dataset is deleted"
)
rdf_store_response = ds.delete_from_triple_store(self.store)
assert rdf_store_response
assert not self.store.ask(query="""where {{
GRAPH <http://localhost:8000/resource/dataset/ton-smits-huis/graph>
{{?s ?p ?o}}
}}"""
)