def index_cached_layers(self):
"""
Index and unindex all layers in the Django cache (Index all layers who have been checked).
"""
from hypermap.aggregator.models import Layer
if SEARCH_TYPE == 'solr':
from hypermap.aggregator.solr import SolrHypermap
solrobject = SolrHypermap()
else:
from hypermap.aggregator.elasticsearch_client import ESHypermap
from elasticsearch import helpers
es_client = ESHypermap()
layers_cache = cache.get('layers')
deleted_layers_cache = cache.get('deleted_layers')
# 1. added layers cache
if layers_cache:
layers_list = list(layers_cache)
LOGGER.debug('There are %s layers in cache: %s' % (len(layers_list), layers_list))
batch_size = settings.REGISTRY_SEARCH_BATCH_SIZE
batch_lists = [layers_list[i:i+batch_size] for i in range(0, len(layers_list), batch_size)]
for batch_list_ids in batch_lists:
layers = Layer.objects.filter(id__in=batch_list_ids)
if batch_size > len(layers):
batch_size = len(layers)
LOGGER.debug('Syncing %s/%s layers to %s: %s' % (batch_size, len(layers_cache), layers, SEARCH_TYPE))
try:
# SOLR
if SEARCH_TYPE == 'solr':
success, layers_errors_ids = solrobject.layers_to_solr(layers)
if success:
# remove layers from cache here
layers_cache = layers_cache.difference(set(batch_list_ids))
LOGGER.debug('Removing layers with id %s from cache' % batch_list_ids)
cache.set('layers', layers_cache)
# ES
elif SEARCH_TYPE == 'elasticsearch':
with_bulk, success = True, False
layers_to_index = [es_client.layer_to_es(layer, with_bulk) for layer in layers]
message = helpers.bulk(es_client.es, layers_to_index)
# Check that all layers where indexed...if not, don't clear cache.
# TODO: Check why es does not index all layers at first.
len_indexed_layers = message[0]
if len_indexed_layers == len(layers):
LOGGER.debug('%d layers indexed successfully' % (len_indexed_layers))
success = True
if success:
# remove layers from cache here
layers_cache = layers_cache.difference(set(batch_list_ids))
cache.set('layers', layers_cache)
else:
raise Exception("Incorrect SEARCH_TYPE=%s" % SEARCH_TYPE)
except Exception as e:
LOGGER.error('Layers were NOT indexed correctly')
LOGGER.error(e, exc_info=True)
else:
LOGGER.debug('No cached layers to add in search engine.')
# 2. deleted layers cache
if deleted_layers_cache:
layers_list = list(deleted_layers_cache)
LOGGER.debug('There are %s layers in cache for deleting: %s' % (len(layers_list), layers_list))
# TODO implement me: batch layer index deletion
for layer_id in layers_list:
# SOLR
if SEARCH_TYPE == 'solr':
if Layer.objects.filter(pk=layer_id).exists():
layer = Layer.objects.get(id=layer_id)
unindex_layer(layer.id, use_cache=False)
deleted_layers_cache = deleted_layers_cache.difference(set([layer_id]))
cache.set('deleted_layers', deleted_layers_cache)
else:
# TODO implement me
raise NotImplementedError
else:
LOGGER.debug('No cached layers to remove in search engine.')
def index_cached_layers(self):
"""
Index and unindex all layers in the Django cache (Index all layers who have been checked).
"""
from hypermap.aggregator.models import Layer
if SEARCH_TYPE == 'solr':
from hypermap.aggregator.solr import SolrHypermap
solrobject = SolrHypermap()
else:
from hypermap.aggregator.elasticsearch_client import ESHypermap
from elasticsearch import helpers
es_client = ESHypermap()
layers_cache = cache.get('layers')
deleted_layers_cache = cache.get('deleted_layers')
# 1. added layers cache
if layers_cache:
layers_list = list(layers_cache)
LOGGER.debug('There are %s layers in cache: %s' % (len(layers_list), layers_list))
batch_size = settings.REGISTRY_SEARCH_BATCH_SIZE
batch_lists = [layers_list[i:i+batch_size] for i in range(0, len(layers_list), batch_size)]
for batch_list_ids in batch_lists:
layers = Layer.objects.filter(id__in=batch_list_ids)
if batch_size > len(layers):
batch_size = len(layers)
LOGGER.debug('Syncing %s/%s layers to %s: %s' % (batch_size, len(layers_cache), layers, SEARCH_TYPE))
try:
# SOLR
if SEARCH_TYPE == 'solr':
success, layers_errors_ids = solrobject.layers_to_solr(layers)
if success:
# remove layers from cache here
layers_cache = layers_cache.difference(set(batch_list_ids))
LOGGER.debug('Removing layers with id %s from cache' % batch_list_ids)
cache.set('layers', layers_cache)
# ES
elif SEARCH_TYPE == 'elasticsearch':
with_bulk, success = True, False
layers_to_index = [es_client.layer_to_es(layer, with_bulk) for layer in layers]
message = helpers.bulk(es_client.es, layers_to_index)
# Check that all layers where indexed...if not, don't clear cache.
# TODO: Check why es does not index all layers at first.
len_indexed_layers = message[0]
if len_indexed_layers == len(layers):
LOGGER.debug('%d layers indexed successfully' % (len_indexed_layers))
success = True
if success:
# remove layers from cache here
layers_cache = layers_cache.difference(set(batch_list_ids))
cache.set('layers', layers_cache)
else:
raise Exception("Incorrect SEARCH_TYPE=%s" % SEARCH_TYPE)
except Exception as e:
LOGGER.error('Layers were NOT indexed correctly')
LOGGER.error(e, exc_info=True)
else:
LOGGER.debug('No cached layers to add in search engine.')
# 2. deleted layers cache
if deleted_layers_cache:
layers_list = list(deleted_layers_cache)
LOGGER.debug('There are %s layers in cache for deleting: %s' % (len(layers_list), layers_list))
# TODO implement me: batch layer index deletion
for layer_id in layers_list:
# SOLR
if SEARCH_TYPE == 'solr':
if Layer.objects.filter(pk=layer_id).exists():
layer = Layer.objects.get(id=layer_id)
unindex_layer(layer.id, use_cache=False)
deleted_layers_cache = deleted_layers_cache.difference(set([layer_id]))
cache.set('deleted_layers', deleted_layers_cache)
else:
# TODO implement me
raise NotImplementedError
else:
LOGGER.debug('No cached layers to remove in search engine.')
def index_cached_layers(self):
"""
Index all layers in the Django cache (Index all layers who have been checked).
"""
from hypermap.aggregator.models import Layer
from hypermap.aggregator.models import TaskError
if SEARCH_TYPE == 'solr':
from hypermap.aggregator.solr import SolrHypermap
solrobject = SolrHypermap()
else:
from hypermap.aggregator.elasticsearch_client import ESHypermap
from elasticsearch import helpers
es_client = ESHypermap()
layers_cache = cache.get('layers')
if layers_cache:
layers_list = list(layers_cache)
LOGGER.debug('There are %s layers in cache: %s' % (len(layers_list), layers_list))
batch_size = settings.REGISTRY_SEARCH_BATCH_SIZE
batch_lists = [layers_list[i:i+batch_size] for i in range(0, len(layers_list), batch_size)]
for batch_list_ids in batch_lists:
layers = Layer.objects.filter(id__in=batch_list_ids)
if batch_size > len(layers):
batch_size = len(layers)
LOGGER.debug('Syncing %s/%s layers to %s: %s' % (batch_size, len(layers_cache), layers, SEARCH_TYPE))
try:
if SEARCH_TYPE == 'solr':
success, message = solrobject.layers_to_solr(layers)
elif SEARCH_TYPE == 'elasticsearch':
with_bulk, success = True, False
layers_to_index = [es_client.layer_to_es(layer, with_bulk) for layer in layers]
message = helpers.bulk(es_client.es, layers_to_index)
# Check that all layers where indexed...if not, don't clear cache.
# TODO: Check why es does not index all layers at first.
len_indexed_layers = message[0]
if len_indexed_layers == len(layers):
LOGGER.debug('%d layers indexed successfully' % (len_indexed_layers))
success = True
else:
raise Exception("Incorrect SEARCH_TYPE=%s" % SEARCH_TYPE)
if success:
# remove layers from cache here
layers_cache = layers_cache.difference(set(batch_list_ids))
cache.set('layers', layers_cache)
else:
task_error = TaskError(
task_name=self.name,
args=batch_list_ids,
message=message
)
task_error.save()
except Exception as e:
LOGGER.error('Layers were NOT indexed correctly')
LOGGER.error(e, exc_info=True)
else:
LOGGER.debug('No cached layers.')