def unindex(self, obj):
index = index_name()
if index is None:
# portal no longer there
return
uid = api.content.get_uuid(obj)
if es_config.use_celery:
unindex_content.delay(
index=index,
doc_type='content',
uid=uid,
timeout=es_config.request_timeout,
)
else:
es = get_ingest_client()
if es is None:
logger.warning('No ElasticSearch client available.', )
return
try:
es.delete(
index=index,
doc_type='content',
id=uid,
request_timeout=es_config.request_timeout,
)
except Exception:
logger.exception('unindexing of {0} failed'.format(uid))
def index(self, obj, attributes=None):
query_blocker.block()
es = get_ingest_client()
if es is None:
logger.warning('No ElasticSearch client available.', )
query_blocker.unblock()
return
try:
self._check_for_ingest_pipeline(es)
self._check_for_mapping(es) # will also create the index
except TransportError:
logger.exception(
'ElasticSearch connection failed for {0}'.format(
obj.absolute_url(), ), )
query_blocker.unblock()
return
parent = aq_parent(obj)
portal = api.portal.get()
if aq_base(portal) is aq_base(parent):
self._check_and_add_portal_to_index(portal)
if es_config.use_celery:
path = '/'.join([p for p in obj.getPhysicalPath() if p != ''])
index_content.delay(path)
else:
es_kwargs = self.get_payload(obj)
try:
es.index(**es_kwargs)
except Exception:
uid = api.content.get_uuid(obj)
logger.exception('indexing of {0} failed.'.format(uid, ), )
import Globals
if Globals.DevelopmentMode:
logger.debug(pformat(es_kwargs, indent=2))
query_blocker.unblock()
def index(self, obj, attributes=None):
start = time.time()
query_blocker.block()
es = get_ingest_client()
if es is None:
logger.warning('No ElasticSearch client available.', )
return
self._check_for_ingest_pipeline(es)
self._check_for_mapping(es) # will also create the index
try:
serializer = getMultiAdapter((obj, getRequest()), ISerializeToJson)
except ComponentLookupError:
logger.exception(
'Abort ElasticSearch Indexing for {0}'.format(
obj.absolute_url(), ), )
return
try:
data = serializer()
except ComponentLookupError:
logger.exception(
'Abort ElasticSearch Indexing for {0}'.format(
obj.absolute_url(), ), )
return
logging.info("TOOK after serializing: {0:2.3f}".format(time.time() -
start))
self._reduce_data(data)
self._expand_rid(obj, data)
self._expand_binary_data(obj, data)
self._auto_mapping(es, obj, data)
uid = api.content.get_uuid(obj)
es_kwargs = dict(
index=index_name(),
doc_type='content',
id=uid,
pipeline=self._es_pipeline_name,
body=data,
)
parent = aq_parent(obj)
portal = api.portal.get()
if aq_base(portal) is aq_base(parent):
self._check_and_add_portal_to_index(portal)
# annotations = IAnnotations(portal)
# es_kwargs['parent'] = annotations[ES_PORTAL_UUID_KEY]
pass
else:
# es_kwargs['parent'] = api.content.get_uuid(parent)
pass
logging.info("TOOK after preprocessing: {0:2.3f}".format(time.time() -
start))
try:
es.index(**es_kwargs)
except Exception:
logger.exception(
'indexing of {0} failed.\n{1}'.format(
uid,
pformat(es_kwargs, indent=2),
), )
query_blocker.unblock()
logging.info("TOOK overall: {0:2.3f}".format(time.time() - start))
def index_content(path, url):
logger.warning('Indexing {}'.format(path))
es = get_ingest_client()
if es is None:
logger.warning('Elasticsearch client not found for indexing.')
return
site = getSite()
obj = site.unrestrictedTraverse(path)
indexer = queryUtility(IIndexQueueProcessor, name='collective.es.index')
data = indexer.get_payload(obj)
data['body']['@id'] = url
es.index(**data)
def unindex(self, obj):
es = get_ingest_client()
if es is None:
logger.warning('No ElasticSearch client available.', )
return
uid = api.content.get_uuid(obj)
try:
es.delete(
index=index_name(),
doc_type='content',
id=uid,
)
except Exception:
logger.exception('unindexing of {0} failed'.format(uid))
def unindex_content(index, doc_type, uid, timeout):
logger.warning('Unindexing {}'.format(uid))
es = get_ingest_client()
if es is None:
logger.warning('Elasticsearch client not found for indexing.')
return
try:
es.delete(index=index,
doc_type=doc_type,
id=uid,
request_timeout=timeout)
except NotFoundError:
logger.warning('Content already unindexed.')
pass
except Exception:
logger.exception('unindexing of {0} failed.'.format(uid, ), )
def index_content(path):
logger.warning('Indexing {}'.format(path))
es = get_ingest_client()
if es is None:
logger.warning('Elasticsearch client not found for indexing.')
return
site = getSite()
obj = site.unrestrictedTraverse(path)
indexer = queryUtility(IIndexQueueProcessor, name='collective.es.index')
data = indexer.get_payload(obj)
try:
es.index(**data)
except Exception:
logger.exception(
'indexing of {0} failed.'.format(
path,
),
)
def _check_and_add_portal_to_index(self, portal):
# at first portal is not in ES!
# Also, portal has no UUID. bad enough. so for es we give it one.
# If portal has our UUID we assume it is also indexed already
annotations = IAnnotations(portal)
if ES_PORTAL_UUID_KEY in annotations:
# looks like we're indexed.
return
annotations[ES_PORTAL_UUID_KEY] = uid = uuid.uuid4().hex
serializer = getMultiAdapter((portal, getRequest()), ISerializeToJson)
data = serializer()
self._reduce_data(data)
es_kwargs = dict(
index=index_name(),
doc_type='content', # why do we still need it in ES6+?
id=uid,
body=data,
)
es = get_ingest_client()
try:
es.index(**es_kwargs)
except Exception:
logger.exception('indexing of {0} failed'.format(uid))