def _get_mapping(self, es): request = getRequest() mapping = getattr(request, CACHE_ATTRIBUTE, None) if mapping is not None: return mapping try: mapping = es.indices.get_mapping(index=index_name()) except TransportError as e: if e.status_code == 404: self._create_index(es) mapping = es.indices.get_mapping(index=index_name()) else: raise setattr(request, CACHE_ATTRIBUTE, mapping) return mapping
def get_payload(self, obj): try: serializer = getMultiAdapter((obj, getRequest()), ISerializeToJson) except ComponentLookupError: logger.exception( 'Abort ElasticSearch Indexing for {0}'.format( obj.absolute_url(), ), ) query_blocker.unblock() return try: data = serializer() except ComponentLookupError: logger.exception( 'Abort ElasticSearch Indexing for {0}'.format( obj.absolute_url(), ), ) query_blocker.unblock() return self._reduce_data(data) if HAS_ARCHETYPES: self._fix_at_fields(obj, data) self._expand_rid(obj, data) self._expand_binary_data(obj, data) uid = api.content.get_uuid(obj) es_kwargs = dict( index=index_name(), doc_type='content', id=uid, pipeline=self._es_pipeline_name, body=data, request_timeout=es_config.request_timeout, ) return es_kwargs
def unindex(self, obj): index = index_name() if index is None: # portal no longer there return uid = api.content.get_uuid(obj) if es_config.use_celery: unindex_content.delay( index=index, doc_type='content', uid=uid, timeout=es_config.request_timeout, ) else: es = get_ingest_client() if es is None: logger.warning('No ElasticSearch client available.', ) return try: es.delete( index=index, doc_type='content', id=uid, request_timeout=es_config.request_timeout, ) except Exception: logger.exception('unindexing of {0} failed'.format(uid))
def index(self, obj, attributes=None): start = time.time() query_blocker.block() es = get_ingest_client() if es is None: logger.warning('No ElasticSearch client available.', ) return self._check_for_ingest_pipeline(es) self._check_for_mapping(es) # will also create the index try: serializer = getMultiAdapter((obj, getRequest()), ISerializeToJson) except ComponentLookupError: logger.exception( 'Abort ElasticSearch Indexing for {0}'.format( obj.absolute_url(), ), ) return try: data = serializer() except ComponentLookupError: logger.exception( 'Abort ElasticSearch Indexing for {0}'.format( obj.absolute_url(), ), ) return logging.info("TOOK after serializing: {0:2.3f}".format(time.time() - start)) self._reduce_data(data) self._expand_rid(obj, data) self._expand_binary_data(obj, data) self._auto_mapping(es, obj, data) uid = api.content.get_uuid(obj) es_kwargs = dict( index=index_name(), doc_type='content', id=uid, pipeline=self._es_pipeline_name, body=data, ) parent = aq_parent(obj) portal = api.portal.get() if aq_base(portal) is aq_base(parent): self._check_and_add_portal_to_index(portal) # annotations = IAnnotations(portal) # es_kwargs['parent'] = annotations[ES_PORTAL_UUID_KEY] pass else: # es_kwargs['parent'] = api.content.get_uuid(parent) pass logging.info("TOOK after preprocessing: {0:2.3f}".format(time.time() - start)) try: es.index(**es_kwargs) except Exception: logger.exception( 'indexing of {0} failed.\n{1}'.format( uid, pformat(es_kwargs, indent=2), ), ) query_blocker.unblock() logging.info("TOOK overall: {0:2.3f}".format(time.time() - start))
def _setup_mapping(self, es): es.indices.put_mapping( doc_type='content', index=index_name(), body=INITIAL_MAPPING, ) request = getRequest() setattr(request, CACHE_ATTRIBUTE, None)
def _apply_index(self, request): """Apply the index to query parameters given in 'request'. The argument should be a mapping object. If the request does not contain the needed parameters, then None is returned. If the request contains a parameter with the name of the column and this parameter is either a Record or a class instance then it is assumed that the parameters of this index are passed as attribute (Note: this is the recommended way to pass parameters since Zope 2.4) Otherwise two objects are returned. The first object is a ResultSet containing the record numbers of the matching records. The second object is a tuple containing the names of all data fields used. """ if query_blocker.blocked: return record = parseIndexRequest(request, self.id) if record.keys is None: return None template_params = { 'keys': record.keys, } query_body = self._apply_template(template_params) logger.info(query_body) es_kwargs = dict( index=index_name(), body=query_body, size=BATCH_SIZE, scroll='1m', _source_include=['rid'], ) es = get_query_client() result = es.search(**es_kwargs) # initial return value, other batches to be applied def score(record): return int(10000 * float(record['_score'])) retval = IIBTree() for r in result['hits']['hits']: retval[r['_source']['rid']] = score(r) total = result['hits']['total'] if total > BATCH_SIZE: sid = result['_scroll_id'] counter = BATCH_SIZE while counter < total: result = es.scroll(scroll_id=sid, scroll='1m') for record in result['hits']['hits']: retval[record['_source']['rid']] = score(record) counter += BATCH_SIZE return retval, (self.id,)
def numObjects(self): """Return the number of indexed objects.""" es = get_query_client() search = Search(using=es, index=index_name()) try: return len(list(search.scan())) except Exception: logger.exception('ElasticSearch "count" query failed') return 'Problem getting all documents count from ElasticSearch!'
def numObjects(self): """Return the number of indexed objects.""" es_kwargs = dict( index=index_name(), body={'query': {'match_all': {}}}, ) es = get_query_client() try: return es.count(**es_kwargs)['count'] except Exception: logger.exception('ElasticSearch "count" query failed') return 'Problem getting all documents count from ElasticSearch!'
def _auto_mapping(self, es, obj, data): mappings = self._get_mapping(es) old_map = mappings[index_name()]['mappings'] old_map = old_map.get('content', {}).get('properties', {}) new_map = {} for key in data: if key in old_map: continue # figure out field type value = data[key] for pytype in MAPPING_TYPE_MAP: if isinstance(value, pytype): new_map[key] = MAPPING_TYPE_MAP[pytype] break for record in INGEST_PIPELINES['processors']: name = record['attachment']['target_field'] if name not in old_map: new_map[name] = { 'type': 'nested', 'properties': { 'content': MAPPING_TYPE_MAP[basestring], 'content_length': MAPPING_TYPE_MAP[int], 'content_type': MAPPING_TYPE_MAP[basestring], 'language': MAPPING_TYPE_MAP[basestring], }, } if not new_map: return new_map = { 'content': { 'properties': new_map, }, } es.indices.put_mapping( doc_type='content', index=index_name(), body=new_map, ) request = getRequest() setattr(request, CACHE_ATTRIBUTE, None)
def unindex(self, obj): es = get_ingest_client() if es is None: logger.warning('No ElasticSearch client available.', ) return uid = api.content.get_uuid(obj) try: es.delete( index=index_name(), doc_type='content', id=uid, ) except Exception: logger.exception('unindexing of {0} failed'.format(uid))
def index(self, obj, attributes=None): index = index_name() if index is None: # Not configured yet. return query_blocker.block() es = get_ingest_client() if es is None: logger.warning('No ElasticSearch client available.', ) query_blocker.unblock() return try: self._check_for_ingest_pipeline(es) self._check_for_mapping(es) # will also create the index except TransportError: logger.exception( 'ElasticSearch connection failed for {0}'.format( obj.absolute_url(), ), ) query_blocker.unblock() return parent = aq_parent(obj) portal = api.portal.get() if aq_base(portal) is aq_base(parent): self._check_and_add_portal_to_index(portal) if es_config.use_celery: path = '/'.join([p for p in obj.getPhysicalPath() if p != '']) index_content.delay(path, obj.absolute_url()) else: es_kwargs = self.get_payload(obj) try: es.index(**es_kwargs) except Exception: uid = api.content.get_uuid(obj) logger.exception('indexing of {0} failed.'.format(uid, ), ) import Globals if Globals.DevelopmentMode: logger.debug(pformat(es_kwargs, indent=2)) query_blocker.unblock()
def _check_and_add_portal_to_index(self, portal): # at first portal is not in ES! # Also, portal has no UUID. bad enough. so for es we give it one. # If portal has our UUID we assume it is also indexed already annotations = IAnnotations(portal) if ES_PORTAL_UUID_KEY in annotations: # looks like we're indexed. return annotations[ES_PORTAL_UUID_KEY] = uid = uuid.uuid4().hex serializer = getMultiAdapter((portal, getRequest()), ISerializeToJson) data = serializer() self._reduce_data(data) es_kwargs = dict( index=index_name(), doc_type='content', # why do we still need it in ES6+? id=uid, body=data, ) es = get_ingest_client() try: es.index(**es_kwargs) except Exception: logger.exception('indexing of {0} failed'.format(uid))
def _es_pipeline_name(self): return 'attachment_ingest_{0}'.format(index_name())
def _apply_index(self, request): """Apply the index to query parameters given in 'request'. The argument should be a mapping object. If the request does not contain the needed parameters, then None is returned. If the request contains a parameter with the name of the column and this parameter is either a Record or a class instance then it is assumed that the parameters of this index are passed as attribute (Note: this is the recommended way to pass parameters since Zope 2.4) Otherwise two objects are returned. The first object is a ResultSet containing the record numbers of the matching records. The second object is a tuple containing the names of all data fields used. """ config = get_configuration() timeout = getattr(config, 'request_timeout', 20) search_fields = getattr(config, 'search_fields', None) if not search_fields: search_fields = SEARCH_FIELDS search_fields = search_fields.split() logger.info(search_fields) if query_blocker.blocked: return record = parseIndexRequest(request, self.id) if record.keys is None: return None es = get_query_client() search = Search(using=es, index=index_name()) search = search.params(request_timeout=timeout) search = search.sort('rid', '_id') search = search.source(include='rid') query_string = record.keys[0].decode('utf8') logger.info(query_string) if '*' in query_string: query_string = query_string.replace('*', ' ') query_string = query_string.strip() search = search.query('simple_query_string', query=query_string, fields=search_fields) results_count = search.count() search = search.params(request_timeout=timeout, size=BATCH_SIZE, track_scores=True) # setup highlighting for field in search_fields: name = field.split('^')[0] if name == 'title': # title shows up in results anyway continue search = search.highlight(name, fragment_size=FRAGMENT_SIZE) # initial return value, other batches to be applied retval = IIBTree() highlights = OOBTree() last_seen = None count = 0 batch_count = results_count / BATCH_SIZE if results_count % BATCH_SIZE != 0: batch_count = batch_count + 1 for i in xrange(batch_count): if last_seen is not None: search = search.update_from_dict({'search_after': last_seen}) try: results = search.execute(ignore_cache=True) except TransportError: # No es client, return empty results logger.exception('ElasticSearch client not available.') return IIBTree(), (self.id, ) for r in results: rid = getattr(r, 'rid', None) if rid is not None: retval[rid] = int(10000 * float(r.meta.score)) # Index query returns only rids, so we need # to save highlights for later use highlight_list = [] if getattr(r.meta, 'highlight', None) is not None: for key in dir(r.meta.highlight): highlight_list.extend(r.meta.highlight[key]) highlights[r.meta.id] = highlight_list last_seen = [rid, r.meta.id] count = count + 1 # store highlights try: annotations = IAnnotations(self.REQUEST) annotations[HIGHLIGHT_KEY] = highlights except TypeError: # maybe we are in a test pass return retval, (self.id, )
def _apply_index(self, request): """Apply the index to query parameters given in 'request'. The argument should be a mapping object. If the request does not contain the needed parameters, then None is returned. If the request contains a parameter with the name of the column and this parameter is either a Record or a class instance then it is assumed that the parameters of this index are passed as attribute (Note: this is the recommended way to pass parameters since Zope 2.4) Otherwise two objects are returned. The first object is a ResultSet containing the record numbers of the matching records. The second object is a tuple containing the names of all data fields used. """ config = get_configuration() timeout = getattr(config, 'request_timeout', 20) search_fields = getattr(config, 'search_fields', None) if not search_fields: search_fields = SEARCH_FIELDS search_fields = search_fields.split() if query_blocker.blocked: return record = parseIndexRequest(request, self.id) if record.keys is None: return None es = get_query_client() search = Search(using=es, index=index_name()) search = search.params( request_timeout=timeout, size=BATCH_SIZE, preserve_order=True, ) search = search.source(include='rid') query_string = record.keys[0] if query_string and query_string.startswith('*'): # plone.app.querystring contains op sends a leading *, remove it query_string = query_string[1:] search = search.query('simple_query_string', query=query_string, fields=search_fields) # setup highlighting for field in search_fields: name = field.split('^')[0] if name == 'title': # title shows up in results anyway continue search = search.highlight(name, fragment_size=FRAGMENT_SIZE) try: result = search.scan() except TransportError: # No es client, return empty results logger.exception('ElasticSearch client not available.') return IIBTree(), (self.id, ) # initial return value, other batches to be applied retval = IIBTree() highlights = OOBTree() for r in result: if getattr(r, 'rid', None) is None: # something was indexed with no rid. Ignore for now. # this is only for highlights, so no big deal if we # skip one continue retval[r.rid] = int(10000 * float(r.meta.score)) # Index query returns only rids, so we need # to save highlights for later use highlight_list = [] if getattr(r.meta, 'highlight', None) is not None: for key in dir(r.meta.highlight): highlight_list.extend(r.meta.highlight[key]) highlights[r.meta.id] = highlight_list # store highlights try: annotations = IAnnotations(self.REQUEST) annotations[HIGHLIGHT_KEY] = highlights except TypeError: # maybe we are in a test pass return retval, (self.id, )
class ElasticSearchIndexQueueProcessor(object): """ a queue processor for ElasticSearch""" @property def _es_pipeline_name(self): return 'attachment_ingest_{0}'.format(index_name()) def _create_index(seld, es): es.indices.create(index=index_name()) @ram.cache(lambda *args: index_name()) def _check_for_ingest_pipeline(self, es): # do we have the ingest pipeline? try: es.ingest.get_pipeline(self._es_pipeline_name) except NotFoundError: es.ingest.put_pipeline(self._es_pipeline_name, INGEST_PIPELINES) def _check_for_mapping(self, es): if not self._get_mapping(es): raise ValueError('Can not fetch mapping.') def _get_mapping(self, es): request = getRequest() mapping = getattr(request, CACHE_ATTRIBUTE, None) if mapping is not None: return mapping try: mapping = es.indices.get_mapping(index=index_name()) except TransportError as e: if e.status_code == 404: self._create_index(es) mapping = es.indices.get_mapping(index=index_name()) else: raise setattr(request, CACHE_ATTRIBUTE, mapping) return mapping def _auto_mapping(self, es, obj, data): mappings = self._get_mapping(es) old_map = mappings[index_name()]['mappings'] old_map = old_map.get('content', {}).get('properties', {}) new_map = {} for key in data: if key in old_map: continue # figure out field type value = data[key] for pytype in MAPPING_TYPE_MAP: if isinstance(value, pytype): new_map[key] = MAPPING_TYPE_MAP[pytype] break for record in INGEST_PIPELINES['processors']: name = record['attachment']['target_field'] if name not in old_map: new_map[name] = { 'type': 'nested', 'properties': { 'content': MAPPING_TYPE_MAP[basestring], 'content_length': MAPPING_TYPE_MAP[int], 'content_type': MAPPING_TYPE_MAP[basestring], 'language': MAPPING_TYPE_MAP[basestring], }, } if not new_map: return new_map = { 'content': { 'properties': new_map, }, } es.indices.put_mapping( doc_type='content', index=index_name(), body=new_map, ) request = getRequest() setattr(request, CACHE_ATTRIBUTE, None) def _check_and_add_portal_to_index(self, portal): # at first portal is not in ES! # Also, portal has no UUID. bad enough. so for es we give it one. # If portal has our UUID we assume it is also indexed already annotations = IAnnotations(portal) if ES_PORTAL_UUID_KEY in annotations: # looks like we're indexed. return annotations[ES_PORTAL_UUID_KEY] = uid = uuid.uuid4().hex serializer = getMultiAdapter((portal, getRequest()), ISerializeToJson) data = serializer() self._reduce_data(data) es_kwargs = dict( index=index_name(), doc_type='content', # why do we still need it in ES6+? id=uid, body=data, ) es = get_ingest_client() try: es.index(**es_kwargs) except Exception: logger.exception('indexing of {0} failed'.format(uid)) def _reduce_data(self, data): for key in KEYS_TO_REMOVE: if key in data: del data[key] def _iterate_binary_fields(self, obj, data): for record in INGEST_PIPELINES['processors']: yield record['attachment']['field'] def _expand_binary_data(self, obj, data): for fieldname in self._iterate_binary_fields(obj, data): if fieldname not in data: continue field = getattr(obj, fieldname, None) if field is None: continue data[fieldname + '_meta'] = data[fieldname] if IBlobby.providedBy(field): with field.open() as fh: data[fieldname] = base64.b64encode(fh.read()) elif IRichTextValue.providedBy(field): data[fieldname] = base64.b64encode( data[fieldname + '_meta']['data'].encode('utf8'), ) def _expand_rid(self, obj, data): cat = api.portal.get_tool('portal_catalog') path = '/'.join(obj.getPhysicalPath()) data['rid'] = cat.getrid(path) def index(self, obj, attributes=None): start = time.time() query_blocker.block() es = get_ingest_client() if es is None: logger.warning('No ElasticSearch client available.', ) return self._check_for_ingest_pipeline(es) self._check_for_mapping(es) # will also create the index try: serializer = getMultiAdapter((obj, getRequest()), ISerializeToJson) except ComponentLookupError: logger.exception( 'Abort ElasticSearch Indexing for {0}'.format( obj.absolute_url(), ), ) return try: data = serializer() except ComponentLookupError: logger.exception( 'Abort ElasticSearch Indexing for {0}'.format( obj.absolute_url(), ), ) return logging.info("TOOK after serializing: {0:2.3f}".format(time.time() - start)) self._reduce_data(data) self._expand_rid(obj, data) self._expand_binary_data(obj, data) self._auto_mapping(es, obj, data) uid = api.content.get_uuid(obj) es_kwargs = dict( index=index_name(), doc_type='content', id=uid, pipeline=self._es_pipeline_name, body=data, ) parent = aq_parent(obj) portal = api.portal.get() if aq_base(portal) is aq_base(parent): self._check_and_add_portal_to_index(portal) # annotations = IAnnotations(portal) # es_kwargs['parent'] = annotations[ES_PORTAL_UUID_KEY] pass else: # es_kwargs['parent'] = api.content.get_uuid(parent) pass logging.info("TOOK after preprocessing: {0:2.3f}".format(time.time() - start)) try: es.index(**es_kwargs) except Exception: logger.exception( 'indexing of {0} failed.\n{1}'.format( uid, pformat(es_kwargs, indent=2), ), ) query_blocker.unblock() logging.info("TOOK overall: {0:2.3f}".format(time.time() - start)) def reindex(self, obj, attributes=None, update_metadata=1): self.index(obj, attributes) def unindex(self, obj): es = get_ingest_client() if es is None: logger.warning('No ElasticSearch client available.', ) return uid = api.content.get_uuid(obj) try: es.delete( index=index_name(), doc_type='content', id=uid, ) except Exception: logger.exception('unindexing of {0} failed'.format(uid)) def begin(self): pass def commit(self, wait=None): pass def abort(self): pass
def _create_index(seld, es): es.indices.create(index=index_name())
def search(self): es = get_query_client() s = Search(doc_type=self.doc_types, index=index_name(), using=es) s = s.params(size=BATCH_SIZE) return s.response_class(FacetedResponse)
def _create_index(self, es): es.indices.create(index=index_name()) self._setup_mapping(es)
class ElasticSearchIndexQueueProcessor(object): """ a queue processor for ElasticSearch""" @property def _es_pipeline_name(self): return 'attachment_ingest_{0}'.format(index_name()) def _create_index(self, es): es.indices.create(index=index_name()) self._setup_mapping(es) @ram.cache(lambda *args: index_name()) def _check_for_ingest_pipeline(self, es): # do we have the ingest pipeline? try: es.ingest.get_pipeline(self._es_pipeline_name) except NotFoundError: es.ingest.put_pipeline(self._es_pipeline_name, INGEST_PIPELINES) def _check_for_mapping(self, es): if not self._get_mapping(es): raise ValueError('Can not fetch mapping.') def _get_mapping(self, es): request = getRequest() mapping = getattr(request, CACHE_ATTRIBUTE, None) if mapping is not None: return mapping try: mapping = es.indices.get_mapping(index=index_name()) except TransportError as e: if e.status_code == 404: self._create_index(es) mapping = es.indices.get_mapping(index=index_name()) else: raise setattr(request, CACHE_ATTRIBUTE, mapping) return mapping def _setup_mapping(self, es): es.indices.put_mapping( doc_type='content', index=index_name(), body=INITIAL_MAPPING, ) request = getRequest() setattr(request, CACHE_ATTRIBUTE, None) def _check_and_add_portal_to_index(self, portal): # at first portal is not in ES! # Also, portal has no UUID. bad enough. so for es we give it one. # If portal has our UUID we assume it is also indexed already annotations = IAnnotations(portal) if ES_PORTAL_UUID_KEY in annotations: # looks like we're indexed. return annotations[ES_PORTAL_UUID_KEY] = uid = uuid.uuid4().hex serializer = getMultiAdapter((portal, getRequest()), ISerializeToJson) data = serializer() self._reduce_data(data) es_kwargs = dict( index=index_name(), doc_type='content', # why do we still need it in ES6+? id=uid, body=data, ) es = get_ingest_client() try: es.index(**es_kwargs) except Exception: logger.exception('indexing of {0} failed'.format(uid)) def _fix_at_fields(self, obj, data): # unlike dexterity, AT text fields use same serializer as rich text, # which wrecks our index mapping if IBaseContent.providedBy(obj): for field in AT_SIMPLE_TEXT_FIELDS: if field in data and isinstance(data[field], dict): data[field] = data[field]['data'] def _reduce_data(self, data): for key in KEYS_TO_REMOVE: if key in data: del data[key] def _iterate_binary_fields(self, obj, data): for record in INGEST_PIPELINES['processors']: if 'attachment' not in record: continue yield record['attachment']['field'] def _expand_binary_data(self, obj, data): max_size = es_config.max_blobsize for fieldname in self._iterate_binary_fields(obj, data): if fieldname not in data: data[fieldname] = None continue field = getattr(obj, fieldname, None) if field is None: data[fieldname] = None continue data[fieldname + '_meta'] = data[fieldname] if IBlobby.providedBy(field): with field.open() as fh: data[fieldname] = base64.b64encode(fh.read()) elif IRichTextValue.providedBy(field): data[fieldname] = base64.b64encode( data[fieldname + '_meta']['data'].encode('utf8'), ) if max_size and len(data[fieldname]) > max_size: data[fieldname] = None del data[fieldname + '_meta'] logger.info( 'File too big for ElasticSearch Indexing: {0}'.format( obj.absolute_url(), ), ) def _expand_rid(self, obj, data): cat = api.portal.get_tool('portal_catalog') path = '/'.join(obj.getPhysicalPath()) data['rid'] = cat.getrid(path) def get_payload(self, obj): try: serializer = getMultiAdapter((obj, getRequest()), ISerializeToJson) except ComponentLookupError: logger.exception( 'Abort ElasticSearch Indexing for {0}'.format( obj.absolute_url(), ), ) query_blocker.unblock() return try: data = serializer() except ComponentLookupError: logger.exception( 'Abort ElasticSearch Indexing for {0}'.format( obj.absolute_url(), ), ) query_blocker.unblock() return self._reduce_data(data) if HAS_ARCHETYPES: self._fix_at_fields(obj, data) self._expand_rid(obj, data) self._expand_binary_data(obj, data) uid = api.content.get_uuid(obj) es_kwargs = dict( index=index_name(), doc_type='content', id=uid, pipeline=self._es_pipeline_name, body=data, request_timeout=es_config.request_timeout, ) return es_kwargs def index(self, obj, attributes=None): query_blocker.block() es = get_ingest_client() if es is None: logger.warning('No ElasticSearch client available.', ) query_blocker.unblock() return try: self._check_for_ingest_pipeline(es) self._check_for_mapping(es) # will also create the index except TransportError: logger.exception( 'ElasticSearch connection failed for {0}'.format( obj.absolute_url(), ), ) query_blocker.unblock() return parent = aq_parent(obj) portal = api.portal.get() if aq_base(portal) is aq_base(parent): self._check_and_add_portal_to_index(portal) if es_config.use_celery: path = '/'.join([p for p in obj.getPhysicalPath() if p != '']) index_content.delay(path) else: es_kwargs = self.get_payload(obj) try: es.index(**es_kwargs) except Exception: uid = api.content.get_uuid(obj) logger.exception('indexing of {0} failed.'.format(uid, ), ) import Globals if Globals.DevelopmentMode: logger.debug(pformat(es_kwargs, indent=2)) query_blocker.unblock() def reindex(self, obj, attributes=None, update_metadata=1): self.index(obj, attributes) def unindex(self, obj): index = index_name() if index is None: # portal no longer there return uid = api.content.get_uuid(obj) if es_config.use_celery: unindex_content.delay( index=index, doc_type='content', uid=uid, timeout=es_config.request_timeout, ) else: es = get_ingest_client() if es is None: logger.warning('No ElasticSearch client available.', ) return try: es.delete( index=index, doc_type='content', id=uid, request_timeout=es_config.request_timeout, ) except Exception: logger.exception('unindexing of {0} failed'.format(uid)) def begin(self): pass def commit(self, wait=None): pass def abort(self): pass