def load_item(self, doc): # Recursively index associated models like attachments for model in doc.traverse(): model_body = json_encoder.encode( JsonLDSerializer().serialize(model)) log.debug('ElasticsearchUpsertLoader indexing document id: %s' % model.get_ori_identifier()) # Update document elasticsearch.update( id=model.get_short_identifier(), index=self.index_name, body={ 'doc': json.loads(model_body), 'doc_as_upsert': True, }, ) if 'enricher_task' in model: # The value seems to be enriched so add to resolver url_doc = { 'ori_identifier': model.get_short_identifier(), 'original_url': model.original_url, 'file_name': model.name, } if 'content_type' in model: url_doc['content_type'] = model.content_type # Update if already exists elasticsearch.index(index=settings.RESOLVER_URL_INDEX, id=get_sha1_hash(model.original_url), body=url_doc)
def load_item( self, combined_object_id, object_id, combined_index_doc, doc ): if combined_index_doc == {}: log.info('Empty document ....') return log.info('Indexing documents...') elasticsearch.update( index=settings.COMBINED_INDEX, doc_type=self.doc_type, id=combined_object_id, body={ 'doc': combined_index_doc, 'doc_as_upsert': True }) # Index documents into new index elasticsearch.update( index=self.index_name, doc_type=self.doc_type, body={ 'doc': doc, 'doc_as_upsert': True }, id=object_id) self._create_resolvable_media_urls(doc)
def load_item(self, combined_object_id, object_id, combined_index_doc, doc): if combined_index_doc == {}: log.info('Empty document ....') return log.info('Indexing documents...') elasticsearch.update(index=settings.COMBINED_INDEX, doc_type=self.doc_type, id=combined_object_id, body={ 'doc': combined_index_doc['doc'], 'doc_as_upsert': combined_index_doc.get( 'doc_as_upsert', False) }) # Index documents into new index elasticsearch.update(index=self.index_name, doc_type=self.doc_type, body={ 'doc': doc['doc'], 'doc_as_upsert': combined_index_doc.get( 'doc_as_upsert', False) }, id=object_id)
def process(self, model, model_body): elasticsearch.update( id=model.get_short_identifier(), index=self.index_name, body={ 'doc': json.loads(model_body), 'doc_as_upsert': True, }, )
def load_item(self, doc): body = json_encoder.encode(JsonLDSerializer().serialize(doc)) if doc == {}: log.info('Empty document ....') return log.info('Indexing document id: %s' % doc.get_ori_identifier()) # Index documents into new index elasticsearch.update( id=doc.get_short_identifier(), index=self.index_name, doc_type=doc_type(doc.verbose_name()), body={'doc': body}, )
def load_item(self, doc): body = json_encoder.encode(JsonLDSerializer().serialize(doc)) if doc == {}: log.info('Empty document ....') return log.info('Indexing document id: %s' % doc.get_ori_identifier()) # Index documents into new index elasticsearch.update( id=doc.get_ori_identifier(), index=self.index_name, doc_type=doc_type(doc.verbose_name()), body={'doc': body}, )
def load_item(self, combined_object_id, object_id, combined_index_doc, doc): if combined_index_doc == {}: log.info('Empty document ....') return log.info('Indexing documents...') elasticsearch.update(index=self.combined_index_name, doc_type=self.doc_type, id=combined_object_id, body={'doc': combined_index_doc['doc']}, request_timeout=600) # Index documents into new index elasticsearch.update(index=self.index_name, doc_type=self.doc_type, body={'doc': doc['doc']}, id=object_id, request_timeout=600)
def load_item(self, doc): # Recursively index associated models like attachments for model in doc.traverse(): model_body = json_encoder.encode( JsonLDSerializer().serialize(model)) if doc == {}: log.info('Empty document ....') return log.debug( 'ElasticsearchUpdateOnlyLoader indexing document id: %s' % model.get_ori_identifier()) # Index document into new index elasticsearch.update( id=model.get_short_identifier(), index=self.index_name, body={'doc': json.loads(model_body)}, )