def load_item(self, doc):
# Recursively index associated models like attachments
for model in doc.traverse():
model_body = json_encoder.encode(
JsonLDSerializer().serialize(model))
log.debug('ElasticsearchUpsertLoader indexing document id: %s' %
model.get_ori_identifier())
# Update document
elasticsearch.update(
id=model.get_short_identifier(),
index=self.index_name,
body={
'doc': json.loads(model_body),
'doc_as_upsert': True,
},
)
if 'enricher_task' in model:
# The value seems to be enriched so add to resolver
url_doc = {
'ori_identifier': model.get_short_identifier(),
'original_url': model.original_url,
'file_name': model.name,
}
if 'content_type' in model:
url_doc['content_type'] = model.content_type
# Update if already exists
elasticsearch.index(index=settings.RESOLVER_URL_INDEX,
id=get_sha1_hash(model.original_url),
body=url_doc)
def load_item(
self, combined_object_id, object_id, combined_index_doc, doc
):
if combined_index_doc == {}:
log.info('Empty document ....')
return
log.info('Indexing documents...')
elasticsearch.update(
index=settings.COMBINED_INDEX, doc_type=self.doc_type,
id=combined_object_id, body={
'doc': combined_index_doc,
'doc_as_upsert': True
})
# Index documents into new index
elasticsearch.update(
index=self.index_name, doc_type=self.doc_type,
body={
'doc': doc,
'doc_as_upsert': True
}, id=object_id)
self._create_resolvable_media_urls(doc)
def load_item(self, combined_object_id, object_id, combined_index_doc,
doc):
if combined_index_doc == {}:
log.info('Empty document ....')
return
log.info('Indexing documents...')
elasticsearch.update(index=settings.COMBINED_INDEX,
doc_type=self.doc_type,
id=combined_object_id,
body={
'doc':
combined_index_doc['doc'],
'doc_as_upsert':
combined_index_doc.get(
'doc_as_upsert', False)
})
# Index documents into new index
elasticsearch.update(index=self.index_name,
doc_type=self.doc_type,
body={
'doc':
doc['doc'],
'doc_as_upsert':
combined_index_doc.get(
'doc_as_upsert', False)
},
id=object_id)
def process(self, model, model_body):
elasticsearch.update(
id=model.get_short_identifier(),
index=self.index_name,
body={
'doc': json.loads(model_body),
'doc_as_upsert': True,
},
)
def load_item(self, doc):
body = json_encoder.encode(JsonLDSerializer().serialize(doc))
if doc == {}:
log.info('Empty document ....')
return
log.info('Indexing document id: %s' % doc.get_ori_identifier())
# Index documents into new index
elasticsearch.update(
id=doc.get_short_identifier(),
index=self.index_name,
doc_type=doc_type(doc.verbose_name()),
body={'doc': body},
)
def load_item(self, doc):
body = json_encoder.encode(JsonLDSerializer().serialize(doc))
if doc == {}:
log.info('Empty document ....')
return
log.info('Indexing document id: %s' % doc.get_ori_identifier())
# Index documents into new index
elasticsearch.update(
id=doc.get_ori_identifier(),
index=self.index_name,
doc_type=doc_type(doc.verbose_name()),
body={'doc': body},
)
def load_item(self, combined_object_id, object_id, combined_index_doc,
doc):
if combined_index_doc == {}:
log.info('Empty document ....')
return
log.info('Indexing documents...')
elasticsearch.update(index=self.combined_index_name,
doc_type=self.doc_type,
id=combined_object_id,
body={'doc': combined_index_doc['doc']},
request_timeout=600)
# Index documents into new index
elasticsearch.update(index=self.index_name,
doc_type=self.doc_type,
body={'doc': doc['doc']},
id=object_id,
request_timeout=600)
def load_item(self, doc):
# Recursively index associated models like attachments
for model in doc.traverse():
model_body = json_encoder.encode(
JsonLDSerializer().serialize(model))
if doc == {}:
log.info('Empty document ....')
return
log.debug(
'ElasticsearchUpdateOnlyLoader indexing document id: %s' %
model.get_ori_identifier())
# Index document into new index
elasticsearch.update(
id=model.get_short_identifier(),
index=self.index_name,
body={'doc': json.loads(model_body)},
)
