def kafka_topic(self):
"""if set, this will use a kafka feed instead of couch for the pillow"""
from corehq.apps.change_feed.document_types import get_doc_meta_object_from_document
from corehq.apps.change_feed.topics import get_topic
meta = get_doc_meta_object_from_document(
self.document_class().to_json())
return get_topic(meta)
def populate_change_metadata(change, data_source_type, data_source_name):
from corehq.apps.change_feed.exceptions import MissingMetaInformationError
from corehq.apps.change_feed.document_types import (
change_meta_from_doc_meta_and_document,
change_meta_from_hard_delete,
get_doc_meta_object_from_document,
)
if change.metadata:
return
document = change.get_document()
if change.deleted:
change.metadata = change_meta_from_hard_delete(document,
data_source_type,
data_source_name)
return
try:
doc_meta = get_doc_meta_object_from_document(document)
change_meta = change_meta_from_doc_meta_and_document(
doc_meta=doc_meta,
document=document,
data_source_type=data_source_type,
data_source_name=data_source_name,
doc_id=change.id,
)
except MissingMetaInformationError:
pass
else:
change.metadata = change_meta
def test_document_meta(self, raw_doc, expected_primary_type, expected_subtype=None,
expected_domain=None, expected_deletion=False):
doc_meta = get_doc_meta_object_from_document(raw_doc)
self.assertEqual(expected_primary_type, doc_meta.primary_type)
self.assertEqual(expected_subtype, doc_meta.subtype)
self.assertEqual(expected_domain, doc_meta.domain)
self.assertEqual(expected_deletion, doc_meta.is_deletion)
def _doc_to_changes(self, doc):
# creates a change object for the last form submission
# for the user to each of their apps.
# this allows us to reindex for the app status report
# without reindexing all forms.
changes = []
forms = get_last_forms_by_app(doc['_id'])
for form in forms:
doc_meta = get_doc_meta_object_from_document(form)
change_meta = change_meta_from_doc_meta_and_document(
doc_meta=doc_meta,
document=form,
data_source_type='elasticsearch',
data_source_name='hqforms',
)
changes.append(
Change(
id=change_meta.document_id,
sequence_id=None,
document=form,
deleted=change_meta.is_deletion,
metadata=change_meta,
document_store=None,
))
return changes
def test_document_meta(self,
raw_doc,
expected_subtype=None,
expected_domain=None,
expected_deletion=False):
doc_meta = get_doc_meta_object_from_document(raw_doc)
self.assertEqual(expected_subtype, doc_meta.subtype)
self.assertEqual(expected_domain, doc_meta.domain)
self.assertEqual(expected_deletion, doc_meta.is_deletion)
def test_change_from_doc_success(self, doc, expected_id):
change_meta = change_meta_from_doc(doc, 'dummy-data-source', 'dummy-data-source-name')
doc_meta = get_doc_meta_object_from_document(doc)
self.assertEqual(expected_id, change_meta.document_id)
self.assertEqual('dummy-data-source', change_meta.data_source_type)
self.assertEqual('dummy-data-source-name', change_meta.data_source_name)
self.assertEqual(doc_meta.raw_doc_type, change_meta.document_type)
self.assertEqual(doc_meta.subtype, change_meta.document_subtype)
self.assertEqual(doc_meta.domain, change_meta.domain)
self.assertEqual(doc_meta.is_deletion, change_meta.is_deletion)
def test_change_from_doc_success(self, doc, expected_id):
change_meta = change_meta_from_doc(doc, 'dummy-data-source', 'dummy-data-source-name')
doc_meta = get_doc_meta_object_from_document(doc)
self.assertEqual(expected_id, change_meta.document_id)
self.assertEqual('dummy-data-source', change_meta.data_source_type)
self.assertEqual('dummy-data-source-name', change_meta.data_source_name)
self.assertEqual(doc_meta.raw_doc_type, change_meta.document_type)
self.assertEqual(doc_meta.subtype, change_meta.document_subtype)
self.assertEqual(doc_meta.domain, change_meta.domain)
self.assertEqual(doc_meta.is_deletion, change_meta.is_deletion)
def IntraHealthFormFluffPillow(delete_filtered=False):
return get_multi_fluff_pillow(
indicator_classes=[
TauxDeSatisfactionFluff,
CouvertureFluff,
RecapPassageFluff,
IntraHealthFluff,
TauxDeRuptureFluff,
LivraisonFluff,
],
name='IntraHealthFormFluff',
kafka_topic=get_topic(get_doc_meta_object_from_document(XFormInstance().to_json())),
delete_filtered=delete_filtered
)
def M4ChangeFormFluffPillow(delete_filtered=False):
return get_multi_fluff_pillow(
indicator_classes=[
AncHmisCaseFluff,
LdHmisCaseFluff,
ImmunizationHmisCaseFluff,
ProjectIndicatorsCaseFluff,
McctMonthlyAggregateFormFluff,
AllHmisCaseFluff,
],
name='M4ChangeFormFluff',
kafka_topic=get_topic(get_doc_meta_object_from_document(XFormInstance().to_json())),
delete_filtered=delete_filtered
)
def _doc_to_change(doc, data_source_type, data_source_name):
doc_meta = get_doc_meta_object_from_document(doc)
change_meta = change_meta_from_doc_meta_and_document(
doc_meta=doc_meta,
document=doc,
data_source_type=data_source_type,
data_source_name=data_source_name,
)
return Change(
id=change_meta.document_id,
sequence_id=None,
document=doc,
deleted=change_meta.is_deletion,
metadata=change_meta,
document_store=None,
)
def _doc_to_change(doc, data_source_type, data_source_name):
doc_meta = get_doc_meta_object_from_document(doc)
change_meta = change_meta_from_doc_meta_and_document(
doc_meta=doc_meta,
document=doc,
data_source_type=data_source_type,
data_source_name=data_source_name,
)
return Change(
id=change_meta.document_id,
sequence_id=None,
document=doc,
deleted=change_meta.is_deletion,
metadata=change_meta,
document_store=None,
)
def process_change(self, change):
from corehq.apps.change_feed.document_types import get_doc_meta_object_from_document
if self.change_filter_fn and self.change_filter_fn(change):
return
if change.deleted and change.id:
doc = change.get_document()
if doc and doc.get('doc_type'):
current_meta = get_doc_meta_object_from_document(doc)
if current_meta.is_deletion:
self._delete_doc_if_exists(change.id)
else:
self._delete_doc_if_exists(change.id)
return
with self._datadog_timing('extract'):
doc = change.get_document()
ensure_document_exists(change)
ensure_matched_revisions(change, doc)
with self._datadog_timing('transform'):
if doc is None or (self.doc_filter_fn and self.doc_filter_fn(doc)):
return
if doc.get('doc_type') is not None and doc['doc_type'].endswith("-Deleted"):
self._delete_doc_if_exists(change.id)
return
# prepare doc for es
doc_ready_to_save = self.doc_transform_fn(doc)
# send it across
with self._datadog_timing('load'):
send_to_elasticsearch(
index_info=self.index_info,
doc_type=self.index_info.type,
doc_id=change.id,
es_getter=self.es_getter,
name='ElasticProcessor',
data=doc_ready_to_save,
)
def process_change(self, pillow_instance, change):
try:
document = change.get_document()
doc_meta = get_doc_meta_object_from_document(document)
change_meta = change_meta_from_doc_meta_and_document(
doc_meta=doc_meta,
document=document,
data_source_type=self._data_source_type,
data_source_name=self._data_source_name,
doc_id=change.id,
)
except MissingMetaInformationError:
pass
else:
# change.deleted is used for hard deletions whereas change_meta.is_deletion is for soft deletions.
# from the consumer's perspective both should be counted as deletions so just "or" them
# note: it is strange and hard to reproduce that the couch changes feed is providing a "doc"
# along with a hard deletion, but it is doing that in the wild so we might as well support it.
change_meta.is_deletion = change_meta.is_deletion or change.deleted
self._producer.send_change(get_topic(doc_meta), change_meta)
def process_change(self, pillow_instance, change):
try:
document = change.get_document()
doc_meta = get_doc_meta_object_from_document(document)
change_meta = change_meta_from_doc_meta_and_document(
doc_meta=doc_meta,
document=document,
data_source_type=self._data_source_type,
data_source_name=self._data_source_name,
doc_id=change.id,
)
except MissingMetaInformationError:
pass
else:
# change.deleted is used for hard deletions whereas change_meta.is_deletion is for soft deletions.
# from the consumer's perspective both should be counted as deletions so just "or" them
# note: it is strange and hard to reproduce that the couch changes feed is providing a "doc"
# along with a hard deletion, but it is doing that in the wild so we might as well support it.
change_meta.is_deletion = change_meta.is_deletion or change.deleted
self._producer.send_change(get_topic(doc_meta), change_meta)
def populate_change_metadata(change, data_source_type, data_source_name):
from corehq.apps.change_feed.exceptions import MissingMetaInformationError
from corehq.apps.change_feed.document_types import get_doc_meta_object_from_document
from corehq.apps.change_feed.document_types import change_meta_from_doc_meta_and_document
if change.metadata:
return
try:
document = change.get_document()
doc_meta = get_doc_meta_object_from_document(document)
change_meta = change_meta_from_doc_meta_and_document(
doc_meta=doc_meta,
document=document,
data_source_type=data_source_type,
data_source_name=data_source_name,
doc_id=change.id,
)
except MissingMetaInformationError:
pass
else:
change.metadata = change_meta
def _doc_to_changes(self, doc):
# creates a change object for the last form submission
# for the user to each of their apps.
# this allows us to reindex for the app status report
# without reindexing all forms.
changes = []
forms = get_last_forms_by_app(doc['_id'])
for form in forms:
doc_meta = get_doc_meta_object_from_document(form)
change_meta = change_meta_from_doc_meta_and_document(
doc_meta=doc_meta,
document=form,
data_source_type='elasticsearch',
data_source_name='hqforms',
)
changes.append(Change(
id=change_meta.document_id,
sequence_id=None,
document=form,
deleted=change_meta.is_deletion,
metadata=change_meta,
document_store=None,
))
return changes
def _is_deleted(change):
doc = change.get_document()
if doc and doc.get('doc_type'):
return get_doc_meta_object_from_document(doc).is_deletion
elif change.deleted:
return bool(change.id)
def kafka_topic(self):
"""if set, this will use a kafka feed instead of couch for the pillow"""
from corehq.apps.change_feed.document_types import get_doc_meta_object_from_document
from corehq.apps.change_feed.topics import get_topic
meta = get_doc_meta_object_from_document(self.document_class().to_json())
return get_topic(meta)
