def kafka_topic(self): """if set, this will use a kafka feed instead of couch for the pillow""" from corehq.apps.change_feed.document_types import get_doc_meta_object_from_document from corehq.apps.change_feed.topics import get_topic meta = get_doc_meta_object_from_document( self.document_class().to_json()) return get_topic(meta)
def populate_change_metadata(change, data_source_type, data_source_name): from corehq.apps.change_feed.exceptions import MissingMetaInformationError from corehq.apps.change_feed.document_types import ( change_meta_from_doc_meta_and_document, change_meta_from_hard_delete, get_doc_meta_object_from_document, ) if change.metadata: return document = change.get_document() if change.deleted: change.metadata = change_meta_from_hard_delete(document, data_source_type, data_source_name) return try: doc_meta = get_doc_meta_object_from_document(document) change_meta = change_meta_from_doc_meta_and_document( doc_meta=doc_meta, document=document, data_source_type=data_source_type, data_source_name=data_source_name, doc_id=change.id, ) except MissingMetaInformationError: pass else: change.metadata = change_meta
def test_document_meta(self, raw_doc, expected_primary_type, expected_subtype=None, expected_domain=None, expected_deletion=False): doc_meta = get_doc_meta_object_from_document(raw_doc) self.assertEqual(expected_primary_type, doc_meta.primary_type) self.assertEqual(expected_subtype, doc_meta.subtype) self.assertEqual(expected_domain, doc_meta.domain) self.assertEqual(expected_deletion, doc_meta.is_deletion)
def _doc_to_changes(self, doc): # creates a change object for the last form submission # for the user to each of their apps. # this allows us to reindex for the app status report # without reindexing all forms. changes = [] forms = get_last_forms_by_app(doc['_id']) for form in forms: doc_meta = get_doc_meta_object_from_document(form) change_meta = change_meta_from_doc_meta_and_document( doc_meta=doc_meta, document=form, data_source_type='elasticsearch', data_source_name='hqforms', ) changes.append( Change( id=change_meta.document_id, sequence_id=None, document=form, deleted=change_meta.is_deletion, metadata=change_meta, document_store=None, )) return changes
def test_document_meta(self, raw_doc, expected_subtype=None, expected_domain=None, expected_deletion=False): doc_meta = get_doc_meta_object_from_document(raw_doc) self.assertEqual(expected_subtype, doc_meta.subtype) self.assertEqual(expected_domain, doc_meta.domain) self.assertEqual(expected_deletion, doc_meta.is_deletion)
def test_change_from_doc_success(self, doc, expected_id): change_meta = change_meta_from_doc(doc, 'dummy-data-source', 'dummy-data-source-name') doc_meta = get_doc_meta_object_from_document(doc) self.assertEqual(expected_id, change_meta.document_id) self.assertEqual('dummy-data-source', change_meta.data_source_type) self.assertEqual('dummy-data-source-name', change_meta.data_source_name) self.assertEqual(doc_meta.raw_doc_type, change_meta.document_type) self.assertEqual(doc_meta.subtype, change_meta.document_subtype) self.assertEqual(doc_meta.domain, change_meta.domain) self.assertEqual(doc_meta.is_deletion, change_meta.is_deletion)
def IntraHealthFormFluffPillow(delete_filtered=False): return get_multi_fluff_pillow( indicator_classes=[ TauxDeSatisfactionFluff, CouvertureFluff, RecapPassageFluff, IntraHealthFluff, TauxDeRuptureFluff, LivraisonFluff, ], name='IntraHealthFormFluff', kafka_topic=get_topic(get_doc_meta_object_from_document(XFormInstance().to_json())), delete_filtered=delete_filtered )
def M4ChangeFormFluffPillow(delete_filtered=False): return get_multi_fluff_pillow( indicator_classes=[ AncHmisCaseFluff, LdHmisCaseFluff, ImmunizationHmisCaseFluff, ProjectIndicatorsCaseFluff, McctMonthlyAggregateFormFluff, AllHmisCaseFluff, ], name='M4ChangeFormFluff', kafka_topic=get_topic(get_doc_meta_object_from_document(XFormInstance().to_json())), delete_filtered=delete_filtered )
def _doc_to_change(doc, data_source_type, data_source_name): doc_meta = get_doc_meta_object_from_document(doc) change_meta = change_meta_from_doc_meta_and_document( doc_meta=doc_meta, document=doc, data_source_type=data_source_type, data_source_name=data_source_name, ) return Change( id=change_meta.document_id, sequence_id=None, document=doc, deleted=change_meta.is_deletion, metadata=change_meta, document_store=None, )
def process_change(self, change): from corehq.apps.change_feed.document_types import get_doc_meta_object_from_document if self.change_filter_fn and self.change_filter_fn(change): return if change.deleted and change.id: doc = change.get_document() if doc and doc.get('doc_type'): current_meta = get_doc_meta_object_from_document(doc) if current_meta.is_deletion: self._delete_doc_if_exists(change.id) else: self._delete_doc_if_exists(change.id) return with self._datadog_timing('extract'): doc = change.get_document() ensure_document_exists(change) ensure_matched_revisions(change, doc) with self._datadog_timing('transform'): if doc is None or (self.doc_filter_fn and self.doc_filter_fn(doc)): return if doc.get('doc_type') is not None and doc['doc_type'].endswith("-Deleted"): self._delete_doc_if_exists(change.id) return # prepare doc for es doc_ready_to_save = self.doc_transform_fn(doc) # send it across with self._datadog_timing('load'): send_to_elasticsearch( index_info=self.index_info, doc_type=self.index_info.type, doc_id=change.id, es_getter=self.es_getter, name='ElasticProcessor', data=doc_ready_to_save, )
def process_change(self, pillow_instance, change): try: document = change.get_document() doc_meta = get_doc_meta_object_from_document(document) change_meta = change_meta_from_doc_meta_and_document( doc_meta=doc_meta, document=document, data_source_type=self._data_source_type, data_source_name=self._data_source_name, doc_id=change.id, ) except MissingMetaInformationError: pass else: # change.deleted is used for hard deletions whereas change_meta.is_deletion is for soft deletions. # from the consumer's perspective both should be counted as deletions so just "or" them # note: it is strange and hard to reproduce that the couch changes feed is providing a "doc" # along with a hard deletion, but it is doing that in the wild so we might as well support it. change_meta.is_deletion = change_meta.is_deletion or change.deleted self._producer.send_change(get_topic(doc_meta), change_meta)
def populate_change_metadata(change, data_source_type, data_source_name): from corehq.apps.change_feed.exceptions import MissingMetaInformationError from corehq.apps.change_feed.document_types import get_doc_meta_object_from_document from corehq.apps.change_feed.document_types import change_meta_from_doc_meta_and_document if change.metadata: return try: document = change.get_document() doc_meta = get_doc_meta_object_from_document(document) change_meta = change_meta_from_doc_meta_and_document( doc_meta=doc_meta, document=document, data_source_type=data_source_type, data_source_name=data_source_name, doc_id=change.id, ) except MissingMetaInformationError: pass else: change.metadata = change_meta
def _doc_to_changes(self, doc): # creates a change object for the last form submission # for the user to each of their apps. # this allows us to reindex for the app status report # without reindexing all forms. changes = [] forms = get_last_forms_by_app(doc['_id']) for form in forms: doc_meta = get_doc_meta_object_from_document(form) change_meta = change_meta_from_doc_meta_and_document( doc_meta=doc_meta, document=form, data_source_type='elasticsearch', data_source_name='hqforms', ) changes.append(Change( id=change_meta.document_id, sequence_id=None, document=form, deleted=change_meta.is_deletion, metadata=change_meta, document_store=None, )) return changes
def _is_deleted(change): doc = change.get_document() if doc and doc.get('doc_type'): return get_doc_meta_object_from_document(doc).is_deletion elif change.deleted: return bool(change.id)
def kafka_topic(self): """if set, this will use a kafka feed instead of couch for the pillow""" from corehq.apps.change_feed.document_types import get_doc_meta_object_from_document from corehq.apps.change_feed.topics import get_topic meta = get_doc_meta_object_from_document(self.document_class().to_json()) return get_topic(meta)