def _process_kafka_change(producer, error): change_metadata = error.change_object.metadata producer.send_change( get_topic_for_doc_type(change_metadata.document_type, change_metadata.data_source_type), change_metadata) PillowError.objects.filter(doc_id=error.doc_id).delete()
def process_pillow_retry(error_doc): pillow_name_or_class = error_doc.pillow try: pillow = get_pillow_by_name(pillow_name_or_class) except PillowNotFoundError: pillow = None if not pillow: notify_error(( "Could not find pillowtop class '%s' while attempting a retry. " "If this pillow was recently deleted then this will be automatically cleaned up eventually. " "If not, then this should be looked into." ) % pillow_name_or_class) try: error_doc.total_attempts = const.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF + 1 error_doc.save() finally: return change = error_doc.change_object delete_all_for_doc = False try: change_metadata = change.metadata if change_metadata: document_store = get_document_store( data_source_type=change_metadata.data_source_type, data_source_name=change_metadata.data_source_name, domain=change_metadata.domain, load_source="pillow_retry", ) change.document_store = document_store if isinstance(pillow.get_change_feed(), CouchChangeFeed): pillow.process_change(change) else: if change_metadata.data_source_type in ('couch', 'sql'): data_source_type = change_metadata.data_source_type else: # legacy metadata will have other values for non-sql # can remove this once all legacy errors have been processed data_source_type = 'sql' producer.send_change( get_topic_for_doc_type( change_metadata.document_type, data_source_type ), change_metadata ) delete_all_for_doc = True except Exception: ex_type, ex_value, ex_tb = sys.exc_info() error_doc.add_attempt(ex_value, ex_tb) error_doc.save() else: if delete_all_for_doc: PillowError.objects.filter(doc_id=error_doc.doc_id).delete() else: error_doc.delete()
def process_change(self, change): populate_change_metadata(change, self._data_source_type, self._data_source_name) if change.metadata: change_meta = change.metadata # change.deleted is used for hard deletions whereas change_meta.is_deletion is for soft deletions. # from the consumer's perspective both should be counted as deletions so just "or" them # note: it is strange and hard to reproduce that the couch changes feed is providing a "doc" # along with a hard deletion, but it is doing that in the wild so we might as well support it. change_meta.is_deletion = change_meta.is_deletion or change.deleted if change_meta.is_deletion: # If a change has been hard deleted, set a default topic because we may # not be able to retrieve its correct doc type topic = get_topic_for_doc_type(change_meta.document_type, self._data_source_type, self._default_topic) else: topic = get_topic_for_doc_type(change_meta.document_type, self._data_source_type) self._producer.send_change(topic, change_meta)
def process_change(self, change): populate_change_metadata(change, self._data_source_type, self._data_source_name) if change.metadata: change_meta = change.metadata topic = get_topic_for_doc_type(change_meta.document_type, self._data_source_type) # change.deleted is used for hard deletions whereas change_meta.is_deletion is for soft deletions. # from the consumer's perspective both should be counted as deletions so just "or" them # note: it is strange and hard to reproduce that the couch changes feed is providing a "doc" # along with a hard deletion, but it is doing that in the wild so we might as well support it. change_meta.is_deletion = change_meta.is_deletion or change.deleted self._producer.send_change(topic, change_meta)
def process_pillow_retry(error_doc): pillow_name_or_class = error_doc.pillow try: pillow = get_pillow_by_name(pillow_name_or_class) except PillowNotFoundError: pillow = None if not pillow: notify_error(( "Could not find pillowtop class '%s' while attempting a retry. " "If this pillow was recently deleted then this will be automatically cleaned up eventually. " "If not, then this should be looked into.") % pillow_name_or_class) try: error_doc.total_attempts = const.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF + 1 error_doc.save() finally: return change = error_doc.change_object delete_all_for_doc = False try: change_metadata = change.metadata if change_metadata: document_store = get_document_store( data_source_type=change_metadata.data_source_type, data_source_name=change_metadata.data_source_name, domain=change_metadata.domain, load_source="pillow_retry", ) change.document_store = document_store if isinstance(pillow.get_change_feed(), CouchChangeFeed): pillow.process_change(change) else: if change_metadata.data_source_type in ('couch', 'sql'): data_source_type = change_metadata.data_source_type else: # legacy metadata will have other values for non-sql # can remove this once all legacy errors have been processed data_source_type = 'sql' producer.send_change( get_topic_for_doc_type(change_metadata.document_type, data_source_type), change_metadata) delete_all_for_doc = True except Exception: ex_type, ex_value, ex_tb = sys.exc_info() error_doc.add_attempt(ex_value, ex_tb) error_doc.save() else: if delete_all_for_doc: PillowError.objects.filter(doc_id=error_doc.doc_id).delete() else: error_doc.delete()
def M4ChangeFormFluffPillow(delete_filtered=False): return get_multi_fluff_pillow(indicator_classes=[ AncHmisCaseFluff, LdHmisCaseFluff, ImmunizationHmisCaseFluff, ProjectIndicatorsCaseFluff, McctMonthlyAggregateFormFluff, AllHmisCaseFluff, ], name='M4ChangeFormFluff', kafka_topic=get_topic_for_doc_type( XFormInstance().to_json()['doc_type']), delete_filtered=delete_filtered)
def process_change(self, pillow_instance, change): populate_change_metadata(change, self._data_source_type, self._data_source_name) if change.metadata: change_meta = change.metadata topic = get_topic_for_doc_type(change_meta.document_type, self._data_source_type) # change.deleted is used for hard deletions whereas change_meta.is_deletion is for soft deletions. # from the consumer's perspective both should be counted as deletions so just "or" them # note: it is strange and hard to reproduce that the couch changes feed is providing a "doc" # along with a hard deletion, but it is doing that in the wild so we might as well support it. change_meta.is_deletion = change_meta.is_deletion or change.deleted self._producer.send_change(topic, change_meta)
def M4ChangeFormFluffPillow(delete_filtered=False): return get_multi_fluff_pillow( indicator_classes=[ AncHmisCaseFluff, LdHmisCaseFluff, ImmunizationHmisCaseFluff, ProjectIndicatorsCaseFluff, McctMonthlyAggregateFormFluff, AllHmisCaseFluff, ], name='M4ChangeFormFluff', kafka_topic=get_topic_for_doc_type(XFormInstance().to_json()['doc_type']), delete_filtered=delete_filtered )
def test_get_topic_for_doc_type(self, doc_type, data_source, expected_topic): topic = get_topic_for_doc_type(doc_type, data_source) self.assertEqual(topic, expected_topic)
def kafka_topic(self): """if set, this will use a kafka feed instead of couch for the pillow""" from corehq.apps.change_feed.topics import get_topic_for_doc_type return get_topic_for_doc_type( self.document_class().to_json()['doc_type'])
def publish_change(change_meta): topic = get_topic_for_doc_type(change_meta.document_type, DATA_SOURCE_TYPE) PRODUCER.send_change(topic, change_meta)
def kafka_topic(self): """if set, this will use a kafka feed instead of couch for the pillow""" from corehq.apps.change_feed.topics import get_topic_for_doc_type return get_topic_for_doc_type(self.document_class().to_json()['doc_type'])
def publish_change(change_meta): topic = get_topic_for_doc_type(change_meta.document_type, DATA_SOURCE_TYPE) print(topic, change_meta) PRODUCER.send_change(topic, change_meta)