def _process_kafka_change(producer, error):
change_metadata = error.change_object.metadata
producer.send_change(
get_topic_for_doc_type(change_metadata.document_type,
change_metadata.data_source_type),
change_metadata)
PillowError.objects.filter(doc_id=error.doc_id).delete()
def process_pillow_retry(error_doc):
pillow_name_or_class = error_doc.pillow
try:
pillow = get_pillow_by_name(pillow_name_or_class)
except PillowNotFoundError:
pillow = None
if not pillow:
notify_error((
"Could not find pillowtop class '%s' while attempting a retry. "
"If this pillow was recently deleted then this will be automatically cleaned up eventually. "
"If not, then this should be looked into."
) % pillow_name_or_class)
try:
error_doc.total_attempts = const.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF + 1
error_doc.save()
finally:
return
change = error_doc.change_object
delete_all_for_doc = False
try:
change_metadata = change.metadata
if change_metadata:
document_store = get_document_store(
data_source_type=change_metadata.data_source_type,
data_source_name=change_metadata.data_source_name,
domain=change_metadata.domain,
load_source="pillow_retry",
)
change.document_store = document_store
if isinstance(pillow.get_change_feed(), CouchChangeFeed):
pillow.process_change(change)
else:
if change_metadata.data_source_type in ('couch', 'sql'):
data_source_type = change_metadata.data_source_type
else:
# legacy metadata will have other values for non-sql
# can remove this once all legacy errors have been processed
data_source_type = 'sql'
producer.send_change(
get_topic_for_doc_type(
change_metadata.document_type,
data_source_type
),
change_metadata
)
delete_all_for_doc = True
except Exception:
ex_type, ex_value, ex_tb = sys.exc_info()
error_doc.add_attempt(ex_value, ex_tb)
error_doc.save()
else:
if delete_all_for_doc:
PillowError.objects.filter(doc_id=error_doc.doc_id).delete()
else:
error_doc.delete()
def process_change(self, change):
populate_change_metadata(change, self._data_source_type,
self._data_source_name)
if change.metadata:
change_meta = change.metadata
# change.deleted is used for hard deletions whereas change_meta.is_deletion is for soft deletions.
# from the consumer's perspective both should be counted as deletions so just "or" them
# note: it is strange and hard to reproduce that the couch changes feed is providing a "doc"
# along with a hard deletion, but it is doing that in the wild so we might as well support it.
change_meta.is_deletion = change_meta.is_deletion or change.deleted
if change_meta.is_deletion:
# If a change has been hard deleted, set a default topic because we may
# not be able to retrieve its correct doc type
topic = get_topic_for_doc_type(change_meta.document_type,
self._data_source_type,
self._default_topic)
else:
topic = get_topic_for_doc_type(change_meta.document_type,
self._data_source_type)
self._producer.send_change(topic, change_meta)
def process_change(self, change):
populate_change_metadata(change, self._data_source_type, self._data_source_name)
if change.metadata:
change_meta = change.metadata
topic = get_topic_for_doc_type(change_meta.document_type, self._data_source_type)
# change.deleted is used for hard deletions whereas change_meta.is_deletion is for soft deletions.
# from the consumer's perspective both should be counted as deletions so just "or" them
# note: it is strange and hard to reproduce that the couch changes feed is providing a "doc"
# along with a hard deletion, but it is doing that in the wild so we might as well support it.
change_meta.is_deletion = change_meta.is_deletion or change.deleted
self._producer.send_change(topic, change_meta)
def process_pillow_retry(error_doc):
pillow_name_or_class = error_doc.pillow
try:
pillow = get_pillow_by_name(pillow_name_or_class)
except PillowNotFoundError:
pillow = None
if not pillow:
notify_error((
"Could not find pillowtop class '%s' while attempting a retry. "
"If this pillow was recently deleted then this will be automatically cleaned up eventually. "
"If not, then this should be looked into.") % pillow_name_or_class)
try:
error_doc.total_attempts = const.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF + 1
error_doc.save()
finally:
return
change = error_doc.change_object
delete_all_for_doc = False
try:
change_metadata = change.metadata
if change_metadata:
document_store = get_document_store(
data_source_type=change_metadata.data_source_type,
data_source_name=change_metadata.data_source_name,
domain=change_metadata.domain,
load_source="pillow_retry",
)
change.document_store = document_store
if isinstance(pillow.get_change_feed(), CouchChangeFeed):
pillow.process_change(change)
else:
if change_metadata.data_source_type in ('couch', 'sql'):
data_source_type = change_metadata.data_source_type
else:
# legacy metadata will have other values for non-sql
# can remove this once all legacy errors have been processed
data_source_type = 'sql'
producer.send_change(
get_topic_for_doc_type(change_metadata.document_type,
data_source_type), change_metadata)
delete_all_for_doc = True
except Exception:
ex_type, ex_value, ex_tb = sys.exc_info()
error_doc.add_attempt(ex_value, ex_tb)
error_doc.save()
else:
if delete_all_for_doc:
PillowError.objects.filter(doc_id=error_doc.doc_id).delete()
else:
error_doc.delete()
def M4ChangeFormFluffPillow(delete_filtered=False):
return get_multi_fluff_pillow(indicator_classes=[
AncHmisCaseFluff,
LdHmisCaseFluff,
ImmunizationHmisCaseFluff,
ProjectIndicatorsCaseFluff,
McctMonthlyAggregateFormFluff,
AllHmisCaseFluff,
],
name='M4ChangeFormFluff',
kafka_topic=get_topic_for_doc_type(
XFormInstance().to_json()['doc_type']),
delete_filtered=delete_filtered)
def process_change(self, pillow_instance, change):
populate_change_metadata(change, self._data_source_type,
self._data_source_name)
if change.metadata:
change_meta = change.metadata
topic = get_topic_for_doc_type(change_meta.document_type,
self._data_source_type)
# change.deleted is used for hard deletions whereas change_meta.is_deletion is for soft deletions.
# from the consumer's perspective both should be counted as deletions so just "or" them
# note: it is strange and hard to reproduce that the couch changes feed is providing a "doc"
# along with a hard deletion, but it is doing that in the wild so we might as well support it.
change_meta.is_deletion = change_meta.is_deletion or change.deleted
self._producer.send_change(topic, change_meta)
def M4ChangeFormFluffPillow(delete_filtered=False):
return get_multi_fluff_pillow(
indicator_classes=[
AncHmisCaseFluff,
LdHmisCaseFluff,
ImmunizationHmisCaseFluff,
ProjectIndicatorsCaseFluff,
McctMonthlyAggregateFormFluff,
AllHmisCaseFluff,
],
name='M4ChangeFormFluff',
kafka_topic=get_topic_for_doc_type(XFormInstance().to_json()['doc_type']),
delete_filtered=delete_filtered
)
def test_get_topic_for_doc_type(self, doc_type, data_source, expected_topic):
topic = get_topic_for_doc_type(doc_type, data_source)
self.assertEqual(topic, expected_topic)
def kafka_topic(self):
"""if set, this will use a kafka feed instead of couch for the pillow"""
from corehq.apps.change_feed.topics import get_topic_for_doc_type
return get_topic_for_doc_type(
self.document_class().to_json()['doc_type'])
def publish_change(change_meta):
topic = get_topic_for_doc_type(change_meta.document_type, DATA_SOURCE_TYPE)
PRODUCER.send_change(topic, change_meta)
def kafka_topic(self):
"""if set, this will use a kafka feed instead of couch for the pillow"""
from corehq.apps.change_feed.topics import get_topic_for_doc_type
return get_topic_for_doc_type(self.document_class().to_json()['doc_type'])
def publish_change(change_meta):
topic = get_topic_for_doc_type(change_meta.document_type, DATA_SOURCE_TYPE)
print(topic, change_meta)
PRODUCER.send_change(topic, change_meta)
