def change_from_kafka_message(message):
change_meta = change_meta_from_kafka_message(message.value)
try:
document_store = get_document_store(
data_source_type=change_meta.data_source_type,
data_source_name=change_meta.data_source_name,
domain=change_meta.domain,
load_source="change_feed",
)
except UnknownDocumentStore:
document_store = None
notify_error("Unknown document store: {}".format(
change_meta.data_source_type))
except UnexpectedBackend:
document_store = None
notify_error("Change from incorrect backend",
details=change_meta.to_json())
return Change(
id=change_meta.document_id,
sequence_id=message.offset,
document=None,
deleted=change_meta.is_deletion,
metadata=change_meta,
document_store=document_store,
topic=message.topic,
partition=message.partition,
)
def process_pillow_retry(error_doc_id):
# Redis error logged in get_redis_client
try:
client = cache_core.get_redis_client()
except cache_core.RedisClientError:
return
# Prevent more than one task from processing this error, just in case
# it got enqueued twice.
lock = client.lock(
"pillow-retry-processing-%s" % error_doc_id,
timeout=settings.PILLOW_RETRY_PROCESSING_LOCK_TIMEOUT*60
)
if lock.acquire(blocking=False):
try:
error_doc = PillowError.objects.get(id=error_doc_id)
except PillowError.DoesNotExist:
release_lock(lock, True)
return
pillow_name_or_class = error_doc.pillow
try:
pillow = get_pillow_by_name(pillow_name_or_class)
except PillowNotFoundError:
pillow = None
if not pillow:
notify_error((
"Could not find pillowtop class '%s' while attempting a retry. "
"If this pillow was recently deleted then this will be automatically cleaned up eventually. "
"If not, then this should be looked into."
) % pillow_name_or_class)
try:
error_doc.total_attempts = PillowError.multi_attempts_cutoff() + 1
error_doc.save()
finally:
release_lock(lock, True)
return
change = error_doc.change_object
try:
change_metadata = change.metadata
if change_metadata:
document_store = get_document_store(
data_source_type=change_metadata.data_source_type,
data_source_name=change_metadata.data_source_name,
domain=change_metadata.domain
)
change.document_store = document_store
pillow.process_change(change)
except Exception:
ex_type, ex_value, ex_tb = sys.exc_info()
error_doc.add_attempt(ex_value, ex_tb)
error_doc.queued = False
error_doc.save()
else:
error_doc.delete()
finally:
release_lock(lock, True)
def test_get_document_store(self,
source_type,
source_name,
expected,
sql_domain=False):
with override_settings(TESTS_SHOULD_USE_SQL_BACKEND=sql_domain):
store = get_document_store(source_type, source_name, 'domain')
self.assertEqual(store.__class__, expected)
def process_pillow_retry(error_doc):
pillow_name_or_class = error_doc.pillow
try:
pillow = get_pillow_by_name(pillow_name_or_class)
except PillowNotFoundError:
pillow = None
if not pillow:
notify_error((
"Could not find pillowtop class '%s' while attempting a retry. "
"If this pillow was recently deleted then this will be automatically cleaned up eventually. "
"If not, then this should be looked into."
) % pillow_name_or_class)
try:
error_doc.total_attempts = const.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF + 1
error_doc.save()
finally:
return
change = error_doc.change_object
delete_all_for_doc = False
try:
change_metadata = change.metadata
if change_metadata:
document_store = get_document_store(
data_source_type=change_metadata.data_source_type,
data_source_name=change_metadata.data_source_name,
domain=change_metadata.domain,
load_source="pillow_retry",
)
change.document_store = document_store
if isinstance(pillow.get_change_feed(), CouchChangeFeed):
pillow.process_change(change)
else:
if change_metadata.data_source_type in ('couch', 'sql'):
data_source_type = change_metadata.data_source_type
else:
# legacy metadata will have other values for non-sql
# can remove this once all legacy errors have been processed
data_source_type = 'sql'
producer.send_change(
get_topic_for_doc_type(
change_metadata.document_type,
data_source_type
),
change_metadata
)
delete_all_for_doc = True
except Exception:
ex_type, ex_value, ex_tb = sys.exc_info()
error_doc.add_attempt(ex_value, ex_tb)
error_doc.save()
else:
if delete_all_for_doc:
PillowError.objects.filter(doc_id=error_doc.doc_id).delete()
else:
error_doc.delete()
def process_pillow_retry(error_doc):
pillow_name_or_class = error_doc.pillow
try:
pillow = get_pillow_by_name(pillow_name_or_class)
except PillowNotFoundError:
pillow = None
if not pillow:
notify_error((
"Could not find pillowtop class '%s' while attempting a retry. "
"If this pillow was recently deleted then this will be automatically cleaned up eventually. "
"If not, then this should be looked into.") % pillow_name_or_class)
try:
error_doc.total_attempts = const.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF + 1
error_doc.save()
finally:
return
change = error_doc.change_object
delete_all_for_doc = False
try:
change_metadata = change.metadata
if change_metadata:
document_store = get_document_store(
data_source_type=change_metadata.data_source_type,
data_source_name=change_metadata.data_source_name,
domain=change_metadata.domain,
load_source="pillow_retry",
)
change.document_store = document_store
if isinstance(pillow.get_change_feed(), CouchChangeFeed):
pillow.process_change(change)
else:
if change_metadata.data_source_type in ('couch', 'sql'):
data_source_type = change_metadata.data_source_type
else:
# legacy metadata will have other values for non-sql
# can remove this once all legacy errors have been processed
data_source_type = 'sql'
producer.send_change(
get_topic_for_doc_type(change_metadata.document_type,
data_source_type), change_metadata)
delete_all_for_doc = True
except Exception:
ex_type, ex_value, ex_tb = sys.exc_info()
error_doc.add_attempt(ex_value, ex_tb)
error_doc.save()
else:
if delete_all_for_doc:
PillowError.objects.filter(doc_id=error_doc.doc_id).delete()
else:
error_doc.delete()
def _process_couch_change(pillow, error):
change = error.change_object
change_metadata = change.metadata
if change_metadata:
document_store = get_document_store(
data_source_type=change_metadata.data_source_type,
data_source_name=change_metadata.data_source_name,
domain=change_metadata.domain,
load_source="pillow_retry",
)
change.document_store = document_store
pillow.process_change(change)
error.delete()
def change_from_kafka_message(message):
change_meta = change_meta_from_kafka_message(message.value)
try:
document_store = get_document_store(
data_source_type=change_meta.data_source_type,
data_source_name=change_meta.data_source_name,
domain=change_meta.domain)
except UnknownDocumentStore:
document_store = None
return Change(
id=change_meta.document_id,
sequence_id=message.offset,
document=None,
deleted=change_meta.is_deletion,
metadata=change_meta,
document_store=document_store,
)
def change_from_kafka_message(message):
change_meta = change_meta_from_kafka_message(message.value)
try:
document_store = get_document_store(
data_source_type=change_meta.data_source_type,
data_source_name=change_meta.data_source_name,
domain=change_meta.domain
)
except UnknownDocumentStore:
document_store = None
return Change(
id=change_meta.document_id,
sequence_id=message.offset,
document=None,
deleted=change_meta.is_deletion,
metadata=change_meta,
document_store=document_store,
)
def change_from_kafka_message(message):
change_meta = change_meta_from_kafka_message(message.value)
try:
document_store = get_document_store(
data_source_type=change_meta.data_source_type,
data_source_name=change_meta.data_source_name,
domain=change_meta.domain
)
except UnknownDocumentStore:
document_store = None
notify_error("Unknown document store: {}".format(change_meta.data_source_type))
return Change(
id=change_meta.document_id,
sequence_id=message.offset,
document=None,
deleted=change_meta.is_deletion,
metadata=change_meta,
document_store=document_store,
topic=message.topic,
)
def _change_from_meta(change_meta):
from corehq.apps.change_feed.data_sources import get_document_store
from pillowtop.feed.interface import Change
document_store = get_document_store(
data_source_type=change_meta.data_source_type,
data_source_name=change_meta.data_source_name,
domain=change_meta.domain,
load_source="change_feed",
)
return Change(
id=change_meta.document_id,
sequence_id=None,
document=None,
deleted=change_meta.is_deletion,
metadata=change_meta,
document_store=document_store,
topic=None,
partition=None,
)
def change_from_kafka_message(message):
change_meta = change_meta_from_kafka_message(message.value)
try:
document_store = get_document_store(
data_source_type=change_meta.data_source_type,
data_source_name=change_meta.data_source_name,
domain=change_meta.domain
)
except UnknownDocumentStore:
document_store = None
notify_error("Unknown document store: {}".format(change_meta.data_source_type))
return Change(
id=change_meta.document_id,
sequence_id=message.offset,
document=None,
deleted=change_meta.is_deletion,
metadata=change_meta,
document_store=document_store,
topic=message.topic,
)
def process_pillow_retry(error_doc_id):
try:
error_doc = PillowError.objects.get(id=error_doc_id)
except PillowError.DoesNotExist:
return
pillow_name_or_class = error_doc.pillow
try:
pillow = get_pillow_by_name(pillow_name_or_class)
except PillowNotFoundError:
pillow = None
if not pillow:
notify_error((
"Could not find pillowtop class '%s' while attempting a retry. "
"If this pillow was recently deleted then this will be automatically cleaned up eventually. "
"If not, then this should be looked into.") % pillow_name_or_class)
try:
error_doc.total_attempts = PillowError.multi_attempts_cutoff() + 1
error_doc.save()
finally:
return
change = error_doc.change_object
try:
change_metadata = change.metadata
if change_metadata:
document_store = get_document_store(
data_source_type=change_metadata.data_source_type,
data_source_name=change_metadata.data_source_name,
domain=change_metadata.domain)
change.document_store = document_store
pillow.process_change(change)
except Exception:
ex_type, ex_value, ex_tb = sys.exc_info()
error_doc.add_attempt(ex_value, ex_tb)
error_doc.save()
else:
error_doc.delete()
def test_missing_db(self):
with self.assertRaises(DatabaseNotFound):
get_document_store(data_sources.SOURCE_COUCH, 'baddb', 'domain')
def test_get_document_store(self, source_type, source_name, expected):
store = get_document_store(source_type, source_name, 'domain')
if isinstance(store, DocStoreLoadTracker):
store = store.store
self.assertEqual(store.__class__, expected)
def test_unknown_store(self):
with self.assertRaises(UnknownDocumentStore):
get_document_store(data_sources.SOURCE_SQL, 'badsource', 'domain')
def test_missing_db(self):
with self.assertRaises(DatabaseNotFound):
get_document_store(data_sources.SOURCE_COUCH, 'baddb', 'domain')
def test_get_document_store(self, source_type, source_name, expected, sql_domain=False):
with override_settings(TESTS_SHOULD_USE_SQL_BACKEND=sql_domain):
store = get_document_store(source_type, source_name, 'domain')
if isinstance(store, DocStoreLoadTracker):
store = store.store
self.assertEqual(store.__class__, expected)
def test_unknown_store(self):
with self.assertRaises(UnknownDocumentStore):
get_document_store(data_sources.SOURCE_SQL, 'badsource', 'domain')
