def get_change_feed_pillow_for_db(pillow_id, couch_db, default_topic=None):
"""Generic pillow for inserting Couch documents into Kafka.
Reads from:
- CouchDB
Writes to:
- Kafka
"""
processor = KafkaProcessor(
data_source_type=data_sources.SOURCE_COUCH,
data_source_name=couch_db.dbname,
default_topic=default_topic,
)
change_feed = CouchChangeFeed(couch_db)
checkpoint = PillowCheckpoint(pillow_id, change_feed.sequence_format)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=processor,
change_processed_event_handler=PillowCheckpointEventHandler(
checkpoint=checkpoint,
checkpoint_frequency=100,
),
)
def _make_couch_pillow(couch_db):
from pillowtop.feed.couch import CouchChangeFeed
from pillowtop.processors import LoggingProcessor
from pillowtop.checkpoints.manager import PillowCheckpoint
pillow = FakeConstructedPillow(
name='fake-couch-pillow',
checkpoint=PillowCheckpoint('fake-feed-test-checkpoint', 'text'),
change_feed=CouchChangeFeed(couch_db=couch_db),
processor=LoggingProcessor(),
)
pillow.process_change = MagicMock(return_value=True)
return pillow
def _get_cache_invalidation_pillow(pillow_id, couch_db, couch_filter=None):
"""
Pillow that listens to changes and invalidates the cache whether it's a
a single doc being cached, to a view.
"""
checkpoint = FakeCheckpoint('cache_invalidate_pillow', couch_db)
change_feed = CouchChangeFeed(couch_db, couch_filter=couch_filter)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=CacheInvalidateProcessor(),
)
def get_change_feed_pillow_for_db(pillow_id, couch_db):
processor = KafkaProcessor(
data_source_type=data_sources.SOURCE_COUCH, data_source_name=couch_db.dbname
)
change_feed = CouchChangeFeed(couch_db)
checkpoint = PillowCheckpoint(pillow_id, change_feed.sequence_format)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=processor,
change_processed_event_handler=PillowCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100,
),
)
def get_change_feed_pillow_for_db(pillow_id, couch_db):
kafka_client = get_kafka_client_or_none()
processor = KafkaProcessor(
kafka_client, data_source_type=data_sources.COUCH, data_source_name=couch_db.dbname
)
change_feed = CouchChangeFeed(couch_db, include_docs=True)
checkpoint = PillowCheckpoint(pillow_id, change_feed.sequence_format)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=processor,
change_processed_event_handler=PillowCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100,
),
)
def get_user_sync_history_pillow(pillow_id='UpdateUserSyncHistoryPillow',
**kwargs):
"""
This gets a pillow which iterates through all synclogs
"""
couch_db = SyncLog.get_db()
change_feed = CouchChangeFeed(couch_db, include_docs=True)
checkpoint = PillowCheckpoint('synclog', change_feed.sequence_format)
form_processor = UserSyncHistoryProcessor()
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=form_processor,
change_processed_event_handler=PillowCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100),
)
def _get_mvp_indicator_pillow(pillow_id, processor):
checkpoint = PillowCheckpoint(
'mvp_docs.pillows.{}.{}'.format(pillow_id, get_machine_id()), )
feed = CouchChangeFeed(XFormInstance.get_db(),
include_docs=True,
couch_filter='hqadmin/domains_and_doc_types',
extra_couch_view_params={
'domains': ' '.join(processor.domains),
'doc_types': ' '.join(processor.doc_types),
})
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=feed,
processor=processor,
change_processed_event_handler=PillowCheckpointEventHandler(
checkpoint=checkpoint, checkpoint_frequency=100),
)
def _get_blob_deletion_pillow(pillow_id,
couch_db,
checkpoint=None,
change_feed=None):
if checkpoint is None:
checkpoint = PillowCheckpoint(pillow_id)
if change_feed is None:
change_feed = CouchChangeFeed(couch_db, include_docs=False)
return ConstructedPillow(
name=pillow_id,
checkpoint=checkpoint,
change_feed=change_feed,
processor=BlobDeletionProcessor(get_blob_db(), couch_db.dbname),
change_processed_event_handler=PillowCheckpointEventHandler(
checkpoint=checkpoint,
checkpoint_frequency=KAFKA_CHECKPOINT_FREQUENCY,
),
)