def get_change_feed_pillow_for_db(pillow_id, couch_db, default_topic=None): """Generic pillow for inserting Couch documents into Kafka. Reads from: - CouchDB Writes to: - Kafka """ processor = KafkaProcessor( data_source_type=data_sources.SOURCE_COUCH, data_source_name=couch_db.dbname, default_topic=default_topic, ) change_feed = CouchChangeFeed(couch_db) checkpoint = PillowCheckpoint(pillow_id, change_feed.sequence_format) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=processor, change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, ), )
def _make_couch_pillow(couch_db): from pillowtop.feed.couch import CouchChangeFeed from pillowtop.processors import LoggingProcessor from pillowtop.checkpoints.manager import PillowCheckpoint pillow = FakeConstructedPillow( name='fake-couch-pillow', checkpoint=PillowCheckpoint('fake-feed-test-checkpoint', 'text'), change_feed=CouchChangeFeed(couch_db=couch_db), processor=LoggingProcessor(), ) pillow.process_change = MagicMock(return_value=True) return pillow
def _get_cache_invalidation_pillow(pillow_id, couch_db, couch_filter=None): """ Pillow that listens to changes and invalidates the cache whether it's a a single doc being cached, to a view. """ checkpoint = FakeCheckpoint('cache_invalidate_pillow', couch_db) change_feed = CouchChangeFeed(couch_db, couch_filter=couch_filter) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=CacheInvalidateProcessor(), )
def get_change_feed_pillow_for_db(pillow_id, couch_db): processor = KafkaProcessor( data_source_type=data_sources.SOURCE_COUCH, data_source_name=couch_db.dbname ) change_feed = CouchChangeFeed(couch_db) checkpoint = PillowCheckpoint(pillow_id, change_feed.sequence_format) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=processor, change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, ), )
def get_change_feed_pillow_for_db(pillow_id, couch_db): kafka_client = get_kafka_client_or_none() processor = KafkaProcessor( kafka_client, data_source_type=data_sources.COUCH, data_source_name=couch_db.dbname ) change_feed = CouchChangeFeed(couch_db, include_docs=True) checkpoint = PillowCheckpoint(pillow_id, change_feed.sequence_format) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=processor, change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, ), )
def get_user_sync_history_pillow(pillow_id='UpdateUserSyncHistoryPillow', **kwargs): """ This gets a pillow which iterates through all synclogs """ couch_db = SyncLog.get_db() change_feed = CouchChangeFeed(couch_db, include_docs=True) checkpoint = PillowCheckpoint('synclog', change_feed.sequence_format) form_processor = UserSyncHistoryProcessor() return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=form_processor, change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100), )
def _get_mvp_indicator_pillow(pillow_id, processor): checkpoint = PillowCheckpoint( 'mvp_docs.pillows.{}.{}'.format(pillow_id, get_machine_id()), ) feed = CouchChangeFeed(XFormInstance.get_db(), include_docs=True, couch_filter='hqadmin/domains_and_doc_types', extra_couch_view_params={ 'domains': ' '.join(processor.domains), 'doc_types': ' '.join(processor.doc_types), }) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=feed, processor=processor, change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100), )
def _get_blob_deletion_pillow(pillow_id, couch_db, checkpoint=None, change_feed=None): if checkpoint is None: checkpoint = PillowCheckpoint(pillow_id) if change_feed is None: change_feed = CouchChangeFeed(couch_db, include_docs=False) return ConstructedPillow( name=pillow_id, checkpoint=checkpoint, change_feed=change_feed, processor=BlobDeletionProcessor(get_blob_db(), couch_db.dbname), change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=KAFKA_CHECKPOINT_FREQUENCY, ), )