def __init__(self, pillow_checkpoint_id=UCR_CHECKPOINT_ID): # todo: this will need to not be hard-coded if we ever split out forms and cases into their own domains couch_db = CachedCouchDB(CommCareCase.get_db().uri, readonly=False) checkpoint = PillowCheckpoint(get_django_checkpoint_store(), pillow_checkpoint_id) super(ConfigurableIndicatorPillow, self).__init__(couch_db=couch_db, checkpoint=checkpoint) self.bootstrapped = False self.last_bootstrapped = datetime.utcnow()
def get_default_couch_db_change_feed_pillow(): default_couch_db = CachedCouchDB(CommCareCase.get_db().uri, readonly=False) kafka_client = get_kafka_client_or_none() return ChangeFeedPillow( couch_db=default_couch_db, kafka=kafka_client, checkpoint=PillowCheckpoint(get_django_checkpoint_store(), 'default-couch-change-feed') )
def get_demo_case_consumer_pillow(): checkpoint = PillowCheckpoint( get_django_checkpoint_store(), 'kafka-demo-case-pillow-checkpoint', ) return ConstructedPillow( name='KafkaCaseConsumerPillow', document_store=None, checkpoint=checkpoint, change_feed=KafkaChangeFeed(topic=topics.CASE, group_id='demo-case-group'), processor=NoopProcessor(), change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=KAFKA_CHECKPOINT_FREQUENCY, ), )
def get_sql_case_to_elasticsearch_pillow(): checkpoint = PillowCheckpoint(get_django_checkpoint_store(), "sql-cases-to-elasticsearch") case_processor = ElasticProcessor( elasticseach=get_es_new(), index_meta=ElasticsearchIndexMeta(index=CASE_INDEX, type=CASE_ES_TYPE), doc_prep_fn=prepare_sql_case_json_for_elasticsearch, ) return ConstructedPillow( name="SqlCaseToElasticsearchPillow", document_store=None, checkpoint=checkpoint, change_feed=KafkaChangeFeed(topic=topics.CASE_SQL, group_id="sql-cases-to-es"), processor=case_processor, change_processed_event_handler=PillowCheckpointEventHandler(checkpoint=checkpoint, checkpoint_frequency=100), )
def get_demo_python_pillow_consumer(): checkpoint = PillowCheckpoint( get_django_checkpoint_store(), 'kafka-demo-python-pillow-checkpoint', ) def arbitrary_filter(change): # just to prove that filters work - only process data from domains starting with # letters between "b" and "f" return 'b' < change.metadata.domain < 'f' return LoggingPythonPillow( couch_db=CachedCouchDB(CommCareCase.get_db().uri, readonly=False), checkpoint=checkpoint, change_feed=KafkaChangeFeed(topic=topics.CASE, group_id='demo-python-pillow-group'), python_filter=arbitrary_filter, )
def get_sql_xform_to_elasticsearch_pillow(): checkpoint = PillowCheckpoint( get_django_checkpoint_store(), 'sql-xforms-to-elasticsearch', ) form_processor = ElasticProcessor( elasticseach=get_es_new(), index_meta=ElasticsearchIndexMeta(index=XFORM_INDEX, type=XFORM_ES_TYPE), doc_prep_fn=prepare_sql_form_json_for_elasticsearch ) return ConstructedPillow( name='SqlXFormToElasticsearchPillow', document_store=None, checkpoint=checkpoint, change_feed=KafkaChangeFeed(topic=topics.FORM_SQL, group_id='sql-forms-to-es'), processor=form_processor, change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, ), )
def get_user_groups_db_kafka_pillow(): # note: this is temporarily using ConstructedPillow as a test. If it is successful we should # flip the main one over as well user_groups_couch_db = couch_config.get_db_for_class(CommCareUser) pillow_name = 'UserGroupsDbKafkaPillow' kafka_client = get_kafka_client_or_none() processor = KafkaProcessor( kafka_client, data_source_type=data_sources.COUCH, data_source_name=user_groups_couch_db.dbname ) checkpoint = PillowCheckpoint(get_django_checkpoint_store(), pillow_name) return ConstructedPillow( name=pillow_name, document_store=None, # because we're using include_docs we can be explicit about not using this checkpoint=checkpoint, change_feed=CouchChangeFeed(user_groups_couch_db, include_docs=True), processor=processor, change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=100, ), )
def get_blob_deletion_pillow(): """Get blob deletion pillow for the main couch database Using the KafkaChangeFeed ties this to the main couch database. """ checkpoint = PillowCheckpoint( get_django_checkpoint_store(), 'kafka-blob-deletion-pillow-checkpoint', ) return ConstructedPillow( name='BlobDeletionPillow', document_store=None, checkpoint=checkpoint, change_feed=KafkaChangeFeed(topic=topics.META, group_id='blob-deletion-group'), processor=BlobDeletionProcessor(get_blob_db(), get_db(None).dbname), change_processed_event_handler=PillowCheckpointEventHandler( checkpoint=checkpoint, checkpoint_frequency=KAFKA_CHECKPOINT_FREQUENCY, ), )