示例#1
0
 def __init__(self, pillow_checkpoint_id=UCR_CHECKPOINT_ID):
     # todo: this will need to not be hard-coded if we ever split out forms and cases into their own domains
     couch_db = CachedCouchDB(CommCareCase.get_db().uri, readonly=False)
     checkpoint = PillowCheckpoint(get_django_checkpoint_store(), pillow_checkpoint_id)
     super(ConfigurableIndicatorPillow, self).__init__(couch_db=couch_db, checkpoint=checkpoint)
     self.bootstrapped = False
     self.last_bootstrapped = datetime.utcnow()
示例#2
0
def get_default_couch_db_change_feed_pillow():
    default_couch_db = CachedCouchDB(CommCareCase.get_db().uri, readonly=False)
    kafka_client = get_kafka_client_or_none()
    return ChangeFeedPillow(
        couch_db=default_couch_db,
        kafka=kafka_client,
        checkpoint=PillowCheckpoint(get_django_checkpoint_store(), 'default-couch-change-feed')
    )
示例#3
0
def get_demo_case_consumer_pillow():
    checkpoint = PillowCheckpoint(
        get_django_checkpoint_store(),
        'kafka-demo-case-pillow-checkpoint',
    )
    return ConstructedPillow(
        name='KafkaCaseConsumerPillow',
        document_store=None,
        checkpoint=checkpoint,
        change_feed=KafkaChangeFeed(topic=topics.CASE, group_id='demo-case-group'),
        processor=NoopProcessor(),
        change_processed_event_handler=PillowCheckpointEventHandler(
            checkpoint=checkpoint, checkpoint_frequency=KAFKA_CHECKPOINT_FREQUENCY,
        ),
    )
示例#4
0
def get_sql_case_to_elasticsearch_pillow():
    checkpoint = PillowCheckpoint(get_django_checkpoint_store(), "sql-cases-to-elasticsearch")
    case_processor = ElasticProcessor(
        elasticseach=get_es_new(),
        index_meta=ElasticsearchIndexMeta(index=CASE_INDEX, type=CASE_ES_TYPE),
        doc_prep_fn=prepare_sql_case_json_for_elasticsearch,
    )
    return ConstructedPillow(
        name="SqlCaseToElasticsearchPillow",
        document_store=None,
        checkpoint=checkpoint,
        change_feed=KafkaChangeFeed(topic=topics.CASE_SQL, group_id="sql-cases-to-es"),
        processor=case_processor,
        change_processed_event_handler=PillowCheckpointEventHandler(checkpoint=checkpoint, checkpoint_frequency=100),
    )
示例#5
0
def get_demo_python_pillow_consumer():
    checkpoint = PillowCheckpoint(
        get_django_checkpoint_store(),
        'kafka-demo-python-pillow-checkpoint',
    )

    def arbitrary_filter(change):
        # just to prove that filters work - only process data from domains starting with
        # letters between "b" and "f"
        return 'b' < change.metadata.domain < 'f'

    return LoggingPythonPillow(
        couch_db=CachedCouchDB(CommCareCase.get_db().uri, readonly=False),
        checkpoint=checkpoint,
        change_feed=KafkaChangeFeed(topic=topics.CASE, group_id='demo-python-pillow-group'),
        python_filter=arbitrary_filter,
    )
示例#6
0
def get_sql_xform_to_elasticsearch_pillow():
    checkpoint = PillowCheckpoint(
        get_django_checkpoint_store(),
        'sql-xforms-to-elasticsearch',
    )
    form_processor = ElasticProcessor(
        elasticseach=get_es_new(),
        index_meta=ElasticsearchIndexMeta(index=XFORM_INDEX, type=XFORM_ES_TYPE),
        doc_prep_fn=prepare_sql_form_json_for_elasticsearch
    )
    return ConstructedPillow(
        name='SqlXFormToElasticsearchPillow',
        document_store=None,
        checkpoint=checkpoint,
        change_feed=KafkaChangeFeed(topic=topics.FORM_SQL, group_id='sql-forms-to-es'),
        processor=form_processor,
        change_processed_event_handler=PillowCheckpointEventHandler(
            checkpoint=checkpoint, checkpoint_frequency=100,
        ),
    )
示例#7
0
def get_user_groups_db_kafka_pillow():
    # note: this is temporarily using ConstructedPillow as a test. If it is successful we should
    # flip the main one over as well
    user_groups_couch_db = couch_config.get_db_for_class(CommCareUser)
    pillow_name = 'UserGroupsDbKafkaPillow'
    kafka_client = get_kafka_client_or_none()
    processor = KafkaProcessor(
        kafka_client, data_source_type=data_sources.COUCH, data_source_name=user_groups_couch_db.dbname
    )
    checkpoint = PillowCheckpoint(get_django_checkpoint_store(), pillow_name)
    return ConstructedPillow(
        name=pillow_name,
        document_store=None,  # because we're using include_docs we can be explicit about not using this
        checkpoint=checkpoint,
        change_feed=CouchChangeFeed(user_groups_couch_db, include_docs=True),
        processor=processor,
        change_processed_event_handler=PillowCheckpointEventHandler(
            checkpoint=checkpoint, checkpoint_frequency=100,
        ),
    )
示例#8
0
def get_blob_deletion_pillow():
    """Get blob deletion pillow for the main couch database

    Using the KafkaChangeFeed ties this to the main couch database.
    """
    checkpoint = PillowCheckpoint(
        get_django_checkpoint_store(),
        'kafka-blob-deletion-pillow-checkpoint',
    )
    return ConstructedPillow(
        name='BlobDeletionPillow',
        document_store=None,
        checkpoint=checkpoint,
        change_feed=KafkaChangeFeed(topic=topics.META, group_id='blob-deletion-group'),
        processor=BlobDeletionProcessor(get_blob_db(), get_db(None).dbname),
        change_processed_event_handler=PillowCheckpointEventHandler(
            checkpoint=checkpoint,
            checkpoint_frequency=KAFKA_CHECKPOINT_FREQUENCY,
        ),
    )