def publish_ledger_v2_deleted(domain, case_id, section_id, entry_id): from corehq.form_processor.parsers.ledgers.helpers import UniqueLedgerReference ref = UniqueLedgerReference(case_id=case_id, section_id=section_id, entry_id=entry_id) producer.send_change(topics.LEDGER, change_meta_from_ledger_v2(ref, domain, True))
def test_unknown_user_pillow(self): FormProcessorTestUtils.delete_all_xforms() user_id = 'test-unknown-user' metadata = TestFormMetadata(domain=TEST_DOMAIN, user_id='test-unknown-user') form = get_form_ready_to_save(metadata) FormProcessorInterface(domain=TEST_DOMAIN).save_processed_models([form]) # send to kafka topic = topics.FORM_SQL if settings.TESTS_SHOULD_USE_SQL_BACKEND else topics.FORM since = self._get_kafka_seq() producer.send_change(topic, _form_to_change_meta(form)) # send to elasticsearch pillow = get_unknown_users_pillow() pillow.process_changes(since=since, forever=False) self.elasticsearch.indices.refresh(self.index_info.index) # the default query doesn't include unknown users so should have no results self.assertEqual(0, UserES().run().total) # clear the default filters which hide unknown users user_es = UserES().remove_default_filters() results = user_es.run() self.assertEqual(1, results.total) user_doc = results.hits[0] self.assertEqual(TEST_DOMAIN, user_doc['domain']) self.assertEqual(user_id, user_doc['_id']) self.assertEqual('UnknownUser', user_doc['doc_type'])
def handle(self, stale_data_in_es_file, delimiter, skip_domains, *args, **options): changes = _iter_changes(stale_data_in_es_file, skip_domains, delimiter=delimiter) for topic, meta in changes: producer.send_change(topic, meta)
def publish_case_saved(case, send_post_save_signal=True): """ Publish the change to kafka and run case post-save signals. """ producer.send_change(topics.CASE_SQL, change_meta_from_sql_case(case)) if send_post_save_signal: sql_case_post_save.send(case.__class__, case=case)
def test_unknown_user_pillow(self): FormProcessorTestUtils.delete_all_xforms() user_id = 'test-unknown-user' metadata = TestFormMetadata(domain=TEST_DOMAIN, user_id='test-unknown-user') form = get_form_ready_to_save(metadata) FormProcessorInterface(domain=TEST_DOMAIN).save_processed_models( [form]) # send to kafka topic = topics.FORM_SQL if settings.TESTS_SHOULD_USE_SQL_BACKEND else topics.FORM since = self._get_kafka_seq() producer.send_change(topic, _form_to_change_meta(form)) # send to elasticsearch pillow = get_xform_pillow() pillow.process_changes(since=since, forever=False) self.elasticsearch.indices.refresh(self.index_info.index) # the default query doesn't include unknown users so should have no results self.assertEqual(0, UserES().run().total) # clear the default filters which hide unknown users user_es = UserES().remove_default_filters() results = user_es.run() self.assertEqual(1, results.total) user_doc = results.hits[0] self.assertEqual(TEST_DOMAIN, user_doc['domain']) self.assertEqual(user_id, user_doc['_id']) self.assertEqual('UnknownUser', user_doc['doc_type'])
def test_xform_pillow_couch(self): form = self._make_form() kafka_seq = self._get_kafka_seq() producer.send_change(topics.FORM, doc_to_change(form.to_json()).metadata) self.assertFalse(self.app.has_submissions) self.pillow.process_changes(since=kafka_seq, forever=False) self.assertTrue(Application.get(self.app._id).has_submissions)
def test_xform_pillow_couch(self): form = self._make_form() kafka_seq = self._get_kafka_seq() producer.send_change(topics.FORM, doc_to_change(form.to_json()).metadata) self.assertFalse(self.app.has_submissions) self.pillow.process_changes(since=kafka_seq, forever=False) self.assertTrue(Application.get(self.app._id).has_submissions)
def handle(self, pillow, **options): self.pillow = pillow for errors in self.get_next_errors(): for error in errors: if error.change_object.metadata: producer.send_change( error.change_object.metadata.data_source_type, error.change_object.metadata)
def publish_case_deleted(domain, case_id): producer.send_change(topics.CASE_SQL, ChangeMeta( document_id=case_id, data_source_type=data_sources.CASE_SQL, data_source_name='case-sql', # todo: this isn't really needed. document_type='CommCareCase-Deleted', domain=domain, is_deletion=True, ))
def publish_form_deleted(domain, form_id): producer.send_change(topics.FORM_SQL, ChangeMeta( document_id=form_id, data_source_type=data_sources.FORM_SQL, data_source_name='form-sql', document_type='XFormInstance-Deleted', domain=domain, is_deletion=True, ))
def _publish_ledgers_from_form(domain, form): from corehq.form_processor.parsers.ledgers.form import get_all_stock_report_helpers_from_form unique_references = { transaction.ledger_reference for helper in get_all_stock_report_helpers_from_form(form) for transaction in helper.transactions } for ledger_reference in unique_references: producer.send_change(topics.LEDGER, _change_meta_from_ledger_reference(domain, ledger_reference))
def process_pillow_retry(error_doc): pillow_name_or_class = error_doc.pillow try: pillow = get_pillow_by_name(pillow_name_or_class) except PillowNotFoundError: pillow = None if not pillow: notify_error(( "Could not find pillowtop class '%s' while attempting a retry. " "If this pillow was recently deleted then this will be automatically cleaned up eventually. " "If not, then this should be looked into." ) % pillow_name_or_class) try: error_doc.total_attempts = const.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF + 1 error_doc.save() finally: return change = error_doc.change_object delete_all_for_doc = False try: change_metadata = change.metadata if change_metadata: document_store = get_document_store( data_source_type=change_metadata.data_source_type, data_source_name=change_metadata.data_source_name, domain=change_metadata.domain, load_source="pillow_retry", ) change.document_store = document_store if isinstance(pillow.get_change_feed(), CouchChangeFeed): pillow.process_change(change) else: if change_metadata.data_source_type in ('couch', 'sql'): data_source_type = change_metadata.data_source_type else: # legacy metadata will have other values for non-sql # can remove this once all legacy errors have been processed data_source_type = 'sql' producer.send_change( get_topic_for_doc_type( change_metadata.document_type, data_source_type ), change_metadata ) delete_all_for_doc = True except Exception: ex_type, ex_value, ex_tb = sys.exc_info() error_doc.add_attempt(ex_value, ex_tb) error_doc.save() else: if delete_all_for_doc: PillowError.objects.filter(doc_id=error_doc.doc_id).delete() else: error_doc.delete()
def _publish_ledgers_from_form(domain, form): from corehq.form_processor.parsers.ledgers.form import get_all_stock_report_helpers_from_form unique_references = { transaction.ledger_reference for helper in get_all_stock_report_helpers_from_form(form) for transaction in helper.transactions } for ledger_reference in unique_references: producer.send_change(topics.LEDGER, _change_meta_from_ledger_reference(domain, ledger_reference))
def publish_case_deleted(domain, case_id): producer.send_change(topics.CASE_SQL, ChangeMeta( document_id=case_id, data_source_type=data_sources.CASE_SQL, data_source_name='case-sql', # todo: this isn't really needed. document_type='CommCareCase-Deleted', domain=domain, is_deletion=True, ))
def publish_form_deleted(domain, form_id): producer.send_change(topics.FORM_SQL, ChangeMeta( document_id=form_id, data_source_type=data_sources.FORM_SQL, data_source_name='form-sql', document_type='XFormInstance-Deleted', domain=domain, is_deletion=True, ))
def publish_case_deleted(domain, case_id): producer.send_change(topics.CASE_SQL, ChangeMeta( document_id=case_id, data_source_type=data_sources.SOURCE_SQL, data_source_name=data_sources.CASE_SQL, document_type='CommCareCase-Deleted', domain=domain, is_deletion=True, ))
def handle(self, pillow, **options): self.pillow = pillow for errors in self.get_next_errors(): for error in errors: if error.change_object.metadata: producer.send_change( error.change_object.metadata.data_source_type, error.change_object.metadata )
def publish_location_saved(domain, location_id, is_deletion=False): change_meta = ChangeMeta( document_id=location_id, data_source_type='location', data_source_name='location', document_type=LOCATION_DOC_TYPE, domain=domain, is_deletion=is_deletion, ) producer.send_change(topics.LOCATION, change_meta)
def publish_location_saved(domain, location_id, is_deletion=False): change_meta = ChangeMeta( document_id=location_id, data_source_type='location', data_source_name='location', document_type=LOCATION_DOC_TYPE, domain=domain, is_deletion=is_deletion, ) producer.send_change(topics.LOCATION, change_meta)
def publish_case_deleted(domain, case_id): producer.send_change(topics.CASE_SQL, ChangeMeta( document_id=case_id, data_source_type=data_sources.CASE_SQL, data_source_name='case-sql', # todo: this isn't really needed. document_type='CommCareCaseSql', # todo: should this be the same as the couch models? document_subtype=None, # todo: does this need a value? domain=domain, is_deletion=True, ))
def publish_location_saved(domain, location_id, is_deletion=False): from corehq.apps.change_feed import data_sources change_meta = ChangeMeta( document_id=location_id, data_source_type=data_sources.SOURCE_SQL, data_source_name=data_sources.LOCATION, document_type=LOCATION_DOC_TYPE, domain=domain, is_deletion=is_deletion, ) producer.send_change(topics.LOCATION, change_meta)
def publish_case_deleted(domain, case_id): producer.send_change( topics.CASE_SQL, ChangeMeta( document_id=case_id, data_source_type=data_sources.SOURCE_SQL, data_source_name=data_sources.CASE_SQL, document_type='CommCareCase-Deleted', domain=domain, is_deletion=True, ))
def process_pillow_retry(error_doc): pillow_name_or_class = error_doc.pillow try: pillow = get_pillow_by_name(pillow_name_or_class) except PillowNotFoundError: pillow = None if not pillow: notify_error(( "Could not find pillowtop class '%s' while attempting a retry. " "If this pillow was recently deleted then this will be automatically cleaned up eventually. " "If not, then this should be looked into.") % pillow_name_or_class) try: error_doc.total_attempts = const.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF + 1 error_doc.save() finally: return change = error_doc.change_object delete_all_for_doc = False try: change_metadata = change.metadata if change_metadata: document_store = get_document_store( data_source_type=change_metadata.data_source_type, data_source_name=change_metadata.data_source_name, domain=change_metadata.domain, load_source="pillow_retry", ) change.document_store = document_store if isinstance(pillow.get_change_feed(), CouchChangeFeed): pillow.process_change(change) else: if change_metadata.data_source_type in ('couch', 'sql'): data_source_type = change_metadata.data_source_type else: # legacy metadata will have other values for non-sql # can remove this once all legacy errors have been processed data_source_type = 'sql' producer.send_change( get_topic_for_doc_type(change_metadata.document_type, data_source_type), change_metadata) delete_all_for_doc = True except Exception: ex_type, ex_value, ex_tb = sys.exc_info() error_doc.add_attempt(ex_value, ex_tb) error_doc.save() else: if delete_all_for_doc: PillowError.objects.filter(doc_id=error_doc.doc_id).delete() else: error_doc.delete()
def test_kafka_user_pillow_deletion(self): user = self._make_and_test_user_kafka_pillow('test-kafka-user_deletion') # soft delete user.retire() # send to kafka since = get_topic_offset(topics.COMMCARE_USER) producer.send_change(topics.COMMCARE_USER, _user_to_change_meta(user)) # send to elasticsearch pillow = get_user_pillow_old(skip_ucr=True) pillow.process_changes(since=since, forever=False) self.elasticsearch.indices.refresh(self.index_info.index) self.assertEqual(0, UserES().run().total)
def test_pillow_deletion(self): user_id, group = self.test_pillow() group.soft_delete() # send to kafka since = get_topic_offset(topics.GROUP) producer.send_change(topics.GROUP, _group_to_change_meta(group.to_json())) pillow = get_group_pillow() pillow.process_changes(since=since, forever=False) # confirm removed in elasticsearch self.es_client.indices.refresh(USER_INDEX) _assert_es_user_and_groups(self, self.es_client, user_id, [], [])
def _make_and_test_user_kafka_pillow(self, username): # make a user user = CommCareUser.create(TEST_DOMAIN, username, 'secret') # send to kafka since = get_current_kafka_seq(document_types.COMMCARE_USER) producer.send_change(document_types.COMMCARE_USER, _user_to_change_meta(user)) # send to elasticsearch pillow = get_user_pillow() pillow.process_changes(since={document_types.COMMCARE_USER: since}, forever=False) self.elasticsearch.indices.refresh(self.index_info.index) self._verify_user_in_es(username) return user
def _make_and_test_user_kafka_pillow(self, username): # make a user user = CommCareUser.create(TEST_DOMAIN, username, 'secret', None, None) # send to kafka since = get_topic_offset(topics.COMMCARE_USER) producer.send_change(topics.COMMCARE_USER, _user_to_change_meta(user)) # send to elasticsearch pillow = get_user_pillow_old() pillow.process_changes(since=since, forever=False) self.elasticsearch.indices.refresh(self.index_info.index) self._verify_user_in_es(username) return user
def test_pillow_deletion(self): user_id, group = self.test_pillow() group.soft_delete() # send to kafka since = get_current_kafka_seq(GROUP) producer.send_change(GROUP, _group_to_change_meta(group.to_json())) pillow = get_group_to_user_pillow() pillow.process_changes(since=since, forever=False) # confirm removed in elasticsearch self.es_client.indices.refresh(USER_INDEX) _assert_es_user_and_groups(self, self.es_client, user_id, [], [])
def test_kafka_user_pillow_deletion(self): user = self._make_and_test_user_kafka_pillow('test-kafka-user_deletion') # soft delete user.retire() # send to kafka since = get_topic_offset(topics.COMMCARE_USER) producer.send_change(topics.COMMCARE_USER, _user_to_change_meta(user)) # send to elasticsearch pillow = get_user_pillow_old(skip_ucr=True) pillow.process_changes(since=since, forever=False) self.elasticsearch.indices.refresh(self.index_info.index) self.assertEqual(0, UserES().run().total)
def test_kafka_user_pillow_deletion(self): user = self._make_and_test_user_kafka_pillow('test-kafka-user_deletion') # soft delete user.doc_type = '{}{}'.format(user.doc_type, DELETED_SUFFIX) user.save() # send to kafka since = get_current_kafka_seq(document_types.COMMCARE_USER) producer.send_change(document_types.COMMCARE_USER, _user_to_change_meta(user)) # send to elasticsearch pillow = get_user_pillow() pillow.process_changes(since={document_types.COMMCARE_USER: since}, forever=False) self.elasticsearch.indices.refresh(self.index_info.index) self.assertEqual(0, UserES().run().total)
def _publish_ledgers_from_form(domain, form): from corehq.form_processor.parsers.ledgers.form import get_all_stock_report_helpers_from_form unique_references = { transaction.ledger_reference for helper in get_all_stock_report_helpers_from_form(form) for transaction in helper.transactions } if form.to_json()['backend_id'] == 'sql': data_source_name = data_sources.LEDGER_V2 else: data_source_name = data_sources.LEDGER_V1 for ledger_reference in unique_references: meta = _change_meta_from_ledger_reference(domain, ledger_reference, data_source_name) producer.send_change(topics.LEDGER, meta)
def test_kafka_user_pillow_deletion(self): user = self._make_and_test_user_kafka_pillow( 'test-kafka-user_deletion') # soft delete user.doc_type = '{}{}'.format(user.doc_type, DELETED_SUFFIX) user.save() # send to kafka since = get_topic_offset(topics.COMMCARE_USER) producer.send_change(topics.COMMCARE_USER, _user_to_change_meta(user)) # send to elasticsearch pillow = get_user_pillow() pillow.process_changes(since=since, forever=False) self.elasticsearch.indices.refresh(self.index_info.index) self.assertEqual(0, UserES().run().total)
def test_kafka_domain_pillow_deletions(self): # run the other test to ensure domain is created and in ES self.test_kafka_domain_pillow() domain_obj = Domain.get_by_name('domain-pillowtest-kafka') domain_obj.doc_type = 'Domain-DUPLICATE' # send to kafka since = get_current_kafka_seq(document_types.DOMAIN) producer.send_change(document_types.DOMAIN, _domain_to_change_meta(domain_obj)) # send to elasticsearch pillow = get_domain_kafka_to_elasticsearch_pillow() pillow.process_changes(since={document_types.DOMAIN: since}, forever=False) self.elasticsearch.indices.refresh(self.index_info.index) # ensure removed from ES self.assertEqual(0, DomainES().run().total)
def _test_process_doc_from_couch(self, datetime_mock, pillow): datetime_mock.utcnow.return_value = self.fake_time_now sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now) # make sure case is in DB case = CommCareCase.wrap(sample_doc) with drop_connected_signals(case_post_save): case.save() # send to kafka since = self.pillow.get_change_feed().get_latest_offsets() producer.send_change(topics.CASE, doc_to_change(sample_doc).metadata) # run pillow and check changes pillow.process_changes(since=since, forever=False) self._check_sample_doc_state(expected_indicators) case.delete()
def test_kafka_domain_pillow_deletions(self): # run the other test to ensure domain is created and in ES self.test_kafka_domain_pillow() domain_obj = Domain.get_by_name('domain-pillowtest-kafka') domain_obj.doc_type = 'Domain-DUPLICATE' # send to kafka since = get_topic_offset(topics.DOMAIN) producer.send_change(topics.DOMAIN, _domain_to_change_meta(domain_obj)) # send to elasticsearch pillow = get_domain_kafka_to_elasticsearch_pillow() pillow.process_changes(since=since, forever=False) self.elasticsearch.indices.refresh(self.index_info.index) # ensure removed from ES self.assertEqual(0, DomainES().run().total)
def test_kafka_domain_pillow(self): # make a domain domain_name = 'domain-pillowtest-kafka' with drop_connected_signals(commcare_domain_post_save): domain = create_domain(domain_name) # send to kafka since = get_topic_offset(topics.DOMAIN) producer.send_change(topics.DOMAIN, _domain_to_change_meta(domain)) # send to elasticsearch pillow = get_domain_kafka_to_elasticsearch_pillow() pillow.process_changes(since=since, forever=False) self.elasticsearch.indices.refresh(self.index_info.index) # verify there self._verify_domain_in_es(domain_name)
def test_kafka_domain_pillow(self): # make a domain domain_name = 'domain-pillowtest-kafka' with drop_connected_signals(commcare_domain_post_save): domain = create_domain(domain_name) # send to kafka since = get_current_kafka_seq(document_types.DOMAIN) producer.send_change(document_types.DOMAIN, _domain_to_change_meta(domain)) # send to elasticsearch pillow = get_domain_kafka_to_elasticsearch_pillow() pillow.process_changes(since={document_types.DOMAIN: since}, forever=False) self.elasticsearch.indices.refresh(self.index_info.index) # verify there self._verify_domain_in_es(domain_name)
def test_case_search_pillow(self): consumer = get_test_kafka_consumer(topics.CASE) kafka_seq = self._get_kafka_seq() case = self._make_case(case_properties={'foo': 'bar'}) producer.send_change(topics.CASE, doc_to_change(case.to_json()).metadata) # confirm change made it to kafka message = next(consumer) change_meta = change_meta_from_kafka_message(message.value) self.assertEqual(case.case_id, change_meta.document_id) self.assertEqual(self.domain, change_meta.domain) # enable case search for domain with patch('corehq.pillows.case_search.domain_needs_search_index', new=MagicMock(return_value=True)) as fake_case_search_enabled_for_domain: # send to elasticsearch self.pillow.process_changes(since=kafka_seq, forever=False) fake_case_search_enabled_for_domain.assert_called_with(self.domain) self._assert_case_in_es(self.domain, case)
def test_pillow(self): user_id = uuid.uuid4().hex domain = 'dbtest-group-user' _create_es_user(self.es_client, user_id, domain) _assert_es_user_and_groups(self, self.es_client, user_id, None, None) # create and save a group group = Group(domain=domain, name='g1', users=[user_id]) group.save() # send to kafka since = get_topic_offset(topics.GROUP) producer.send_change(topics.GROUP, _group_to_change_meta(group.to_json())) # process using pillow pillow = get_group_pillow() pillow.process_changes(since=since, forever=False) # confirm updated in elasticsearch self.es_client.indices.refresh(USER_INDEX) _assert_es_user_and_groups(self, self.es_client, user_id, [group._id], [group.name]) return user_id, group
def test_pillow(self): user_id = uuid.uuid4().hex domain = 'dbtest-group-user' _create_es_user(self.es_client, user_id, domain) _assert_es_user_and_groups(self, self.es_client, user_id, None, None) # create and save a group group = Group(domain=domain, name='g1', users=[user_id]) group.save() # send to kafka since = get_current_kafka_seq(GROUP) producer.send_change(GROUP, _group_to_change_meta(group.to_json())) # process using pillow pillow = get_group_to_user_pillow() pillow.process_changes(since=since, forever=False) # confirm updated in elasticsearch self.es_client.indices.refresh(USER_INDEX) _assert_es_user_and_groups(self, self.es_client, user_id, [group._id], [group.name]) return user_id, group
def test_kafka_group_pillow(self): domain = uuid.uuid4().hex user_id = uuid.uuid4().hex # make a group group = Group(domain=domain, name='g1', users=[user_id]) group.save() # send to kafka since = get_topic_offset(GROUP) change_meta = change_meta_from_doc( document=group.to_json(), data_source_type=data_sources.COUCH, data_source_name=Group.get_db().dbname, ) producer.send_change(GROUP, change_meta) # send to elasticsearch pillow = get_group_pillow() pillow.process_changes(since={GROUP: since}, forever=False) self.elasticsearch.indices.refresh(GROUP_INDEX_INFO.index) # verify there self._verify_group_in_es(group)
def test_kafka_group_pillow(self): domain = uuid.uuid4().hex user_id = uuid.uuid4().hex # make a group group = Group(domain=domain, name='g1', users=[user_id]) group.save() # send to kafka since = get_topic_offset(topics.GROUP) change_meta = change_meta_from_doc( document=group.to_json(), data_source_type=data_sources.SOURCE_COUCH, data_source_name=Group.get_db().dbname, ) producer.send_change(topics.GROUP, change_meta) # send to elasticsearch pillow = get_group_pillow() pillow.process_changes(since=since, forever=False) self.elasticsearch.indices.refresh(GROUP_INDEX_INFO.index) # verify there self._verify_group_in_es(group)
def _publish_cases_for_sql(domain, case_records): records_with_types = filter(lambda r: r.doc_subtype, case_records) records_with_no_types = filter(lambda r: not r.doc_subtype, case_records) # if we already had a type just publish as-is for record in records_with_types: producer.send_change( topics.CASE_SQL, _change_meta_for_sql_case(domain, record.doc_id, record.doc_subtype)) # else lookup the type from the database for record_chunk in chunked(records_with_no_types, 10000): # databases will contain a mapping of shard database ids to case_ids in that DB id_chunk = [r.doc_id for r in record_chunk] databases = ShardAccessor.get_docs_by_database(id_chunk) for db, doc_ids in databases.items(): results = CommCareCaseSQL.objects.using(db).filter( case_id__in=doc_ids, ).values_list('case_id', 'type') # make sure we found the same number of IDs assert len(results) == len(doc_ids) for case_id, case_type in results: producer.send_change( topics.CASE_SQL, _change_meta_for_sql_case(domain, case_id, case_type))
def _produce_changes(self, count): for i in range(count): meta = ChangeMeta(document_id=uuid.uuid4().hex, data_source_type='dummy-type', data_source_name='dummy-name') producer.send_change(topics.CASE, meta)
def publish_synclog_saved(synclog): from corehq.apps.change_feed import topics producer.send_change(topics.SYNCLOG_SQL, change_meta_from_synclog(synclog))
def publish_ledger_v1_saved(stock_state, deleted=False): producer.send_change(topics.LEDGER, change_meta_from_ledger_v1(stock_state, deleted))
def publish_form_saved(form): producer.send_change(topics.FORM_SQL, change_meta_from_sql_form(form))
def publish_ledger_v2_saved(ledger_value): producer.send_change( topics.LEDGER, change_meta_from_ledger_v2(ledger_value.ledger_reference, ledger_value.domain))
def publish_case_saved(case): producer.send_change(topics.CASE_SQL, change_meta_from_sql_case(case))
def publish_form_saved(form): producer.send_change(topics.FORM_SQL, change_meta_from_sql_form(form))
def publish_ledger_v2_saved(ledger_value): producer.send_change(topics.LEDGER, change_meta_from_ledger_v2(ledger_value))
def _publish_forms_for_sql(domain, form_records): for record in form_records: producer.send_change(topics.FORM_SQL, _change_meta_for_sql_form_record(domain, record))
class IndicatorPillowTest(TestCase): @classmethod def setUpClass(cls): super(IndicatorPillowTest, cls).setUpClass() cls.config = get_sample_data_source() cls.config.save() cls.adapter = get_indicator_adapter(cls.config) cls.adapter.build_table() cls.fake_time_now = datetime(2015, 4, 24, 12, 30, 8, 24886) cls.pillow = get_kafka_ucr_pillow() @classmethod def tearDownClass(cls): cls.config.delete() cls.adapter.drop_table() super(IndicatorPillowTest, cls).tearDownClass() def tearDown(self): self.adapter.clear_table() @patch('corehq.apps.userreports.specs.datetime') def _check_sample_doc_state(self, expected_indicators, datetime_mock): datetime_mock.utcnow.return_value = self.fake_time_now self.adapter.refresh_table() self.assertEqual(1, self.adapter.get_query_object().count()) row = self.adapter.get_query_object()[0] for k in row.keys(): v = getattr(row, k) if isinstance(expected_indicators[k], decimal.Decimal): self.assertAlmostEqual(expected_indicators[k], v) else: self.assertEqual( expected_indicators[k], v, 'mismatched property: {} (expected {}, was {})'.format( k, expected_indicators[k], v ) ) def test_stale_rebuild(self): # rebuild indicators in another test will save this later_config = DataSourceConfiguration.get(self.config._id) later_config.save() self.assertNotEqual(self.config._rev, later_config._rev) with self.assertRaises(StaleRebuildError): self.pillow.rebuild_table(get_indicator_adapter(self.config)) @patch('corehq.apps.userreports.specs.datetime') def test_change_transport(self, datetime_mock): datetime_mock.utcnow.return_value = self.fake_time_now sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now) self.pillow.process_change(doc_to_change(sample_doc)) self._check_sample_doc_state(expected_indicators) @patch('corehq.apps.userreports.specs.datetime') def test_rebuild_indicators(self, datetime_mock): datetime_mock.utcnow.return_value = self.fake_time_now sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now) CommCareCase.get_db().save_doc(sample_doc) self.addCleanup(lambda id: CommCareCase.get_db().delete_doc(id), sample_doc['_id']) rebuild_indicators(self.config._id) self._check_sample_doc_state(expected_indicators) def test_bad_integer_datatype(self): bad_ints = ['a', '', None] for bad_value in bad_ints: self.pillow.process_change(doc_to_change({ '_id': uuid.uuid4().hex, 'doc_type': 'CommCareCase', 'domain': 'user-reports', 'type': 'ticket', 'priority': bad_value })) self.adapter.refresh_table() # make sure we saved rows to the table for everything self.assertEqual(len(bad_ints), self.adapter.get_query_object().count()) @patch('corehq.apps.userreports.specs.datetime') def test_basic_doc_processing(self, datetime_mock): datetime_mock.utcnow.return_value = self.fake_time_now sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now) self.pillow.process_change(doc_to_change(sample_doc)) self._check_sample_doc_state(expected_indicators) @patch('corehq.apps.userreports.specs.datetime') def test_not_relevant_to_domain(self, datetime_mock): datetime_mock.utcnow.return_value = self.fake_time_now sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now) sample_doc['domain'] = 'not-this-domain' self.pillow.process_change(doc_to_change(sample_doc)) self.adapter.refresh_table() self.assertEqual(0, self.adapter.get_query_object().count()) @patch('corehq.apps.userreports.specs.datetime') def test_process_doc_from_couch(self, datetime_mock): datetime_mock.utcnow.return_value = self.fake_time_now sample_doc, expected_indicators = get_sample_doc_and_indicators(self.fake_time_now) # make sure case is in DB case = CommCareCase.wrap(sample_doc) with drop_connected_signals(case_post_save): case.save() # send to kafka since = self.pillow.get_change_feed().get_latest_offsets() producer.send_change(topics.CASE, doc_to_change(sample_doc).metadata) # run pillow and check changes self.pillow.process_changes(since=since, forever=False) self._check_sample_doc_state(expected_indicators) case.delete()
def publish_synclog_saved(synclog): from corehq.apps.change_feed import topics producer.send_change(topics.SYNCLOG_SQL, change_meta_from_synclog(synclog))
def _produce_changes(self, count): for i in range(count): meta = ChangeMeta(document_id=uuid.uuid4().hex, data_source_type='dummy-type', data_source_name='dummy-name') producer.send_change(topics.CASE, meta)
def publish_stub_change(topic): meta = ChangeMeta(document_id=uuid.uuid4().hex, data_source_type='dummy-type', data_source_name='dummy-name') producer.send_change(topic, meta) return meta
def publish_ledger_v1_saved(stock_state): producer.send_change(topics.LEDGER, change_meta_from_ledger_v1(stock_state))
def publish_sms_saved(sms): if do_publish(): producer.send_change(topics.SMS, change_meta_from_sms(sms))
def publish_sms_saved(sms): if do_publish(): producer.send_change(topics.SMS, change_meta_from_sms(sms))