def test(self):
document = {
'_id': 'test-id',
'doc_type': 'CommCareCase',
'type': 'mother',
'domain': 'kafka-test-domain',
}
change = Change(id='test-id', sequence_id='3', document=document)
populate_change_metadata(change, SOURCE_COUCH, 'test_commcarehq')
with patch('pillow_retry.api.get_pillow_by_name', return_value=self.pillow):
# first change creates error
message = 'test retry 1'
self.pillow.process_change = MagicMock(side_effect=TestException(message))
self.pillow.process_with_error_handling(change, PillowRuntimeContext(changes_seen=0))
errors = self._check_errors(1, message)
# second attempt updates error
with process_pillow_changes(self.pillow):
process_pillow_retry(errors[0])
errors = self._check_errors(2)
# third attempt successful
self.pillow.process_change = self.original_process_change
with process_pillow_changes(self.pillow):
process_pillow_retry(errors[0])
errors = list(PillowError.objects.filter(pillow=self.pillow.pillow_id).all())
self.assertEqual(0, len(errors))
self.assertEqual(1, self.processor.count)
def _create_case_and_sync_to_es(self):
case_id = uuid.uuid4().hex
case_name = 'case-name-{}'.format(uuid.uuid4().hex)
with process_pillow_changes('case-pillow'):
with process_pillow_changes('DefaultChangeFeedPillow'):
create_and_save_a_case(self.domain, case_id, case_name)
self.elasticsearch.indices.refresh(CASE_INDEX_INFO.index)
return case_id, case_name
def _create_case_and_sync_to_es(self):
case_id = uuid.uuid4().hex
case_name = 'case-name-{}'.format(uuid.uuid4().hex)
with process_pillow_changes('case-pillow', {'skip_ucr': True}):
with process_pillow_changes('DefaultChangeFeedPillow'):
create_and_save_a_case(self.domain, case_id, case_name)
self.elasticsearch.indices.refresh(CASE_INDEX_INFO.alias)
return case_id, case_name
def create_form_and_sync_to_es(received_on):
with process_pillow_changes('xform-pillow', {'skip_ucr': True}):
with process_pillow_changes('DefaultChangeFeedPillow'):
metadata = TestFormMetadata(domain=cls.domain, app_id=cls.app_id,
xmlns=cls.xmlns, received_on=received_on)
form = get_form_ready_to_save(metadata, is_db_test=True)
form_processor = FormProcessorInterface(domain=cls.domain)
form_processor.save_processed_models([form])
return form
def _create_form_and_sync_to_es(self):
with process_pillow_changes('xform-pillow', {'skip_ucr': True}):
with process_pillow_changes('DefaultChangeFeedPillow'):
metadata = TestFormMetadata(domain=self.domain)
form = get_form_ready_to_save(metadata, is_db_test=True)
form_processor = FormProcessorInterface(domain=self.domain)
form_processor.save_processed_models([form])
self.elasticsearch.indices.refresh(XFORM_INDEX_INFO.index)
return form, metadata
def create_form_and_sync_to_es(received_on):
with process_pillow_changes('xform-pillow', {'skip_ucr': True}):
with process_pillow_changes('DefaultChangeFeedPillow'):
metadata = TestFormMetadata(domain=cls.domain, app_id=cls.app_id,
xmlns=cls.xmlns, received_on=received_on)
form = get_form_ready_to_save(metadata, is_db_test=True)
form_processor = FormProcessorInterface(domain=cls.domain)
form_processor.save_processed_models([form])
return form
def test_case_is_published(self):
with process_pillow_changes(self.case_pillow):
with process_pillow_changes('DefaultChangeFeedPillow'):
case = create_and_save_a_case(self.domain, case_id=uuid.uuid4().hex, case_name='test case')
self.assertEqual(1, len(self.processor.changes_seen))
change_meta = self.processor.changes_seen[0].metadata
self.assertEqual(case.case_id, change_meta.document_id)
self.assertEqual(self.domain, change_meta.domain)
def test_form_is_published(self):
with process_pillow_changes(self.form_pillow):
with process_pillow_changes('DefaultChangeFeedPillow'):
form = create_and_save_a_form(self.domain)
self.assertEqual(1, len(self.processor.changes_seen))
change_meta = self.processor.changes_seen[0].metadata
self.assertEqual(form.form_id, change_meta.document_id)
self.assertEqual(self.domain, change_meta.domain)
def test_form_soft_deletions(self):
form = create_and_save_a_form(self.domain)
with process_pillow_changes(self.form_pillow):
with process_pillow_changes('DefaultChangeFeedPillow'):
form.soft_delete()
self.assertEqual(1, len(self.processor.changes_seen))
change_meta = self.processor.changes_seen[0].metadata
self.assertEqual(form.form_id, change_meta.document_id)
self.assertTrue(change_meta.is_deletion)
def test_archive_last_form(self):
initial_amounts = [(p._id, float(100)) for p in self.products]
self.submit_xml_form(
balance_submission(initial_amounts),
timestamp=datetime.utcnow() + timedelta(-30)
)
final_amounts = [(p._id, float(50)) for i, p in enumerate(self.products)]
second_form_id = self.submit_xml_form(balance_submission(final_amounts))
ledger_accessors = LedgerAccessors(self.domain.name)
def _assert_initial_state():
if should_use_sql_backend(self.domain):
self.assertEqual(3, len(self._get_all_ledger_transactions(Q(form_id=second_form_id))))
else:
self.assertEqual(1, StockReport.objects.filter(form_id=second_form_id).count())
# 6 = 3 stockonhand and 3 inferred consumption txns
self.assertEqual(6, StockTransaction.objects.filter(report__form_id=second_form_id).count())
ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id)
self.assertEqual(3, len(ledger_values))
for lv in ledger_values:
self.assertEqual(50, lv.stock_on_hand)
self.assertEqual(
round(float(lv.daily_consumption), 2),
1.67
)
# check initial setup
_assert_initial_state()
# archive and confirm commtrack data is deleted
form = FormAccessors(self.domain.name).get_form(second_form_id)
with process_pillow_changes('LedgerToElasticsearchPillow'):
form.archive()
if should_use_sql_backend(self.domain):
self.assertEqual(0, len(self._get_all_ledger_transactions(Q(form_id=second_form_id))))
else:
self.assertEqual(0, StockReport.objects.filter(form_id=second_form_id).count())
self.assertEqual(0, StockTransaction.objects.filter(report__form_id=second_form_id).count())
ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id)
self.assertEqual(3, len(ledger_values))
for state in ledger_values:
# balance should be reverted to 100 in the StockState
self.assertEqual(100, int(state.stock_on_hand))
# consumption should be none since there will only be 1 data point
self.assertIsNone(state.daily_consumption)
# unarchive and confirm commtrack data is restored
with process_pillow_changes('LedgerToElasticsearchPillow'):
form.unarchive()
_assert_initial_state()
def test_archive_last_form(self):
initial_amounts = [(p._id, float(100)) for p in self.products]
self.submit_xml_form(
balance_submission(initial_amounts),
timestamp=datetime.utcnow() + timedelta(-30)
)
final_amounts = [(p._id, float(50)) for i, p in enumerate(self.products)]
second_form_id = self.submit_xml_form(balance_submission(final_amounts))
ledger_accessors = LedgerAccessors(self.domain.name)
def _assert_initial_state():
if should_use_sql_backend(self.domain):
self.assertEqual(3, len(self._get_all_ledger_transactions(Q(form_id=second_form_id))))
else:
self.assertEqual(1, StockReport.objects.filter(form_id=second_form_id).count())
# 6 = 3 stockonhand and 3 inferred consumption txns
self.assertEqual(6, StockTransaction.objects.filter(report__form_id=second_form_id).count())
ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id)
self.assertEqual(3, len(ledger_values))
for lv in ledger_values:
self.assertEqual(50, lv.stock_on_hand)
self.assertEqual(
round(float(lv.daily_consumption), 2),
1.67
)
# check initial setup
_assert_initial_state()
# archive and confirm commtrack data is deleted
form = FormAccessors(self.domain.name).get_form(second_form_id)
with process_pillow_changes('LedgerToElasticsearchPillow'):
form.archive()
if should_use_sql_backend(self.domain):
self.assertEqual(0, len(self._get_all_ledger_transactions(Q(form_id=second_form_id))))
else:
self.assertEqual(0, StockReport.objects.filter(form_id=second_form_id).count())
self.assertEqual(0, StockTransaction.objects.filter(report__form_id=second_form_id).count())
ledger_values = ledger_accessors.get_ledger_values_for_case(self.sp.case_id)
self.assertEqual(3, len(ledger_values))
for state in ledger_values:
# balance should be reverted to 100 in the StockState
self.assertEqual(100, int(state.stock_on_hand))
# consumption should be none since there will only be 1 data point
self.assertIsNone(state.daily_consumption)
# unarchive and confirm commtrack data is restored
with process_pillow_changes('LedgerToElasticsearchPillow'):
form.unarchive()
_assert_initial_state()
def test_case_soft_deletion(self):
case_id, case_name = self._create_case_and_sync_to_es()
# verify there
results = CaseES().run()
self.assertEqual(1, results.total)
# soft delete the case
with process_pillow_changes('CaseToElasticsearchPillow'):
with process_pillow_changes('DefaultChangeFeedPillow'):
CaseAccessors(self.domain).soft_delete_cases([case_id])
self.elasticsearch.indices.refresh(CASE_INDEX_INFO.index)
# ensure not there anymore
results = CaseES().run()
self.assertEqual(0, results.total)
def test_case_soft_deletion(self):
case_id, case_name = self._create_case_and_sync_to_es()
# verify there
results = CaseES().run()
self.assertEqual(1, results.total)
# soft delete the case
with process_pillow_changes('case-pillow', {'skip_ucr': True}):
with process_pillow_changes('DefaultChangeFeedPillow'):
CaseAccessors(self.domain).soft_delete_cases([case_id])
self.elasticsearch.indices.refresh(CASE_INDEX_INFO.index)
# ensure not there anymore
results = CaseES().run()
self.assertEqual(0, results.total)
def test_global_registry_ucr(self):
"""Globally accessible UCR get's data from all participating domains regardless
of grants.
"""
config = get_sample_registry_data_source(
domain=self.registry_owner_domain,
registry_slug=self.registry_1.slug,
globally_accessible=True)
config.save()
self.addCleanup(config.delete)
adapter = get_indicator_adapter(config)
adapter.build_table()
self.addCleanup(adapter.drop_table)
pillow = get_kafka_ucr_registry_pillow(ucr_configs=[config],
processor_chunk_size=100)
expected = {}
with process_pillow_changes(pillow):
for domain in self.participants:
case = create_and_save_a_case(domain,
uuid.uuid4().hex,
f"name-{domain}",
case_type="ticket",
case_properties={
"category": "bug",
"tags": "easy-win public",
"is_starred": "yes",
"estimate": "2.3",
"priority": "4",
})
expected[case.case_id] = domain
def test_form_soft_deletion(self):
form, metadata = self._create_form_and_sync_to_es()
# verify there
results = FormES().run()
self.assertEqual(1, results.total)
# soft delete the form
with process_pillow_changes('xform-pillow', {'skip_ucr': True}):
with process_pillow_changes('DefaultChangeFeedPillow'):
FormAccessors(self.domain).soft_delete_forms([form.form_id])
self.elasticsearch.indices.refresh(XFORM_INDEX_INFO.index)
# ensure not there anymore
results = FormES().run()
self.assertEqual(0, results.total)
def setUp(self):
super(CasePillowTest, self).setUp()
self.process_case_changes = process_pillow_changes('DefaultChangeFeedPillow')
self.process_case_changes.add_pillow('case-pillow', {'skip_ucr': True})
FormProcessorTestUtils.delete_all_cases()
with trap_extra_setup(ConnectionError):
self.elasticsearch = get_es_new()
initialize_index_and_mapping(self.elasticsearch, CASE_INDEX_INFO)
initialize_index_and_mapping(self.elasticsearch, CASE_SEARCH_INDEX_INFO)
def setUpClass(cls):
super().setUpClass()
cls.process_form_changes = process_pillow_changes(
'DefaultChangeFeedPillow')
cls.process_form_changes.add_pillow(cls.pillow_id, {'skip_ucr': True})
cls.user = CommCareUser.create(cls.domain, cls.username, cls.password)
cls.metadata = TestFormMetadata(
domain=cls.domain,
user_id=cls.user._id,
)
def test_pre_set_defaults(self):
set_default_monthly_consumption_for_domain(self.domain, 5 * 30)
with process_pillow_changes('LedgerToElasticsearchPillow'):
self.report(25, 0)
state = StockState.objects.get(
section_id='stock',
case_id=self.sp.case_id,
product_id=self.products[0]._id,
)
self.assertEqual(5, float(state.get_daily_consumption()))
def setUpClass(cls):
super(SchedulingRecipientTest, cls).setUpClass()
cls.domain_obj = create_domain(cls.domain)
cls.location_types = setup_location_types(cls.domain, ['country', 'state', 'city'])
cls.country_location = make_loc('usa', domain=cls.domain, type='country')
cls.state_location = make_loc('ma', domain=cls.domain, type='state', parent=cls.country_location)
cls.city_location = make_loc('boston', domain=cls.domain, type='city', parent=cls.state_location)
cls.mobile_user = CommCareUser.create(cls.domain, 'mobile', 'abc')
cls.mobile_user.set_location(cls.city_location)
cls.mobile_user2 = CommCareUser.create(cls.domain, 'mobile2', 'abc')
cls.mobile_user2.set_location(cls.state_location)
cls.mobile_user3 = CommCareUser.create(cls.domain, 'mobile3', 'abc')
cls.mobile_user3.user_data['role'] = 'pharmacist'
cls.mobile_user3.save()
cls.mobile_user4 = CommCareUser.create(cls.domain, 'mobile4', 'abc')
cls.mobile_user4.user_data['role'] = 'nurse'
cls.mobile_user4.save()
cls.mobile_user5 = CommCareUser.create(cls.domain, 'mobile5', 'abc')
cls.mobile_user5.user_data['role'] = ['nurse', 'pharmacist']
cls.mobile_user5.save()
cls.web_user = WebUser.create(cls.domain, 'web', 'abc')
cls.web_user2 = WebUser.create(cls.domain, 'web2', 'abc')
cls.web_user2.user_data['role'] = 'nurse'
cls.web_user2.save()
cls.group = Group(domain=cls.domain, users=[cls.mobile_user.get_id])
cls.group.save()
cls.group2 = Group(
domain=cls.domain,
users=[
cls.mobile_user.get_id,
cls.mobile_user3.get_id,
cls.mobile_user4.get_id,
cls.mobile_user5.get_id,
]
)
cls.group2.save()
cls.case_group = CommCareCaseGroup(domain=cls.domain)
cls.case_group.save()
cls.process_pillow_changes = process_pillow_changes('DefaultChangeFeedPillow')
cls.process_pillow_changes.add_pillow(get_case_messaging_sync_pillow())
def test_none_with_no_defaults(self):
# need to submit something to have a state initialized
with process_pillow_changes('LedgerToElasticsearchPillow'):
self.report(25, 0)
state = StockState.objects.get(
section_id='stock',
case_id=self.sp.case_id,
product_id=self.products[0]._id,
)
self.assertEqual(None, state.get_daily_consumption())
def test_stock_state(self):
with process_pillow_changes('LedgerToElasticsearchPillow'):
self.report(25, 5)
self.report(10, 0)
state = StockState.objects.get(
section_id='stock',
case_id=self.sp.case_id,
product_id=self.products[0]._id,
)
self.assertEqual(10, state.stock_on_hand)
self.assertEqual(3.0, state.get_daily_consumption())
def setUpClass(cls):
super(KafkaPublishingTest, cls).setUpClass()
cls.processor = TestProcessor()
cls.form_pillow = ConstructedPillow(
name='test-kafka-form-feed',
checkpoint=None,
change_feed=KafkaChangeFeed(topics=[topics.FORM_SQL],
client_id='test-kafka-form-feed'),
processor=cls.processor)
cls.case_pillow = ConstructedPillow(
name='test-kafka-case-feed',
checkpoint=None,
change_feed=KafkaChangeFeed(topics=[topics.CASE_SQL],
client_id='test-kafka-case-feed'),
processor=cls.processor)
cls.process_form_changes = process_pillow_changes(
'DefaultChangeFeedPillow')
cls.process_form_changes.add_pillow(cls.form_pillow)
cls.process_case_changes = process_pillow_changes(
'DefaultChangeFeedPillow')
cls.process_case_changes.add_pillow(cls.case_pillow)
def test_registry_ucr(self):
expected = {}
with process_pillow_changes(self.pillow):
for domain in self.participants:
case = create_and_save_a_case(domain,
uuid.uuid4().hex,
f"name-{domain}",
case_type="ticket",
case_properties={
"category": "bug",
"tags": "easy-win public",
"is_starred": "yes",
"estimate": "2.3",
"priority": "4",
})
if domain != self.participator_3:
# 3 does not grant either of the others access
expected[case.case_id] = domain
def setUpClass(cls):
super(StockStateTest, cls).setUpClass()
cls.domain_obj = util.bootstrap_domain(cls.domain)
util.bootstrap_location_types(cls.domain)
util.bootstrap_products(cls.domain)
cls.ct_settings = CommtrackConfig.for_domain(cls.domain)
cls.ct_settings.use_auto_consumption = True
cls.ct_settings.consumption_config = ConsumptionConfig(
min_transactions=0,
min_window=0,
optimal_window=60,
min_periods=0,
)
cls.ct_settings.save()
cls.loc = util.make_loc('loc1', domain=cls.domain)
cls.sp = cls.loc.linked_supply_point()
cls.products = sorted(Product.by_domain(cls.domain), key=lambda p: p._id)
cls.process_ledger_changes = process_pillow_changes('LedgerToElasticsearchPillow')
def test_case_is_published(self):
with process_pillow_changes(self.case_pillow):
with process_pillow_changes('DefaultChangeFeedPillow'):
case = create_and_save_a_case(self.domain,
case_id=uuid.uuid4().hex,
case_name='test case')
self.assertEqual(1, len(self.processor.changes_seen))
change_meta = self.processor.changes_seen[0].metadata
self.assertEqual(case.case_id, change_meta.document_id)
self.assertEqual(self.domain, change_meta.domain)
def test_case_deletions(self):
case = create_and_save_a_case(self.domain,
case_id=uuid.uuid4().hex,
case_name='test case')
with process_pillow_changes(self.case_pillow):
with process_pillow_changes('DefaultChangeFeedPillow'):
case.soft_delete()
self.assertEqual(1, len(self.processor.changes_seen))
change_meta = self.processor.changes_seen[0].metadata
self.assertEqual(case.case_id, change_meta.document_id)
self.assertTrue(change_meta.is_deletion)
@use_sql_backend
class KafkaPublishingTestSQL(KafkaPublishingTest):
pass
def setUpClass(cls):
super(CommTrackSubmissionTest, cls).setUpClass()
cls.process_legder_changes = process_pillow_changes(
'LedgerToElasticsearchPillow')
def setUpClass(cls):
super(CasePillowTest, cls).setUpClass()
cls.process_case_changes = process_pillow_changes(
'DefaultChangeFeedPillow')
cls.process_case_changes.add_pillow('case-pillow', {'skip_ucr': True})
def setUpClass(cls):
super(XFormPillowTest, cls).setUpClass()
cls.process_form_changes = process_pillow_changes('DefaultChangeFeedPillow')
cls.process_form_changes.add_pillow('xform-pillow', {'skip_ucr': True})
def setUpClass(cls):
super(SchedulingRecipientTest, cls).setUpClass()
cls.domain_obj = create_domain(cls.domain)
cls.location_types = setup_location_types(cls.domain,
['country', 'state', 'city'])
cls.country_location = make_loc('usa',
domain=cls.domain,
type='country')
cls.state_location = make_loc('ma',
domain=cls.domain,
type='state',
parent=cls.country_location)
cls.city_location = make_loc('boston',
domain=cls.domain,
type='city',
parent=cls.state_location)
cls.mobile_user = CommCareUser.create(cls.domain, 'mobile', 'abc',
None, None)
cls.mobile_user.set_location(cls.city_location)
cls.mobile_user2 = CommCareUser.create(cls.domain, 'mobile2', 'abc',
None, None)
cls.mobile_user2.set_location(cls.state_location)
cls.mobile_user3 = CommCareUser.create(cls.domain,
'mobile3',
'abc',
None,
None,
metadata={
'role': 'pharmacist',
})
cls.mobile_user3.save()
cls.mobile_user4 = CommCareUser.create(cls.domain,
'mobile4',
'abc',
None,
None,
metadata={
'role': 'nurse',
})
cls.mobile_user4.save()
cls.mobile_user5 = CommCareUser.create(cls.domain,
'mobile5',
'abc',
None,
None,
metadata={
'role':
['nurse', 'pharmacist'],
})
cls.mobile_user5.save()
full_username = normalize_username('mobile', cls.domain)
cls.full_mobile_user = CommCareUser.create(cls.domain, full_username,
'abc', None, None)
cls.definition = CustomDataFieldsDefinition(
domain=cls.domain, field_type=UserFieldsView.field_type)
cls.definition.save()
cls.definition.set_fields([
Field(
slug='role',
label='Role',
),
])
cls.definition.save()
cls.profile = CustomDataFieldsProfile(
name='nurse_profile',
fields={'role': ['nurse']},
definition=cls.definition,
)
cls.profile.save()
cls.mobile_user6 = CommCareUser.create(cls.domain,
'mobile6',
'abc',
None,
None,
metadata={
PROFILE_SLUG:
cls.profile.id,
})
cls.mobile_user5.save()
cls.web_user = WebUser.create(cls.domain, 'web', 'abc', None, None)
cls.web_user2 = WebUser.create(cls.domain,
'web2',
'abc',
None,
None,
metadata={
'role': 'nurse',
})
cls.web_user2.save()
cls.group = Group(domain=cls.domain, users=[cls.mobile_user.get_id])
cls.group.save()
cls.group2 = Group(domain=cls.domain,
users=[
cls.mobile_user.get_id,
cls.mobile_user3.get_id,
cls.mobile_user4.get_id,
cls.mobile_user5.get_id,
cls.mobile_user6.get_id,
])
cls.group2.save()
cls.case_group = CommCareCaseGroup(domain=cls.domain)
cls.case_group.save()
cls.process_pillow_changes = process_pillow_changes(
'DefaultChangeFeedPillow')
cls.process_pillow_changes.add_pillow(get_case_messaging_sync_pillow())
