def handle(self, domains, file_name, **options):
blob_db = get_blob_db()
with open(file_name, 'w', encoding='utf-8') as csv_file:
field_names = ['domain', 'archived', 'form_id', 'received_on']
csv_writer = csv.DictWriter(csv_file, field_names)
csv_writer.writeheader()
for domain in domains:
self.stdout.write("Handling domain %s" % domain)
form_db = FormAccessors(domain)
form_ids = form_db.get_all_form_ids_in_domain()
form_ids.extend(
form_db.get_all_form_ids_in_domain('XFormArchived'))
for form in with_progress_bar(form_db.iter_forms(form_ids),
len(form_ids)):
if isinstance(form, CouchForm):
meta = form.blobs.get(ATTACHMENT_NAME)
if not meta or not blob_db.exists(key=meta.key):
self.write_row(csv_writer, domain,
form.is_archived, form.received_on,
form.form_id)
elif isinstance(form, XFormInstanceSQL):
meta = form.get_attachment_meta(ATTACHMENT_NAME)
if not meta or not blob_db.exists(key=meta.key):
self.write_row(csv_writer, domain,
form.is_archived, form.received_on,
form.form_id)
else:
raise Exception("not sure how we got here")
def handle(self, domain, folder_path, **options):
if os.path.exists(folder_path):
if not os.path.isdir(folder_path):
raise CommandError(
'Folder path must be the path to a directory')
else:
os.mkdir(folder_path)
form_accessors = FormAccessors(domain)
form_ids = form_accessors.get_all_form_ids_in_domain()
for form in form_accessors.iter_forms(form_ids):
form_path = os.path.join(folder_path, form.form_id)
if not os.path.exists(form_path):
os.mkdir(form_path)
form_meta = FormMetadata(
user_id=form.user_id,
received_on=form.received_on,
app_id=form.app_id,
build_id=form.build_id,
attachments=list(form.attachments.keys()),
auth_context=form.auth_context,
)
with open(os.path.join(form_path, 'metadata.json'), 'w') as meta:
json.dump(form_meta.to_json(), meta)
xml = form.get_xml()
with open(os.path.join(form_path, 'form.xml'), 'w') as f:
f.write(xml)
for name, meta in form.attachments.items():
with open(os.path.join(form_path, name), 'w') as f:
f.write(form.get_attachment(name))
def handle(self, domain, folder_path, **options):
if os.path.exists(folder_path):
if not os.path.isdir(folder_path):
raise CommandError('Folder path must be the path to a directory')
else:
os.mkdir(folder_path)
form_accessors = FormAccessors(domain)
form_ids = form_accessors.get_all_form_ids_in_domain()
for form in form_accessors.iter_forms(form_ids):
form_path = os.path.join(folder_path, form.form_id)
if not os.path.exists(form_path):
os.mkdir(form_path)
form_meta = FormMetadata(
user_id=form.user_id,
received_on=form.received_on,
app_id=form.app_id,
build_id=form.build_id,
attachments=form.attachments.keys(),
auth_context=form.auth_context,
)
with open(os.path.join(form_path, 'metadata.json'), 'w') as meta:
json.dump(form_meta.to_json(), meta)
xml = form.get_xml()
with open(os.path.join(form_path, 'form.xml'), 'w') as f:
f.write(xml)
for name, meta in form.attachments.items():
with open(os.path.join(form_path, name), 'w') as f:
f.write(form.get_attachment(name))
def _delete_all_forms(domain_name):
logger.info('Deleting forms...')
form_accessor = FormAccessors(domain_name)
form_ids = form_accessor.get_all_form_ids_in_domain()
for form_id_chunk in chunked(
with_progress_bar(form_ids, stream=silence_during_tests()), 500):
form_accessor.soft_delete_forms(list(form_id_chunk))
logger.info('Deleting forms complete.')
def _delete_all_forms(domain_name):
logger.info('Deleting forms...')
form_accessor = FormAccessors(domain_name)
form_ids = list(itertools.chain(*[
form_accessor.get_all_form_ids_in_domain(doc_type=doc_type)
for doc_type in doc_type_to_state
]))
for form_id_chunk in chunked(with_progress_bar(form_ids, stream=silence_during_tests()), 500):
form_accessor.soft_delete_forms(list(form_id_chunk))
logger.info('Deleting forms complete.')
def _delete_all_forms(domain_name):
logger.info('Deleting forms...')
form_accessor = FormAccessors(domain_name)
form_ids = list(itertools.chain(*[
form_accessor.get_all_form_ids_in_domain(doc_type=doc_type)
for doc_type in doc_type_to_state
]))
for form_id_chunk in chunked(with_progress_bar(form_ids, stream=silence_during_tests()), 500):
form_accessor.soft_delete_forms(list(form_id_chunk))
logger.info('Deleting forms complete.')
def handle(self, domains, file_name, **options):
blob_db = get_blob_db()
with open(file_name, 'w', encoding='utf-8') as csv_file:
field_names = ['domain', 'archived', 'form_id', 'received_on']
csv_writer = csv.DictWriter(csv_file, field_names)
csv_writer.writeheader()
for domain in domains:
self.stdout.write("Handling domain %s" % domain)
form_db = FormAccessors(domain)
form_ids = form_db.get_all_form_ids_in_domain()
form_ids.extend(form_db.get_all_form_ids_in_domain('XFormArchived'))
for form in with_progress_bar(form_db.iter_forms(form_ids), len(form_ids)):
if isinstance(form, CouchForm):
meta = form.blobs.get(ATTACHMENT_NAME)
if not meta or not blob_db.exists(key=meta.key):
self.write_row(csv_writer, domain, form.is_archived, form.received_on, form.form_id)
elif isinstance(form, XFormInstanceSQL):
meta = form.get_attachment_meta(ATTACHMENT_NAME)
if not meta or not blob_db.exists(key=meta.key):
self.write_row(csv_writer, domain, form.is_archived, form.received_on, form.form_id)
else:
raise Exception("not sure how we got here")
class ReadonlyFormDocumentStore(ReadOnlyDocumentStore):
def __init__(self, domain):
self.domain = domain
self.form_accessors = FormAccessors(domain=domain)
def get_document(self, doc_id):
try:
return add_couch_properties_to_sql_form_json(self.form_accessors.get_form(doc_id).to_json())
except (XFormNotFound, BlobError) as e:
raise DocumentNotFoundError(e)
def iter_document_ids(self, last_id=None):
# todo: support last_id
return iter(self.form_accessors.get_all_form_ids_in_domain())
def iter_documents(self, ids):
for wrapped_form in self.form_accessors.iter_forms(ids):
yield add_couch_properties_to_sql_form_json(wrapped_form.to_json())
class ReadonlyFormDocumentStore(ReadOnlyDocumentStore):
def __init__(self, domain):
self.domain = domain
self.form_accessors = FormAccessors(domain=domain)
def get_document(self, doc_id):
try:
return self.form_accessors.get_form(doc_id).to_json(include_attachments=True)
except (XFormNotFound, BlobError) as e:
raise DocumentNotFoundError(e)
def iter_document_ids(self, last_id=None):
# todo: support last_id
return iter(self.form_accessors.get_all_form_ids_in_domain())
def iter_documents(self, ids):
for wrapped_form in self.form_accessors.iter_forms(ids):
yield wrapped_form.to_json()
class KafkaPublishingTest(TestCase):
domain = 'kafka-publishing-test'
def setUp(self):
super(KafkaPublishingTest, self).setUp()
FormProcessorTestUtils.delete_all_cases_forms_ledgers()
self.form_accessors = FormAccessors(domain=self.domain)
self.processor = TestProcessor()
self.form_pillow = ConstructedPillow(
name='test-kafka-form-feed',
checkpoint=None,
change_feed=KafkaChangeFeed(topics=[topics.FORM, topics.FORM_SQL],
group_id='test-kafka-form-feed'),
processor=self.processor)
self.case_pillow = ConstructedPillow(
name='test-kafka-case-feed',
checkpoint=None,
change_feed=KafkaChangeFeed(topics=[topics.CASE, topics.CASE_SQL],
group_id='test-kafka-case-feed'),
processor=self.processor)
self.ledger_pillow = ConstructedPillow(
name='test-kafka-ledger-feed',
checkpoint=None,
change_feed=KafkaChangeFeed(topics=[topics.LEDGER],
group_id='test-kafka-ledger-feed'),
processor=self.processor)
def tearDown(self):
FormProcessorTestUtils.delete_all_cases_forms_ledgers()
def test_form_is_published(self):
with process_pillow_changes(self.form_pillow):
with process_pillow_changes('DefaultChangeFeedPillow'):
form = create_and_save_a_form(self.domain)
self.assertEqual(1, len(self.processor.changes_seen))
change_meta = self.processor.changes_seen[0].metadata
self.assertEqual(form.form_id, change_meta.document_id)
self.assertEqual(self.domain, change_meta.domain)
def test_duplicate_form_published(self):
form_id = uuid.uuid4().hex
form_xml = get_simple_form_xml(form_id)
orig_form = submit_form_locally(form_xml, domain=self.domain).xform
self.assertEqual(form_id, orig_form.form_id)
self.assertEqual(1,
len(self.form_accessors.get_all_form_ids_in_domain()))
with process_pillow_changes(self.form_pillow):
with process_pillow_changes('DefaultChangeFeedPillow'):
# post an exact duplicate
dupe_form = submit_form_locally(form_xml,
domain=self.domain).xform
self.assertTrue(dupe_form.is_duplicate)
self.assertNotEqual(form_id, dupe_form.form_id)
if should_use_sql_backend(self.domain):
self.assertEqual(form_id, dupe_form.orig_id)
# make sure changes made it to kafka
dupe_form_meta = self.processor.changes_seen[0].metadata
self.assertEqual(dupe_form.form_id, dupe_form_meta.document_id)
self.assertEqual(dupe_form.domain, dupe_form.domain)
if should_use_sql_backend(self.domain):
# sql domains also republish the original form to ensure that if the server crashed
# in the processing of the form the first time that it is still sent to kafka
orig_form_meta = self.processor.changes_seen[1].metadata
self.assertEqual(orig_form.form_id, orig_form_meta.document_id)
self.assertEqual(self.domain, orig_form_meta.domain)
self.assertEqual(dupe_form.domain, dupe_form.domain)
def test_form_soft_deletions(self):
form = create_and_save_a_form(self.domain)
with process_pillow_changes(self.form_pillow):
with process_pillow_changes('DefaultChangeFeedPillow'):
form.soft_delete()
self.assertEqual(1, len(self.processor.changes_seen))
change_meta = self.processor.changes_seen[0].metadata
self.assertEqual(form.form_id, change_meta.document_id)
self.assertTrue(change_meta.is_deletion)
def test_case_is_published(self):
with process_pillow_changes(self.case_pillow):
with process_pillow_changes('DefaultChangeFeedPillow'):
case = create_and_save_a_case(self.domain,
case_id=uuid.uuid4().hex,
case_name='test case')
self.assertEqual(1, len(self.processor.changes_seen))
change_meta = self.processor.changes_seen[0].metadata
self.assertEqual(case.case_id, change_meta.document_id)
self.assertEqual(self.domain, change_meta.domain)
def form_ids_in_domain(domain):
form_accessor = FormAccessors(domain)
for doc_type in DOC_TYPES:
form_ids = form_accessor.get_all_form_ids_in_domain(doc_type=doc_type)
for form_id in form_ids:
yield form_id
class KafkaPublishingTest(OverridableSettingsTestMixin, TestCase):
domain = 'kafka-publishing-test'
def setUp(self):
FormProcessorTestUtils.delete_all_sql_forms()
FormProcessorTestUtils.delete_all_sql_cases()
self.form_accessors = FormAccessors(domain=self.domain)
def test_form_is_published(self):
kafka_consumer = get_test_kafka_consumer(topics.FORM_SQL)
form = create_and_save_a_form(self.domain)
message = kafka_consumer.next()
change_meta = change_meta_from_kafka_message(message.value)
self.assertEqual(form.form_id, change_meta.document_id)
self.assertEqual(self.domain, change_meta.domain)
def test_case_is_published(self):
kafka_consumer = get_test_kafka_consumer(topics.CASE_SQL)
case = create_and_save_a_case(self.domain, case_id=uuid.uuid4().hex, case_name='test case')
change_meta = change_meta_from_kafka_message(kafka_consumer.next().value)
self.assertEqual(case.case_id, change_meta.document_id)
self.assertEqual(self.domain, change_meta.domain)
def test_duplicate_form_published(self):
form_id = uuid.uuid4().hex
form_xml = get_simple_form_xml(form_id)
orig_form = submit_form_locally(form_xml, domain=self.domain)[1]
self.assertEqual(form_id, orig_form.form_id)
self.assertEqual(1, len(self.form_accessors.get_all_form_ids_in_domain()))
form_consumer = get_test_kafka_consumer(topics.FORM_SQL)
# post an exact duplicate
dupe_form = submit_form_locally(form_xml, domain=self.domain)[1]
self.assertTrue(dupe_form.is_duplicate)
self.assertNotEqual(form_id, dupe_form.form_id)
self.assertEqual(form_id, dupe_form.orig_id)
# make sure changes made it to kafka
# first the dupe
dupe_form_meta = change_meta_from_kafka_message(form_consumer.next().value)
self.assertEqual(dupe_form.form_id, dupe_form_meta.document_id)
# then the original form
orig_form_meta = change_meta_from_kafka_message(form_consumer.next().value)
self.assertEqual(orig_form.form_id, orig_form_meta.document_id)
self.assertEqual(self.domain, orig_form_meta.domain)
def test_duplicate_case_published(self):
case_id = uuid.uuid4().hex
form_xml = get_simple_form_xml(uuid.uuid4().hex, case_id)
submit_form_locally(form_xml, domain=self.domain)[1]
self.assertEqual(1, len(CaseAccessors(self.domain).get_case_ids_in_domain()))
case_consumer = get_test_kafka_consumer(topics.CASE_SQL)
dupe_form = submit_form_locally(form_xml, domain=self.domain)[1]
self.assertTrue(dupe_form.is_duplicate)
# check the case was republished
case_meta = change_meta_from_kafka_message(case_consumer.next().value)
self.assertEqual(case_id, case_meta.document_id)
self.assertEqual(self.domain, case_meta.domain)
def test_duplicate_ledger_published(self):
# setup products and case
product_a = make_product(self.domain, 'A Product', 'prodcode_a')
product_b = make_product(self.domain, 'B Product', 'prodcode_b')
case_id = uuid.uuid4().hex
form_xml = get_simple_form_xml(uuid.uuid4().hex, case_id)
submit_form_locally(form_xml, domain=self.domain)[1]
# submit ledger data
balances = (
(product_a._id, 100),
(product_b._id, 50),
)
ledger_blocks = [
get_single_balance_block(case_id, prod_id, balance)
for prod_id, balance in balances
]
form = submit_case_blocks(ledger_blocks, self.domain)
# submit duplicate
ledger_consumer = get_test_kafka_consumer(topics.LEDGER)
dupe_form = submit_form_locally(form.get_xml(), domain=self.domain)[1]
self.assertTrue(dupe_form.is_duplicate)
# confirm republished
ledger_meta_a = change_meta_from_kafka_message(ledger_consumer.next().value)
ledger_meta_b = change_meta_from_kafka_message(ledger_consumer.next().value)
format_id = lambda product_id: '{}/{}/{}'.format(case_id, 'stock', product_id)
expected_ids = {format_id(product_a._id), format_id(product_b._id)}
for meta in [ledger_meta_a, ledger_meta_b]:
self.assertTrue(meta.document_id in expected_ids)
expected_ids.remove(meta.document_id)
self.assertEqual(self.domain, meta.domain)
# cleanup
product_a.delete()
product_b.delete()
class ReprocessSubmissionStubTests(TestCase):
@classmethod
def setUpClass(cls):
super(ReprocessSubmissionStubTests, cls).setUpClass()
cls.domain = uuid.uuid4().hex
cls.product = SQLProduct.objects.create(domain=cls.domain,
product_id='product1',
name='product1')
@classmethod
def tearDownClass(cls):
cls.product.delete()
super(ReprocessSubmissionStubTests, cls).tearDownClass()
def setUp(self):
super(ReprocessSubmissionStubTests, self).setUp()
self.factory = CaseFactory(domain=self.domain)
self.formdb = FormAccessors(self.domain)
self.casedb = CaseAccessors(self.domain)
self.ledgerdb = LedgerAccessors(self.domain)
def tearDown(self):
FormProcessorTestUtils.delete_all_cases_forms_ledgers(self.domain)
super(ReprocessSubmissionStubTests, self).tearDown()
def test_reprocess_unfinished_submission_case_create(self):
case_id = uuid.uuid4().hex
with _patch_save_to_raise_error(self):
self.factory.create_or_update_cases([
CaseStructure(case_id=case_id,
attrs={
'case_type': 'parent',
'create': True
})
])
stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain,
saved=False).all()
self.assertEqual(1, len(stubs))
# form that was saved before case error raised
normal_form_ids = self.formdb.get_all_form_ids_in_domain(
'XFormInstance')
self.assertEqual(0, len(normal_form_ids))
# shows error form (duplicate of form that was saved before case error)
# this is saved becuase the saving was assumed to be atomic so if there was any error it's assumed
# the form didn't get saved
# we don't really care about this form in this test
error_forms = self.formdb.get_forms_by_type('XFormError', 10)
self.assertEqual(1, len(error_forms))
self.assertIsNone(error_forms[0].orig_id)
self.assertEqual(error_forms[0].form_id, stubs[0].xform_id)
self.assertEqual(0,
len(self.casedb.get_case_ids_in_domain(self.domain)))
result = reprocess_unfinished_stub(stubs[0])
self.assertEqual(1, len(result.cases))
case_ids = self.casedb.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
self.assertEqual(case_id, case_ids[0])
with self.assertRaises(UnfinishedSubmissionStub.DoesNotExist):
UnfinishedSubmissionStub.objects.get(pk=stubs[0].pk)
def test_reprocess_unfinished_submission_case_update(self):
case_id = uuid.uuid4().hex
form_ids = []
form_ids.append(
submit_case_blocks(
CaseBlock(case_id=case_id, create=True,
case_type='box').as_string(),
self.domain)[0].form_id)
with _patch_save_to_raise_error(self):
submit_case_blocks(
CaseBlock(case_id=case_id, update={
'prop': 'a'
}).as_string(), self.domain)
stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain,
saved=False).all()
self.assertEqual(1, len(stubs))
form_ids.append(stubs[0].xform_id)
# submit second form with case update
form_ids.append(
submit_case_blocks(
CaseBlock(case_id=case_id, update={
'prop': 'b'
}).as_string(), self.domain)[0].form_id)
case = self.casedb.get_case(case_id)
self.assertEqual(2, len(case.xform_ids))
self.assertEqual('b', case.get_case_property('prop'))
result = reprocess_unfinished_stub(stubs[0])
self.assertEqual(1, len(result.cases))
self.assertEqual(0, len(result.ledgers))
case = self.casedb.get_case(case_id)
self.assertEqual('b', case.get_case_property(
'prop')) # should be property value from most recent form
self.assertEqual(3, len(case.xform_ids))
self.assertEqual(form_ids, case.xform_ids)
with self.assertRaises(UnfinishedSubmissionStub.DoesNotExist):
UnfinishedSubmissionStub.objects.get(pk=stubs[0].pk)
def test_reprocess_unfinished_submission_ledger_create(self):
from corehq.apps.commtrack.tests.util import get_single_balance_block
case_id = uuid.uuid4().hex
self.factory.create_or_update_cases([
CaseStructure(case_id=case_id,
attrs={
'case_type': 'parent',
'create': True
})
])
with _patch_save_to_raise_error(self):
submit_case_blocks(
get_single_balance_block(case_id, 'product1', 100),
self.domain)
stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain,
saved=False).all()
self.assertEqual(1, len(stubs))
ledgers = self.ledgerdb.get_ledger_values_for_case(case_id)
self.assertEqual(0, len(ledgers))
case = self.casedb.get_case(case_id)
self.assertEqual(1, len(case.xform_ids))
ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case(
case_id)
self.assertEqual(0, len(ledger_transactions))
result = reprocess_unfinished_stub(stubs[0])
self.assertEqual(1, len(result.cases))
self.assertEqual(1, len(result.ledgers))
ledgers = self.ledgerdb.get_ledger_values_for_case(case_id)
self.assertEqual(1, len(ledgers))
ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case(
case_id)
self.assertEqual(1, len(ledger_transactions))
# case still only has 2 transactions
case = self.casedb.get_case(case_id)
self.assertEqual(2, len(case.xform_ids))
if should_use_sql_backend(self.domain):
self.assertTrue(case.actions[1].is_ledger_transaction)
def test_reprocess_unfinished_submission_ledger_rebuild(self):
from corehq.apps.commtrack.tests.util import get_single_balance_block
case_id = uuid.uuid4().hex
form_ids = []
form_ids.append(
submit_case_blocks([
CaseBlock(case_id=case_id, create=True,
case_type='shop').as_string(),
get_single_balance_block(case_id, 'product1', 100),
], self.domain)[0].form_id)
with _patch_save_to_raise_error(self):
submit_case_blocks(
get_single_balance_block(case_id, 'product1', 50), self.domain)
stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain,
saved=False).all()
self.assertEqual(1, len(stubs))
form_ids.append(stubs[0].xform_id)
# submit another form afterwards
form_ids.append(
submit_case_blocks(
get_single_balance_block(case_id, 'product1', 25),
self.domain)[0].form_id)
ledgers = self.ledgerdb.get_ledger_values_for_case(case_id)
self.assertEqual(1, len(ledgers))
self.assertEqual(25, ledgers[0].balance)
ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case(
case_id)
if should_use_sql_backend(self.domain):
self.assertEqual(2, len(ledger_transactions))
else:
# includes extra consumption transaction
self.assertEqual(3, len(ledger_transactions))
# should rebuild ledger transactions
result = reprocess_unfinished_stub(stubs[0])
self.assertEqual(1, len(result.cases))
self.assertEqual(1, len(result.ledgers))
ledgers = self.ledgerdb.get_ledger_values_for_case(case_id)
self.assertEqual(1, len(ledgers)) # still only 1
self.assertEqual(25, ledgers[0].balance)
ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case(
case_id)
if should_use_sql_backend(self.domain):
self.assertEqual(3, len(ledger_transactions))
# make sure transactions are in correct order
self.assertEqual(form_ids,
[trans.form_id for trans in ledger_transactions])
self.assertEqual(100, ledger_transactions[0].updated_balance)
self.assertEqual(100, ledger_transactions[0].delta)
self.assertEqual(50, ledger_transactions[1].updated_balance)
self.assertEqual(-50, ledger_transactions[1].delta)
self.assertEqual(25, ledger_transactions[2].updated_balance)
self.assertEqual(-25, ledger_transactions[2].delta)
else:
self.assertEqual(3, len(ledger_transactions))
self.assertEqual(
form_ids,
[trans.report.form_id for trans in ledger_transactions])
self.assertEqual(100, ledger_transactions[0].stock_on_hand)
self.assertEqual(50, ledger_transactions[1].stock_on_hand)
self.assertEqual(25, ledger_transactions[2].stock_on_hand)
def test_fire_signals(self):
from corehq.apps.receiverwrapper.tests.test_submit_errors import failing_signal_handler
case_id = uuid.uuid4().hex
form_id = uuid.uuid4().hex
with failing_signal_handler('signal death'):
submit_case_blocks(CaseBlock(case_id=case_id,
create=True,
case_type='box').as_string(),
self.domain,
form_id=form_id)
form = self.formdb.get_form(form_id)
with catch_signal(
successful_form_received) as form_handler, catch_signal(
case_post_save) as case_handler:
submit_form_locally(
instance=form.get_xml(),
domain=self.domain,
)
case = self.casedb.get_case(case_id)
if should_use_sql_backend(self.domain):
self.assertEqual(form, form_handler.call_args[1]['xform'])
self.assertEqual(case, case_handler.call_args[1]['case'])
else:
signal_form = form_handler.call_args[1]['xform']
self.assertEqual(form.form_id, signal_form.form_id)
self.assertEqual(form.get_rev, signal_form.get_rev)
signal_case = case_handler.call_args[1]['case']
self.assertEqual(case.case_id, signal_case.case_id)
self.assertEqual(case.get_rev, signal_case.get_rev)
class KafkaPublishingTest(TestCase):
domain = 'kafka-publishing-test'
def setUp(self):
super(KafkaPublishingTest, self).setUp()
FormProcessorTestUtils.delete_all_cases_forms_ledgers()
self.form_accessors = FormAccessors(domain=self.domain)
self.processor = TestProcessor()
self.form_pillow = ConstructedPillow(
name='test-kafka-form-feed',
checkpoint=None,
change_feed=KafkaChangeFeed(topics=[topics.FORM, topics.FORM_SQL], group_id='test-kafka-form-feed'),
processor=self.processor
)
self.case_pillow = ConstructedPillow(
name='test-kafka-case-feed',
checkpoint=None,
change_feed=KafkaChangeFeed(topics=[topics.CASE, topics.CASE_SQL], group_id='test-kafka-case-feed'),
processor=self.processor
)
self.ledger_pillow = ConstructedPillow(
name='test-kafka-ledger-feed',
checkpoint=None,
change_feed=KafkaChangeFeed(topics=[topics.LEDGER], group_id='test-kafka-ledger-feed'),
processor=self.processor
)
def tearDown(self):
FormProcessorTestUtils.delete_all_cases_forms_ledgers()
@run_with_all_backends
def test_form_is_published(self):
with process_kafka_changes(self.form_pillow):
with process_couch_changes('DefaultChangeFeedPillow'):
form = create_and_save_a_form(self.domain)
self.assertEqual(1, len(self.processor.changes_seen))
change_meta = self.processor.changes_seen[0].metadata
self.assertEqual(form.form_id, change_meta.document_id)
self.assertEqual(self.domain, change_meta.domain)
@run_with_all_backends
def test_duplicate_form_published(self):
form_id = uuid.uuid4().hex
form_xml = get_simple_form_xml(form_id)
orig_form = submit_form_locally(form_xml, domain=self.domain)[1]
self.assertEqual(form_id, orig_form.form_id)
self.assertEqual(1, len(self.form_accessors.get_all_form_ids_in_domain()))
with process_kafka_changes(self.form_pillow):
with process_couch_changes('DefaultChangeFeedPillow'):
# post an exact duplicate
dupe_form = submit_form_locally(form_xml, domain=self.domain)[1]
self.assertTrue(dupe_form.is_duplicate)
self.assertNotEqual(form_id, dupe_form.form_id)
if should_use_sql_backend(self.domain):
self.assertEqual(form_id, dupe_form.orig_id)
# make sure changes made it to kafka
dupe_form_meta = self.processor.changes_seen[0].metadata
self.assertEqual(dupe_form.form_id, dupe_form_meta.document_id)
self.assertEqual(dupe_form.domain, dupe_form.domain)
if should_use_sql_backend(self.domain):
# sql domains also republish the original form to ensure that if the server crashed
# in the processing of the form the first time that it is still sent to kafka
orig_form_meta = self.processor.changes_seen[1].metadata
self.assertEqual(orig_form.form_id, orig_form_meta.document_id)
self.assertEqual(self.domain, orig_form_meta.domain)
self.assertEqual(dupe_form.domain, dupe_form.domain)
@run_with_all_backends
def test_form_soft_deletions(self):
form = create_and_save_a_form(self.domain)
with process_kafka_changes(self.form_pillow):
with process_couch_changes('DefaultChangeFeedPillow'):
form.soft_delete()
self.assertEqual(1, len(self.processor.changes_seen))
change_meta = self.processor.changes_seen[0].metadata
self.assertEqual(form.form_id, change_meta.document_id)
self.assertTrue(change_meta.is_deletion)
@run_with_all_backends
def test_case_is_published(self):
with process_kafka_changes(self.case_pillow):
with process_couch_changes('DefaultChangeFeedPillow'):
case = create_and_save_a_case(self.domain, case_id=uuid.uuid4().hex, case_name='test case')
self.assertEqual(1, len(self.processor.changes_seen))
change_meta = self.processor.changes_seen[0].metadata
self.assertEqual(case.case_id, change_meta.document_id)
self.assertEqual(self.domain, change_meta.domain)
@run_with_all_backends
def test_case_deletions(self):
case = create_and_save_a_case(self.domain, case_id=uuid.uuid4().hex, case_name='test case')
with process_kafka_changes(self.case_pillow):
with process_couch_changes('DefaultChangeFeedPillow'):
case.soft_delete()
self.assertEqual(1, len(self.processor.changes_seen))
change_meta = self.processor.changes_seen[0].metadata
self.assertEqual(case.case_id, change_meta.document_id)
self.assertTrue(change_meta.is_deletion)
def test_duplicate_case_published(self):
# this test only runs on sql because it's handling a sql-specific edge case where duplicate
# form submissions should cause cases to be resubmitted.
# see: http://manage.dimagi.com/default.asp?228463 for context
case_id = uuid.uuid4().hex
form_xml = get_simple_form_xml(uuid.uuid4().hex, case_id)
submit_form_locally(form_xml, domain=self.domain)[1]
self.assertEqual(1, len(CaseAccessors(self.domain).get_case_ids_in_domain()))
with process_kafka_changes(self.case_pillow):
with process_couch_changes('DefaultChangeFeedPillow'):
dupe_form = submit_form_locally(form_xml, domain=self.domain)[1]
self.assertTrue(dupe_form.is_duplicate)
# check the case was republished
self.assertEqual(1, len(self.processor.changes_seen))
case_meta = self.processor.changes_seen[0].metadata
self.assertEqual(case_id, case_meta.document_id)
self.assertEqual(self.domain, case_meta.domain)
def test_duplicate_ledger_published(self):
# this test also only runs on the sql backend for reasons described in test_duplicate_case_published
# setup products and case
product_a = make_product(self.domain, 'A Product', 'prodcode_a')
product_b = make_product(self.domain, 'B Product', 'prodcode_b')
case_id = uuid.uuid4().hex
form_xml = get_simple_form_xml(uuid.uuid4().hex, case_id)
submit_form_locally(form_xml, domain=self.domain)[1]
# submit ledger data
balances = (
(product_a._id, 100),
(product_b._id, 50),
)
ledger_blocks = [
get_single_balance_block(case_id, prod_id, balance)
for prod_id, balance in balances
]
form = submit_case_blocks(ledger_blocks, self.domain)[0]
# submit duplicate
with process_kafka_changes(self.ledger_pillow):
with process_couch_changes('DefaultChangeFeedPillow'):
dupe_form = submit_form_locally(form.get_xml(), domain=self.domain)[1]
self.assertTrue(dupe_form.is_duplicate)
# confirm republished
ledger_meta_a = self.processor.changes_seen[0].metadata
ledger_meta_b = self.processor.changes_seen[1].metadata
format_id = lambda product_id: '{}/{}/{}'.format(case_id, 'stock', product_id)
expected_ids = {format_id(product_a._id), format_id(product_b._id)}
for meta in [ledger_meta_a, ledger_meta_b]:
self.assertTrue(meta.document_id in expected_ids)
expected_ids.remove(meta.document_id)
self.assertEqual(self.domain, meta.domain)
# cleanup
product_a.delete()
product_b.delete()
class ReprocessSubmissionStubTests(TestCase):
@classmethod
def setUpClass(cls):
super(ReprocessSubmissionStubTests, cls).setUpClass()
cls.domain = uuid.uuid4().hex
cls.product = SQLProduct.objects.create(domain=cls.domain, product_id='product1', name='product1')
@classmethod
def tearDownClass(cls):
cls.product.delete()
super(ReprocessSubmissionStubTests, cls).tearDownClass()
def setUp(self):
super(ReprocessSubmissionStubTests, self).setUp()
self.factory = CaseFactory(domain=self.domain)
self.formdb = FormAccessors(self.domain)
self.casedb = CaseAccessors(self.domain)
self.ledgerdb = LedgerAccessors(self.domain)
def tearDown(self):
FormProcessorTestUtils.delete_all_cases_forms_ledgers(self.domain)
super(ReprocessSubmissionStubTests, self).tearDown()
def test_reprocess_unfinished_submission_case_create(self):
case_id = uuid.uuid4().hex
with _patch_save_to_raise_error(self):
self.factory.create_or_update_cases([
CaseStructure(case_id=case_id, attrs={'case_type': 'parent', 'create': True})
])
stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, saved=False).all()
self.assertEqual(1, len(stubs))
# form that was saved before case error raised
normal_form_ids = self.formdb.get_all_form_ids_in_domain('XFormInstance')
self.assertEqual(0, len(normal_form_ids))
# shows error form (duplicate of form that was saved before case error)
# this is saved becuase the saving was assumed to be atomic so if there was any error it's assumed
# the form didn't get saved
# we don't really care about this form in this test
error_forms = self.formdb.get_forms_by_type('XFormError', 10)
self.assertEqual(1, len(error_forms))
self.assertIsNone(error_forms[0].orig_id)
self.assertEqual(error_forms[0].form_id, stubs[0].xform_id)
self.assertEqual(0, len(self.casedb.get_case_ids_in_domain(self.domain)))
result = reprocess_unfinished_stub(stubs[0])
self.assertEqual(1, len(result.cases))
case_ids = self.casedb.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
self.assertEqual(case_id, case_ids[0])
with self.assertRaises(UnfinishedSubmissionStub.DoesNotExist):
UnfinishedSubmissionStub.objects.get(pk=stubs[0].pk)
def test_reprocess_unfinished_submission_case_update(self):
case_id = uuid.uuid4().hex
form_ids = []
form_ids.append(submit_case_blocks(
CaseBlock(case_id=case_id, create=True, case_type='box').as_string().decode('utf-8'),
self.domain
)[0].form_id)
with _patch_save_to_raise_error(self):
submit_case_blocks(
CaseBlock(case_id=case_id, update={'prop': 'a'}).as_string().decode('utf-8'),
self.domain
)
stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, saved=False).all()
self.assertEqual(1, len(stubs))
form_ids.append(stubs[0].xform_id)
# submit second form with case update
form_ids.append(submit_case_blocks(
CaseBlock(case_id=case_id, update={'prop': 'b'}).as_string().decode('utf-8'),
self.domain
)[0].form_id)
case = self.casedb.get_case(case_id)
self.assertEqual(2, len(case.xform_ids))
self.assertEqual('b', case.get_case_property('prop'))
result = reprocess_unfinished_stub(stubs[0])
self.assertEqual(1, len(result.cases))
self.assertEqual(0, len(result.ledgers))
case = self.casedb.get_case(case_id)
self.assertEqual('b', case.get_case_property('prop')) # should be property value from most recent form
self.assertEqual(3, len(case.xform_ids))
self.assertEqual(form_ids, case.xform_ids)
with self.assertRaises(UnfinishedSubmissionStub.DoesNotExist):
UnfinishedSubmissionStub.objects.get(pk=stubs[0].pk)
def test_reprocess_unfinished_submission_ledger_create(self):
from corehq.apps.commtrack.tests.util import get_single_balance_block
case_id = uuid.uuid4().hex
self.factory.create_or_update_cases([
CaseStructure(case_id=case_id, attrs={'case_type': 'parent', 'create': True})
])
with _patch_save_to_raise_error(self):
submit_case_blocks(
get_single_balance_block(case_id, 'product1', 100),
self.domain
)
stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, saved=False).all()
self.assertEqual(1, len(stubs))
ledgers = self.ledgerdb.get_ledger_values_for_case(case_id)
self.assertEqual(0, len(ledgers))
case = self.casedb.get_case(case_id)
self.assertEqual(1, len(case.xform_ids))
ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case(case_id)
self.assertEqual(0, len(ledger_transactions))
result = reprocess_unfinished_stub(stubs[0])
self.assertEqual(1, len(result.cases))
self.assertEqual(1, len(result.ledgers))
ledgers = self.ledgerdb.get_ledger_values_for_case(case_id)
self.assertEqual(1, len(ledgers))
ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case(case_id)
self.assertEqual(1, len(ledger_transactions))
# case still only has 2 transactions
case = self.casedb.get_case(case_id)
self.assertEqual(2, len(case.xform_ids))
if should_use_sql_backend(self.domain):
self.assertTrue(case.actions[1].is_ledger_transaction)
def test_reprocess_unfinished_submission_ledger_rebuild(self):
from corehq.apps.commtrack.tests.util import get_single_balance_block
case_id = uuid.uuid4().hex
form_ids = []
form_ids.append(submit_case_blocks(
[
CaseBlock(case_id=case_id, create=True, case_type='shop').as_string().decode('utf-8'),
get_single_balance_block(case_id, 'product1', 100),
],
self.domain
)[0].form_id)
with _patch_save_to_raise_error(self):
submit_case_blocks(
get_single_balance_block(case_id, 'product1', 50),
self.domain
)
stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, saved=False).all()
self.assertEqual(1, len(stubs))
form_ids.append(stubs[0].xform_id)
# submit another form afterwards
form_ids.append(submit_case_blocks(
get_single_balance_block(case_id, 'product1', 25),
self.domain
)[0].form_id)
ledgers = self.ledgerdb.get_ledger_values_for_case(case_id)
self.assertEqual(1, len(ledgers))
self.assertEqual(25, ledgers[0].balance)
ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case(case_id)
if should_use_sql_backend(self.domain):
self.assertEqual(2, len(ledger_transactions))
else:
# includes extra consumption transaction
self.assertEqual(3, len(ledger_transactions))
# should rebuild ledger transactions
result = reprocess_unfinished_stub(stubs[0])
self.assertEqual(1, len(result.cases))
self.assertEqual(1, len(result.ledgers))
ledgers = self.ledgerdb.get_ledger_values_for_case(case_id)
self.assertEqual(1, len(ledgers)) # still only 1
self.assertEqual(25, ledgers[0].balance)
ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case(case_id)
if should_use_sql_backend(self.domain):
self.assertEqual(3, len(ledger_transactions))
# make sure transactions are in correct order
self.assertEqual(form_ids, [trans.form_id for trans in ledger_transactions])
self.assertEqual(100, ledger_transactions[0].updated_balance)
self.assertEqual(100, ledger_transactions[0].delta)
self.assertEqual(50, ledger_transactions[1].updated_balance)
self.assertEqual(-50, ledger_transactions[1].delta)
self.assertEqual(25, ledger_transactions[2].updated_balance)
self.assertEqual(-25, ledger_transactions[2].delta)
else:
self.assertEqual(3, len(ledger_transactions))
self.assertEqual(form_ids, [trans.report.form_id for trans in ledger_transactions])
self.assertEqual(100, ledger_transactions[0].stock_on_hand)
self.assertEqual(50, ledger_transactions[1].stock_on_hand)
self.assertEqual(25, ledger_transactions[2].stock_on_hand)
def test_fire_signals(self):
from corehq.apps.receiverwrapper.tests.test_submit_errors import failing_signal_handler
case_id = uuid.uuid4().hex
form_id = uuid.uuid4().hex
with failing_signal_handler('signal death'):
submit_case_blocks(
CaseBlock(case_id=case_id, create=True, case_type='box').as_string().decode('utf-8'),
self.domain,
form_id=form_id
)
form = self.formdb.get_form(form_id)
with catch_signal(successful_form_received) as form_handler, catch_signal(case_post_save) as case_handler:
submit_form_locally(
instance=form.get_xml(),
domain=self.domain,
)
case = self.casedb.get_case(case_id)
if should_use_sql_backend(self.domain):
self.assertEqual(form, form_handler.call_args[1]['xform'])
self.assertEqual(case, case_handler.call_args[1]['case'])
else:
signal_form = form_handler.call_args[1]['xform']
self.assertEqual(form.form_id, signal_form.form_id)
self.assertEqual(form.get_rev, signal_form.get_rev)
signal_case = case_handler.call_args[1]['case']
self.assertEqual(case.case_id, signal_case.case_id)
self.assertEqual(case.get_rev, signal_case.get_rev)
class Command(BaseCommand):
help = """Expects 4 arguments in order : domain app_id version_number test_run
ex: ./manage.py purge_forms_and_cases testapp c531daeece0633738c9a3676a13e3d4f 88 yes
domain is included with app_id to ensure the user knows what app to delete
version_number to delete data accumulated by versions BEFORE this : integer
test_run should be yes(case-sensitive) for a test_run and anything otherwise
though deletion would be re-confirmed so dont panic
"""
def add_arguments(self, parser):
parser.add_argument('domain')
parser.add_argument('app_id')
parser.add_argument('version_number', type=int)
parser.add_argument('test_run')
def __init__(self):
super(Command, self).__init__()
self.case_ids = set()
self.filtered_xform_ids, self.xform_ids = [], []
self.xform_writer, self.case_writer = None, None
self.forms_accessor, self.case_accessors = None, None
self.domain, self.app_id, self.version_number, self.test_run = None, None, None, None
self.version_mapping = dict()
def setup(self):
self.xform_writer = csv.writer(open(XFORM_FILENAME, 'w+b'))
self.xform_writer.writerow(XFORM_HEADER)
self.case_writer = csv.writer(open(CASE_FILE_NAME, 'w+b'))
self.case_writer.writerow(CASE_HEADER)
self.forms_accessor = FormAccessors(self.domain)
self.case_accessors = CaseAccessors(self.domain)
def ensure_prerequisites(self, domain, app_id, version_number, test_run):
self.domain = domain
self.app_id = app_id
self.version_number = version_number
self.test_run = test_run == 'yes'
_notify_parsed_args(domain, app_id, version_number, test_run)
app = Application.get(self.app_id)
if app.domain != self.domain:
raise CommandError('Domain not same as from app id')
self.setup()
def handle(self, domain, app_id, version_number, test_run, **options):
self.ensure_prerequisites(domain, app_id, version_number, test_run)
self.xform_ids = self.forms_accessor.get_all_form_ids_in_domain()
self.iterate_forms_and_collect_case_ids()
_print_final_debug_info(self.xform_ids, self.filtered_xform_ids, self.case_ids)
if self.data_to_delete() and self.delete_permitted():
self.delete_forms_and_cases()
print("Process Completed!! Keep copy of files %s, %s" % (XFORM_FILENAME, CASE_FILE_NAME))
else:
print('Process Finished w/o Changes..')
def iterate_forms_and_collect_case_ids(self):
print("Iterating Through %s XForms and Collecting Case Ids" % len(self.xform_ids))
for xform in self.forms_accessor.iter_forms(self.xform_ids):
# Get app version by fetching app corresponding to xform build_id since xform.form
# does not have updated app version unless form was updated for that version
app_version_built_with = self.get_xform_build_version(xform)
if app_version_built_with and app_version_built_with < self.version_number:
_print_form_details(xform, self.xform_writer, app_version_built_with)
self.ensure_valid_xform(xform)
self.filtered_xform_ids.append(xform.form_id)
self.case_ids = self.case_ids.union(get_case_ids_from_form(xform))
else:
print('skipping xform id: %s' % xform.form_id)
if self.case_ids:
self.print_case_details()
def get_xform_build_version(self, xform):
version_from_mapping = None
if xform.build_id:
version_from_mapping = self.version_mapping.get(xform.build_id, None)
if not version_from_mapping:
try:
get_app_version = get_app(self.domain, xform.build_id).version
except Http404:
get_app_version = None
if get_app_version:
version_from_mapping = int(get_app_version)
self.version_mapping[xform.build_id] = version_from_mapping
return version_from_mapping
def ensure_valid_xform(self, xform):
if xform.app_id != self.app_id and xform.domain != self.domain:
_raise_xform_domain_mismatch(xform)
def print_case_details(self):
for case in self.case_accessors.iter_cases(self.case_ids):
_print_case_details(case, self.case_writer)
def delete_permitted(self):
return not self.test_run and are_you_sure()
def data_to_delete(self):
return len(self.filtered_xform_ids) != 0 or len(self.case_ids) != 0
def delete_forms_and_cases(self):
print('Proceeding with deleting forms and cases')
self.forms_accessor.soft_delete_forms(self.filtered_xform_ids)
self.case_accessors.soft_delete_cases(list(self.case_ids))
class Command(BaseCommand):
help = """Expects 4 arguments in order : domain app_id version_number test_run
ex: ./manage.py purge_forms_and_cases testapp c531daeece0633738c9a3676a13e3d4f 88 yes
domain is included with app_id to ensure the user knows what app to delete
version_number to delete data accumulated by versions BEFORE this : integer
test_run should be yes(case-sensitive) for a test_run and anything otherwise
though deletion would be re-confirmed so dont panic
"""
def add_arguments(self, parser):
parser.add_argument('domain')
parser.add_argument('app_id')
parser.add_argument('version_number', type=int)
parser.add_argument('test_run')
def __init__(self):
super(Command, self).__init__()
self.case_ids = set()
self.filtered_xform_ids, self.xform_ids = [], []
self.xform_writer, self.case_writer = None, None
self.forms_accessor, self.case_accessors = None, None
self.domain, self.app_id, self.version_number, self.test_run = None, None, None, None
self.version_mapping = dict()
def setup(self):
self.xform_writer = csv.writer(open(XFORM_FILENAME, 'w+b'))
self.xform_writer.writerow(XFORM_HEADER)
self.case_writer = csv.writer(open(CASE_FILE_NAME, 'w+b'))
self.case_writer.writerow(CASE_HEADER)
self.forms_accessor = FormAccessors(self.domain)
self.case_accessors = CaseAccessors(self.domain)
def ensure_prerequisites(self, domain, app_id, version_number, test_run):
NotAllowed.check(domain)
self.domain = domain
self.app_id = app_id
self.version_number = version_number
self.test_run = test_run == 'yes'
_notify_parsed_args(domain, app_id, version_number, test_run)
app = Application.get(self.app_id)
if app.domain != self.domain:
raise CommandError('Domain not same as from app id')
self.setup()
def handle(self, domain, app_id, version_number, test_run, **options):
self.ensure_prerequisites(domain, app_id, version_number, test_run)
self.xform_ids = self.forms_accessor.get_all_form_ids_in_domain()
self.iterate_forms_and_collect_case_ids()
_print_final_debug_info(self.xform_ids, self.filtered_xform_ids,
self.case_ids)
if self.data_to_delete() and self.delete_permitted():
self.delete_forms_and_cases()
print("Process Completed!! Keep copy of files %s, %s" %
(XFORM_FILENAME, CASE_FILE_NAME))
else:
print('Process Finished w/o Changes..')
def iterate_forms_and_collect_case_ids(self):
print("Iterating Through %s XForms and Collecting Case Ids" %
len(self.xform_ids))
for xform in self.forms_accessor.iter_forms(self.xform_ids):
# Get app version by fetching app corresponding to xform build_id since xform.form
# does not have updated app version unless form was updated for that version
app_version_built_with = self.get_xform_build_version(xform)
if app_version_built_with and app_version_built_with < self.version_number:
_print_form_details(xform, self.xform_writer,
app_version_built_with)
self.ensure_valid_xform(xform)
self.filtered_xform_ids.append(xform.form_id)
self.case_ids = self.case_ids.union(
get_case_ids_from_form(xform))
else:
print('skipping xform id: %s' % xform.form_id)
if self.case_ids:
self.print_case_details()
def get_xform_build_version(self, xform):
version_from_mapping = None
if xform.build_id:
version_from_mapping = self.version_mapping.get(
xform.build_id, None)
if not version_from_mapping:
try:
get_app_version = get_app(self.domain,
xform.build_id).version
except Http404:
get_app_version = None
if get_app_version:
version_from_mapping = int(get_app_version)
self.version_mapping[xform.build_id] = version_from_mapping
return version_from_mapping
def ensure_valid_xform(self, xform):
if xform.app_id != self.app_id and xform.domain != self.domain:
_raise_xform_domain_mismatch(xform)
def print_case_details(self):
for case in self.case_accessors.iter_cases(self.case_ids):
_print_case_details(case, self.case_writer)
def delete_permitted(self):
return not self.test_run and are_you_sure()
def data_to_delete(self):
return len(self.filtered_xform_ids) != 0 or len(self.case_ids) != 0
def delete_forms_and_cases(self):
print('Proceeding with deleting forms and cases')
self.forms_accessor.soft_delete_forms(self.filtered_xform_ids)
self.case_accessors.soft_delete_cases(list(self.case_ids))
