def handle(self, domains, file_name, **options): blob_db = get_blob_db() with open(file_name, 'w', encoding='utf-8') as csv_file: field_names = ['domain', 'archived', 'form_id', 'received_on'] csv_writer = csv.DictWriter(csv_file, field_names) csv_writer.writeheader() for domain in domains: self.stdout.write("Handling domain %s" % domain) form_db = FormAccessors(domain) form_ids = form_db.get_all_form_ids_in_domain() form_ids.extend( form_db.get_all_form_ids_in_domain('XFormArchived')) for form in with_progress_bar(form_db.iter_forms(form_ids), len(form_ids)): if isinstance(form, CouchForm): meta = form.blobs.get(ATTACHMENT_NAME) if not meta or not blob_db.exists(key=meta.key): self.write_row(csv_writer, domain, form.is_archived, form.received_on, form.form_id) elif isinstance(form, XFormInstanceSQL): meta = form.get_attachment_meta(ATTACHMENT_NAME) if not meta or not blob_db.exists(key=meta.key): self.write_row(csv_writer, domain, form.is_archived, form.received_on, form.form_id) else: raise Exception("not sure how we got here")
def handle(self, domain, folder_path, **options): if os.path.exists(folder_path): if not os.path.isdir(folder_path): raise CommandError( 'Folder path must be the path to a directory') else: os.mkdir(folder_path) form_accessors = FormAccessors(domain) form_ids = form_accessors.get_all_form_ids_in_domain() for form in form_accessors.iter_forms(form_ids): form_path = os.path.join(folder_path, form.form_id) if not os.path.exists(form_path): os.mkdir(form_path) form_meta = FormMetadata( user_id=form.user_id, received_on=form.received_on, app_id=form.app_id, build_id=form.build_id, attachments=list(form.attachments.keys()), auth_context=form.auth_context, ) with open(os.path.join(form_path, 'metadata.json'), 'w') as meta: json.dump(form_meta.to_json(), meta) xml = form.get_xml() with open(os.path.join(form_path, 'form.xml'), 'w') as f: f.write(xml) for name, meta in form.attachments.items(): with open(os.path.join(form_path, name), 'w') as f: f.write(form.get_attachment(name))
def handle(self, domain, folder_path, **options): if os.path.exists(folder_path): if not os.path.isdir(folder_path): raise CommandError('Folder path must be the path to a directory') else: os.mkdir(folder_path) form_accessors = FormAccessors(domain) form_ids = form_accessors.get_all_form_ids_in_domain() for form in form_accessors.iter_forms(form_ids): form_path = os.path.join(folder_path, form.form_id) if not os.path.exists(form_path): os.mkdir(form_path) form_meta = FormMetadata( user_id=form.user_id, received_on=form.received_on, app_id=form.app_id, build_id=form.build_id, attachments=form.attachments.keys(), auth_context=form.auth_context, ) with open(os.path.join(form_path, 'metadata.json'), 'w') as meta: json.dump(form_meta.to_json(), meta) xml = form.get_xml() with open(os.path.join(form_path, 'form.xml'), 'w') as f: f.write(xml) for name, meta in form.attachments.items(): with open(os.path.join(form_path, name), 'w') as f: f.write(form.get_attachment(name))
def _delete_all_forms(domain_name): logger.info('Deleting forms...') form_accessor = FormAccessors(domain_name) form_ids = form_accessor.get_all_form_ids_in_domain() for form_id_chunk in chunked( with_progress_bar(form_ids, stream=silence_during_tests()), 500): form_accessor.soft_delete_forms(list(form_id_chunk)) logger.info('Deleting forms complete.')
def _delete_all_forms(domain_name): logger.info('Deleting forms...') form_accessor = FormAccessors(domain_name) form_ids = list(itertools.chain(*[ form_accessor.get_all_form_ids_in_domain(doc_type=doc_type) for doc_type in doc_type_to_state ])) for form_id_chunk in chunked(with_progress_bar(form_ids, stream=silence_during_tests()), 500): form_accessor.soft_delete_forms(list(form_id_chunk)) logger.info('Deleting forms complete.')
def handle(self, domains, file_name, **options): blob_db = get_blob_db() with open(file_name, 'w', encoding='utf-8') as csv_file: field_names = ['domain', 'archived', 'form_id', 'received_on'] csv_writer = csv.DictWriter(csv_file, field_names) csv_writer.writeheader() for domain in domains: self.stdout.write("Handling domain %s" % domain) form_db = FormAccessors(domain) form_ids = form_db.get_all_form_ids_in_domain() form_ids.extend(form_db.get_all_form_ids_in_domain('XFormArchived')) for form in with_progress_bar(form_db.iter_forms(form_ids), len(form_ids)): if isinstance(form, CouchForm): meta = form.blobs.get(ATTACHMENT_NAME) if not meta or not blob_db.exists(key=meta.key): self.write_row(csv_writer, domain, form.is_archived, form.received_on, form.form_id) elif isinstance(form, XFormInstanceSQL): meta = form.get_attachment_meta(ATTACHMENT_NAME) if not meta or not blob_db.exists(key=meta.key): self.write_row(csv_writer, domain, form.is_archived, form.received_on, form.form_id) else: raise Exception("not sure how we got here")
class ReadonlyFormDocumentStore(ReadOnlyDocumentStore): def __init__(self, domain): self.domain = domain self.form_accessors = FormAccessors(domain=domain) def get_document(self, doc_id): try: return add_couch_properties_to_sql_form_json(self.form_accessors.get_form(doc_id).to_json()) except (XFormNotFound, BlobError) as e: raise DocumentNotFoundError(e) def iter_document_ids(self, last_id=None): # todo: support last_id return iter(self.form_accessors.get_all_form_ids_in_domain()) def iter_documents(self, ids): for wrapped_form in self.form_accessors.iter_forms(ids): yield add_couch_properties_to_sql_form_json(wrapped_form.to_json())
class ReadonlyFormDocumentStore(ReadOnlyDocumentStore): def __init__(self, domain): self.domain = domain self.form_accessors = FormAccessors(domain=domain) def get_document(self, doc_id): try: return self.form_accessors.get_form(doc_id).to_json(include_attachments=True) except (XFormNotFound, BlobError) as e: raise DocumentNotFoundError(e) def iter_document_ids(self, last_id=None): # todo: support last_id return iter(self.form_accessors.get_all_form_ids_in_domain()) def iter_documents(self, ids): for wrapped_form in self.form_accessors.iter_forms(ids): yield wrapped_form.to_json()
class KafkaPublishingTest(TestCase): domain = 'kafka-publishing-test' def setUp(self): super(KafkaPublishingTest, self).setUp() FormProcessorTestUtils.delete_all_cases_forms_ledgers() self.form_accessors = FormAccessors(domain=self.domain) self.processor = TestProcessor() self.form_pillow = ConstructedPillow( name='test-kafka-form-feed', checkpoint=None, change_feed=KafkaChangeFeed(topics=[topics.FORM, topics.FORM_SQL], group_id='test-kafka-form-feed'), processor=self.processor) self.case_pillow = ConstructedPillow( name='test-kafka-case-feed', checkpoint=None, change_feed=KafkaChangeFeed(topics=[topics.CASE, topics.CASE_SQL], group_id='test-kafka-case-feed'), processor=self.processor) self.ledger_pillow = ConstructedPillow( name='test-kafka-ledger-feed', checkpoint=None, change_feed=KafkaChangeFeed(topics=[topics.LEDGER], group_id='test-kafka-ledger-feed'), processor=self.processor) def tearDown(self): FormProcessorTestUtils.delete_all_cases_forms_ledgers() def test_form_is_published(self): with process_pillow_changes(self.form_pillow): with process_pillow_changes('DefaultChangeFeedPillow'): form = create_and_save_a_form(self.domain) self.assertEqual(1, len(self.processor.changes_seen)) change_meta = self.processor.changes_seen[0].metadata self.assertEqual(form.form_id, change_meta.document_id) self.assertEqual(self.domain, change_meta.domain) def test_duplicate_form_published(self): form_id = uuid.uuid4().hex form_xml = get_simple_form_xml(form_id) orig_form = submit_form_locally(form_xml, domain=self.domain).xform self.assertEqual(form_id, orig_form.form_id) self.assertEqual(1, len(self.form_accessors.get_all_form_ids_in_domain())) with process_pillow_changes(self.form_pillow): with process_pillow_changes('DefaultChangeFeedPillow'): # post an exact duplicate dupe_form = submit_form_locally(form_xml, domain=self.domain).xform self.assertTrue(dupe_form.is_duplicate) self.assertNotEqual(form_id, dupe_form.form_id) if should_use_sql_backend(self.domain): self.assertEqual(form_id, dupe_form.orig_id) # make sure changes made it to kafka dupe_form_meta = self.processor.changes_seen[0].metadata self.assertEqual(dupe_form.form_id, dupe_form_meta.document_id) self.assertEqual(dupe_form.domain, dupe_form.domain) if should_use_sql_backend(self.domain): # sql domains also republish the original form to ensure that if the server crashed # in the processing of the form the first time that it is still sent to kafka orig_form_meta = self.processor.changes_seen[1].metadata self.assertEqual(orig_form.form_id, orig_form_meta.document_id) self.assertEqual(self.domain, orig_form_meta.domain) self.assertEqual(dupe_form.domain, dupe_form.domain) def test_form_soft_deletions(self): form = create_and_save_a_form(self.domain) with process_pillow_changes(self.form_pillow): with process_pillow_changes('DefaultChangeFeedPillow'): form.soft_delete() self.assertEqual(1, len(self.processor.changes_seen)) change_meta = self.processor.changes_seen[0].metadata self.assertEqual(form.form_id, change_meta.document_id) self.assertTrue(change_meta.is_deletion) def test_case_is_published(self): with process_pillow_changes(self.case_pillow): with process_pillow_changes('DefaultChangeFeedPillow'): case = create_and_save_a_case(self.domain, case_id=uuid.uuid4().hex, case_name='test case') self.assertEqual(1, len(self.processor.changes_seen)) change_meta = self.processor.changes_seen[0].metadata self.assertEqual(case.case_id, change_meta.document_id) self.assertEqual(self.domain, change_meta.domain)
def form_ids_in_domain(domain): form_accessor = FormAccessors(domain) for doc_type in DOC_TYPES: form_ids = form_accessor.get_all_form_ids_in_domain(doc_type=doc_type) for form_id in form_ids: yield form_id
class KafkaPublishingTest(OverridableSettingsTestMixin, TestCase): domain = 'kafka-publishing-test' def setUp(self): FormProcessorTestUtils.delete_all_sql_forms() FormProcessorTestUtils.delete_all_sql_cases() self.form_accessors = FormAccessors(domain=self.domain) def test_form_is_published(self): kafka_consumer = get_test_kafka_consumer(topics.FORM_SQL) form = create_and_save_a_form(self.domain) message = kafka_consumer.next() change_meta = change_meta_from_kafka_message(message.value) self.assertEqual(form.form_id, change_meta.document_id) self.assertEqual(self.domain, change_meta.domain) def test_case_is_published(self): kafka_consumer = get_test_kafka_consumer(topics.CASE_SQL) case = create_and_save_a_case(self.domain, case_id=uuid.uuid4().hex, case_name='test case') change_meta = change_meta_from_kafka_message(kafka_consumer.next().value) self.assertEqual(case.case_id, change_meta.document_id) self.assertEqual(self.domain, change_meta.domain) def test_duplicate_form_published(self): form_id = uuid.uuid4().hex form_xml = get_simple_form_xml(form_id) orig_form = submit_form_locally(form_xml, domain=self.domain)[1] self.assertEqual(form_id, orig_form.form_id) self.assertEqual(1, len(self.form_accessors.get_all_form_ids_in_domain())) form_consumer = get_test_kafka_consumer(topics.FORM_SQL) # post an exact duplicate dupe_form = submit_form_locally(form_xml, domain=self.domain)[1] self.assertTrue(dupe_form.is_duplicate) self.assertNotEqual(form_id, dupe_form.form_id) self.assertEqual(form_id, dupe_form.orig_id) # make sure changes made it to kafka # first the dupe dupe_form_meta = change_meta_from_kafka_message(form_consumer.next().value) self.assertEqual(dupe_form.form_id, dupe_form_meta.document_id) # then the original form orig_form_meta = change_meta_from_kafka_message(form_consumer.next().value) self.assertEqual(orig_form.form_id, orig_form_meta.document_id) self.assertEqual(self.domain, orig_form_meta.domain) def test_duplicate_case_published(self): case_id = uuid.uuid4().hex form_xml = get_simple_form_xml(uuid.uuid4().hex, case_id) submit_form_locally(form_xml, domain=self.domain)[1] self.assertEqual(1, len(CaseAccessors(self.domain).get_case_ids_in_domain())) case_consumer = get_test_kafka_consumer(topics.CASE_SQL) dupe_form = submit_form_locally(form_xml, domain=self.domain)[1] self.assertTrue(dupe_form.is_duplicate) # check the case was republished case_meta = change_meta_from_kafka_message(case_consumer.next().value) self.assertEqual(case_id, case_meta.document_id) self.assertEqual(self.domain, case_meta.domain) def test_duplicate_ledger_published(self): # setup products and case product_a = make_product(self.domain, 'A Product', 'prodcode_a') product_b = make_product(self.domain, 'B Product', 'prodcode_b') case_id = uuid.uuid4().hex form_xml = get_simple_form_xml(uuid.uuid4().hex, case_id) submit_form_locally(form_xml, domain=self.domain)[1] # submit ledger data balances = ( (product_a._id, 100), (product_b._id, 50), ) ledger_blocks = [ get_single_balance_block(case_id, prod_id, balance) for prod_id, balance in balances ] form = submit_case_blocks(ledger_blocks, self.domain) # submit duplicate ledger_consumer = get_test_kafka_consumer(topics.LEDGER) dupe_form = submit_form_locally(form.get_xml(), domain=self.domain)[1] self.assertTrue(dupe_form.is_duplicate) # confirm republished ledger_meta_a = change_meta_from_kafka_message(ledger_consumer.next().value) ledger_meta_b = change_meta_from_kafka_message(ledger_consumer.next().value) format_id = lambda product_id: '{}/{}/{}'.format(case_id, 'stock', product_id) expected_ids = {format_id(product_a._id), format_id(product_b._id)} for meta in [ledger_meta_a, ledger_meta_b]: self.assertTrue(meta.document_id in expected_ids) expected_ids.remove(meta.document_id) self.assertEqual(self.domain, meta.domain) # cleanup product_a.delete() product_b.delete()
class ReprocessSubmissionStubTests(TestCase): @classmethod def setUpClass(cls): super(ReprocessSubmissionStubTests, cls).setUpClass() cls.domain = uuid.uuid4().hex cls.product = SQLProduct.objects.create(domain=cls.domain, product_id='product1', name='product1') @classmethod def tearDownClass(cls): cls.product.delete() super(ReprocessSubmissionStubTests, cls).tearDownClass() def setUp(self): super(ReprocessSubmissionStubTests, self).setUp() self.factory = CaseFactory(domain=self.domain) self.formdb = FormAccessors(self.domain) self.casedb = CaseAccessors(self.domain) self.ledgerdb = LedgerAccessors(self.domain) def tearDown(self): FormProcessorTestUtils.delete_all_cases_forms_ledgers(self.domain) super(ReprocessSubmissionStubTests, self).tearDown() def test_reprocess_unfinished_submission_case_create(self): case_id = uuid.uuid4().hex with _patch_save_to_raise_error(self): self.factory.create_or_update_cases([ CaseStructure(case_id=case_id, attrs={ 'case_type': 'parent', 'create': True }) ]) stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, saved=False).all() self.assertEqual(1, len(stubs)) # form that was saved before case error raised normal_form_ids = self.formdb.get_all_form_ids_in_domain( 'XFormInstance') self.assertEqual(0, len(normal_form_ids)) # shows error form (duplicate of form that was saved before case error) # this is saved becuase the saving was assumed to be atomic so if there was any error it's assumed # the form didn't get saved # we don't really care about this form in this test error_forms = self.formdb.get_forms_by_type('XFormError', 10) self.assertEqual(1, len(error_forms)) self.assertIsNone(error_forms[0].orig_id) self.assertEqual(error_forms[0].form_id, stubs[0].xform_id) self.assertEqual(0, len(self.casedb.get_case_ids_in_domain(self.domain))) result = reprocess_unfinished_stub(stubs[0]) self.assertEqual(1, len(result.cases)) case_ids = self.casedb.get_case_ids_in_domain() self.assertEqual(1, len(case_ids)) self.assertEqual(case_id, case_ids[0]) with self.assertRaises(UnfinishedSubmissionStub.DoesNotExist): UnfinishedSubmissionStub.objects.get(pk=stubs[0].pk) def test_reprocess_unfinished_submission_case_update(self): case_id = uuid.uuid4().hex form_ids = [] form_ids.append( submit_case_blocks( CaseBlock(case_id=case_id, create=True, case_type='box').as_string(), self.domain)[0].form_id) with _patch_save_to_raise_error(self): submit_case_blocks( CaseBlock(case_id=case_id, update={ 'prop': 'a' }).as_string(), self.domain) stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, saved=False).all() self.assertEqual(1, len(stubs)) form_ids.append(stubs[0].xform_id) # submit second form with case update form_ids.append( submit_case_blocks( CaseBlock(case_id=case_id, update={ 'prop': 'b' }).as_string(), self.domain)[0].form_id) case = self.casedb.get_case(case_id) self.assertEqual(2, len(case.xform_ids)) self.assertEqual('b', case.get_case_property('prop')) result = reprocess_unfinished_stub(stubs[0]) self.assertEqual(1, len(result.cases)) self.assertEqual(0, len(result.ledgers)) case = self.casedb.get_case(case_id) self.assertEqual('b', case.get_case_property( 'prop')) # should be property value from most recent form self.assertEqual(3, len(case.xform_ids)) self.assertEqual(form_ids, case.xform_ids) with self.assertRaises(UnfinishedSubmissionStub.DoesNotExist): UnfinishedSubmissionStub.objects.get(pk=stubs[0].pk) def test_reprocess_unfinished_submission_ledger_create(self): from corehq.apps.commtrack.tests.util import get_single_balance_block case_id = uuid.uuid4().hex self.factory.create_or_update_cases([ CaseStructure(case_id=case_id, attrs={ 'case_type': 'parent', 'create': True }) ]) with _patch_save_to_raise_error(self): submit_case_blocks( get_single_balance_block(case_id, 'product1', 100), self.domain) stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, saved=False).all() self.assertEqual(1, len(stubs)) ledgers = self.ledgerdb.get_ledger_values_for_case(case_id) self.assertEqual(0, len(ledgers)) case = self.casedb.get_case(case_id) self.assertEqual(1, len(case.xform_ids)) ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case( case_id) self.assertEqual(0, len(ledger_transactions)) result = reprocess_unfinished_stub(stubs[0]) self.assertEqual(1, len(result.cases)) self.assertEqual(1, len(result.ledgers)) ledgers = self.ledgerdb.get_ledger_values_for_case(case_id) self.assertEqual(1, len(ledgers)) ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case( case_id) self.assertEqual(1, len(ledger_transactions)) # case still only has 2 transactions case = self.casedb.get_case(case_id) self.assertEqual(2, len(case.xform_ids)) if should_use_sql_backend(self.domain): self.assertTrue(case.actions[1].is_ledger_transaction) def test_reprocess_unfinished_submission_ledger_rebuild(self): from corehq.apps.commtrack.tests.util import get_single_balance_block case_id = uuid.uuid4().hex form_ids = [] form_ids.append( submit_case_blocks([ CaseBlock(case_id=case_id, create=True, case_type='shop').as_string(), get_single_balance_block(case_id, 'product1', 100), ], self.domain)[0].form_id) with _patch_save_to_raise_error(self): submit_case_blocks( get_single_balance_block(case_id, 'product1', 50), self.domain) stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, saved=False).all() self.assertEqual(1, len(stubs)) form_ids.append(stubs[0].xform_id) # submit another form afterwards form_ids.append( submit_case_blocks( get_single_balance_block(case_id, 'product1', 25), self.domain)[0].form_id) ledgers = self.ledgerdb.get_ledger_values_for_case(case_id) self.assertEqual(1, len(ledgers)) self.assertEqual(25, ledgers[0].balance) ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case( case_id) if should_use_sql_backend(self.domain): self.assertEqual(2, len(ledger_transactions)) else: # includes extra consumption transaction self.assertEqual(3, len(ledger_transactions)) # should rebuild ledger transactions result = reprocess_unfinished_stub(stubs[0]) self.assertEqual(1, len(result.cases)) self.assertEqual(1, len(result.ledgers)) ledgers = self.ledgerdb.get_ledger_values_for_case(case_id) self.assertEqual(1, len(ledgers)) # still only 1 self.assertEqual(25, ledgers[0].balance) ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case( case_id) if should_use_sql_backend(self.domain): self.assertEqual(3, len(ledger_transactions)) # make sure transactions are in correct order self.assertEqual(form_ids, [trans.form_id for trans in ledger_transactions]) self.assertEqual(100, ledger_transactions[0].updated_balance) self.assertEqual(100, ledger_transactions[0].delta) self.assertEqual(50, ledger_transactions[1].updated_balance) self.assertEqual(-50, ledger_transactions[1].delta) self.assertEqual(25, ledger_transactions[2].updated_balance) self.assertEqual(-25, ledger_transactions[2].delta) else: self.assertEqual(3, len(ledger_transactions)) self.assertEqual( form_ids, [trans.report.form_id for trans in ledger_transactions]) self.assertEqual(100, ledger_transactions[0].stock_on_hand) self.assertEqual(50, ledger_transactions[1].stock_on_hand) self.assertEqual(25, ledger_transactions[2].stock_on_hand) def test_fire_signals(self): from corehq.apps.receiverwrapper.tests.test_submit_errors import failing_signal_handler case_id = uuid.uuid4().hex form_id = uuid.uuid4().hex with failing_signal_handler('signal death'): submit_case_blocks(CaseBlock(case_id=case_id, create=True, case_type='box').as_string(), self.domain, form_id=form_id) form = self.formdb.get_form(form_id) with catch_signal( successful_form_received) as form_handler, catch_signal( case_post_save) as case_handler: submit_form_locally( instance=form.get_xml(), domain=self.domain, ) case = self.casedb.get_case(case_id) if should_use_sql_backend(self.domain): self.assertEqual(form, form_handler.call_args[1]['xform']) self.assertEqual(case, case_handler.call_args[1]['case']) else: signal_form = form_handler.call_args[1]['xform'] self.assertEqual(form.form_id, signal_form.form_id) self.assertEqual(form.get_rev, signal_form.get_rev) signal_case = case_handler.call_args[1]['case'] self.assertEqual(case.case_id, signal_case.case_id) self.assertEqual(case.get_rev, signal_case.get_rev)
class KafkaPublishingTest(TestCase): domain = 'kafka-publishing-test' def setUp(self): super(KafkaPublishingTest, self).setUp() FormProcessorTestUtils.delete_all_cases_forms_ledgers() self.form_accessors = FormAccessors(domain=self.domain) self.processor = TestProcessor() self.form_pillow = ConstructedPillow( name='test-kafka-form-feed', checkpoint=None, change_feed=KafkaChangeFeed(topics=[topics.FORM, topics.FORM_SQL], group_id='test-kafka-form-feed'), processor=self.processor ) self.case_pillow = ConstructedPillow( name='test-kafka-case-feed', checkpoint=None, change_feed=KafkaChangeFeed(topics=[topics.CASE, topics.CASE_SQL], group_id='test-kafka-case-feed'), processor=self.processor ) self.ledger_pillow = ConstructedPillow( name='test-kafka-ledger-feed', checkpoint=None, change_feed=KafkaChangeFeed(topics=[topics.LEDGER], group_id='test-kafka-ledger-feed'), processor=self.processor ) def tearDown(self): FormProcessorTestUtils.delete_all_cases_forms_ledgers() @run_with_all_backends def test_form_is_published(self): with process_kafka_changes(self.form_pillow): with process_couch_changes('DefaultChangeFeedPillow'): form = create_and_save_a_form(self.domain) self.assertEqual(1, len(self.processor.changes_seen)) change_meta = self.processor.changes_seen[0].metadata self.assertEqual(form.form_id, change_meta.document_id) self.assertEqual(self.domain, change_meta.domain) @run_with_all_backends def test_duplicate_form_published(self): form_id = uuid.uuid4().hex form_xml = get_simple_form_xml(form_id) orig_form = submit_form_locally(form_xml, domain=self.domain)[1] self.assertEqual(form_id, orig_form.form_id) self.assertEqual(1, len(self.form_accessors.get_all_form_ids_in_domain())) with process_kafka_changes(self.form_pillow): with process_couch_changes('DefaultChangeFeedPillow'): # post an exact duplicate dupe_form = submit_form_locally(form_xml, domain=self.domain)[1] self.assertTrue(dupe_form.is_duplicate) self.assertNotEqual(form_id, dupe_form.form_id) if should_use_sql_backend(self.domain): self.assertEqual(form_id, dupe_form.orig_id) # make sure changes made it to kafka dupe_form_meta = self.processor.changes_seen[0].metadata self.assertEqual(dupe_form.form_id, dupe_form_meta.document_id) self.assertEqual(dupe_form.domain, dupe_form.domain) if should_use_sql_backend(self.domain): # sql domains also republish the original form to ensure that if the server crashed # in the processing of the form the first time that it is still sent to kafka orig_form_meta = self.processor.changes_seen[1].metadata self.assertEqual(orig_form.form_id, orig_form_meta.document_id) self.assertEqual(self.domain, orig_form_meta.domain) self.assertEqual(dupe_form.domain, dupe_form.domain) @run_with_all_backends def test_form_soft_deletions(self): form = create_and_save_a_form(self.domain) with process_kafka_changes(self.form_pillow): with process_couch_changes('DefaultChangeFeedPillow'): form.soft_delete() self.assertEqual(1, len(self.processor.changes_seen)) change_meta = self.processor.changes_seen[0].metadata self.assertEqual(form.form_id, change_meta.document_id) self.assertTrue(change_meta.is_deletion) @run_with_all_backends def test_case_is_published(self): with process_kafka_changes(self.case_pillow): with process_couch_changes('DefaultChangeFeedPillow'): case = create_and_save_a_case(self.domain, case_id=uuid.uuid4().hex, case_name='test case') self.assertEqual(1, len(self.processor.changes_seen)) change_meta = self.processor.changes_seen[0].metadata self.assertEqual(case.case_id, change_meta.document_id) self.assertEqual(self.domain, change_meta.domain) @run_with_all_backends def test_case_deletions(self): case = create_and_save_a_case(self.domain, case_id=uuid.uuid4().hex, case_name='test case') with process_kafka_changes(self.case_pillow): with process_couch_changes('DefaultChangeFeedPillow'): case.soft_delete() self.assertEqual(1, len(self.processor.changes_seen)) change_meta = self.processor.changes_seen[0].metadata self.assertEqual(case.case_id, change_meta.document_id) self.assertTrue(change_meta.is_deletion) def test_duplicate_case_published(self): # this test only runs on sql because it's handling a sql-specific edge case where duplicate # form submissions should cause cases to be resubmitted. # see: http://manage.dimagi.com/default.asp?228463 for context case_id = uuid.uuid4().hex form_xml = get_simple_form_xml(uuid.uuid4().hex, case_id) submit_form_locally(form_xml, domain=self.domain)[1] self.assertEqual(1, len(CaseAccessors(self.domain).get_case_ids_in_domain())) with process_kafka_changes(self.case_pillow): with process_couch_changes('DefaultChangeFeedPillow'): dupe_form = submit_form_locally(form_xml, domain=self.domain)[1] self.assertTrue(dupe_form.is_duplicate) # check the case was republished self.assertEqual(1, len(self.processor.changes_seen)) case_meta = self.processor.changes_seen[0].metadata self.assertEqual(case_id, case_meta.document_id) self.assertEqual(self.domain, case_meta.domain) def test_duplicate_ledger_published(self): # this test also only runs on the sql backend for reasons described in test_duplicate_case_published # setup products and case product_a = make_product(self.domain, 'A Product', 'prodcode_a') product_b = make_product(self.domain, 'B Product', 'prodcode_b') case_id = uuid.uuid4().hex form_xml = get_simple_form_xml(uuid.uuid4().hex, case_id) submit_form_locally(form_xml, domain=self.domain)[1] # submit ledger data balances = ( (product_a._id, 100), (product_b._id, 50), ) ledger_blocks = [ get_single_balance_block(case_id, prod_id, balance) for prod_id, balance in balances ] form = submit_case_blocks(ledger_blocks, self.domain)[0] # submit duplicate with process_kafka_changes(self.ledger_pillow): with process_couch_changes('DefaultChangeFeedPillow'): dupe_form = submit_form_locally(form.get_xml(), domain=self.domain)[1] self.assertTrue(dupe_form.is_duplicate) # confirm republished ledger_meta_a = self.processor.changes_seen[0].metadata ledger_meta_b = self.processor.changes_seen[1].metadata format_id = lambda product_id: '{}/{}/{}'.format(case_id, 'stock', product_id) expected_ids = {format_id(product_a._id), format_id(product_b._id)} for meta in [ledger_meta_a, ledger_meta_b]: self.assertTrue(meta.document_id in expected_ids) expected_ids.remove(meta.document_id) self.assertEqual(self.domain, meta.domain) # cleanup product_a.delete() product_b.delete()
class ReprocessSubmissionStubTests(TestCase): @classmethod def setUpClass(cls): super(ReprocessSubmissionStubTests, cls).setUpClass() cls.domain = uuid.uuid4().hex cls.product = SQLProduct.objects.create(domain=cls.domain, product_id='product1', name='product1') @classmethod def tearDownClass(cls): cls.product.delete() super(ReprocessSubmissionStubTests, cls).tearDownClass() def setUp(self): super(ReprocessSubmissionStubTests, self).setUp() self.factory = CaseFactory(domain=self.domain) self.formdb = FormAccessors(self.domain) self.casedb = CaseAccessors(self.domain) self.ledgerdb = LedgerAccessors(self.domain) def tearDown(self): FormProcessorTestUtils.delete_all_cases_forms_ledgers(self.domain) super(ReprocessSubmissionStubTests, self).tearDown() def test_reprocess_unfinished_submission_case_create(self): case_id = uuid.uuid4().hex with _patch_save_to_raise_error(self): self.factory.create_or_update_cases([ CaseStructure(case_id=case_id, attrs={'case_type': 'parent', 'create': True}) ]) stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, saved=False).all() self.assertEqual(1, len(stubs)) # form that was saved before case error raised normal_form_ids = self.formdb.get_all_form_ids_in_domain('XFormInstance') self.assertEqual(0, len(normal_form_ids)) # shows error form (duplicate of form that was saved before case error) # this is saved becuase the saving was assumed to be atomic so if there was any error it's assumed # the form didn't get saved # we don't really care about this form in this test error_forms = self.formdb.get_forms_by_type('XFormError', 10) self.assertEqual(1, len(error_forms)) self.assertIsNone(error_forms[0].orig_id) self.assertEqual(error_forms[0].form_id, stubs[0].xform_id) self.assertEqual(0, len(self.casedb.get_case_ids_in_domain(self.domain))) result = reprocess_unfinished_stub(stubs[0]) self.assertEqual(1, len(result.cases)) case_ids = self.casedb.get_case_ids_in_domain() self.assertEqual(1, len(case_ids)) self.assertEqual(case_id, case_ids[0]) with self.assertRaises(UnfinishedSubmissionStub.DoesNotExist): UnfinishedSubmissionStub.objects.get(pk=stubs[0].pk) def test_reprocess_unfinished_submission_case_update(self): case_id = uuid.uuid4().hex form_ids = [] form_ids.append(submit_case_blocks( CaseBlock(case_id=case_id, create=True, case_type='box').as_string().decode('utf-8'), self.domain )[0].form_id) with _patch_save_to_raise_error(self): submit_case_blocks( CaseBlock(case_id=case_id, update={'prop': 'a'}).as_string().decode('utf-8'), self.domain ) stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, saved=False).all() self.assertEqual(1, len(stubs)) form_ids.append(stubs[0].xform_id) # submit second form with case update form_ids.append(submit_case_blocks( CaseBlock(case_id=case_id, update={'prop': 'b'}).as_string().decode('utf-8'), self.domain )[0].form_id) case = self.casedb.get_case(case_id) self.assertEqual(2, len(case.xform_ids)) self.assertEqual('b', case.get_case_property('prop')) result = reprocess_unfinished_stub(stubs[0]) self.assertEqual(1, len(result.cases)) self.assertEqual(0, len(result.ledgers)) case = self.casedb.get_case(case_id) self.assertEqual('b', case.get_case_property('prop')) # should be property value from most recent form self.assertEqual(3, len(case.xform_ids)) self.assertEqual(form_ids, case.xform_ids) with self.assertRaises(UnfinishedSubmissionStub.DoesNotExist): UnfinishedSubmissionStub.objects.get(pk=stubs[0].pk) def test_reprocess_unfinished_submission_ledger_create(self): from corehq.apps.commtrack.tests.util import get_single_balance_block case_id = uuid.uuid4().hex self.factory.create_or_update_cases([ CaseStructure(case_id=case_id, attrs={'case_type': 'parent', 'create': True}) ]) with _patch_save_to_raise_error(self): submit_case_blocks( get_single_balance_block(case_id, 'product1', 100), self.domain ) stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, saved=False).all() self.assertEqual(1, len(stubs)) ledgers = self.ledgerdb.get_ledger_values_for_case(case_id) self.assertEqual(0, len(ledgers)) case = self.casedb.get_case(case_id) self.assertEqual(1, len(case.xform_ids)) ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case(case_id) self.assertEqual(0, len(ledger_transactions)) result = reprocess_unfinished_stub(stubs[0]) self.assertEqual(1, len(result.cases)) self.assertEqual(1, len(result.ledgers)) ledgers = self.ledgerdb.get_ledger_values_for_case(case_id) self.assertEqual(1, len(ledgers)) ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case(case_id) self.assertEqual(1, len(ledger_transactions)) # case still only has 2 transactions case = self.casedb.get_case(case_id) self.assertEqual(2, len(case.xform_ids)) if should_use_sql_backend(self.domain): self.assertTrue(case.actions[1].is_ledger_transaction) def test_reprocess_unfinished_submission_ledger_rebuild(self): from corehq.apps.commtrack.tests.util import get_single_balance_block case_id = uuid.uuid4().hex form_ids = [] form_ids.append(submit_case_blocks( [ CaseBlock(case_id=case_id, create=True, case_type='shop').as_string().decode('utf-8'), get_single_balance_block(case_id, 'product1', 100), ], self.domain )[0].form_id) with _patch_save_to_raise_error(self): submit_case_blocks( get_single_balance_block(case_id, 'product1', 50), self.domain ) stubs = UnfinishedSubmissionStub.objects.filter(domain=self.domain, saved=False).all() self.assertEqual(1, len(stubs)) form_ids.append(stubs[0].xform_id) # submit another form afterwards form_ids.append(submit_case_blocks( get_single_balance_block(case_id, 'product1', 25), self.domain )[0].form_id) ledgers = self.ledgerdb.get_ledger_values_for_case(case_id) self.assertEqual(1, len(ledgers)) self.assertEqual(25, ledgers[0].balance) ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case(case_id) if should_use_sql_backend(self.domain): self.assertEqual(2, len(ledger_transactions)) else: # includes extra consumption transaction self.assertEqual(3, len(ledger_transactions)) # should rebuild ledger transactions result = reprocess_unfinished_stub(stubs[0]) self.assertEqual(1, len(result.cases)) self.assertEqual(1, len(result.ledgers)) ledgers = self.ledgerdb.get_ledger_values_for_case(case_id) self.assertEqual(1, len(ledgers)) # still only 1 self.assertEqual(25, ledgers[0].balance) ledger_transactions = self.ledgerdb.get_ledger_transactions_for_case(case_id) if should_use_sql_backend(self.domain): self.assertEqual(3, len(ledger_transactions)) # make sure transactions are in correct order self.assertEqual(form_ids, [trans.form_id for trans in ledger_transactions]) self.assertEqual(100, ledger_transactions[0].updated_balance) self.assertEqual(100, ledger_transactions[0].delta) self.assertEqual(50, ledger_transactions[1].updated_balance) self.assertEqual(-50, ledger_transactions[1].delta) self.assertEqual(25, ledger_transactions[2].updated_balance) self.assertEqual(-25, ledger_transactions[2].delta) else: self.assertEqual(3, len(ledger_transactions)) self.assertEqual(form_ids, [trans.report.form_id for trans in ledger_transactions]) self.assertEqual(100, ledger_transactions[0].stock_on_hand) self.assertEqual(50, ledger_transactions[1].stock_on_hand) self.assertEqual(25, ledger_transactions[2].stock_on_hand) def test_fire_signals(self): from corehq.apps.receiverwrapper.tests.test_submit_errors import failing_signal_handler case_id = uuid.uuid4().hex form_id = uuid.uuid4().hex with failing_signal_handler('signal death'): submit_case_blocks( CaseBlock(case_id=case_id, create=True, case_type='box').as_string().decode('utf-8'), self.domain, form_id=form_id ) form = self.formdb.get_form(form_id) with catch_signal(successful_form_received) as form_handler, catch_signal(case_post_save) as case_handler: submit_form_locally( instance=form.get_xml(), domain=self.domain, ) case = self.casedb.get_case(case_id) if should_use_sql_backend(self.domain): self.assertEqual(form, form_handler.call_args[1]['xform']) self.assertEqual(case, case_handler.call_args[1]['case']) else: signal_form = form_handler.call_args[1]['xform'] self.assertEqual(form.form_id, signal_form.form_id) self.assertEqual(form.get_rev, signal_form.get_rev) signal_case = case_handler.call_args[1]['case'] self.assertEqual(case.case_id, signal_case.case_id) self.assertEqual(case.get_rev, signal_case.get_rev)
class Command(BaseCommand): help = """Expects 4 arguments in order : domain app_id version_number test_run ex: ./manage.py purge_forms_and_cases testapp c531daeece0633738c9a3676a13e3d4f 88 yes domain is included with app_id to ensure the user knows what app to delete version_number to delete data accumulated by versions BEFORE this : integer test_run should be yes(case-sensitive) for a test_run and anything otherwise though deletion would be re-confirmed so dont panic """ def add_arguments(self, parser): parser.add_argument('domain') parser.add_argument('app_id') parser.add_argument('version_number', type=int) parser.add_argument('test_run') def __init__(self): super(Command, self).__init__() self.case_ids = set() self.filtered_xform_ids, self.xform_ids = [], [] self.xform_writer, self.case_writer = None, None self.forms_accessor, self.case_accessors = None, None self.domain, self.app_id, self.version_number, self.test_run = None, None, None, None self.version_mapping = dict() def setup(self): self.xform_writer = csv.writer(open(XFORM_FILENAME, 'w+b')) self.xform_writer.writerow(XFORM_HEADER) self.case_writer = csv.writer(open(CASE_FILE_NAME, 'w+b')) self.case_writer.writerow(CASE_HEADER) self.forms_accessor = FormAccessors(self.domain) self.case_accessors = CaseAccessors(self.domain) def ensure_prerequisites(self, domain, app_id, version_number, test_run): self.domain = domain self.app_id = app_id self.version_number = version_number self.test_run = test_run == 'yes' _notify_parsed_args(domain, app_id, version_number, test_run) app = Application.get(self.app_id) if app.domain != self.domain: raise CommandError('Domain not same as from app id') self.setup() def handle(self, domain, app_id, version_number, test_run, **options): self.ensure_prerequisites(domain, app_id, version_number, test_run) self.xform_ids = self.forms_accessor.get_all_form_ids_in_domain() self.iterate_forms_and_collect_case_ids() _print_final_debug_info(self.xform_ids, self.filtered_xform_ids, self.case_ids) if self.data_to_delete() and self.delete_permitted(): self.delete_forms_and_cases() print("Process Completed!! Keep copy of files %s, %s" % (XFORM_FILENAME, CASE_FILE_NAME)) else: print('Process Finished w/o Changes..') def iterate_forms_and_collect_case_ids(self): print("Iterating Through %s XForms and Collecting Case Ids" % len(self.xform_ids)) for xform in self.forms_accessor.iter_forms(self.xform_ids): # Get app version by fetching app corresponding to xform build_id since xform.form # does not have updated app version unless form was updated for that version app_version_built_with = self.get_xform_build_version(xform) if app_version_built_with and app_version_built_with < self.version_number: _print_form_details(xform, self.xform_writer, app_version_built_with) self.ensure_valid_xform(xform) self.filtered_xform_ids.append(xform.form_id) self.case_ids = self.case_ids.union(get_case_ids_from_form(xform)) else: print('skipping xform id: %s' % xform.form_id) if self.case_ids: self.print_case_details() def get_xform_build_version(self, xform): version_from_mapping = None if xform.build_id: version_from_mapping = self.version_mapping.get(xform.build_id, None) if not version_from_mapping: try: get_app_version = get_app(self.domain, xform.build_id).version except Http404: get_app_version = None if get_app_version: version_from_mapping = int(get_app_version) self.version_mapping[xform.build_id] = version_from_mapping return version_from_mapping def ensure_valid_xform(self, xform): if xform.app_id != self.app_id and xform.domain != self.domain: _raise_xform_domain_mismatch(xform) def print_case_details(self): for case in self.case_accessors.iter_cases(self.case_ids): _print_case_details(case, self.case_writer) def delete_permitted(self): return not self.test_run and are_you_sure() def data_to_delete(self): return len(self.filtered_xform_ids) != 0 or len(self.case_ids) != 0 def delete_forms_and_cases(self): print('Proceeding with deleting forms and cases') self.forms_accessor.soft_delete_forms(self.filtered_xform_ids) self.case_accessors.soft_delete_cases(list(self.case_ids))
class Command(BaseCommand): help = """Expects 4 arguments in order : domain app_id version_number test_run ex: ./manage.py purge_forms_and_cases testapp c531daeece0633738c9a3676a13e3d4f 88 yes domain is included with app_id to ensure the user knows what app to delete version_number to delete data accumulated by versions BEFORE this : integer test_run should be yes(case-sensitive) for a test_run and anything otherwise though deletion would be re-confirmed so dont panic """ def add_arguments(self, parser): parser.add_argument('domain') parser.add_argument('app_id') parser.add_argument('version_number', type=int) parser.add_argument('test_run') def __init__(self): super(Command, self).__init__() self.case_ids = set() self.filtered_xform_ids, self.xform_ids = [], [] self.xform_writer, self.case_writer = None, None self.forms_accessor, self.case_accessors = None, None self.domain, self.app_id, self.version_number, self.test_run = None, None, None, None self.version_mapping = dict() def setup(self): self.xform_writer = csv.writer(open(XFORM_FILENAME, 'w+b')) self.xform_writer.writerow(XFORM_HEADER) self.case_writer = csv.writer(open(CASE_FILE_NAME, 'w+b')) self.case_writer.writerow(CASE_HEADER) self.forms_accessor = FormAccessors(self.domain) self.case_accessors = CaseAccessors(self.domain) def ensure_prerequisites(self, domain, app_id, version_number, test_run): NotAllowed.check(domain) self.domain = domain self.app_id = app_id self.version_number = version_number self.test_run = test_run == 'yes' _notify_parsed_args(domain, app_id, version_number, test_run) app = Application.get(self.app_id) if app.domain != self.domain: raise CommandError('Domain not same as from app id') self.setup() def handle(self, domain, app_id, version_number, test_run, **options): self.ensure_prerequisites(domain, app_id, version_number, test_run) self.xform_ids = self.forms_accessor.get_all_form_ids_in_domain() self.iterate_forms_and_collect_case_ids() _print_final_debug_info(self.xform_ids, self.filtered_xform_ids, self.case_ids) if self.data_to_delete() and self.delete_permitted(): self.delete_forms_and_cases() print("Process Completed!! Keep copy of files %s, %s" % (XFORM_FILENAME, CASE_FILE_NAME)) else: print('Process Finished w/o Changes..') def iterate_forms_and_collect_case_ids(self): print("Iterating Through %s XForms and Collecting Case Ids" % len(self.xform_ids)) for xform in self.forms_accessor.iter_forms(self.xform_ids): # Get app version by fetching app corresponding to xform build_id since xform.form # does not have updated app version unless form was updated for that version app_version_built_with = self.get_xform_build_version(xform) if app_version_built_with and app_version_built_with < self.version_number: _print_form_details(xform, self.xform_writer, app_version_built_with) self.ensure_valid_xform(xform) self.filtered_xform_ids.append(xform.form_id) self.case_ids = self.case_ids.union( get_case_ids_from_form(xform)) else: print('skipping xform id: %s' % xform.form_id) if self.case_ids: self.print_case_details() def get_xform_build_version(self, xform): version_from_mapping = None if xform.build_id: version_from_mapping = self.version_mapping.get( xform.build_id, None) if not version_from_mapping: try: get_app_version = get_app(self.domain, xform.build_id).version except Http404: get_app_version = None if get_app_version: version_from_mapping = int(get_app_version) self.version_mapping[xform.build_id] = version_from_mapping return version_from_mapping def ensure_valid_xform(self, xform): if xform.app_id != self.app_id and xform.domain != self.domain: _raise_xform_domain_mismatch(xform) def print_case_details(self): for case in self.case_accessors.iter_cases(self.case_ids): _print_case_details(case, self.case_writer) def delete_permitted(self): return not self.test_run and are_you_sure() def data_to_delete(self): return len(self.filtered_xform_ids) != 0 or len(self.case_ids) != 0 def delete_forms_and_cases(self): print('Proceeding with deleting forms and cases') self.forms_accessor.soft_delete_forms(self.filtered_xform_ids) self.case_accessors.soft_delete_cases(list(self.case_ids))