def __init__(self, instance=None, attachments=None, auth_context=None, domain=None, app_id=None, build_id=None, path=None, location=None, submit_ip=None, openrosa_headers=None, last_sync_token=None, received_on=None, date_header=None, partial_submission=False): assert domain, domain assert instance, instance assert not isinstance(instance, HttpRequest), instance self.domain = domain self.app_id = app_id self.build_id = build_id # get_location has good default self.location = location or couchforms.get_location() self.received_on = received_on self.date_header = date_header self.submit_ip = submit_ip self.last_sync_token = last_sync_token self.openrosa_headers = openrosa_headers or {} self.instance = instance self.attachments = attachments or {} self.auth_context = auth_context or DefaultAuthContext() self.path = path self.interface = FormProcessorInterface(domain) self.formdb = FormAccessors(domain) self.partial_submission = partial_submission
def get_form_ready_to_save(metadata, is_db_test=False): from corehq.form_processor.parsers.form import process_xform_xml from corehq.form_processor.utils import get_simple_form_xml, convert_xform_to_json from corehq.form_processor.interfaces.processor import FormProcessorInterface from corehq.form_processor.models import Attachment assert metadata is not None metadata.domain = metadata.domain or uuid.uuid4().hex form_id = uuid.uuid4().hex form_xml = get_simple_form_xml(form_id=form_id, metadata=metadata) if is_db_test: wrapped_form = process_xform_xml(metadata.domain, form_xml).submitted_form else: interface = FormProcessorInterface(domain=metadata.domain) form_json = convert_xform_to_json(form_xml) wrapped_form = interface.new_xform(form_json) wrapped_form.domain = metadata.domain interface.store_attachments(wrapped_form, [ Attachment( name='form.xml', raw_content=form_xml, content_type='text/xml') ]) wrapped_form.received_on = metadata.received_on wrapped_form.app_id = metadata.app_id return wrapped_form
def _handle_id_conflict(xform, domain): """ For id conflicts, we check if the files contain exactly the same content, If they do, we just log this as a dupe. If they don't, we deprecate the previous form and overwrite it with the new form's contents. :returns: A two-tuple: `(<new form>, <duplicate form or None>)` The new form may have a different `form_id` than `xform.form_id`. """ assert domain conflict_id = xform.form_id interface = FormProcessorInterface(domain) if interface.is_duplicate(conflict_id, domain): # It looks like a duplicate/edit in the same domain so pursue that workflow. logging.info('Handling duplicate doc id %s for domain %s', conflict_id, domain) return _handle_duplicate(xform) else: # the same form was submitted to two domains, or a form was submitted with # an ID that belonged to a different doc type. these are likely developers # manually testing or broken API users. just resubmit with a generated ID. interface.assign_new_id(xform) logging.info('Reassigned doc id from %s to %s', conflict_id, xform.form_id) return xform, None
def __init__(self, instance=None, attachments=None, auth_context=None, domain=None, app_id=None, build_id=None, path=None, location=None, submit_ip=None, openrosa_headers=None, last_sync_token=None, received_on=None, date_header=None, partial_submission=False, case_db=None, force_logs=False): assert domain, "'domain' is required" assert instance, instance assert not isinstance(instance, HttpRequest), instance self.domain = domain self.app_id = app_id self.build_id = build_id # get_location has good default self.location = location or couchforms.get_location() self.received_on = received_on self.date_header = date_header self.submit_ip = submit_ip self.last_sync_token = last_sync_token self.openrosa_headers = openrosa_headers or {} self.instance = instance self.attachments = attachments or {} self.auth_context = auth_context or DefaultAuthContext() self.path = path self.interface = FormProcessorInterface(domain) self.formdb = FormAccessors(domain) self.partial_submission = partial_submission # always None except in the case where a system form is being processed as part of another submission # e.g. for closing extension cases self.case_db = case_db if case_db: assert case_db.domain == domain self.force_logs = force_logs self.is_openrosa_version3 = self.openrosa_headers.get(OPENROSA_VERSION_HEADER, '') == OPENROSA_VERSION_3 self.track_load = form_load_counter("form_submission", domain)
def __init__(self, domain=None, strip_history=False, deleted_ok=False, lock=False, wrap=True, initial=None, xforms=None): self._populate_from_initial(initial) self.domain = domain self.cached_xforms = xforms if xforms is not None else [] self.strip_history = strip_history self.deleted_ok = deleted_ok self.lock = lock self.wrap = wrap if self.lock and not self.wrap: raise ValueError( 'Currently locking only supports explicitly wrapping cases!') self.locks = [] self._changed = set() # this is used to allow casedb to be re-entrant. Each new context pushes the parent context locks # onto this stack and restores them when the context exits self.lock_stack = [] self.processor_interface = FormProcessorInterface(self.domain)
def _create_form_and_sync_to_es(self): with self.process_form_changes: form = get_form_ready_to_save(self.metadata, is_db_test=True) form_processor = FormProcessorInterface(domain=self.domain) form_processor.save_processed_models([form]) self.elasticsearch.indices.refresh(XFORM_INDEX_INFO.index) return form, self.metadata
class CaseDbCacheCouchOnlyTest(TestCase): def setUp(self): super(CaseDbCacheCouchOnlyTest, self).setUp() self.interface = FormProcessorInterface() def testDocTypeCheck(self): id = uuid.uuid4().hex CommCareCase.get_db().save_doc({ "_id": id, "doc_type": "AintNoCasesHere" }) doc_back = CommCareCase.get_db().get(id) self.assertEqual("AintNoCasesHere", doc_back['doc_type']) cache = CaseDbCacheCouch() try: cache.get(id) self.fail('doc type security check failed to raise exception') except IllegalCaseId: pass def testStripHistory(self): case_ids = _make_some_cases(3) history_cache = self.interface.casedb_cache() for i, id in enumerate(case_ids): self.assertFalse(history_cache.in_cache(id)) case = history_cache.get(id) self.assertEqual(str(i), case.my_index) self.assertTrue(len(case.actions) > 0) nohistory_cache = self.interface.casedb_cache(strip_history=True) for i, id in enumerate(case_ids): self.assertFalse(nohistory_cache.in_cache(id)) case = nohistory_cache.get(id) self.assertEqual(str(i), case.my_index) self.assertTrue(len(case.actions) == 0) more_case_ids = _make_some_cases(3) history_cache.populate(more_case_ids) nohistory_cache.populate(more_case_ids) for i, id in enumerate(more_case_ids): self.assertTrue(history_cache.in_cache(id)) case = history_cache.get(id) self.assertEqual(str(i), case.my_index) self.assertTrue(len(case.actions) > 0) for i, id in enumerate(more_case_ids): self.assertTrue(nohistory_cache.in_cache(id)) case = nohistory_cache.get(id) self.assertEqual(str(i), case.my_index) self.assertTrue(len(case.actions) == 0) def test_nowrap(self): case_ids = _make_some_cases(1) cache = self.interface.casedb_cache(wrap=False) case = cache.get(case_ids[0]) self.assertTrue(isinstance(case, dict)) self.assertFalse(isinstance(case, CommCareCase))
def _perfom_post_save_actions(form, save=True): interface = FormProcessorInterface(form.domain) cache = interface.casedb_cache( domain=form.domain, lock=False, deleted_ok=True, xforms=[form], load_src="reprocess_form_post_save", ) with cache as casedb: case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb) case_models = case_stock_result.case_models if interface.use_sql_domain: forms = ProcessedForms(form, None) stock_result = case_stock_result.stock_result try: FormProcessorSQL.publish_changes_to_kafka(forms, case_models, stock_result) except Exception: error_message = "Error publishing to kafka" return ReprocessingResult(form, None, None, error_message) try: save and SubmissionPost.do_post_save_actions(casedb, [form], case_stock_result) except PostSaveError: error_message = "Error performing post save operations" return ReprocessingResult(form, None, None, error_message) return ReprocessingResult(form, case_models, None, None)
def safe_hard_delete(case): """ Hard delete a case - by deleting the case itself as well as all forms associated with it permanently from the database. Will fail hard if the case has any reverse indices or if any of the forms associated with the case also touch other cases. This is used primarily for cleaning up system cases/actions (e.g. the location delegate case). """ if not settings.UNIT_TESTING: from corehq.apps.commtrack.const import USER_LOCATION_OWNER_MAP_TYPE if not (case.is_deleted or case.type == USER_LOCATION_OWNER_MAP_TYPE): raise CommCareCaseError("Attempt to hard delete a live case whose type isn't white listed") if case.reverse_indices: raise CommCareCaseError("You can't hard delete a case that has other dependencies ({})!".format(case.case_id)) interface = FormProcessorInterface(case.domain) forms = interface.get_case_forms(case.case_id) for form in forms: case_updates = get_case_updates(form) if any([c.id != case.case_id for c in case_updates]): raise CommCareCaseError("You can't hard delete a case that has shared forms with other cases!") interface.hard_delete_case_and_forms(case, forms)
def _perfom_post_save_actions(form, save=True): interface = FormProcessorInterface(form.domain) cache = interface.casedb_cache( domain=form.domain, lock=False, deleted_ok=True, xforms=[form], load_src="reprocess_form_post_save", ) with cache as casedb: case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb) case_models = case_stock_result.case_models if interface.use_sql_domain: forms = ProcessedForms(form, None) stock_result = case_stock_result.stock_result try: FormProcessorSQL.publish_changes_to_kafka( forms, case_models, stock_result) except Exception: error_message = "Error publishing to kafka" return ReprocessingResult(form, None, None, error_message) try: save and SubmissionPost.do_post_save_actions( casedb, [form], case_stock_result) except PostSaveError: error_message = "Error performing post save operations" return ReprocessingResult(form, None, None, error_message) return ReprocessingResult(form, case_models, None, None)
def _handle_duplicate(new_doc, instance): """ Handle duplicate xforms and xform editing ('deprecation') existing doc *must* be validated as an XFormInstance in the right domain and *must* include inline attachments """ interface = FormProcessorInterface(new_doc.domain) conflict_id = new_doc.form_id existing_doc = FormAccessors(new_doc.domain).get_with_attachments(conflict_id) existing_md5 = existing_doc.xml_md5() new_md5 = hashlib.md5(instance).hexdigest() if existing_md5 != new_md5: # if the form contents are not the same: # - "Deprecate" the old form by making a new document with the same contents # but a different ID and a doc_type of XFormDeprecated # - Save the new instance to the previous document to preserve the ID existing_doc, new_doc = apply_deprecation(existing_doc, new_doc, interface) return FormProcessingResult(new_doc, existing_doc) else: # follow standard dupe handling, which simply saves a copy of the form # but a new doc_id, and a doc_type of XFormDuplicate duplicate = interface.deduplicate_xform(new_doc) return FormProcessingResult(duplicate)
def _handle_duplicate(new_doc): """ Handle duplicate xforms and xform editing ('deprecation') existing doc *must* be validated as an XFormInstance in the right domain and *must* include inline attachments """ interface = FormProcessorInterface(new_doc.domain) conflict_id = new_doc.form_id existing_doc = FormAccessors( new_doc.domain).get_with_attachments(conflict_id) existing_md5 = existing_doc.xml_md5() new_md5 = new_doc.xml_md5() if existing_md5 != new_md5: # if the form contents are not the same: # - "Deprecate" the old form by making a new document with the same contents # but a different ID and a doc_type of XFormDeprecated # - Save the new instance to the previous document to preserve the ID existing_doc, new_doc = apply_deprecation(existing_doc, new_doc, interface) return FormProcessingResult(new_doc, existing_doc) else: # follow standard dupe handling, which simply saves a copy of the form # but a new doc_id, and a doc_type of XFormDuplicate duplicate = interface.deduplicate_xform(new_doc) return FormProcessingResult(duplicate)
def test_xform_pillow_sql(self): consumer = get_test_kafka_consumer(topics.FORM_SQL) # have to get the seq id before the change is processed kafka_seq = consumer.offsets()['fetch'][(topics.FORM_SQL, 0)] metadata = TestFormMetadata(domain=self.domain) form = get_form_ready_to_save(metadata, is_db_test=True) form_processor = FormProcessorInterface(domain=self.domain) form_processor.save_processed_models([form]) # confirm change made it to kafka message = consumer.next() change_meta = change_meta_from_kafka_message(message.value) self.assertEqual(form.form_id, change_meta.document_id) self.assertEqual(self.domain, change_meta.domain) # send to elasticsearch sql_pillow = get_sql_xform_to_elasticsearch_pillow() sql_pillow.process_changes(since=kafka_seq, forever=False) self.elasticsearch.indices.refresh(self.pillow.es_index) # confirm change made it to elasticserach results = FormES().run() self.assertEqual(1, results.total) form_doc = results.hits[0] self.assertEqual(self.domain, form_doc['domain']) self.assertEqual(metadata.xmlns, form_doc['xmlns']) self.assertEqual('XFormInstance', form_doc['doc_type'])
def get_related_cases(initial_cases, domain, strip_history=False, search_up=True): """ Gets the flat list of related cases based on a starting list. Walks all the referenced indexes recursively. If search_up is True, all cases and their parent cases are returned. If search_up is False, all cases and their child cases are returned. """ if not initial_cases: return {} # infer whether to wrap or not based on whether the initial list is wrapped or not # initial_cases may be a list or a set wrap = isinstance(next(iter(initial_cases)), CommCareCase) # todo: should assert that domain exists here but this breaks tests case_db = FormProcessorInterface(domain).casedb_cache( domain=domain, strip_history=strip_history, deleted_ok=True, wrap=wrap, initial=initial_cases ) def indices(case): return case['indices'] if search_up else get_reverse_indices_json(domain, case['_id']) relevant_cases = {} relevant_deleted_case_ids = [] cases_to_process = list(case for case in initial_cases) directly_referenced_indices = itertools.chain( *[[index['referenced_id'] for index in indices(case)] for case in initial_cases] ) case_db.populate(directly_referenced_indices) def process_cases(cases): new_relations = set() for case in cases: if case and case['_id'] not in relevant_cases: relevant_cases[case['_id']] = case if case['doc_type'] == 'CommCareCase-Deleted': relevant_deleted_case_ids.append(case['_id']) new_relations.update(index['referenced_id'] for index in indices(case)) if new_relations: case_db.populate(new_relations) return [case_db.get(related_case) for related_case in new_relations] while cases_to_process: cases_to_process = process_cases(cases_to_process) if relevant_deleted_case_ids: logging.info('deleted cases included in footprint (restore): %s' % ( ', '.join(relevant_deleted_case_ids) )) return relevant_cases
def create_form_and_sync_to_es(received_on): with process_kafka_changes('XFormToElasticsearchPillow'): with process_couch_changes('DefaultChangeFeedPillow'): metadata = TestFormMetadata(domain=cls.domain, app_id=cls.app_id, xmlns=cls.xmlns, received_on=received_on) form = get_form_ready_to_save(metadata, is_db_test=True) form_processor = FormProcessorInterface(domain=cls.domain) form_processor.save_processed_models([form]) return form
def _handle_duplicate(new_doc): """ Handle duplicate xforms and xform editing ('deprecation') existing doc *must* be validated as an XFormInstance in the right domain and *must* include inline attachments :returns: A two-tuple: `(<new form>, <duplicate form or None>)` The new form may have a different `form_id` than `new_doc.form_id`. """ interface = FormProcessorInterface(new_doc.domain) conflict_id = new_doc.form_id try: existing_doc = FormAccessors( new_doc.domain).get_with_attachments(conflict_id) except ResourceNotFound: # Original form processing failed but left behind a form doc with no # attachments. It's safe to delete this now since we're going to re-process # the form anyway. from couchforms.models import XFormInstance XFormInstance.get_db().delete_doc(conflict_id) return new_doc, None existing_md5 = existing_doc.xml_md5() new_md5 = new_doc.xml_md5() if existing_md5 != new_md5: _soft_assert = soft_assert(to='{}@{}.com'.format('skelly', 'dimagi'), exponential_backoff=False) if new_doc.xmlns != existing_doc.xmlns: # if the XMLNS has changed this probably isn't a form edit # it could be a UUID clash (yes we've had that before) # Assign a new ID to the form and process as normal + notify_admins xform = interface.assign_new_id(new_doc) _soft_assert( False, "Potential UUID clash", { 'incoming_form_id': conflict_id, 'existing_form_id': existing_doc.form_id, 'new_form_id': xform.form_id, 'incoming_xmlns': new_doc.xmlns, 'existing_xmlns': existing_doc.xmlns, 'domain': new_doc.domain, }) return xform, None else: # if the form contents are not the same: # - "Deprecate" the old form by making a new document with the same contents # but a different ID and a doc_type of XFormDeprecated # - Save the new instance to the previous document to preserve the ID existing_doc, new_doc = apply_deprecation(existing_doc, new_doc, interface) return new_doc, existing_doc else: # follow standard dupe handling, which simply saves a copy of the form # but a new doc_id, and a doc_type of XFormDuplicate duplicate = interface.deduplicate_xform(new_doc) return duplicate, existing_doc
def _create_form_and_sync_to_es(self): with process_pillow_changes('xform-pillow', {'skip_ucr': True}): with process_pillow_changes('DefaultChangeFeedPillow'): metadata = TestFormMetadata(domain=self.domain) form = get_form_ready_to_save(metadata, is_db_test=True) form_processor = FormProcessorInterface(domain=self.domain) form_processor.save_processed_models([form]) self.elasticsearch.indices.refresh(XFORM_INDEX_INFO.index) return form, metadata
def create_form_and_sync_to_es(received_on): with process_pillow_changes('xform-pillow', {'skip_ucr': True}): with process_pillow_changes('DefaultChangeFeedPillow'): metadata = TestFormMetadata(domain=cls.domain, app_id=cls.app_id, xmlns=cls.xmlns, received_on=received_on) form = get_form_ready_to_save(metadata, is_db_test=True) form_processor = FormProcessorInterface(domain=cls.domain) form_processor.save_processed_models([form]) return form
def test_sync_log_invalidation_bug(self): sync_log = FormProcessorInterface().sync_log_model(user_id="6dac4940-913e-11e0-9d4b-005056aa7fb5") sync_log.save() self.addCleanup(FormProcessorTestUtils.delete_all_sync_logs) _, case = self._doCreateCaseWithMultimedia() # this used to fail before we fixed http://manage.dimagi.com/default.asp?158373 self._doSubmitUpdateWithMultimedia(new_attachments=["commcare_logo_file"], removes=[], sync_token=sync_log._id)
def process_stock(xforms, case_db=None): """ process the commtrack xml constructs in an incoming submission """ if not case_db: case_db = FormProcessorInterface(xforms[0].domain).casedb_cache() else: assert isinstance(case_db, AbstractCaseDbCache) sorted_forms = sorted(xforms, key=lambda f: 0 if f.is_deprecated else 1) stock_report_helpers = [] case_action_intents = [] for xform in sorted_forms: actions_for_form = get_stock_actions(xform) stock_report_helpers += actions_for_form.stock_report_helpers case_action_intents += actions_for_form.case_action_intents # omitted: normalize_transactions (used for bulk requisitions?) # validate the parsed transactions for stock_report_helper in stock_report_helpers: stock_report_helper.validate() relevant_cases = [] # touch every case for proper ota restore logic syncing to be preserved for action_intent in case_action_intents: case_id = action_intent.case_id case = case_db.get(action_intent.case_id) relevant_cases.append(case) if case is None: raise IllegalCaseId( _('Ledger transaction references invalid Case ID "{}"') .format(case_id)) if action_intent.is_deprecation: # just remove the old stock actions for the form from the case case.actions = [ a for a in case.actions if not (a.xform_id == action_intent.form_id and a.action_type == CASE_ACTION_COMMTRACK) ] else: case_action = action_intent.action # hack: clear the sync log id so this modification always counts # since consumption data could change server-side case_action.sync_log_id = '' case.actions.append(case_action) if action_intent.form_id not in case.xform_ids: case.xform_ids.append(action_intent.form_id) case_db.mark_changed(case) return StockProcessingResult( xform=sorted_forms[-1], relevant_cases=relevant_cases, stock_report_helpers=stock_report_helpers, )
class CaseDbCacheTest(TestCase): """ Tests the functionality of the CaseDbCache object """ def setUp(self): super(CaseDbCacheTest, self).setUp() self.interface = FormProcessorInterface() @run_with_all_backends def testDomainCheck(self): id = uuid.uuid4().hex post_case_blocks( [CaseBlock(create=True, case_id=id, user_id='some-user').as_xml()], {'domain': 'good-domain'}) bad_cache = self.interface.casedb_cache(domain='bad-domain') try: bad_cache.get(id) self.fail('domain security check failed to raise exception') except IllegalCaseId: pass good_cache = self.interface.casedb_cache(domain='good-domain') case = good_cache.get(id) self.assertEqual( 'some-user', case.user_id) # just sanity check it's the right thing def testDocTypeCheck(self): id = uuid.uuid4().hex CommCareCase.get_db().save_doc({ "_id": id, "doc_type": "AintNoCasesHere" }) doc_back = CommCareCase.get_db().get(id) self.assertEqual("AintNoCasesHere", doc_back['doc_type']) cache = CaseDbCacheCouch() try: cache.get(id) self.fail('doc type security check failed to raise exception') except IllegalCaseId: pass @run_with_all_backends def testGetPopulatesCache(self): case_ids = _make_some_cases(3) cache = self.interface.casedb_cache() for id in case_ids: self.assertFalse(cache.in_cache(id)) for i, id in enumerate(case_ids): case = cache.get(id) self.assertEqual(str(i), case.dynamic_case_properties()['my_index']) for id in case_ids: self.assertTrue(cache.in_cache(id))
def acquire_lock_for_xform(xform_id): from corehq.form_processor.interfaces.processor import FormProcessorInterface # this is high, but I want to test if MVP conflicts disappear lock = FormProcessorInterface().xform_model.get_obj_lock_by_id(xform_id, timeout_seconds=2 * 60) try: lock.acquire() except RedisError: lock = None return lock
def _save_migrated_models(domain, sql_form, case_stock_result): """ See SubmissionPost.save_processed_models for ~what this should do. However, note that that function does some things that this one shouldn't, e.g. process ownership cleanliness flags. """ interface = FormProcessorInterface(domain) interface.save_processed_models([sql_form], case_stock_result.case_models, case_stock_result.stock_result) case_stock_result.case_result.close_extensions()
def test_cant_own_case(self): interface = FormProcessorInterface() _, _, [case] = interface.submit_form_locally(ALICE_XML, ALICE_DOMAIN) response, form, cases = interface.submit_form_locally(EVE_XML, EVE_DOMAIN) self.assertIn('IllegalCaseId', response.content) self.assertFalse(hasattr(case, 'plan_to_buy_gun')) _, _, [case] = interface.submit_form_locally(ALICE_UPDATE_XML, ALICE_DOMAIN) self.assertEqual(case.plan_to_buy_gun, 'no')
def test_xform_locked(self): from corehq.form_processor.interfaces.processor import FormProcessorInterface form_id = 'ad38211be256653bceac8e2156475664' proc = FormProcessorInterface(self.domain) lock = proc.acquire_lock_for_xform(form_id) try: _, response = self._submit('simple_form.xml') finally: lock.release() self.assertEqual(response.status_code, 423)
def get_models_to_save(self): processor = FormProcessorInterface(domain=self.domain).ledger_processor ledger_db = LedgerDB(processor=processor) update_results = [] for stock_report_helper in self.stock_report_helpers: this_result = processor.get_models_to_update(stock_report_helper, ledger_db) if this_result: update_results.append(this_result) return update_results
def rows(self): data = SMS.by_domain(self.domain, start_date=self.datespan.startdate_utc, end_date=self.datespan.enddate_utc).exclude( direction=OUTGOING, processed=False).order_by('date') if self.show_only_survey_traffic(): data = data.filter(xforms_session_couch_id__isnull=False) result = [] direction_map = { INCOMING: _("Incoming"), OUTGOING: _("Outgoing"), } message_bank_messages = get_message_bank(self.domain, for_comparing=True) case_cache = FormProcessorInterface(self.domain).casedb_cache( domain=self.domain, strip_history=False, deleted_ok=True) user_cache = UserCache() for message in data: # Add metadata from the message bank if it has not been added already if (message.direction == OUTGOING) and ( not message.fri_message_bank_lookup_completed): add_metadata(message, message_bank_messages) if message.couch_recipient_doc_type == "CommCareCase": recipient = case_cache.get(message.couch_recipient) else: recipient = user_cache.get(message.couch_recipient) if message.chat_user_id: sender = user_cache.get(message.chat_user_id) else: sender = None study_arm = None if message.couch_recipient_doc_type == "CommCareCase": study_arm = case_cache.get( message.couch_recipient).get_case_property("study_arm") timestamp = ServerTime(message.date).user_time( self.timezone).done() result.append([ self._fmt(self._participant_id(recipient)), self._fmt(study_arm or "-"), self._fmt(self._originator(message, recipient, sender)), self._fmt_timestamp(timestamp), self._fmt(message.text), self._fmt(message.fri_id or "-"), self._fmt(direction_map.get(message.direction, "-")), ]) return result
def __init__(self, submitted_form, existing_duplicate=None): self.submitted_form = submitted_form self.existing_duplicate = existing_duplicate if submitted_form.is_duplicate: assert existing_duplicate is None if existing_duplicate: assert existing_duplicate.is_deprecated self.interface = FormProcessorInterface(self.submitted_form.domain)
def test_sync_log_invalidation_bug(self): sync_log = FormProcessorInterface().sync_log_model( user_id='6dac4940-913e-11e0-9d4b-005056aa7fb5' ) sync_log.save() self.addCleanup(FormProcessorTestUtils.delete_all_sync_logs) _, case = self._doCreateCaseWithMultimedia() # this used to fail before we fixed http://manage.dimagi.com/default.asp?158373 self._doSubmitUpdateWithMultimedia(new_attachments=['commcare_logo_file'], removes=[], sync_token=sync_log._id)
def testOutOfOrderSubmissions(self): dir = os.path.join(os.path.dirname(__file__), "data", "ordering") interface = FormProcessorInterface() for fname in ('update_oo.xml', 'create_oo.xml'): with open(os.path.join(dir, fname), "rb") as f: xml_data = f.read() interface.submit_form_locally(xml_data) case = interface.case_model.get('30bc51f6-3247-4966-b4ae-994f572e85fe') self.assertEqual('from the update form', case.pupdate) self.assertEqual('from the create form', case.pcreate) self.assertEqual('overridden by the update form', case.pboth)
def _get_or_update_cases(xforms, case_db): """ Given an xform document, update any case blocks found within it, returning a dictionary mapping the case ids affected to the couch case document objects """ domain = getattr(case_db, 'domain', None) touched_cases = FormProcessorInterface(domain).get_cases_from_forms(case_db, xforms) _validate_indices(case_db, touched_cases.values()) return CaseProcessingResult( domain, [update.case for update in touched_cases.values()], )
def _perfom_post_save_actions(form, save=True): interface = FormProcessorInterface(form.domain) cache = interface.casedb_cache( domain=form.domain, lock=False, deleted_ok=True, xforms=[form] ) with cache as casedb: case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb) try: save and SubmissionPost.do_post_save_actions(casedb, [form], case_stock_result) except PostSaveError: error_message = "Error performing post save operations" return ReprocessingResult(form, None, None, error_message) return ReprocessingResult(form, case_stock_result.case_models, None, None)
def _get_submission_error(domain, instance_xml, error, auth_context): """ Handle's a hard failure from posting a form to couch. :returns: xform error instance with raw xml as attachment """ try: message = str(error) except UnicodeDecodeError: message = str(str(error), encoding='utf-8') xform = FormProcessorInterface(domain).submission_error_form_instance(instance_xml, message) xform.auth_context = auth_context return FormProcessingResult(xform)
def setUp(self): super(RebuildStockStateTest, self).setUp() self.domain = 'asldkjf-domain' self.case = CaseFactory(domain=self.domain).create_case() self.product = make_product(self.domain, 'Product Name', 'prodcode') self._stock_state_key = dict(section_id='stock', case_id=self.case.case_id, product_id=self.product.get_id) self.unique_reference = UniqueLedgerReference( case_id=self.case.case_id, section_id='stock', entry_id=self.product.get_id) self.ledger_processor = FormProcessorInterface( self.domain).ledger_processor
def process_stock(xforms, case_db=None): """ process the commtrack xml constructs in an incoming submission """ if not case_db: case_db = FormProcessorInterface(xforms[0].domain).casedb_cache( domain=xforms[0].domain, load_src="process_stock", ) else: assert isinstance(case_db, AbstractCaseDbCache) stock_report_helpers = [] case_action_intents = [] sorted_forms = sorted(xforms, key=lambda f: 0 if f.is_deprecated else 1) for xform in sorted_forms: try: actions_for_form = get_stock_actions(xform) stock_report_helpers += actions_for_form.stock_report_helpers case_action_intents += actions_for_form.case_action_intents except MissingFormXml: if not xform.is_deprecated: raise # validate the parsed transactions for stock_report_helper in stock_report_helpers: stock_report_helper.validate() relevant_cases = mark_cases_changed(case_action_intents, case_db) return StockProcessingResult( xform=sorted_forms[-1], relevant_cases=relevant_cases, stock_report_helpers=stock_report_helpers, )
def populate_models(self): self.populated = True interface = FormProcessorInterface(domain=self.domain) processor = interface.ledger_processor ledger_db = interface.ledger_db for helper in self.stock_report_helpers: assert helper.domain == self.domain normal_helpers = [ srh for srh in self.stock_report_helpers if not srh.deprecated ] deprecated_helpers = [ srh for srh in self.stock_report_helpers if srh.deprecated ] models_result = processor.get_models_to_update(self.xform.form_id, normal_helpers, deprecated_helpers, ledger_db) self.models_to_save, self.models_to_delete = models_result self.cases_with_deprecated_transactions = { trans.case_id for srh in deprecated_helpers for trans in srh.transactions }
class CaseDbCacheCouchOnlyTest(TestCase): def setUp(self): super(CaseDbCacheCouchOnlyTest, self).setUp() self.interface = FormProcessorInterface() def testDocTypeCheck(self): id = uuid.uuid4().hex CommCareCase.get_db().save_doc({ "_id": id, "doc_type": "AintNoCasesHere" }) doc_back = CommCareCase.get_db().get(id) self.assertEqual("AintNoCasesHere", doc_back['doc_type']) cache = CaseDbCacheCouch() try: cache.get(id) self.fail('doc type security check failed to raise exception') except IllegalCaseId: pass def test_nowrap(self): case_ids = _make_some_cases(1) cache = self.interface.casedb_cache(wrap=False) case = cache.get(case_ids[0]) self.assertTrue(isinstance(case, dict)) self.assertFalse(isinstance(case, CommCareCase))
def test_unknown_user_pillow(self): FormProcessorTestUtils.delete_all_xforms() user_id = 'test-unknown-user' metadata = TestFormMetadata(domain=TEST_DOMAIN, user_id='test-unknown-user') form = get_form_ready_to_save(metadata) FormProcessorInterface(domain=TEST_DOMAIN).save_processed_models( [form]) # send to kafka topic = topics.FORM_SQL if settings.TESTS_SHOULD_USE_SQL_BACKEND else topics.FORM since = self._get_kafka_seq() producer.send_change(topic, _form_to_change_meta(form)) # send to elasticsearch pillow = get_xform_pillow() pillow.process_changes(since=since, forever=False) self.elasticsearch.indices.refresh(self.index_info.index) # the default query doesn't include unknown users so should have no results self.assertEqual(0, UserES().run().total) # clear the default filters which hide unknown users user_es = UserES().remove_default_filters() results = user_es.run() self.assertEqual(1, results.total) user_doc = results.hits[0] self.assertEqual(TEST_DOMAIN, user_doc['domain']) self.assertEqual(user_id, user_doc['_id']) self.assertEqual('UnknownUser', user_doc['doc_type'])
def __init__(self, instance=None, attachments=None, auth_context=None, domain=None, app_id=None, build_id=None, path=None, location=None, submit_ip=None, openrosa_headers=None, last_sync_token=None, received_on=None, date_header=None, partial_submission=False, case_db=None): assert domain, "'domain' is required" assert instance, instance assert not isinstance(instance, HttpRequest), instance self.domain = domain self.app_id = app_id self.build_id = build_id # get_location has good default self.location = location or couchforms.get_location() self.received_on = received_on self.date_header = date_header self.submit_ip = submit_ip self.last_sync_token = last_sync_token self.openrosa_headers = openrosa_headers or {} self.instance = instance self.attachments = attachments or {} self.auth_context = auth_context or DefaultAuthContext() self.path = path self.interface = FormProcessorInterface(domain) self.formdb = FormAccessors(domain) self.partial_submission = partial_submission # always None except in the case where a system form is being processed as part of another submission # e.g. for closing extension cases self.case_db = case_db if case_db: assert case_db.domain == domain self.is_openrosa_version3 = self.openrosa_headers.get(OPENROSA_VERSION_HEADER, '') == OPENROSA_VERSION_3 self.track_load = form_load_counter("form_submission", domain)
def setUp(self): super(StockReportDomainTest, self).setUp() self.domain = _get_name_for_domain() self.ledger_processor = FormProcessorInterface( domain=self.domain).ledger_processor create_domain(self.domain) transactions_flat = [] self.transactions = {} for case, c_bal in self.case_ids.items(): for section, s_bal in self.section_ids.items(): for product, p_bal in self.product_ids.items(): bal = c_bal + s_bal + p_bal transactions_flat.append( StockTransactionHelper(case_id=case, section_id=section, product_id=product, action='soh', quantity=bal, timestamp=datetime.utcnow())) self.transactions.setdefault(case, {}).setdefault( section, {})[product] = bal self.new_stock_report, self.form = self.create_report( transactions_flat) self._create_models_for_stock_report_helper(self.form, self.new_stock_report)
def populate_models(self): self.populated = True interface = FormProcessorInterface(domain=self.domain) processor = interface.ledger_processor ledger_db = interface.ledger_db track_load = ledger_load_counter("process_stock", self.domain) normal_helpers = [] deprecated_helpers = [] for helper in self.stock_report_helpers: assert helper.domain == self.domain if not helper.deprecated: normal_helpers.append(helper) else: deprecated_helpers.append(helper) track_load(len(helper.transactions)) models_result = processor.get_models_to_update( self.xform.form_id, normal_helpers, deprecated_helpers, ledger_db ) self.models_to_save, self.models_to_delete = models_result self.cases_with_deprecated_transactions = { trans.case_id for srh in deprecated_helpers for trans in srh.transactions }
def setUp(self): super(CaseObjectCacheTest, self).setUp() self.domain = Domain.get_or_create_with_name(TEST_DOMAIN_NAME, is_active=True) self.user = WebUser.create(TEST_DOMAIN_NAME, TEST_USER, TEST_PASSWORD, None, None) self.user.set_role(self.domain.name, 'admin') self.user.save() self.interface = FormProcessorInterface(TEST_DOMAIN_NAME)
class FormProcessingResult(object): def __init__(self, submitted_form, existing_duplicate=None): self.submitted_form = submitted_form self.existing_duplicate = existing_duplicate if submitted_form.is_duplicate: assert existing_duplicate is None if existing_duplicate: assert existing_duplicate.is_deprecated self.interface = FormProcessorInterface(self.submitted_form.domain) def _get_form_lock(self, form_id): return self.interface.acquire_lock_for_xform(form_id) def get_locked_forms(self): if self.existing_duplicate: # Lock docs with their original ID's (before they got switched during deprecation) new_id = self.existing_duplicate.form_id old_id = self.existing_duplicate.orig_id return MultiLockManager([ LockManager(self.submitted_form, self._get_form_lock(new_id)), LockManager(self.existing_duplicate, self._get_form_lock(old_id)), ]) else: return MultiLockManager([ LockManager(self.submitted_form, self._get_form_lock(self.submitted_form.form_id)) ])
def _migrate_form(domain, couch_form): """ This copies the couch form into a new sql form but does not save it. See form_processor.parsers.form._create_new_xform and SubmissionPost._set_submission_properties for what this should do. """ interface = FormProcessorInterface(domain) form_data = couch_form.form with force_phone_timezones_should_be_processed(): adjust_datetimes(form_data) sql_form = interface.new_xform(form_data) sql_form.form_id = couch_form.form_id # some legacy forms don't have ID's so are assigned random ones if sql_form.xmlns is None: sql_form.xmlns = '' return _copy_form_properties(domain, sql_form, couch_form)
def _get_or_update_cases(xforms, case_db): """ Given an xform document, update any case blocks found within it, returning a dictionary mapping the case ids affected to the couch case document objects """ domain = getattr(case_db, 'domain', None) touched_cases = FormProcessorInterface(domain).get_cases_from_forms(case_db, xforms) _validate_indices(case_db, [case_update_meta.case for case_update_meta in touched_cases.values()]) dirtiness_flags = _get_all_dirtiness_flags_from_cases(case_db, touched_cases) extensions_to_close = get_all_extensions_to_close(domain, touched_cases.values()) return CaseProcessingResult( domain, [update.case for update in touched_cases.values()], dirtiness_flags, extensions_to_close )
def get_form_ready_to_save(metadata, is_db_test=False): from corehq.form_processor.parsers.form import process_xform_xml from corehq.form_processor.utils import get_simple_form_xml, convert_xform_to_json from corehq.form_processor.interfaces.processor import FormProcessorInterface assert metadata is not None metadata.domain = metadata.domain or uuid.uuid4().hex form_id = uuid.uuid4().hex form_xml = get_simple_form_xml(form_id=form_id, metadata=metadata) if is_db_test: wrapped_form = process_xform_xml(metadata.domain, form_xml).submitted_form else: form_json = convert_xform_to_json(form_xml) wrapped_form = FormProcessorInterface(domain=metadata.domain).new_xform(form_json) wrapped_form.domain = metadata.domain wrapped_form.received_on = metadata.received_on return wrapped_form
def test_various_encodings(self): tests = ( ('utf-8', u'हिन्दी चट्टानों'), ('UTF-8', u'हिन्दी चट्टानों'), ('ASCII', 'hello'), ) file_path = os.path.join(os.path.dirname(__file__), "data", "encoding.xml") with open(file_path, "rb") as f: xml_template = f.read() for encoding, value in tests: xml_data = xml_template.format( encoding=encoding, form_id=uuid.uuid4().hex, sample_value=value.encode(encoding), ) xform = FormProcessorInterface().post_xform(xml_data) self.assertEqual(value, xform.form_data['test']) elem = xform.get_xml_element() self.assertEqual(value, elem.find('{http://commcarehq.org/couchforms-tests}test').text)
def _handle_id_conflict(instance, xform, domain): """ For id conflicts, we check if the files contain exactly the same content, If they do, we just log this as a dupe. If they don't, we deprecate the previous form and overwrite it with the new form's contents. """ assert domain conflict_id = xform.form_id interface = FormProcessorInterface(domain) if interface.is_duplicate(conflict_id, domain): # It looks like a duplicate/edit in the same domain so pursue that workflow. return _handle_duplicate(xform, instance) else: # the same form was submitted to two domains, or a form was submitted with # an ID that belonged to a different doc type. these are likely developers # manually testing or broken API users. just resubmit with a generated ID. xform = interface.assign_new_id(xform) return FormProcessingResult(xform)
def _create_new_xform(domain, instance_xml, attachments=None): """ create but do not save an XFormInstance from an xform payload (xml_string) optionally set the doc _id to a predefined value (_id) return doc _id of the created doc `process` is transformation to apply to the form right before saving This is to avoid having to save multiple times If xml_string is bad xml - raise couchforms.XMLSyntaxError :param domain: """ from corehq.form_processor.interfaces.processor import FormProcessorInterface interface = FormProcessorInterface(domain) assert attachments is not None form_data = convert_xform_to_json(instance_xml) if not form_data.get('@xmlns'): raise MissingXMLNSError("Form is missing a required field: XMLNS") adjust_datetimes(form_data) xform = interface.new_xform(form_data) xform.domain = domain # Maps all attachments to uniform format and adds form.xml to list before storing attachments = map( lambda a: Attachment(name=a[0], raw_content=a[1], content_type=a[1].content_type), attachments.items() ) attachments.append(Attachment(name='form.xml', raw_content=instance_xml, content_type='text/xml')) interface.store_attachments(xform, attachments) result = LockedFormProcessingResult(xform) with ReleaseOnError(result.lock): if interface.is_duplicate(xform.form_id): raise DuplicateError(xform) return result
def _get_case_and_ledger_updates(domain, sql_form): """ Get a CaseStockProcessingResult with the appropriate cases and ledgers to be saved. See SubmissionPost.process_xforms_for_cases and methods it calls for the equivalent section of the form-processing code. """ from corehq.apps.commtrack.processing import process_stock interface = FormProcessorInterface(domain) assert sql_form.domain xforms = [sql_form] with interface.casedb_cache( domain=domain, lock=False, deleted_ok=True, xforms=xforms, load_src="couchsqlmigration", ) as case_db: touched_cases = FormProcessorInterface(domain).get_cases_from_forms(case_db, xforms) extensions_to_close = get_all_extensions_to_close(domain, list(touched_cases.values())) case_result = CaseProcessingResult( domain, [update.case for update in touched_cases.values()], [], # ignore dirtiness_flags, extensions_to_close ) for case in case_result.cases: case_db.post_process_case(case, sql_form) case_db.mark_changed(case) stock_result = process_stock(xforms, case_db) cases = case_db.get_cases_for_saving(sql_form.received_on) stock_result.populate_models() return CaseStockProcessingResult( case_result=case_result, case_models=cases, stock_result=stock_result, )
class MultiCaseTest(TestCase): def setUp(self): self.domain = "gigglyfoo" self.interface = FormProcessorInterface() delete_all_xforms() delete_all_cases() def testParallel(self): file_path = os.path.join(os.path.dirname(__file__), "data", "multicase", "parallel_cases.xml") with open(file_path, "rb") as f: xml_data = f.read() _, form, cases = self.interface.submit_form_locally(xml_data, domain=self.domain) self.assertEqual(4, len(cases)) self._check_ids(form, cases) def testMixed(self): file_path = os.path.join(os.path.dirname(__file__), "data", "multicase", "mixed_cases.xml") with open(file_path, "rb") as f: xml_data = f.read() _, form, cases = self.interface.submit_form_locally(xml_data, domain=self.domain) self.assertEqual(4, len(cases)) self._check_ids(form, cases) def testCasesInRepeats(self): file_path = os.path.join(os.path.dirname(__file__), "data", "multicase", "case_in_repeats.xml") with open(file_path, "rb") as f: xml_data = f.read() _, form, cases = self.interface.submit_form_locally(xml_data, domain=self.domain) self.assertEqual(3, len(cases)) self._check_ids(form, cases) def _check_ids(self, form, cases): for case in cases: ids = self.interface.case_model.get_case_xform_ids(case.case_id) self.assertEqual(1, len(ids)) self.assertEqual(form._id, ids[0])
def setUp(self): super(RebuildStockStateTest, self).setUp() self.domain = 'asldkjf-domain' self.case = CaseFactory(domain=self.domain).create_case() self.product = make_product(self.domain, 'Product Name', 'prodcode') self._stock_state_key = dict( section_id='stock', case_id=self.case.case_id, product_id=self.product.get_id ) self.unique_reference = UniqueLedgerReference( case_id=self.case.case_id, section_id='stock', entry_id=self.product.get_id ) self.ledger_processor = FormProcessorInterface(self.domain).ledger_processor
def get_form_ready_to_save(metadata, is_db_test=False): from corehq.form_processor.parsers.form import process_xform_xml from corehq.form_processor.utils import get_simple_form_xml, convert_xform_to_json from corehq.form_processor.interfaces.processor import FormProcessorInterface from corehq.form_processor.models import Attachment assert metadata is not None metadata.domain = metadata.domain or uuid.uuid4().hex form_id = uuid.uuid4().hex form_xml = get_simple_form_xml(form_id=form_id, metadata=metadata) if is_db_test: wrapped_form = process_xform_xml(metadata.domain, form_xml).submitted_form else: interface = FormProcessorInterface(domain=metadata.domain) form_json = convert_xform_to_json(form_xml) wrapped_form = interface.new_xform(form_json) wrapped_form.domain = metadata.domain interface.store_attachments(wrapped_form, [ Attachment(name='form.xml', raw_content=form_xml, content_type='text/xml') ]) wrapped_form.received_on = metadata.received_on wrapped_form.app_id = metadata.app_id return wrapped_form