def _send_case_to_es(self, domain=None, owner_id=None, user_id=None, case_type=None, opened_on=None, closed_on=None): case = CommCareCaseSQL( case_id=uuid.uuid4().hex, domain=domain or self.domain, owner_id=owner_id or self.owner_id, modified_by=user_id or self.user_id, type=case_type or self.case_type, opened_on=opened_on or datetime.now(), opened_by=user_id or self.user_id, closed_on=closed_on, closed_by=user_id or self.user_id, server_modified_on=datetime.utcnow(), closed=bool(closed_on) ) case.track_create(CaseTransaction( type=CaseTransaction.TYPE_FORM, form_id=uuid.uuid4().hex, case=case, server_date=opened_on, )) es_case = transform_case_for_elasticsearch(case.to_json()) send_to_elasticsearch('cases', es_case) self.es.indices.refresh(CASE_INDEX) return case
def hard_rebuild_case(domain, case_id, detail, lock=True): if lock: # only record metric if locking since otherwise it has been # (most likley) recorded elsewhere case_load_counter("rebuild_case", domain)() case, lock_obj = FormProcessorSQL.get_case_with_lock(case_id, lock=lock) found = bool(case) if not found: case = CommCareCaseSQL(case_id=case_id, domain=domain) if lock: lock_obj = CommCareCaseSQL.get_obj_lock_by_id(case_id) acquire_lock(lock_obj, degrade_gracefully=False) try: assert case.domain == domain, (case.domain, domain) case, rebuild_transaction = FormProcessorSQL._rebuild_case_from_transactions(case, detail) if case.is_deleted and not case.is_saved(): return None case.server_modified_on = rebuild_transaction.server_date CaseAccessorSQL.save_case(case) publish_case_saved(case) return case finally: release_lock(lock_obj, degrade_gracefully=True)
def create_form_for_test(domain, case_id=None, attachments=None, save=True, state=XFormInstanceSQL.NORMAL): """ Create the models directly so that these tests aren't dependent on any other apps. Not testing form processing here anyway. :param case_id: create case with ID if supplied :param attachments: additional attachments dict :param save: if False return the unsaved form :return: form object """ from corehq.form_processor.utils import get_simple_form_xml form_id = uuid4().hex user_id = 'user1' utcnow = datetime.utcnow() form_xml = get_simple_form_xml(form_id, case_id) form = XFormInstanceSQL( form_id=form_id, xmlns='http://openrosa.org/formdesigner/form-processor', received_on=utcnow, user_id=user_id, domain=domain, state=state ) attachments = attachments or {} attachment_tuples = map( lambda a: Attachment(name=a[0], raw_content=a[1], content_type=a[1].content_type), attachments.items() ) attachment_tuples.append(Attachment('form.xml', form_xml, 'text/xml')) FormProcessorSQL.store_attachments(form, attachment_tuples) cases = [] if case_id: case = CommCareCaseSQL( case_id=case_id, domain=domain, type='', owner_id=user_id, opened_on=utcnow, modified_on=utcnow, modified_by=user_id, server_modified_on=utcnow, ) case.track_create(CaseTransaction.form_transaction(case, form)) cases = [case] if save: FormProcessorSQL.save_processed_models(ProcessedForms(form, None), cases) return form
def _get_case(self, case_id): try: if self.lock: try: case, lock = CommCareCaseSQL.get_locked_obj(_id=case_id) except redis.RedisError: case = CommCareCaseSQL.get(case_id) else: self.locks.append(lock) else: case = CommCareCaseSQL.get(case_id) except CommCareCaseSQL.DoesNotExist: return None return case
def get_case_with_lock(case_id, lock=False, strip_history=False, wrap=False): try: if lock: try: return CommCareCaseSQL.get_locked_obj(_id=case_id) except redis.RedisError: case = CaseAccessorSQL.get_case(case_id) else:
def _create_case(self, case_type=None, user_id=None, case_id=None): case_id = case_id or uuid.uuid4().hex user_id = user_id or 'mr_wednesday' utcnow = datetime.utcnow() case = CommCareCaseSQL( case_id=case_id, domain=self.DOMAIN, type=case_type or '', owner_id=user_id, opened_on=utcnow, modified_on=utcnow, modified_by=utcnow, server_modified_on=utcnow ) form = self._create_form(user_id, utcnow) case.track_create(self._create_case_transaction(case, form, utcnow, action_types=[128])) FormProcessorSQL.save_processed_models(ProcessedForms(form, []), [case]) return CaseAccessorSQL.get_case(case_id)
def hard_rebuild_case(domain, case_id, detail, lock=True): case, lock_obj = FormProcessorSQL.get_case_with_lock(case_id, lock=lock) found = bool(case) if not found: case = CommCareCaseSQL(case_id=case_id, domain=domain) if lock: lock_obj = CommCareCaseSQL.get_obj_lock_by_id(case_id) acquire_lock(lock_obj, degrade_gracefully=False) try: assert case.domain == domain, (case.domain, domain) case, rebuild_transaction = FormProcessorSQL._rebuild_case_from_transactions(case, detail) if case.is_deleted and not case.is_saved(): return None case.server_modified_on = rebuild_transaction.server_date CaseAccessorSQL.save_case(case) publish_case_saved(case) return case finally: release_lock(lock_obj, degrade_gracefully=True)
def iter_sql_cases_with_sorted_transactions(domain): sql = f""" SELECT cx.case_id FROM {CommCareCaseSQL._meta.db_table} cx INNER JOIN {CaseTransaction._meta.db_table} tx ON cx.case_id = tx.case_id WHERE cx.domain = %s AND tx.details LIKE %s """ reason = f'%{SortTransactionsRebuild._REASON}%' for dbname in get_db_aliases_for_partitioned_query(): with CommCareCaseSQL.get_cursor_for_partition_db(dbname) as cursor: cursor.execute(sql, [domain, reason]) yield from iter(set(case_id for case_id, in cursor.fetchall()))
def create_form_for_test(domain, case_id=None, attachments=None, save=True, state=XFormInstanceSQL.NORMAL): """ Create the models directly so that these tests aren't dependent on any other apps. Not testing form processing here anyway. :param case_id: create case with ID if supplied :param attachments: additional attachments dict :param save: if False return the unsaved form :return: form object """ from corehq.form_processor.utils import get_simple_form_xml form_id = uuid4().hex user_id = 'user1' utcnow = datetime.utcnow() form_xml = get_simple_form_xml(form_id, case_id) form = XFormInstanceSQL( form_id=form_id, xmlns='http://openrosa.org/formdesigner/form-processor', received_on=utcnow, user_id=user_id, domain=domain, state=state) attachments = attachments or {} attachment_tuples = map( lambda a: Attachment( name=a[0], raw_content=a[1], content_type=a[1].content_type), attachments.items()) attachment_tuples.append(Attachment('form.xml', form_xml, 'text/xml')) FormProcessorSQL.store_attachments(form, attachment_tuples) cases = [] if case_id: case = CommCareCaseSQL( case_id=case_id, domain=domain, type='', owner_id=user_id, opened_on=utcnow, modified_on=utcnow, modified_by=user_id, server_modified_on=utcnow, ) case.track_create(CaseTransaction.form_transaction(case, form)) cases = [case]
def get_case_with_lock(case_id, lock=False, strip_history=False, wrap=False): try: if lock: try: return CommCareCaseSQL.get_locked_obj(_id=case_id) except redis.RedisError: case = CaseAccessorSQL.get_case(case_id) else: case = CaseAccessorSQL.get_case(case_id) except CaseNotFound: return None, None return case, None
def _create_case(domain=None, form_id=None, case_type=None, user_id=None, closed=False): """ Create the models directly so that these tests aren't dependent on any other apps. Not testing form processing here anyway. :return: case_id """ domain = domain or DOMAIN form_id = form_id or uuid.uuid4().hex case_id = uuid.uuid4().hex user_id = user_id or 'user1' utcnow = datetime.utcnow() form = XFormInstanceSQL( form_id=form_id, xmlns='http://openrosa.org/formdesigner/form-processor', received_on=utcnow, user_id=user_id, domain=domain ) cases = [] if case_id: case = CommCareCaseSQL( case_id=case_id, domain=domain, type=case_type or '', owner_id=user_id, opened_on=utcnow, modified_on=utcnow, modified_by=user_id, server_modified_on=utcnow, closed=closed or False ) case.track_create(CaseTransaction.form_transaction(case, form)) cases = [case] FormProcessorSQL.save_processed_models(ProcessedForms(form, None), cases) return CaseAccessorSQL.get_case(case_id)
def _send_case_to_es(self, domain=None, owner_id=None, user_id=None, case_type=None, opened_on=None, closed_on=None): case = CommCareCaseSQL( case_id=uuid.uuid4().hex, domain=domain or self.domain, owner_id=owner_id or self.owner_id, modified_by=user_id or self.user_id, type=case_type or self.case_type, opened_on=opened_on or datetime.now(), opened_by=user_id or self.user_id, closed_on=closed_on, closed_by=user_id or self.user_id, server_modified_on=datetime.utcnow(), closed=bool(closed_on) ) case.track_create(CaseTransaction( case=case, server_date=opened_on, )) change = Change( id=case.case_id, sequence_id='123', document=case.to_json(), ) self.pillow.processor(change, do_set_checkpoint=False) es = get_es_new() es.indices.refresh(CASE_INDEX) return case
def _get_case(self, case_id): try: if self.lock: try: case, lock = CommCareCaseSQL.get_locked_obj(_id=case_id) except redis.RedisError: case = CaseAccessorSQL.get_case(case_id) else: self.locks.append(lock) else: case = CaseAccessorSQL.get_case(case_id) except CaseNotFound: return None return case
def _copy_unprocessed_case(self, change): couch_case = CommCareCase.wrap(change.get_document()) log.debug('Processing doc: {}({})'.format(couch_case['doc_type'], change.id)) try: first_action = couch_case.actions[0] except IndexError: first_action = CommCareCaseAction() sql_case = CommCareCaseSQL( case_id=couch_case.case_id, domain=self.domain, type=couch_case.type or '', name=couch_case.name, owner_id=couch_case.owner_id or couch_case.user_id or '', opened_on=couch_case.opened_on or first_action.date, opened_by=couch_case.opened_by or first_action.user_id, modified_on=couch_case.modified_on, modified_by=couch_case.modified_by or couch_case.user_id or '', server_modified_on=couch_case.server_modified_on, closed=couch_case.closed, closed_on=couch_case.closed_on, closed_by=couch_case.closed_by, deleted=True, deletion_id=couch_case.deletion_id, deleted_on=couch_case.deletion_date, external_id=couch_case.external_id, case_json=couch_case.dynamic_case_properties() ) _migrate_case_actions(couch_case, sql_case) _migrate_case_indices(couch_case, sql_case) _migrate_case_attachments(couch_case, sql_case) try: CaseAccessorSQL.save_case(sql_case) except IntegrityError: # case re-created by form processing so just mark the case as deleted CaseAccessorSQL.soft_delete_cases( self.domain, [sql_case.case_id], sql_case.deleted_on, sql_case.deletion_id ) self.processed_docs += 1 self._log_unprocessed_cases_processed_count(throttled=True)
def _create_case(self, case_type=None, user_id=None, case_id=None): case_id = case_id or uuid.uuid4().hex user_id = user_id or 'mr_wednesday' utcnow = datetime.utcnow() case = CommCareCaseSQL(case_id=case_id, domain=self.DOMAIN, type=case_type or '', owner_id=user_id, opened_on=utcnow, modified_on=utcnow, modified_by=utcnow, server_modified_on=utcnow) form = self._create_form(user_id, utcnow) case.track_create( self._create_case_transaction(case, form, utcnow, action_types=[128]))
def _copy_unprocessed_case(self, doc): couch_case = CommCareCase.wrap(doc) log.debug('Processing doc: %(doc_type)s(%(_id)s)', doc) try: first_action = couch_case.actions[0] except IndexError: first_action = CommCareCaseAction() opened_on = couch_case.opened_on or first_action.date sql_case = CommCareCaseSQL( case_id=couch_case.case_id, domain=self.domain, type=couch_case.type or '', name=couch_case.name, owner_id=couch_case.owner_id or couch_case.user_id or '', opened_on=opened_on, opened_by=couch_case.opened_by or first_action.user_id, modified_on=couch_case.modified_on or opened_on, modified_by=couch_case.modified_by or couch_case.user_id or '', server_modified_on=couch_case.server_modified_on, closed=couch_case.closed, closed_on=couch_case.closed_on, closed_by=couch_case.closed_by, deleted=True, deletion_id=couch_case.deletion_id, deleted_on=couch_case.deletion_date, external_id=couch_case.external_id, case_json=couch_case.dynamic_case_properties()) _migrate_case_actions(couch_case, sql_case) _migrate_case_indices(couch_case, sql_case) _migrate_case_attachments(couch_case, sql_case) try: CaseAccessorSQL.save_case(sql_case) except IntegrityError: # case re-created by form processing so just mark the case as deleted CaseAccessorSQL.soft_delete_cases(self.domain, [sql_case.case_id], sql_case.deleted_on, sql_case.deletion_id) finally: self.case_diff_queue.enqueue(couch_case.case_id)
class TestGetCaseTriggerInfo(TestCase): def setUp(self): self.case_type = CaseType.objects.create(domain=DOMAIN, name='foo') self.resource_type = FHIRResourceType.objects.create( domain=DOMAIN, case_type=self.case_type, name='Foo', ) for name in ('een', 'twee', 'drie'): prop = CaseProperty.objects.create( case_type=self.case_type, name=name, ) FHIRResourceProperty.objects.create( resource_type=self.resource_type, case_property=prop, jsonpath=f'$.{name}', ) def tearDown(self): self.case_type.delete() def test_case_properties(self): case = CommCareCaseSQL(case_id=str(uuid4()), domain=DOMAIN, type='foo', name='bar', case_json={ 'een': 1, 'twee': 2, 'drie': 3, 'vier': 4, 'vyf': 5, 'ses': 6, }) info = get_case_trigger_info(case, self.resource_type) for name in ('een', 'twee', 'drie'): self.assertIn(name, info.extra_fields) for name in ('vier', 'vyf', 'ses'): self.assertNotIn(name, info.extra_fields)
def _copy_unprocessed_cases(self): doc_types = ['CommCareCase-Deleted'] changes = _get_case_iterator(self.domain, doc_types=doc_types).iter_all_changes() for change in self._with_progress(doc_types, changes): couch_case = CommCareCase.wrap(change.get_document()) self.log_debug('Processing doc: {}({})'.format( couch_case['doc_type'], change.id)) try: first_action = couch_case.actions[0] except IndexError: first_action = CommCareCaseAction() sql_case = CommCareCaseSQL( case_id=couch_case.case_id, domain=self.domain, type=couch_case.type or '', name=couch_case.name, owner_id=couch_case.owner_id or couch_case.user_id or '', opened_on=couch_case.opened_on or first_action.date, opened_by=couch_case.opened_by or first_action.user_id, modified_on=couch_case.modified_on, modified_by=couch_case.modified_by or couch_case.user_id or '', server_modified_on=couch_case.server_modified_on, closed=couch_case.closed, closed_on=couch_case.closed_on, closed_by=couch_case.closed_by, deleted=True, deletion_id=couch_case.deletion_id, deleted_on=couch_case.deletion_date, external_id=couch_case.external_id, case_json=couch_case.dynamic_case_properties()) _migrate_case_actions(couch_case, sql_case) _migrate_case_indices(couch_case, sql_case) _migrate_case_attachments(couch_case, sql_case) try: CaseAccessorSQL.save_case(sql_case) except IntegrityError as e: self.log_error("Unable to migrate case:\n{}\n{}".format( couch_case.case_id, e))
def setUpClass(cls): cls.factory = AppFactory(domain=cls.domain) cls.app = cls.factory.app cls.module, cls.basic_form = cls.factory.new_basic_module( 'basic', 'patient') # necessary to render_xform builder = XFormBuilder(cls.basic_form.name) builder.new_question(name='name', label='Name') cls.basic_form.source = builder.tostring( pretty_print=True).decode('utf-8') cls.phone_number = "+919999999999" cls.case_id = uuid.uuid4().hex cls.recipient = None cls.case = CommCareCaseSQL(domain=cls.domain, case_id=cls.case_id, case_json={'language_code': 'fr'}) cls.web_user = WebUser(username='******', _id=uuid.uuid4().hex, language='hin')
def get_databases(): sql_dbs = [ _SQLFormDb(XFormInstanceSQL._meta.db_table, lambda id_: XFormInstanceSQL.get_obj_by_id(id_), XFormInstanceSQL.__name__), _SQLDb( CommCareCaseSQL._meta.db_table, lambda id_: CommCareCaseSQLRawDocSerializer( CommCareCaseSQL.get_obj_by_id(id_)).data, CommCareCaseSQL.__name__), _SQLDb(SQLLocation._meta.db_table, lambda id_: SQLLocation.objects.get(location_id=id_).to_json(), SQLLocation.__name__), ] all_dbs = OrderedDict() couchdbs_by_name = couch_config.all_dbs_by_db_name for dbname in sorted(couchdbs_by_name): all_dbs[dbname] = _CouchDb(couchdbs_by_name[dbname]) for db in sql_dbs: all_dbs[db.dbname] = db return all_dbs
def _create_case(domain=None, form_id=None, case_type=None, user_id=None, closed=False, case_id=None): """ Create the models directly so that these tests aren't dependent on any other apps. Not testing form processing here anyway. :return: CommCareCaseSQL """ domain = domain or DOMAIN form_id = form_id or uuid.uuid4().hex case_id = case_id or uuid.uuid4().hex user_id = user_id or 'user1' utcnow = datetime.utcnow() form = XFormInstanceSQL( form_id=form_id, xmlns='http://openrosa.org/formdesigner/form-processor', received_on=utcnow, user_id=user_id, domain=domain) case = CommCareCaseSQL(case_id=case_id, domain=domain, type=case_type or '', owner_id=user_id, opened_on=utcnow, modified_on=utcnow, modified_by=user_id, server_modified_on=utcnow, closed=closed or False) case.track_create(CaseTransaction.form_transaction(case, form, utcnow)) cases = [case] FormProcessorSQL.save_processed_models(ProcessedForms(form, None), cases) return CaseAccessorSQL.get_case(case_id)
def get_cases_from_forms(case_db, xforms): """Get all cases affected by the forms. Includes new cases, updated cases. """ touched_cases = {} if len(xforms) > 1: domain = xforms[0].domain affected_cases = set() deprecated_form = None for xform in xforms: if xform.is_deprecated: deprecated_form = xform if not (xform.is_deprecated and xform.problem): # don't process deprecatd forms which have errors. # see http://manage.dimagi.com/default.asp?243382 for context. # note that we have to use .problem instead of .is_error because applying # the state=DEPRECATED overrides state=ERROR affected_cases.update( case_update.id for case_update in get_case_updates(xform)) rebuild_detail = FormEditRebuild( deprecated_form_id=deprecated_form.form_id) for case_id in affected_cases: case = case_db.get(case_id) is_creation = False if not case: case = CommCareCaseSQL(domain=domain, case_id=case_id) is_creation = True case_db.set(case_id, case) previous_owner = case.owner_id case, _ = FormProcessorSQL._rebuild_case_from_transactions( case, rebuild_detail, updated_xforms=xforms) if case: touched_cases[case.case_id] = CaseUpdateMetadata( case=case, is_creation=is_creation, previous_owner_id=previous_owner, )
def get(self, record_id): try: return self._getter(record_id) except (XFormNotFound, CaseNotFound, ObjectDoesNotExist): raise ResourceNotFound("missing") _SQL_DBS = OrderedDict((db.dbname, db) for db in [ _Db( XFormInstanceSQL._meta.db_table, lambda id_: XFormInstanceSQLRawDocSerializer(XFormInstanceSQL.get_obj_by_id(id_) ).data, XFormInstanceSQL.__name__), _Db( CommCareCaseSQL._meta.db_table, lambda id_: CommCareCaseSQLRawDocSerializer(CommCareCaseSQL.get_obj_by_id(id_) ).data, CommCareCaseSQL.__name__), _Db(SQLLocation._meta.db_table, lambda id_: SQLLocation.objects.get( location_id=id_).to_json(), SQLLocation.__name__), ]) def get_db_from_db_name(db_name): if db_name in _SQL_DBS: return _SQL_DBS[db_name] elif db_name == couch_config.get_db(None).dbname: # primary db return couch_config.get_db(None) else: return couch_config.get_db(db_name)
def get(self, record_id): try: return self._getter(record_id) except (XFormNotFound, CaseNotFound, ObjectDoesNotExist): raise ResourceNotFound("missing") _SQL_DBS = OrderedDict((db.dbname, db) for db in [ _Db( XFormInstanceSQL._meta.db_table, lambda id_: XFormInstanceSQLRawDocSerializer(XFormInstanceSQL.get_obj_by_id(id_)).data, XFormInstanceSQL.__name__ ), _Db( CommCareCaseSQL._meta.db_table, lambda id_: CommCareCaseSQLRawDocSerializer(CommCareCaseSQL.get_obj_by_id(id_)).data, CommCareCaseSQL.__name__ ), _Db( SQLLocation._meta.db_table, lambda id_: SQLLocation.objects.get(location_id=id_).to_json(), SQLLocation.__name__ ), ]) def get_db_from_db_name(db_name): if db_name in _SQL_DBS: return _SQL_DBS[db_name] elif db_name == couch_config.get_db(None).dbname: # primary db return couch_config.get_db(None)
def get(self, record_id): try: return self._getter(record_id) except (XFormNotFound, CaseNotFound, ObjectDoesNotExist): raise ResourceNotFound("missing") _SQL_DBS = OrderedDict((db.dbname, db) for db in [ _Db( XFormInstanceSQL._meta.db_table, lambda id_: XFormInstanceSQLRawDocSerializer(XFormInstanceSQL.get_obj_by_id(id_)).data, XFormInstanceSQL.__name__ ), _Db( CommCareCaseSQL._meta.db_table, lambda id_: CommCareCaseSQLRawDocSerializer(CommCareCaseSQL.get_obj_by_id(id_)).data, CommCareCaseSQL.__name__ ), _Db( SQLLocation._meta.db_table, lambda id_: SQLLocation.objects.get(location_id=id_).to_json(), SQLLocation.__name__ ), ]) def _get_db_from_db_name(db_name): if db_name in _SQL_DBS: return _SQL_DBS[db_name] elif db_name == couch_config.get_db(None).dbname: # primary db return couch_config.get_db(None)
case_update_meta.case.case_id] = case_update_meta else: logging.error( "XForm %s had a case block that wasn't able to create a case! " "This usually means it had a missing ID" % xform.get_id) return touched_cases @staticmethod def hard_rebuild_case(domain, case_id, detail, lock=True): case, lock_obj = FormProcessorSQL.get_case_with_lock(case_id, lock=lock) found = bool(case) if not found: case = CommCareCaseSQL(case_id=case_id, domain=domain) if lock: lock_obj = CommCareCaseSQL.get_obj_lock_by_id(case_id) acquire_lock(lock_obj, degrade_gracefully=False) try: assert case.domain == domain, (case.domain, domain) case, rebuild_transaction = FormProcessorSQL._rebuild_case_from_transactions( case, detail) if case.is_deleted and not case.is_saved(): return None case.server_modified_on = rebuild_transaction.server_date CaseAccessorSQL.save_case(case) publish_case_saved(case) return case
else: logging.error( "XForm %s had a case block that wasn't able to create a case! " "This usually means it had a missing ID" % xform.get_id ) return touched_cases @staticmethod def hard_rebuild_case(domain, case_id, detail): try: case = CaseAccessorSQL.get_case(case_id) assert case.domain == domain found = True except CaseNotFound: case = CommCareCaseSQL(case_id=case_id, domain=domain) found = False case = FormProcessorSQL._rebuild_case_from_transactions(case, detail) if case.is_deleted and not found: return None CaseAccessorSQL.save_case(case) publish_case_saved(case) return case @staticmethod def _rebuild_case_from_transactions(case, detail, updated_xforms=None): transactions = get_case_transactions(case.case_id, updated_xforms=updated_xforms) strategy = SqlCaseUpdateStrategy(case) rebuild_transaction = CaseTransaction.rebuild_transaction(case, detail)