def test_case_pillow_indicators(self): since = get_current_seq(XFormInstance.get_db()) self._save_doc_to_db('indicator_form.json', XFormInstance) case_id = self._save_doc_to_db('indicator_case.json', CommCareCase) case_instance = CommCareCase.get(case_id) # FormDataInCaseIndicatorDef (For those forgotten properties) forgotten_property = FormDataInCaseIndicatorDefinition.increment_or_create_unique( INDICATOR_TEST_NAMESPACE, INDICATOR_TEST_DOMAIN, slug='club_name', question_id='location.club', case_type='song_tag', xmlns='http://openrosa.org/formdesigner/indicator-create-xmlns', ) forgotten_property.save() self.case_pillow.process_changes(since=since, forever=False) indicator_case = IndicatorCase.get(case_id) self.assertEqual(indicator_case.get_id, case_instance.get_id) self.assertNotEqual(indicator_case.get_db().dbname, case_instance.get_db().dbname) self.assertNotEqual(indicator_case.computed_, {})
def handle_one(self, domain, case_type, chunk_size): self.log('Copying {case_type} cases in {domain}' .format(case_type=case_type, domain=domain)) old_db = CommCareCase.get_db() new_db = IndicatorCase.get_db() assert old_db.uri != new_db.uri # this dbaccessor pulls from old_db case_ids = get_case_ids_in_domain(domain, case_type) self.delete_bad_doc_types(case_ids, chunk_size) case_dict_chunks = chunked(iter_docs(old_db, case_ids, chunk_size), chunk_size) for case_dicts in case_dict_chunks: for case_dict in case_dicts: del case_dict['_rev'] case_dict.pop('_attachments', None) case_dict['doc_type'] = "IndicatorCase" try: results = new_db.bulk_save(case_dicts) except BulkSaveError as error: results = error.results for result in results: if result.get('error') == 'conflict': self.log('- OK: [{id}] is already in the indicator db' .format(id=result.get('id'))) elif 'error' in result: self.log('- ERROR: [{id}] ({result})'.format( id=result.get('id'), result=json.dumps(result) )) else: self.log('- ADDED: [{id}] saved to indicator db'.format( id=result.get('id') ))
def test_case_pillow_indicators(self): self._save_doc_to_db('indicator_form.json', XFormInstance) case_id = self._save_doc_to_db('indicator_case.json', CommCareCase) case_instance = CommCareCase.get(case_id) # FormDataInCaseIndicatorDef (For those forgotten properties) forgotten_property = FormDataInCaseIndicatorDefinition.increment_or_create_unique( INDICATOR_TEST_NAMESPACE, INDICATOR_TEST_DOMAIN, slug='club_name', question_id='location.club', case_type='song_tag', xmlns='http://openrosa.org/formdesigner/indicator-create-xmlns', ) forgotten_property.save() self.case_pillow.process_changes(since=None, forever=False) indicator_case = IndicatorCase.get(case_id) self.assertEqual(indicator_case.get_id, case_instance.get_id) self.assertNotEqual( indicator_case.get_db().dbname, case_instance.get_db().dbname ) self.assertNotEqual(indicator_case.computed_, {})
def delete_bad_doc_types(self, case_ids, chunk_size): """ No view in this db includes CommCareCases, so check manually """ db = IndicatorCase.get_db() case_dict_chunks = chunked(iter_docs(db, case_ids, chunk_size), chunk_size) to_delete = [] for case_dicts in case_dict_chunks: for case_dict in case_dicts: if case_dict['doc_type'] == 'CommCareCase': to_delete.append(case_dict['_id']) elif case_dict['doc_type'] != 'IndicatorCase': raise Exception("Unexpected case type {} found" .format(case_dict['doc_type'])) assert db.uri != CommCareCase.get_db().uri print "Deleting {} docs from db '{}'".format(len(to_delete), db.dbname) iter_bulk_delete(db, to_delete, chunk_size)
def handle_one(self, domain, case_type, chunk_size): self.log('Copying {case_type} cases in {domain}' .format(case_type=case_type, domain=domain)) old_db = CommCareCase.get_db() new_db = IndicatorCase.get_db() assert old_db.uri != new_db.uri case_ids = old_db.view( 'case/all_cases', startkey=["all type", domain, case_type], endkey=["all type", domain, case_type, {}], reduce=False, wrapper=lambda x: x['id'] ).all() self.delete_bad_doc_types(case_ids, chunk_size) case_dict_chunks = chunked(iter_docs(old_db, case_ids, chunk_size), chunk_size) for case_dicts in case_dict_chunks: for case_dict in case_dicts: del case_dict['_rev'] case_dict.pop('_attachments', None) case_dict['doc_type'] = "IndicatorCase" try: results = new_db.bulk_save(case_dicts) except BulkSaveError as error: results = error.results for result in results: if result.get('error') == 'conflict': self.log('- OK: [{id}] is already in the indicator db' .format(id=result.get('id'))) elif 'error' in result: self.log('- ERROR: [{id}] ({result})'.format( id=result.get('id'), result=json.dumps(result) )) else: self.log('- ADDED: [{id}] saved to indicator db'.format( id=result.get('id') ))
def test_case_pillow_indicators(self): self._save_doc_to_db("indicator_form.json", XFormInstance) case_id = self._save_doc_to_db("indicator_case.json", CommCareCase) case_instance = CommCareCase.get(case_id) # FormDataInCaseIndicatorDef (For those forgotten properties) forgotten_property = FormDataInCaseIndicatorDefinition.increment_or_create_unique( INDICATOR_TEST_NAMESPACE, INDICATOR_TEST_DOMAIN, slug="club_name", question_id="location.club", case_type="song_tag", xmlns="http://openrosa.org/formdesigner/indicator-create-xmlns", ) forgotten_property.save() self.case_pillow.run_burst() indicator_case = IndicatorCase.get(case_id) self.assertEqual(indicator_case.get_id, case_instance.get_id) self.assertNotEqual(indicator_case.get_db().dbname, case_instance.get_db().dbname) self.assertNotEqual(indicator_case.computed_, {})
def process_indicators_for_case(namespaces, domain, doc_dict): case_type = doc_dict.get('type') if not case_type: return case_indicator_defs = [] for namespace in namespaces: case_indicator_defs.extend(CaseIndicatorDefinition.get_all( namespace, domain, case_type=case_type )) try: indicator_case = IndicatorCase.wrap_for_indicator_db(doc_dict) indicator_case.update_indicators_in_bulk( case_indicator_defs, logger=pillow_logging, save_on_update=False ) indicator_case.save() except Exception as e: pillow_logging.error( "Error creating for MVP Indicator for form %(form_id)s: " "%(error)s" % { 'form_id': doc_dict['_id'], 'error': e, }, ) raise # Now Update Data From Case to All Related Xforms (ewwww) xform_ids = doc_dict.get('xform_ids', []) if not xform_ids: return for xform_id in xform_ids: related_xform_indicators = [] try: # first try to get the doc from the indicator DB xform_doc = IndicatorXForm.get(xform_id) except ResourceNotFound: # if that fails fall back to the main DB try: xform_dict = XFormInstance.get_db().get(xform_id) xform_doc = IndicatorXForm.wrap_for_indicator_db(xform_dict) related_xform_indicators = get_form_indicators(namespaces, domain, xform_doc.xmlns) except ResourceNotFound: pillow_logging.error( "Could not find an XFormInstance with id %(xform_id)s " "related to Case %(case_id)s" % { 'xform_id': xform_id, 'case_id': doc_dict['_id'], } ) continue if not xform_doc.xmlns: continue for namespace in namespaces: related_xform_indicators.extend( CaseDataInFormIndicatorDefinition.get_all( namespace, domain, xmlns=xform_doc.xmlns ) ) xform_doc.update_indicators_in_bulk( related_xform_indicators, logger=pillow_logging, save_on_update=False ) xform_doc.save()
def process_indicators_for_case(namespaces, domain, doc_dict): case_type = doc_dict.get('type') if not case_type: return case_indicator_defs = [] for namespace in namespaces: case_indicator_defs.extend( CaseIndicatorDefinition.get_all(namespace, domain, case_type=case_type)) try: indicator_case = IndicatorCase.wrap_for_indicator_db(doc_dict) indicator_case.update_indicators_in_bulk(case_indicator_defs, logger=pillow_logging, save_on_update=False) indicator_case.save() except Exception as e: pillow_logging.error( "Error creating for MVP Indicator for form %(form_id)s: " "%(error)s" % { 'form_id': doc_dict['_id'], 'error': e, }, ) raise # Now Update Data From Case to All Related Xforms (ewwww) xform_ids = doc_dict.get('xform_ids', []) if not xform_ids: return for xform_id in xform_ids: related_xform_indicators = [] try: # first try to get the doc from the indicator DB xform_doc = IndicatorXForm.get(xform_id) except ResourceNotFound: # if that fails fall back to the main DB try: xform_dict = XFormInstance.get_db().get(xform_id) xform_doc = IndicatorXForm.wrap_for_indicator_db(xform_dict) related_xform_indicators = get_form_indicators( namespaces, domain, xform_doc.xmlns) except ResourceNotFound: pillow_logging.error( "Could not find an XFormInstance with id %(xform_id)s " "related to Case %(case_id)s" % { 'xform_id': xform_id, 'case_id': doc_dict['_id'], }) continue if not xform_doc.xmlns: continue for namespace in namespaces: related_xform_indicators.extend( CaseDataInFormIndicatorDefinition.get_all( namespace, domain, xmlns=xform_doc.xmlns)) xform_doc.update_indicators_in_bulk(related_xform_indicators, logger=pillow_logging, save_on_update=False) xform_doc.save()
def process_indicators(self, namespaces, domain, doc_dict): case_type = doc_dict.get('type') if not case_type: return case_indicator_defs = [] for namespace in namespaces: case_indicator_defs.extend(CaseIndicatorDefinition.get_all( namespace, domain, case_type=case_type )) if not case_indicator_defs: return try: indicator_case = IndicatorCase.get_or_create_from_dict(doc_dict)[0] indicator_case.update_indicators_in_bulk( case_indicator_defs, logger=pillow_eval_logging ) except Exception as e: pillow_eval_logging.error( "Error creating for MVP Indicator for form %(form_id)s: " "%(error)s" % { 'form_id': doc_dict['_id'], 'error': e, }, ) # Now Update Data From Case to All Related Xforms (ewwww) xform_ids = doc_dict.get('xform_ids', []) if not xform_ids: return for xform_id in xform_ids: try: xform_dict = XFormInstance.get_db().get(xform_id) xform_doc = IndicatorXForm.get_or_create_from_dict(xform_dict)[0] except ResourceNotFound: pillow_eval_logging.error( "Could not find an XFormInstance with id %(xform_id)s " "related to Case %(case_id)s" % { 'xform_id': xform_id, 'case_id': doc_dict['_id'], } ) continue if not xform_doc.xmlns: continue related_xform_defs = [] for namespace in namespaces: related_xform_defs.extend( CaseDataInFormIndicatorDefinition.get_all( namespace, domain, xmlns=xform_doc.xmlns ) ) xform_doc.update_indicators_in_bulk( related_xform_defs, logger=pillow_eval_logging, )