def child_cases(self): from corehq.apps.api.util import case_to_es_case accessor = CaseAccessors(self.domain) return { index.case_id: case_to_es_case(accessor.get_case(index.case_id)) for index in self._reverse_indices }
def parent_cases(self): from corehq.apps.api.util import case_to_es_case accessor = CaseAccessors(self.domain) return { index['identifier']: case_to_es_case(accessor.get_case(index['referenced_id'])) for index in self.indices }
def _update_case(domain, case_id, server_modified_on, last_visit_date=None): accessors = CaseAccessors(domain) case = accessors.get_case(case_id) case.server_modified_on = server_modified_on if last_visit_date: set_case_property_directly(case, 'last_visit_date', last_visit_date.strftime('%Y-%m-%d')) _save_case(domain, case)
def _delete_all_cases(domain_name): logger.info('Deleting cases...') case_accessor = CaseAccessors(domain_name) case_ids = case_accessor.get_case_ids_in_domain() for case_id_chunk in chunked(with_progress_bar(case_ids, stream=silence_during_tests()), 500): case_accessor.soft_delete_cases(list(case_id_chunk)) logger.info('Deleting cases complete.')
def recalculate_stagnant_cases(): domain = 'icds-cas' config_ids = [ 'static-icds-cas-static-ccs_record_cases_monthly_v2', 'static-icds-cas-static-ccs_record_cases_monthly_tableau_v2', 'static-icds-cas-static-child_cases_monthly_v2', ] stagnant_cases = set() for config_id in config_ids: config, is_static = get_datasource_config(config_id, domain) adapter = get_indicator_adapter(config) case_ids = _find_stagnant_cases(adapter) celery_task_logger.info( "Found {} stagnant cases in config {}".format(len(case_ids), config_id) ) stagnant_cases = stagnant_cases.union(set(case_ids)) celery_task_logger.info( "Total number of stagant cases is now {}".format(len(stagnant_cases)) ) case_accessor = CaseAccessors(domain) num_stagnant_cases = len(stagnant_cases) current_case_num = 0 for case_ids in chunked(stagnant_cases, 1000): current_case_num += len(case_ids) cases = case_accessor.get_cases(list(case_ids)) for case in cases: publish_case_saved(case, send_post_save_signal=False) celery_task_logger.info( "Resaved {} / {} cases".format(current_case_num, num_stagnant_cases) )
def test_edit_submissions_simple(self): initial_quantity = 100 form = submit_case_blocks( case_blocks=get_single_balance_block(quantity=initial_quantity, **self._stock_state_key), domain=self.domain, )[0] self._assert_stats(1, initial_quantity, initial_quantity) case_accessors = CaseAccessors(self.domain) case = case_accessors.get_case(self.case.case_id) try: self.assertTrue(any([action.is_ledger_transaction for action in case.actions])) except AttributeError: self.assertTrue('commtrack' in [action.action_type for action in case.actions]) self.assertEqual([form.form_id], case.xform_ids[1:]) # change the value to 50 edit_quantity = 50 submit_case_blocks( case_blocks=get_single_balance_block(quantity=edit_quantity, **self._stock_state_key), domain=self.domain, form_id=form.form_id, ) case = case_accessors.get_case(self.case.case_id) try: # CaseTransaction self.assertTrue(any([action.is_ledger_transaction for action in case.actions])) except AttributeError: # CaseAction self.assertTrue('commtrack' in [action.action_type for action in case.actions]) self._assert_stats(1, edit_quantity, edit_quantity) self.assertEqual([form.form_id], case.xform_ids[1:])
class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument('infile') parser.add_argument('outfile') def handle(self, infile, outfile, *args, **options): self.case_accessor = CaseAccessors('icds-cas') with open(infile, 'r', encoding='utf-8') as old, open(outfile, 'w', encoding='utf-8') as new: reader = csv.reader(old) writer = csv.writer(new) headers = next(reader) writer.writerow(headers) for row in reader: case_id = row[4] hh_id = row[10] if hh_id: person, hh = self.case_accessor.get_cases([case_id, hh_id], ordered=True) else: person = self.case_accessor.get_case(case_id) hh = None if hh: row[18] = hh.get_case_property('name') row[19] = hh.get_case_property('hh_num') row[20] = person.get_case_property('name') writer.writerow(row)
def lookup_case(search_field, search_id, domain, case_type): """ Attempt to find the case in CouchDB by the provided search_field and search_id. Returns a tuple with case (if found) and an error code (if there was an error in lookup). """ found = False case_accessors = CaseAccessors(domain) if search_field == 'case_id': try: case = case_accessors.get_case(search_id) if case.domain == domain and case.type == case_type: found = True except CaseNotFound: pass elif search_field == EXTERNAL_ID: cases_by_type = case_accessors.get_cases_by_external_id(search_id, case_type=case_type) if not cases_by_type: return (None, LookupErrors.NotFound) elif len(cases_by_type) > 1: return (None, LookupErrors.MultipleResults) else: case = cases_by_type[0] found = True if found: return (case, None) else: return (None, LookupErrors.NotFound)
def test_xform_ids_diff(self): case_id = uuid.uuid4().hex submit_case_blocks( CaseBlock( case_id, case_type='migrate', create=True, update={'p1': 1}, ).as_string(), self.domain_name ) submit_case_blocks( CaseBlock( case_id, update={'p2': 2}, ).as_string(), self.domain_name ) case = CaseAccessors(self.domain_name).get_case(case_id) removed_form_id = case.xform_ids.pop(1) case.save() self.assertEqual(1, len(self._get_case_ids())) self._do_migration_and_assert_flags(self.domain_name) self.assertEqual(1, len(self._get_case_ids())) self._compare_diffs([ (u'CommCareCase', FormJsonDiff( diff_type=u'set_mismatch', path=[u'xform_ids', u'[*]'], old_value=u'', new_value=removed_form_id )) ])
def test_archiving_only_form(self): """ Checks that archiving the only form associated with the case archives the case and unarchiving unarchives it. """ case_id = _post_util(create=True, p1='p1-1', p2='p2-1') case_accessors = CaseAccessors(REBUILD_TEST_DOMAIN) case = case_accessors.get_case(case_id) self.assertFalse(case.is_deleted) if should_use_sql_backend(REBUILD_TEST_DOMAIN): self.assertEqual(1, len(case.actions)) else: self.assertEqual(2, len(case.actions)) [form_id] = case.xform_ids form = FormAccessors(REBUILD_TEST_DOMAIN).get_form(form_id) form.archive() case = case_accessors.get_case(case_id) self.assertTrue(case.is_deleted) # should just have the 'rebuild' action self.assertEqual(1, len(case.actions)) self.assertTrue(case.actions[0].is_case_rebuild) form.unarchive() case = case_accessors.get_case(case_id) self.assertFalse(case.is_deleted) self.assertEqual(3, len(case.actions)) self.assertTrue(case.actions[-1].is_case_rebuild)
def get_parent_of_case(domain, case_id, parent_case_type): case_accessor = CaseAccessors(domain) try: if not isinstance(case_id, basestring): case_id = case_id.case_id child_case = case_accessor.get_case(case_id) except CaseNotFound: raise ENikshayCaseNotFound( "Couldn't find case: {}".format(case_id) ) parent_case_ids = [ indexed_case.referenced_id for indexed_case in child_case.indices if indexed_case.referenced_type == parent_case_type ] parent_cases = case_accessor.get_cases(parent_case_ids) open_parent_cases = [ occurrence_case for occurrence_case in parent_cases if not occurrence_case.closed ] if not open_parent_cases: raise ENikshayCaseNotFound( "Couldn't find any open {} cases for id: {}".format(parent_case_type, case_id) ) return open_parent_cases[0]
def _get_direct_dependencies(domain, cases): case_accessor = CaseAccessors(domain) extension_cases = set(case_accessor.get_extension_case_ids(cases)) indexed_cases = set(case_accessor.get_indexed_case_ids(cases)) return DirectDependencies( all=extension_cases | indexed_cases, indexed_cases=indexed_cases, extension_cases=extension_cases )
def case_types(self): """ :return: Set of all case types for the domain excluding the CallCenter case type. """ case_types = CaseAccessors(self.domain).get_case_types() case_types.remove(self.cc_case_type) return case_types
def get_case_by_identifier(domain, identifier): # circular import from corehq.apps.api.es import CaseES case_es = CaseES(domain) case_accessors = CaseAccessors(domain) def _query_by_type(i_type): q = case_es.base_query( terms={ i_type: identifier, }, fields=['_id', i_type], size=1 ) response = case_es.run_query(q) raw_docs = response['hits']['hits'] if raw_docs: return case_accessors.get_case(raw_docs[0]['_id']) # Try by any of the allowed identifiers for identifier_type in ALLOWED_CASE_IDENTIFIER_TYPES: case = _query_by_type(identifier_type) if case is not None: return case # Try by case id try: case_by_id = case_accessors.get_case(identifier) if case_by_id.domain == domain: return case_by_id except (CaseNotFound, KeyError): pass return None
def test_update_adherence_confidence(self): self.create_case_structure() case_accessor = CaseAccessors(self.domain) adherence_dates = [ datetime(2005, 7, 10), datetime(2016, 8, 10), datetime(2016, 8, 11), ] adherence_cases = self.create_adherence_cases(adherence_dates) update_adherence_confidence_level( self.domain, self.person_id, datetime(2016, 8, 10, tzinfo=pytz.UTC), datetime(2016, 8, 11, tzinfo=pytz.UTC), "new_confidence_level", ) adherence_case_ids = [adherence_date.strftime("%Y-%m-%d") for adherence_date in adherence_dates] adherence_cases = {case.case_id: case for case in case_accessor.get_cases(adherence_case_ids)} self.assertEqual( adherence_cases[adherence_case_ids[0]].dynamic_case_properties()['adherence_confidence'], 'medium', ) self.assertEqual( adherence_cases[adherence_case_ids[1]].dynamic_case_properties()['adherence_confidence'], 'new_confidence_level', ) self.assertEqual( adherence_cases[adherence_case_ids[2]].dynamic_case_properties()['adherence_confidence'], 'new_confidence_level', )
def get_cleanliness_flag_from_scratch(domain, owner_id): casedb = CaseAccessors(domain) footprint_info = get_case_footprint_info(domain, owner_id) owned_cases = footprint_info.base_ids cases_to_check = footprint_info.all_ids - owned_cases if cases_to_check: closed_owned_case_ids = set(casedb.get_closed_case_ids_for_owner(owner_id)) cases_to_check = cases_to_check - closed_owned_case_ids - footprint_info.extension_ids # check extension cases that are unowned or owned by others extension_cases_to_check = footprint_info.extension_ids - closed_owned_case_ids - owned_cases while extension_cases_to_check: extension_case = extension_cases_to_check.pop() dependent_cases = set(get_dependent_case_info(domain, [extension_case]).all_ids) unowned_dependent_cases = dependent_cases - owned_cases extension_cases_to_check = extension_cases_to_check - dependent_cases dependent_cases_owned_by_other_owners = { dependent_case.case_id for dependent_case in casedb.get_cases(list(unowned_dependent_cases)) if dependent_case.owner_id != UNOWNED_EXTENSION_OWNER_ID } if dependent_cases_owned_by_other_owners: hint_id = dependent_cases & owned_cases # can't get back from extension case to owned case e.g. host is a child of owned case if hint_id: return CleanlinessFlag(False, hint_id.pop()) if cases_to_check: # it wasn't in any of the open or closed IDs - it must be dirty reverse_index_infos = casedb.get_all_reverse_indices_info(list(cases_to_check)) reverse_index_ids = set([r.case_id for r in reverse_index_infos]) indexed_with_right_owner = (reverse_index_ids & (owned_cases | closed_owned_case_ids)) found_deleted_cases = False while indexed_with_right_owner: hint_id = indexed_with_right_owner.pop() infos_for_this_owner = _get_info_by_case_id(reverse_index_infos, hint_id) for info in infos_for_this_owner: try: case = CaseAccessors(domain).get_case(info.referenced_id) if not case.is_deleted: return CleanlinessFlag(False, hint_id) else: found_deleted_cases = True except ResourceNotFound: # the case doesn't exist - don't use it as a dirty flag found_deleted_cases = True if found_deleted_cases: # if we made it all the way to the end of the loop without returning anything # then the owner was only flagged as dirty due to missing cases, # This implies the owner is still clean. return CleanlinessFlag(True, None) else: # I don't believe code can ever be hit, but if it is we should fail hard # until we can better understand it. raise IllegalCaseId('Owner {} in domain {} has an invalid index reference chain!!'.format( owner_id, domain )) return CleanlinessFlag(True, None)
def set_parent_case(domain, child_case, parent_case): server_modified_on = child_case.server_modified_on set_actual_parent_case(domain, child_case, parent_case) child_case = CaseAccessors(domain).get_case(child_case.case_id) child_case.server_modified_on = server_modified_on _save_case(domain, child_case) return CaseAccessors(domain).get_case(child_case.case_id)
def _get_direct_dependencies(domain, case_ids): assert not isinstance(case_ids, (six.text_type, bytes)) case_accessor = CaseAccessors(domain) extension_cases = set(case_accessor.get_extension_case_ids(case_ids)) indexed_cases = set(case_accessor.get_indexed_case_ids(case_ids)) return DirectDependencies( all=extension_cases | indexed_cases, indexed_cases=indexed_cases, extension_cases=extension_cases )
def cases_referenced_by_xform(xform): """ Returns a list of CommCareCase or CommCareCaseSQL given a JSON representation of an XFormInstance """ assert xform.domain, "Form is missing 'domain'" from corehq.form_processor.interfaces.dbaccessors import CaseAccessors case_ids = get_case_ids_from_form(xform) case_accessor = CaseAccessors(xform.domain) return list(case_accessor.get_cases(list(case_ids)))
class ExplodeLedgersTest(BaseSyncTest): def setUp(self): super(ExplodeLedgersTest, self).setUp() self.case_accessor = CaseAccessors(self.project.name) self.ledger_accessor = LedgerAccessors(self.project.name) self._create_ledgers() def tearDown(self): delete_all_ledgers() delete_all_cases() delete_all_xforms() super(ExplodeLedgersTest, self).tearDown() def _create_ledgers(self): case_type = 'case' case1 = CaseStructure( case_id='case1', attrs={'create': True, 'case_type': case_type}, ) case2 = CaseStructure( case_id='case2', attrs={'create': True, 'case_type': case_type}, ) # case2 will have no ledgers self.ledgers = { 'icecream': Balance( entity_id=case1.case_id, date=datetime(2017, 11, 21, 0, 0, 0, 0), section_id='test', entry=Entry(id='icecream', quantity=4), ), 'blondie': Balance( entity_id=case1.case_id, date=datetime(2017, 11, 21, 0, 0, 0, 0), section_id='test', entry=Entry(id='blondie', quantity=5), ) } self.device.post_changes([case1, case2]) self.device.post_changes(list(self.ledgers.values())) def test_explode_ledgers(self): explode_cases(self.project.name, self.user_id, 5) cases = self.case_accessor.iter_cases(self.case_accessor.get_case_ids_in_domain()) for case in cases: ledger_values = {l.entry_id: l for l in self.ledger_accessor.get_ledger_values_for_case(case.case_id)} if case.case_id == 'case2' or case.get_case_property('cc_exploded_from') == 'case2': self.assertEqual(len(ledger_values), 0) else: self.assertEqual(len(ledger_values), len(self.ledgers)) for id, balance in six.iteritems(self.ledgers): self.assertEqual(ledger_values[id].balance, balance.entry.quantity) self.assertEqual(ledger_values[id].entry_id, balance.entry.id)
def _update_case(domain, case_id, server_modified_on, last_visit_date=None): accessors = CaseAccessors(domain) case = accessors.get_case(case_id) case.server_modified_on = server_modified_on if last_visit_date: set_case_property_directly(case, 'last_visit_date', last_visit_date.strftime('%Y-%m-%d')) if should_use_sql_backend(domain): CaseAccessorSQL.save_case(case) else: # can't call case.save() since it overrides the server_modified_on property CommCareCase.get_db().save_doc(case.to_json())
class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( 'shard', help="db shard to query against", ) parser.add_argument( 'log_file', help="File path for log file", ) def handle(self, shard, log_file, **options): self.domain = 'icds-cas' self.db = shard self.case_accessor = CaseAccessors(self.domain) failed_updates = [] with open(log_file, "w", encoding='utf-8') as fh: fh.write('--------Successful Form Ids----------\n') chunk_num = 1 for orphan_case_chunk in self._get_cases(): print('Currently on chunk {}'.format(chunk_num)) case_tupes = [(case_id, {}, True) for case_id in orphan_case_chunk] try: xform, cases = bulk_update_cases( self.domain, case_tupes, self.__module__) fh.write(xform.form_id + '\n') except LocalSubmissionError as e: print('submission error') print(six.text_type(e)) failed_updates.extend(orphan_case_chunk) except Exception as e: print('unexpected error') print(six.text_type(e)) failed_updates.extend(orphan_case_chunk) chunk_num += 1 fh.write('--------Failed Cases--------------\n') for case_id in failed_updates: fh.write(case_id + '\n') fh.write('--------Logging Complete--------------\n') print('-------------COMPLETE--------------') def _get_cases(self): hh_cases = CommCareCaseSQL.objects.using(self.db).filter(domain=self.domain, type='household', closed=True).values_list('case_id', flat=True) for cases in chunked(hh_cases, 100): related_cases = self.case_accessor.get_reverse_indexed_cases(list(cases)) ccs_cases = self.case_accessor.get_reverse_indexed_cases([case.case_id for case in related_cases]) orphan_cases = {case.case_id for case in related_cases if not case.closed and [c for c in case.cached_indices if c.relationship == 'child']} orphan_cases |= {case.case_id for case in ccs_cases if not case.closed and [c for c in case.cached_indices if c.relationship == 'child']} yield orphan_cases
def get_first_claim(domain, user_id, case_id): """ Returns the first claim by user_id of case_id, or None """ case = CaseAccessors(domain).get_case(case_id) identifier = DEFAULT_CASE_INDEX_IDENTIFIERS[CASE_INDEX_EXTENSION] try: return next((c for c in case.get_subcases(identifier) if c.type == CLAIM_CASE_TYPE and c.owner_id == user_id)) except StopIteration: return None
def test_sync_usercase_custom_user_data_on_create(self): """ Custom user data should be synced when the user is created """ self.user.user_data = { 'completed_training': 'yes', } self.user.save() case = CaseAccessors(TEST_DOMAIN).get_case_by_domain_hq_user_id(self.user._id, USERCASE_TYPE) self.assertIsNotNone(case) self.assertEquals(case.dynamic_case_properties()['completed_training'], 'yes')
class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( 'domain', help="File path for log file", ) parser.add_argument( 'log_file', help="File path for log file", ) def handle(self, domain, log_file, **options): total_cases = CaseES().domain(domain).case_type('household').is_closed().count() self.case_accessor = CaseAccessors(domain) failed_updates = [] with open(log_file, "w", encoding='utf-8') as fh: fh.write('--------Successful Form Ids----------\n') for cases in chunked(with_progress_bar(self._get_cases_to_process(domain), total_cases), 100): related_cases = self._get_related_cases(cases) case_tupes = [(case_id, {}, True) for case_id in related_cases] try: xform, cases = bulk_update_cases( domain, case_tupes, self.__module__) fh.write(xform.form_id + '\n') except LocalSubmissionError as e: print('submission error') print(six.text_type(e)) failed_updates.extend(related_cases) except Exception as e: print('unexpected error') print(six.text_type(e)) failed_updates.extend(related_cases) fh.write('--------Failed Cases--------------\n') for case_id in failed_updates: fh.write(case_id) print('-------------COMPLETE--------------') def _get_cases_to_process(self, domain): from corehq.sql_db.util import get_db_aliases_for_partitioned_query dbs = get_db_aliases_for_partitioned_query() for db in dbs: cases = CommCareCaseSQL.objects.using(db).filter(domain=domain, type='household', closed=True) for case in cases: yield case.case_id def _get_related_cases(self, cases): related_cases = {case.case_id for case in self.case_accessor.get_all_reverse_indices_info(list(cases)) if case.relationship == CommCareCaseIndexSQL.CHILD} related_cases |= {case.case_id for case in self.case_accessor.get_all_reverse_indices_info(list(related_cases)) if case.relationship == CommCareCaseIndexSQL.CHILD} return related_cases
def cases_referenced_by_xform(xform): """ Returns a list of CommCareCase or CommCareCaseSQL given a JSON representation of an XFormInstance """ from corehq.form_processor.backends.couch.dbaccessors import CaseAccessorCouch from corehq.form_processor.interfaces.dbaccessors import CaseAccessors case_ids = get_case_ids_from_form(xform) domain = get_and_check_xform_domain(xform) case_accessor = CaseAccessors(domain) if domain is None: assert case_accessor.db_accessor == CaseAccessorCouch return list(case_accessor.get_cases(list(case_ids)))
def get_adherence_cases_between_dates(domain, person_case_id, start_date, end_date): case_accessor = CaseAccessors(domain) episode = get_open_episode_case_from_person(domain, person_case_id) indexed_cases = case_accessor.get_reverse_indexed_cases([episode.case_id]) open_pertinent_adherence_cases = [ case for case in indexed_cases if not case.closed and case.type == CASE_TYPE_ADHERENCE and (start_date.astimezone(pytz.UTC) <= parse_datetime(case.dynamic_case_properties().get('adherence_date')).astimezone(pytz.UTC) <= end_date.astimezone(pytz.UTC)) ] return open_pertinent_adherence_cases
def _with_case(domain, case_type, last_modified): with drop_connected_signals(case_post_save): case = CaseFactory(domain).create_case(case_type=case_type) _update_case(domain, case.case_id, last_modified) accessors = CaseAccessors(domain) case = accessors.get_case(case.case_id) try: yield case finally: if should_use_sql_backend(domain): CaseAccessorSQL.hard_delete_cases(domain, [case.case_id]) else: case.delete()
def hint_still_valid(domain, hint): """ For a given domain/owner/cleanliness hint check if it's still valid """ casedb = CaseAccessors(domain) try: hint_case = casedb.get_case(hint) hint_owner = hint_case.owner_id except CaseNotFound: # hint was deleted return False dependent_case_ids = set(get_dependent_case_info(domain, [hint]).all_ids) return any([c.owner_id != hint_owner and c.owner_id != UNOWNED_EXTENSION_OWNER_ID for c in casedb.get_cases(list(dependent_case_ids))])
def handle(self, *args, **options): domain = options["domain"] case_id = options["case_id"] case_accessor = CaseAccessors(domain=domain) case = case_accessor.get_case(case_id) if ( not case.is_deleted and raw_input( "\n".join(["Case {} is not already deleted. Are you sure you want to delete it? (y/N)".format(case_id)]) ).lower() != "y" ): sys.exit(0) dependent_case_ids = get_entire_case_network(domain, [case_id]) cases_to_delete = filter(lambda case: not case.is_deleted, case_accessor.get_cases(dependent_case_ids)) if cases_to_delete: with open(options["filename"], "w") as csvfile: writer = csv.writer(csvfile) headers = ["case id", "case type", "owner", "opened by", "app version"] writer.writerow(headers) print headers for case in cases_to_delete: form = FormAccessors(domain=domain).get_form(case.xform_ids[0]) app_version_info = get_app_version_info( domain, form.build_id, form.form_data["@version"], form.metadata ) row = [ case.case_id, case.type, cached_owner_id_to_display(case.owner_id) or case.owner_id, cached_owner_id_to_display(case.opened_by), app_version_info.build_version, ] writer.writerow(row) print row if ( cases_to_delete and raw_input("\n".join(["Delete these {} cases? (y/N)".format(len(cases_to_delete))])).lower() == "y" ): case_accessor.soft_delete_cases([c.case_id for c in cases_to_delete]) print "deleted {} cases".format(len(cases_to_delete)) if cases_to_delete: print "details here: {}".format(options["filename"]) else: print "didn't find any cases to delete"
class Command(BaseCommand): help = """ Finds cases with duplicate IDs and marks all but one of each ID as a duplicate """ logfile_fields = [ # person-case properties always logged 'person_case_id', 'person_name', 'dto_name', 'phi_name', 'owner_id', 'dob', 'phone_number', 'dataset', 'enrolled_in_private', # case-specific properties that may be updated 'case_type', 'case_id', 'name', 'person_id', 'person_id_flat', 'person_id_deprecated', 'person_id_flat_deprecated', 'person_id_at_request', 'person_id_flat_at_request', ] def add_arguments(self, parser): parser.add_argument('domain') parser.add_argument( '--commit', action='store_true', dest='commit', default=False, ) def handle(self, domain, **options): self.domain = domain self.accessor = CaseAccessors(domain) commit = options['commit'] self.id_generator = ReadableIdGenerator(domain, commit) filename = '{}-{}.csv'.format( self.__module__.split('.')[-1], datetime.datetime.now().strftime('%Y-%m-%d_%H.%M.%S')) print("Logging actions to {}".format(filename)) with open(filename, 'w') as f: logfile = csv.DictWriter(f, self.logfile_fields, extrasaction='ignore') logfile.writeheader() print("Finding duplicates") bad_case_stubs = get_duplicated_case_stubs(self.domain, CASE_TYPE_PERSON) bad_cases = self.accessor.iter_cases(stub['case_id'] for stub in bad_case_stubs) print("Processing duplicate cases") for person_case in with_progress_bar(bad_cases, len(bad_case_stubs)): if person_case.get_case_property( 'enrolled_in_private') == 'true': updates = list( filter(None, self.get_private_updates(person_case))) else: updates = list( filter(None, self.get_public_updates(person_case))) person_info = self.get_person_case_info(person_case) for case, update in updates: log = { unidecode(k): unidecode(v) for d in [person_info, update] for k, v in d.items() if v } log['case_type'] = case.type log['case_id'] = case.case_id logfile.writerow(log) if commit: update_tuples = [(case.case_id, update, False) for case, update in updates] bulk_update_cases(self.domain, update_tuples, self.__module__)
def __init__(self, timing_context, restore_state, async_task=None): self.timing_context = timing_context self.restore_state = restore_state self.case_accessor = CaseAccessors(self.restore_state.domain) self.async_task = async_task
if importer_util.is_valid_id(uploaded_owner_id, domain, id_cache): owner_id = uploaded_owner_id id_cache[uploaded_owner_id] = True else: errors.add(ImportErrors.InvalidOwnerId, i + 1, 'owner_id') id_cache[uploaded_owner_id] = False continue else: # if they didn't supply an owner_id mapping, default to current # user owner_id = user_id extras = {} if parent_id: try: parent_case = CaseAccessors(domain).get_case(parent_id) track_load() if parent_case.domain == domain: extras['index'] = { parent_ref: (parent_case.type, parent_id) } except ResourceNotFound: errors.add(ImportErrors.InvalidParentId, i + 1, 'parent_id') continue elif parent_external_id: parent_case, error = importer_util.lookup_case( 'external_id', parent_external_id, domain, parent_type) track_load() if parent_case: extras['index'] = {
class TestFormArchiving(TestCase, TestFileMixin): file_path = ('data', 'sample_xforms') root = os.path.dirname(__file__) def setUp(self): super(TestFormArchiving, self).setUp() self.casedb = CaseAccessors('test-domain') self.formdb = FormAccessors('test-domain') def tearDown(self): FormProcessorTestUtils.delete_all_xforms() FormProcessorTestUtils.delete_all_cases() super(TestFormArchiving, self).tearDown() @run_with_all_backends def testArchive(self): case_id = 'ddb8e2b3-7ce0-43e4-ad45-d7a2eebe9169' xml_data = self.get_xml('basic') response, xform, cases = submit_form_locally( xml_data, 'test-domain', ) self.assertTrue(xform.is_normal) self.assertEqual(0, len(xform.history)) lower_bound = datetime.utcnow() - timedelta(seconds=1) xform.archive(user_id='mr. librarian') upper_bound = datetime.utcnow() + timedelta(seconds=1) xform = self.formdb.get_form(xform.form_id) self.assertTrue(xform.is_archived) case = self.casedb.get_case(case_id) self.assertTrue(case.is_deleted) self.assertEqual(case.xform_ids, []) [archival] = xform.history self.assertTrue(lower_bound <= archival.date <= upper_bound) self.assertEqual('archive', archival.operation) self.assertEqual('mr. librarian', archival.user) lower_bound = datetime.utcnow() - timedelta(seconds=1) xform.unarchive(user_id='mr. researcher') upper_bound = datetime.utcnow() + timedelta(seconds=1) xform = self.formdb.get_form(xform.form_id) self.assertTrue(xform.is_normal) case = self.casedb.get_case(case_id) self.assertFalse(case.is_deleted) self.assertEqual(case.xform_ids, [xform.form_id]) [archival, restoration] = xform.history self.assertTrue(lower_bound <= restoration.date <= upper_bound) self.assertEqual('unarchive', restoration.operation) self.assertEqual('mr. researcher', restoration.user) @run_with_all_backends def testSignal(self): global archive_counter, restore_counter archive_counter = 0 restore_counter = 0 def count_archive(**kwargs): global archive_counter archive_counter += 1 def count_unarchive(**kwargs): global restore_counter restore_counter += 1 xform_archived.connect(count_archive) xform_unarchived.connect(count_unarchive) xml_data = self.get_xml('basic') response, xform, cases = submit_form_locally( xml_data, 'test-domain', ) self.assertEqual(0, archive_counter) self.assertEqual(0, restore_counter) xform.archive() self.assertEqual(1, archive_counter) self.assertEqual(0, restore_counter) xform = self.formdb.get_form(xform.form_id) xform.unarchive() self.assertEqual(1, archive_counter) self.assertEqual(1, restore_counter) @override_settings(TESTS_SHOULD_USE_SQL_BACKEND=True) def testPublishChanges(self): xml_data = self.get_xml('basic') response, xform, cases = submit_form_locally( xml_data, 'test-domain', ) with capture_kafka_changes_context(topics.FORM_SQL) as change_context: with drop_connected_signals(xform_archived): xform.archive() self.assertEqual(1, len(change_context.changes)) self.assertEqual(change_context.changes[0].id, xform.form_id) xform = self.formdb.get_form(xform.form_id) with capture_kafka_changes_context(topics.FORM_SQL) as change_context: with drop_connected_signals(xform_unarchived): xform.unarchive() self.assertEqual(1, len(change_context.changes)) self.assertEqual(change_context.changes[0].id, xform.form_id)
class EditFormTest(TestCase, TestFileMixin): ID = '7H46J37FGH3' domain = 'test-form-edits' file_path = ('data', 'deprecation') root = os.path.dirname(__file__) def setUp(self): super(EditFormTest, self).setUp() self.interface = FormProcessorInterface(self.domain) self.casedb = CaseAccessors(self.domain) self.formdb = FormAccessors(self.domain) def tearDown(self): FormProcessorTestUtils.delete_all_xforms(self.domain) FormProcessorTestUtils.delete_all_cases(self.domain) UnfinishedSubmissionStub.objects.all().delete() super(EditFormTest, self).tearDown() def test_basic_edit(self): original_xml = self.get_xml('original') edit_xml = self.get_xml('edit') xform = post_xform(original_xml, domain=self.domain) self.assertEqual(self.ID, xform.form_id) self.assertTrue(xform.is_normal) self.assertEqual("", xform.form_data['vitals']['height']) self.assertEqual("other", xform.form_data['assessment']['categories']) xform = post_xform(edit_xml, domain=self.domain) self.assertEqual(self.ID, xform.form_id) self.assertTrue(xform.is_normal) self.assertEqual("100", xform.form_data['vitals']['height']) self.assertEqual("Edited Baby!", xform.form_data['assessment']['categories']) deprecated_xform = self.formdb.get_form(xform.deprecated_form_id) self.assertEqual(self.ID, deprecated_xform.orig_id) self.assertNotEqual(self.ID, deprecated_xform.form_id) self.assertTrue(deprecated_xform.is_deprecated) self.assertEqual("", deprecated_xform.form_data['vitals']['height']) self.assertEqual("other", deprecated_xform.form_data['assessment']['categories']) self.assertEqual(xform.received_on, deprecated_xform.received_on) self.assertEqual(xform.deprecated_form_id, deprecated_xform.form_id) self.assertTrue(xform.edited_on > deprecated_xform.received_on) self.assertEqual( deprecated_xform.get_xml(), original_xml ) self.assertEqual(xform.get_xml(), edit_xml) def test_edit_an_error(self): form_id = uuid.uuid4().hex case_block = CaseBlock( create=True, case_id='', # this should cause the submission to error case_type='person', owner_id='some-owner', ) form, _ = submit_case_blocks(case_block.as_string(), domain=self.domain, form_id=form_id) self.assertTrue(form.is_error) self.assertTrue('IllegalCaseId' in form.problem) case_block.case_id = uuid.uuid4().hex form, _ = submit_case_blocks(case_block.as_string(), domain=self.domain, form_id=form_id) self.assertFalse(form.is_error) self.assertEqual(None, getattr(form, 'problem', None)) def test_broken_save(self): """ Test that if the second form submission terminates unexpectedly and the main form isn't saved, then there are no side effects such as the original having been marked as deprecated. """ original_xml = self.get_xml('original') edit_xml = self.get_xml('edit') result = submit_form_locally(original_xml, self.domain) xform = result.xform self.assertEqual(self.ID, xform.form_id) self.assertTrue(xform.is_normal) self.assertEqual(self.domain, xform.domain) self.assertEqual( UnfinishedSubmissionStub.objects.filter(xform_id=self.ID).count(), 0 ) with patch.object(self.interface.processor, 'save_processed_models', side_effect=RequestFailed): with self.assertRaises(RequestFailed): submit_form_locally(edit_xml, self.domain) # it didn't go through, so make sure there are no edits still self.assertIsNone(getattr(xform, 'deprecated_form_id', None)) xform = self.formdb.get_form(self.ID) self.assertIsNotNone(xform) self.assertEqual( UnfinishedSubmissionStub.objects.filter(xform_id=self.ID, saved=False).count(), 1 ) self.assertEqual( UnfinishedSubmissionStub.objects.filter(xform_id=self.ID).count(), 1 ) def test_case_management(self): form_id = uuid.uuid4().hex case_id = uuid.uuid4().hex owner_id = uuid.uuid4().hex case_block = CaseBlock( create=True, case_id=case_id, case_type='person', owner_id=owner_id, update={ 'property': 'original value' } ).as_string() submit_case_blocks(case_block, domain=self.domain, form_id=form_id) # validate some assumptions case = self.casedb.get_case(case_id) self.assertEqual(case.type, 'person') self.assertEqual(case.dynamic_case_properties()['property'], 'original value') self.assertEqual([form_id], case.xform_ids) if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False): self.assertEqual(2, len(case.actions)) for a in case.actions: self.assertEqual(form_id, a.xform_id) # submit a new form with a different case update case_block = CaseBlock( create=True, case_id=case_id, case_type='newtype', owner_id=owner_id, update={ 'property': 'edited value' } ).as_string() xform, _ = submit_case_blocks(case_block, domain=self.domain, form_id=form_id) case = self.casedb.get_case(case_id) self.assertEqual(case.type, 'newtype') self.assertEqual(case.dynamic_case_properties()['property'], 'edited value') self.assertEqual([form_id], case.xform_ids) self.assertEqual(case.server_modified_on, xform.edited_on) if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False): self.assertEqual(2, len(case.actions)) for a in case.actions: self.assertEqual(form_id, a.xform_id) def test_second_edit_fails(self): form_id = uuid.uuid4().hex case_id = uuid.uuid4().hex case_block = CaseBlock( create=True, case_id=case_id, case_type='person', ).as_string() submit_case_blocks(case_block, domain=self.domain, form_id=form_id) # submit an edit form with a bad case update (for example a bad ID) case_block = CaseBlock( create=True, case_id='', case_type='person', ).as_string() submit_case_blocks(case_block, domain=self.domain, form_id=form_id) xform = self.formdb.get_form(form_id) self.assertTrue(xform.is_error) deprecated_xform = self.formdb.get_form(xform.deprecated_form_id) self.assertTrue(deprecated_xform.is_deprecated) def test_case_management_ordering(self): case_id = uuid.uuid4().hex owner_id = uuid.uuid4().hex # create a case case_block = CaseBlock( create=True, case_id=case_id, case_type='person', owner_id=owner_id, ).as_string() create_form_id = submit_case_blocks(case_block, domain=self.domain)[0].form_id # validate that worked case = self.casedb.get_case(case_id) self.assertEqual([create_form_id], case.xform_ids) if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False): self.assertTrue(create_form_id in [a.xform_id for a in case.actions]) for a in case.actions: self.assertEqual(create_form_id, a.xform_id) edit_date = datetime.utcnow() # set some property value case_block = CaseBlock( create=False, case_id=case_id, date_modified=edit_date, update={ 'property': 'first value', } ).as_string() edit_form_id = submit_case_blocks(case_block, domain=self.domain)[0].form_id # validate that worked case = self.casedb.get_case(case_id) self.assertEqual(case.dynamic_case_properties()['property'], 'first value') self.assertEqual([create_form_id, edit_form_id], case.xform_ids) if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False): self.assertTrue(all( form_id in [a.xform_id for a in case.actions] for form_id in [create_form_id, edit_form_id] )) # submit a second (new) form updating the value case_block = CaseBlock( create=False, case_id=case_id, update={ 'property': 'final value', } ).as_string() second_edit_form_id = submit_case_blocks(case_block, domain=self.domain)[0].form_id # validate that worked case = self.casedb.get_case(case_id) self.assertEqual(case.dynamic_case_properties()['property'], 'final value') self.assertEqual([create_form_id, edit_form_id, second_edit_form_id], case.xform_ids) if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False): self.assertTrue(all( form_id in [a.xform_id for a in case.actions] for form_id in [create_form_id, edit_form_id, second_edit_form_id] )) # deprecate the middle edit case_block = CaseBlock( create=False, case_id=case_id, date_modified=edit_date, # need to use the previous edit date for action sort comparisons update={ 'property': 'edited value', 'added_property': 'added value', } ).as_string() submit_case_blocks(case_block, domain=self.domain, form_id=edit_form_id) # ensure that the middle edit stays in the right place and is applied # before the final one case = self.casedb.get_case(case_id) self.assertEqual(case.dynamic_case_properties()['property'], 'final value') self.assertEqual(case.dynamic_case_properties()['added_property'], 'added value') self.assertEqual([create_form_id, edit_form_id, second_edit_form_id], case.xform_ids) if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False): self.assertEqual( [create_form_id, create_form_id, edit_form_id, second_edit_form_id], [a.xform_id for a in case.actions] )
def get(self, request, domain, case_id=None, attachment_id=None): """ https://github.com/dimagi/commcare/wiki/CaseAttachmentAPI max_size The largest size (in bytes) for the attachment max_image_width The largest width in pixels for an an image attachment max_image_height The largest width in pixels for an an image attachment """ if self.request.couch_user.is_web_user() and not can_view_attachments( self.request): return HttpResponseForbidden() if not case_id or not attachment_id: raise Http404 img = self.request.GET.get('img', None) size = self.request.GET.get('size', OBJECT_ORIGINAL) max_width = int(self.request.GET.get('max_image_width', 0)) max_height = int(self.request.GET.get('max_image_height', 0)) max_filesize = int(self.request.GET.get('max_size', 0)) try: CaseAccessors(domain).get_case(case_id) except CaseNotFound: raise Http404 if img is not None: if size == "debug_all": url_base = reverse("api_case_attachment", kwargs={ "domain": self.request.domain, "case_id": case_id, "attachment_id": attachment_id, }) r = HttpResponse(content_type="text/html") r.write('<html><body>') r.write('<ul>') for fsize in IMAGE_SIZE_ORDERING: meta, stream = fetch_case_image( domain, case_id, attachment_id, filesize_limit=max_filesize, width_limit=max_width, height_limit=max_height, fixed_size=fsize) r.write('<li>') r.write('Size: %s<br>' % fsize) r.write("Limit: max_size: %s" % max_filesize) if max_width > 0: r.write(", max_width: %s" % max_width) if max_height > 0: r.write(", max_height: %s" % max_height) r.write("<br>") if meta is not None: r.write('Resolution: %d x %d<br>' % (meta['width'], meta['height'])) r.write('Filesize: %d<br>' % meta['content_length']) url_params = urllib.urlencode({ "img": '1', "size": fsize, "max_size": max_filesize, "max_image_width": max_width, "max_image_height": max_height }) r.write('<img src="%(attach_url)s?%(params)s">' % { "attach_url": url_base, "params": url_params }) else: r.write('Not available') r.write('</li>') r.write('</ul></body></html>') return r else: attachment_meta, attachment_stream = fetch_case_image( domain, case_id, attachment_id, filesize_limit=max_filesize, width_limit=max_width, height_limit=max_height, fixed_size=size) else: cached_attachment = get_cached_case_attachment( domain, case_id, attachment_id) attachment_meta, attachment_stream = cached_attachment.get() if attachment_meta is not None: mime_type = attachment_meta['content_type'] else: mime_type = "plain/text" return StreamingHttpResponse( streaming_content=FileWrapper(attachment_stream), content_type=mime_type)
class ImporterTest(TestCase): def setUp(self): super(ImporterTest, self).setUp() self.domain_obj = create_domain("importer-test") self.domain = self.domain_obj.name self.default_case_type = 'importer-test-casetype' self.couch_user = WebUser.create(None, "test", "foobar", None, None) self.couch_user.add_domain_membership(self.domain, is_admin=True) self.couch_user.save() self.accessor = CaseAccessors(self.domain) self.factory = CaseFactory(domain=self.domain, case_defaults={ 'case_type': self.default_case_type, }) delete_all_cases() def tearDown(self): self.couch_user.delete(deleted_by=None) self.domain_obj.delete() super(ImporterTest, self).tearDown() def _config(self, col_names, search_column=None, case_type=None, search_field='case_id', create_new_cases=True): return ImporterConfig( couch_user_id=self.couch_user._id, case_type=case_type or self.default_case_type, excel_fields=col_names, case_fields=[''] * len(col_names), custom_fields=col_names, search_column=search_column or col_names[0], search_field=search_field, create_new_cases=create_new_cases, ) @run_with_all_backends @patch('corehq.apps.case_importer.tasks.bulk_import_async.update_state') def testImportFileMissing(self, update_state): # by using a made up upload_id, we ensure it's not referencing any real file case_upload = CaseUploadRecord(upload_id=str(uuid.uuid4()), task_id=str(uuid.uuid4())) case_upload.save() res = bulk_import_async.delay(self._config(['anything']), self.domain, case_upload.upload_id) self.assertIsInstance(res.result, Ignore) update_state.assert_called_with( state=states.FAILURE, meta=get_interned_exception( 'Sorry, your session has expired. Please start over and try again.' )) self.assertEqual(0, len(get_case_ids_in_domain(self.domain))) @run_with_all_backends def testImportBasic(self): config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], ['case_id-0', 'age-0', 'sex-0', 'location-0'], ['case_id-1', 'age-1', 'sex-1', 'location-1'], ['case_id-2', 'age-2', 'sex-2', 'location-2'], ['case_id-3', 'age-3', 'sex-3', 'location-3'], ['case_id-4', 'age-4', 'sex-4', 'location-4'], ) res = do_import(file, config, self.domain) self.assertEqual(5, res['created_count']) self.assertEqual(0, res['match_count']) self.assertFalse(res['errors']) self.assertEqual(1, res['num_chunks']) case_ids = self.accessor.get_case_ids_in_domain() cases = list(self.accessor.get_cases(case_ids)) self.assertEqual(5, len(cases)) properties_seen = set() for case in cases: self.assertEqual(self.couch_user._id, case.user_id) self.assertEqual(self.couch_user._id, case.owner_id) self.assertEqual(self.default_case_type, case.type) for prop in ['age', 'sex', 'location']: self.assertTrue(prop in case.get_case_property(prop)) self.assertFalse( case.get_case_property(prop) in properties_seen) properties_seen.add(case.get_case_property(prop)) @run_with_all_backends def testImportNamedColumns(self): config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], ['case_id-0', 'age-0', 'sex-0', 'location-0'], ['case_id-1', 'age-1', 'sex-1', 'location-1'], ['case_id-2', 'age-2', 'sex-2', 'location-2'], ['case_id-3', 'age-3', 'sex-3', 'location-3'], ) res = do_import(file, config, self.domain) self.assertEqual(4, res['created_count']) self.assertEqual(4, len(self.accessor.get_case_ids_in_domain())) @run_with_all_backends def testImportTrailingWhitespace(self): cols = ['case_id', 'age', 'sex\xa0', 'location'] config = self._config(cols) file = make_worksheet_wrapper( ['case_id', 'age', 'sex\xa0', 'location'], ['case_id-0', 'age-0', 'sex\xa0-0', 'location-0'], ) res = do_import(file, config, self.domain) self.assertEqual(1, res['created_count']) case_ids = self.accessor.get_case_ids_in_domain() self.assertEqual(1, len(case_ids)) case = self.accessor.get_case(case_ids[0]) self.assertTrue(bool(case.get_case_property( 'sex'))) # make sure the value also got properly set @run_with_all_backends def testCaseIdMatching(self): # bootstrap a stub case [case] = self.factory.create_or_update_case( CaseStructure(attrs={ 'create': True, 'update': { 'importer_test_prop': 'foo' }, })) self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], [case.case_id, 'age-0', 'sex-0', 'location-0'], [case.case_id, 'age-1', 'sex-1', 'location-1'], [case.case_id, 'age-2', 'sex-2', 'location-2'], ) res = do_import(file, config, self.domain) self.assertEqual(0, res['created_count']) self.assertEqual(3, res['match_count']) self.assertFalse(res['errors']) # shouldn't create any more cases, just the one case_ids = self.accessor.get_case_ids_in_domain() self.assertEqual(1, len(case_ids)) [case] = self.accessor.get_cases(case_ids) for prop in ['age', 'sex', 'location']: self.assertTrue(prop in case.get_case_property(prop)) # shouldn't touch existing properties self.assertEqual('foo', case.get_case_property('importer_test_prop')) @run_with_all_backends def testCaseLookupTypeCheck(self): [case] = self.factory.create_or_update_case( CaseStructure(attrs={ 'create': True, 'case_type': 'nonmatch-type', })) self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], [case.case_id, 'age-0', 'sex-0', 'location-0'], [case.case_id, 'age-1', 'sex-1', 'location-1'], [case.case_id, 'age-2', 'sex-2', 'location-2'], ) res = do_import(file, config, self.domain) # because the type is wrong these shouldn't match self.assertEqual(3, res['created_count']) self.assertEqual(0, res['match_count']) self.assertEqual(4, len(self.accessor.get_case_ids_in_domain())) @run_with_all_backends def testCaseLookupDomainCheck(self): self.factory.domain = 'wrong-domain' [case] = self.factory.create_or_update_case( CaseStructure(attrs={ 'create': True, })) self.assertEqual(0, len(self.accessor.get_case_ids_in_domain())) config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], [case.case_id, 'age-0', 'sex-0', 'location-0'], [case.case_id, 'age-1', 'sex-1', 'location-1'], [case.case_id, 'age-2', 'sex-2', 'location-2'], ) res = do_import(file, config, self.domain) # because the domain is wrong these shouldn't match self.assertEqual(3, res['created_count']) self.assertEqual(0, res['match_count']) self.assertEqual(3, len(self.accessor.get_case_ids_in_domain())) @run_with_all_backends def testExternalIdMatching(self): # bootstrap a stub case external_id = 'importer-test-external-id' [case] = self.factory.create_or_update_case( CaseStructure(attrs={ 'create': True, 'external_id': external_id, })) self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) headers = ['external_id', 'age', 'sex', 'location'] config = self._config(headers, search_field='external_id') file = make_worksheet_wrapper( ['external_id', 'age', 'sex', 'location'], ['importer-test-external-id', 'age-0', 'sex-0', 'location-0'], ['importer-test-external-id', 'age-1', 'sex-1', 'location-1'], ['importer-test-external-id', 'age-2', 'sex-2', 'location-2'], ) res = do_import(file, config, self.domain) self.assertEqual(0, res['created_count']) self.assertEqual(3, res['match_count']) self.assertFalse(res['errors']) # shouldn't create any more cases, just the one self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) @run_with_all_backends def test_external_id_matching_on_create_with_custom_column_name(self): headers = ['id_column', 'age', 'sex', 'location'] external_id = 'external-id-test' config = self._config(headers[1:], search_column='id_column', search_field='external_id') file = make_worksheet_wrapper( ['id_column', 'age', 'sex', 'location'], ['external-id-test', 'age-0', 'sex-0', 'location-0'], ['external-id-test', 'age-1', 'sex-1', 'location-1'], ) res = do_import(file, config, self.domain) self.assertFalse(res['errors']) self.assertEqual(1, res['created_count']) self.assertEqual(1, res['match_count']) case_ids = self.accessor.get_case_ids_in_domain() self.assertEqual(1, len(case_ids)) case = self.accessor.get_case(case_ids[0]) self.assertEqual(external_id, case.external_id) def testNoCreateNew(self): config = self._config(['case_id', 'age', 'sex', 'location'], create_new_cases=False) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], ['case_id-0', 'age-0', 'sex-0', 'location-0'], ['case_id-1', 'age-1', 'sex-1', 'location-1'], ['case_id-2', 'age-2', 'sex-2', 'location-2'], ['case_id-3', 'age-3', 'sex-3', 'location-3'], ['case_id-4', 'age-4', 'sex-4', 'location-4'], ) res = do_import(file, config, self.domain) # no matching and no create new set - should do nothing self.assertEqual(0, res['created_count']) self.assertEqual(0, res['match_count']) self.assertEqual(0, len(get_case_ids_in_domain(self.domain))) def testBlankRows(self): # don't create new cases for rows left blank config = self._config(['case_id', 'age', 'sex', 'location'], create_new_cases=True) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], [None, None, None, None], ['', '', '', ''], ) res = do_import(file, config, self.domain) # no matching and no create new set - should do nothing self.assertEqual(0, res['created_count']) self.assertEqual(0, res['match_count']) self.assertEqual(0, len(get_case_ids_in_domain(self.domain))) @patch('corehq.apps.case_importer.do_import.CASEBLOCK_CHUNKSIZE', 2) def testBasicChunking(self): config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], ['case_id-0', 'age-0', 'sex-0', 'location-0'], ['case_id-1', 'age-1', 'sex-1', 'location-1'], ['case_id-2', 'age-2', 'sex-2', 'location-2'], ['case_id-3', 'age-3', 'sex-3', 'location-3'], ['case_id-4', 'age-4', 'sex-4', 'location-4'], ) res = do_import(file, config, self.domain) # 5 cases in chunks of 2 = 3 chunks self.assertEqual(3, res['num_chunks']) self.assertEqual(5, res['created_count']) self.assertEqual(5, len(get_case_ids_in_domain(self.domain))) @run_with_all_backends def testExternalIdChunking(self): # bootstrap a stub case external_id = 'importer-test-external-id' headers = ['external_id', 'age', 'sex', 'location'] config = self._config(headers, search_field='external_id') file = make_worksheet_wrapper( ['external_id', 'age', 'sex', 'location'], ['importer-test-external-id', 'age-0', 'sex-0', 'location-0'], ['importer-test-external-id', 'age-1', 'sex-1', 'location-1'], ['importer-test-external-id', 'age-2', 'sex-2', 'location-2'], ) # the first one should create the case, and the remaining two should update it res = do_import(file, config, self.domain) self.assertEqual(1, res['created_count']) self.assertEqual(2, res['match_count']) self.assertFalse(res['errors']) self.assertEqual(2, res['num_chunks']) # the lookup causes an extra chunk # should just create the one case case_ids = self.accessor.get_case_ids_in_domain() self.assertEqual(1, len(case_ids)) [case] = self.accessor.get_cases(case_ids) self.assertEqual(external_id, case.external_id) for prop in ['age', 'sex', 'location']: self.assertTrue(prop in case.get_case_property(prop)) @run_with_all_backends def testParentCase(self): headers = ['parent_id', 'name', 'case_id'] config = self._config(headers, create_new_cases=True, search_column='case_id') rows = 3 [parent_case] = self.factory.create_or_update_case( CaseStructure(attrs={'create': True})) self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) file = make_worksheet_wrapper( ['parent_id', 'name', 'case_id'], [parent_case.case_id, 'name-0', 'case_id-0'], [parent_case.case_id, 'name-1', 'case_id-1'], [parent_case.case_id, 'name-2', 'case_id-2'], ) file_missing = make_worksheet_wrapper( ['parent_id', 'name', 'case_id'], ['parent_id-0', 'name-0', 'case_id-0'], ['parent_id-1', 'name-1', 'case_id-1'], ['parent_id-2', 'name-2', 'case_id-2'], ) # Should successfully match on `rows` cases res = do_import(file, config, self.domain) self.assertEqual(rows, res['created_count']) # Should be unable to find parent case on `rows` cases res = do_import(file_missing, config, self.domain) error_column_name = 'parent_id' self.assertEqual( rows, len(res['errors'][exceptions.InvalidParentId.title] [error_column_name]['rows']), "All cases should have missing parent") def import_mock_file(self, rows): config = self._config(rows[0]) xls_file = make_worksheet_wrapper(*rows) return do_import(xls_file, config, self.domain) @run_with_all_backends def testLocationOwner(self): # This is actually testing several different things, but I figure it's # worth it, as each of these tests takes a non-trivial amount of time. non_case_sharing = LocationType.objects.create(domain=self.domain, name='lt1', shares_cases=False) case_sharing = LocationType.objects.create(domain=self.domain, name='lt2', shares_cases=True) location = make_loc('loc-1', 'Loc 1', self.domain, case_sharing.code) make_loc('loc-2', 'Loc 2', self.domain, case_sharing.code) duplicate_loc = make_loc('loc-3', 'Loc 2', self.domain, case_sharing.code) improper_loc = make_loc('loc-4', 'Loc 4', self.domain, non_case_sharing.code) res = self.import_mock_file([ ['case_id', 'name', 'owner_id', 'owner_name'], ['', 'location-owner-id', location.group_id, ''], ['', 'location-owner-code', '', location.site_code], ['', 'location-owner-name', '', location.name], ['', 'duplicate-location-name', '', duplicate_loc.name], ['', 'non-case-owning-name', '', improper_loc.name], ]) case_ids = self.accessor.get_case_ids_in_domain() cases = {c.name: c for c in list(self.accessor.get_cases(case_ids))} self.assertEqual(cases['location-owner-id'].owner_id, location.group_id) self.assertEqual(cases['location-owner-code'].owner_id, location.group_id) self.assertEqual(cases['location-owner-name'].owner_id, location.group_id) error_message = exceptions.DuplicateLocationName.title error_column_name = None self.assertIn(error_message, res['errors']) self.assertEqual( res['errors'][error_message][error_column_name]['rows'], [5]) error_message = exceptions.InvalidOwner.title self.assertIn(error_message, res['errors']) error_column_name = 'owner_name' self.assertEqual( res['errors'][error_message][error_column_name]['rows'], [6]) @run_with_all_backends def test_opened_on(self): case = self.factory.create_case() new_date = '2015-04-30T14:41:53.000000Z' with flag_enabled('BULK_UPLOAD_DATE_OPENED'): self.import_mock_file([['case_id', 'date_opened'], [case.case_id, new_date]]) case = CaseAccessors(self.domain).get_case(case.case_id) self.assertEqual(case.opened_on, PhoneTime(parse_datetime(new_date)).done())
def payload_doc(self, repeat_record): return CaseAccessors(repeat_record.domain).get_case( repeat_record.payload_id)
return (previous_owner_id and previous_owner_id != UNOWNED_EXTENSION_OWNER_ID and previous_owner_id != next_owner_id) def get_all_extensions_to_close(domain, case_updates): extensions_to_close = set() for case_update_meta in case_updates: extensions_to_close = extensions_to_close | get_extensions_to_close( case_update_meta.case, domain) return extensions_to_close def get_extensions_to_close(case, domain): if case.closed and EXTENSION_CASES_SYNC_ENABLED.enabled(domain): return CaseAccessors(domain).get_extension_chain([case.case_id], include_closed=False) else: return set() def is_device_report(doc): """exclude device reports""" device_report_xmlns = "http://code.javarosa.org/devicereport" def _from_form_dict(doc): return isinstance( doc, dict) and "@xmlns" in doc and doc["@xmlns"] == device_report_xmlns def _from_xform_instance(doc):
def obj_get(self, bundle, **kwargs): case_id = kwargs['pk'] try: return CaseAccessors(kwargs['domain']).get_case(case_id) except CaseNotFound: raise object_does_not_exist("CommCareCase", case_id)
def _get_updated_episode(self): self.case_updater.run() return CaseAccessors(self.domain).get_case(self.episode_id)
def couch_doc(self): if self._couch_doc is None: self._couch_doc = CaseAccessors(self.domain).get_case(self._id) return self._couch_doc
class CaseAPIHelper(object): """ Simple config object for querying the APIs """ def __init__(self, domain, status=CASE_STATUS_OPEN, case_type=None, ids_only=False, footprint=False, strip_history=False, filters=None): if status not in [ CASE_STATUS_ALL, CASE_STATUS_CLOSED, CASE_STATUS_OPEN ]: raise ValueError("invalid case status %s" % status) self.domain = domain self.status = status self.case_type = case_type self.ids_only = ids_only self.wrap = not ids_only # if we're just querying IDs we don't need to wrap the docs self.footprint = footprint self.strip_history = strip_history self.filters = filters self.case_accessors = CaseAccessors(self.domain) def _case_results(self, case_id_list): def _filter(res): if self.filters: for path, val in self.filters.items(): actual_val = safe_index(res.case_json, path.split("/")) if actual_val != val: # closed=false => case.closed == False if val in ('null', 'true', 'false'): if actual_val != json.loads(val): return False else: return False return True if self.filters and not self.footprint: base_results = self._populate_results(case_id_list) return filter(_filter, base_results) if self.footprint: initial_case_ids = set(case_id_list) dependent_case_ids = get_dependent_case_info( self.domain, initial_case_ids).all_ids all_case_ids = initial_case_ids | dependent_case_ids else: all_case_ids = case_id_list if self.ids_only: return [ CaseAPIResult(domain=self.domain, id=case_id, id_only=True) for case_id in all_case_ids ] else: return self._populate_results(all_case_ids) def _populate_results(self, case_id_list): if should_use_sql_backend(self.domain): base_results = [ CaseAPIResult(domain=self.domain, couch_doc=case, id_only=self.ids_only) for case in self.case_accessors.iter_cases(case_id_list) ] else: base_results = [ CaseAPIResult(domain=self.domain, couch_doc=case, id_only=self.ids_only) for case in iter_cases( case_id_list, self.strip_history, self.wrap) ] return base_results def get_all(self): status = self.status or CASE_STATUS_ALL if status == CASE_STATUS_ALL: case_ids = self.case_accessors.get_case_ids_in_domain( self.case_type) elif status == CASE_STATUS_OPEN: case_ids = self.case_accessors.get_open_case_ids_in_domain_by_type( self.case_type) else: raise ValueError("Invalid value for 'status': '%s'" % status) return self._case_results(case_ids) def get_owned(self, user_id): try: user = CouchUser.get_by_user_id(user_id, self.domain) except KeyError: user = None try: owner_ids = user.get_owner_ids() except AttributeError: owner_ids = [user_id] closed = { CASE_STATUS_OPEN: False, CASE_STATUS_CLOSED: True, CASE_STATUS_ALL: None, }[self.status] ids = self.case_accessors.get_case_ids_by_owners(owner_ids, closed=closed) return self._case_results(ids)
case_location = get_case_location(case) if not case_location: return [] location_repeaters = defaultdict(list) for repeater in OpenmrsRepeater.by_domain(case.domain): if repeater.location_id: location_repeaters[repeater.location_id].append(repeater) for location_id in reversed(case_location.path): if location_id in location_repeaters: return location_repeaters[location_id] return [] def get_openmrs_location_uuid(domain, case_id): case = CaseAccessors(domain).get_case(case_id) location = get_case_location(case) return location.metadata.get(LOCATION_OPENMRS_UUID) if location else None class CreatePersonAttributeTask(WorkflowTask): def __init__(self, requests, person_uuid, attribute_type_uuid, value): self.requests = requests self.person_uuid = person_uuid self.attribute_type_uuid = attribute_type_uuid self.value = value self.attribute_uuid = None def run(self): response = self.requests.post(
def case(self): try: return CaseAccessors(self.domain).get_case(self.case_id) except CaseNotFound: return None
def setUp(self): super(TestFormArchiving, self).setUp() self.casedb = CaseAccessors('test-domain') self.formdb = FormAccessors('test-domain')
class ImporterTest(TestCase): def setUp(self): super(ImporterTest, self).setUp() self.domain = create_domain("importer-test").name self.default_case_type = 'importer-test-casetype' self.default_headers = ['case_id', 'age', 'sex', 'location'] self.couch_user = WebUser.create(None, "test", "foobar") self.couch_user.add_domain_membership(self.domain, is_admin=True) self.couch_user.save() self.accessor = CaseAccessors(self.domain) self.factory = CaseFactory(domain=self.domain, case_defaults={ 'case_type': self.default_case_type, }) delete_all_cases() def tearDown(self): self.couch_user.delete() delete_all_locations() LocationType.objects.all().delete() super(ImporterTest, self).tearDown() def _config(self, col_names=None, search_column=None, case_type=None, search_field='case_id', named_columns=False, create_new_cases=True, type_fields=None): col_names = col_names or self.default_headers case_type = case_type or self.default_case_type search_column = search_column or col_names[0] type_fields = type_fields if type_fields is not None else [ 'plain' ] * len(col_names) return ImporterConfig( couch_user_id=self.couch_user._id, case_type=case_type, excel_fields=col_names, case_fields=[''] * len(col_names), custom_fields=col_names, type_fields=type_fields, search_column=search_column, search_field=search_field, named_columns=named_columns, create_new_cases=create_new_cases, key_column='', value_column='', ) @run_with_all_backends def testImportNone(self): res = bulk_import_async(self._config(), self.domain, None) self.assertEqual( 'Sorry, your session has expired. Please start over and try again.', unicode(res['errors'])) self.assertEqual(0, len(get_case_ids_in_domain(self.domain))) @run_with_all_backends def testImporterErrors(self): with mock.patch( 'corehq.apps.importer.tasks.importer_util.get_spreadsheet', side_effect=ImporterError()): res = bulk_import_async(self._config(), self.domain, None) self.assertEqual( 'The session containing the file you uploaded has expired - please upload a new one.', unicode(res['errors'])) self.assertEqual(0, len(get_case_ids_in_domain(self.domain))) @run_with_all_backends def testImportBasic(self): config = self._config(self.default_headers) file = MockExcelFile(header_columns=self.default_headers, num_rows=5) res = do_import(file, config, self.domain) self.assertEqual(5, res['created_count']) self.assertEqual(0, res['match_count']) self.assertFalse(res['errors']) self.assertEqual(1, res['num_chunks']) case_ids = self.accessor.get_case_ids_in_domain() cases = list(self.accessor.get_cases(case_ids)) self.assertEqual(5, len(cases)) properties_seen = set() for case in cases: self.assertEqual(self.couch_user._id, case.user_id) self.assertEqual(self.couch_user._id, case.owner_id) self.assertEqual(self.default_case_type, case.type) for prop in self.default_headers[1:]: self.assertTrue(prop in case.get_case_property(prop)) self.assertFalse( case.get_case_property(prop) in properties_seen) properties_seen.add(case.get_case_property(prop)) @run_with_all_backends def testImportNamedColumns(self): config = self._config(self.default_headers, named_columns=True) file = MockExcelFile(header_columns=self.default_headers, num_rows=5) res = do_import(file, config, self.domain) # we create 1 less since we knock off the header column self.assertEqual(4, res['created_count']) self.assertEqual(4, len(self.accessor.get_case_ids_in_domain())) @run_with_all_backends def testImportTrailingWhitespace(self): cols = ['case_id', 'age', u'sex\xa0', 'location'] config = self._config(cols, named_columns=True) file = MockExcelFile(header_columns=cols, num_rows=2) res = do_import(file, config, self.domain) # we create 1 less since we knock off the header column self.assertEqual(1, res['created_count']) case_ids = self.accessor.get_case_ids_in_domain() self.assertEqual(1, len(case_ids)) case = self.accessor.get_case(case_ids[0]) self.assertTrue(bool(case.get_case_property( 'sex'))) # make sure the value also got properly set @run_with_all_backends def testCaseIdMatching(self): # bootstrap a stub case [case] = self.factory.create_or_update_case( CaseStructure(attrs={ 'create': True, 'update': { 'importer_test_prop': 'foo' }, })) self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) config = self._config(self.default_headers) file = MockExcelFile(header_columns=self.default_headers, num_rows=3, row_generator=id_match_generator(case.case_id)) res = do_import(file, config, self.domain) self.assertEqual(0, res['created_count']) self.assertEqual(3, res['match_count']) self.assertFalse(res['errors']) # shouldn't create any more cases, just the one case_ids = self.accessor.get_case_ids_in_domain() self.assertEqual(1, len(case_ids)) [case] = self.accessor.get_cases(case_ids) for prop in self.default_headers[1:]: self.assertTrue(prop in case.get_case_property(prop)) # shouldn't touch existing properties self.assertEqual('foo', case.get_case_property('importer_test_prop')) @run_with_all_backends def testCaseLookupTypeCheck(self): [case] = self.factory.create_or_update_case( CaseStructure(attrs={ 'create': True, 'case_type': 'nonmatch-type', })) self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) config = self._config(self.default_headers) file = MockExcelFile(header_columns=self.default_headers, num_rows=3, row_generator=id_match_generator(case.case_id)) res = do_import(file, config, self.domain) # because the type is wrong these shouldn't match self.assertEqual(3, res['created_count']) self.assertEqual(0, res['match_count']) self.assertEqual(4, len(self.accessor.get_case_ids_in_domain())) @run_with_all_backends def testCaseLookupDomainCheck(self): self.factory.domain = 'wrong-domain' [case] = self.factory.create_or_update_case( CaseStructure(attrs={ 'create': True, })) self.assertEqual(0, len(self.accessor.get_case_ids_in_domain())) config = self._config(self.default_headers) file = MockExcelFile(header_columns=self.default_headers, num_rows=3, row_generator=id_match_generator(case.case_id)) res = do_import(file, config, self.domain) # because the domain is wrong these shouldn't match self.assertEqual(3, res['created_count']) self.assertEqual(0, res['match_count']) self.assertEqual(3, len(self.accessor.get_case_ids_in_domain())) @run_with_all_backends def testExternalIdMatching(self): # bootstrap a stub case external_id = 'importer-test-external-id' [case] = self.factory.create_or_update_case( CaseStructure(attrs={ 'create': True, 'external_id': external_id, })) self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) headers = ['external_id', 'age', 'sex', 'location'] config = self._config(headers, search_field='external_id') file = MockExcelFile(header_columns=headers, num_rows=3, row_generator=id_match_generator(external_id)) res = do_import(file, config, self.domain) self.assertEqual(0, res['created_count']) self.assertEqual(3, res['match_count']) self.assertFalse(res['errors']) # shouldn't create any more cases, just the one self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) @run_with_all_backends def test_external_id_matching_on_create_with_custom_column_name(self): headers = ['id_column', 'age', 'sex', 'location'] external_id = 'external-id-test' config = self._config(headers[1:], search_column='id_column', search_field='external_id') file = MockExcelFile(header_columns=headers, num_rows=2, row_generator=id_match_generator(external_id)) res = do_import(file, config, self.domain) self.assertEqual(1, res['created_count']) self.assertEqual(1, res['match_count']) self.assertFalse(res['errors']) case_ids = self.accessor.get_case_ids_in_domain() self.assertEqual(1, len(case_ids)) case = self.accessor.get_case(case_ids[0]) self.assertEqual(external_id, case.external_id) def testNoCreateNew(self): config = self._config(self.default_headers, create_new_cases=False) file = MockExcelFile(header_columns=self.default_headers, num_rows=5) res = do_import(file, config, self.domain) # no matching and no create new set - should do nothing self.assertEqual(0, res['created_count']) self.assertEqual(0, res['match_count']) self.assertEqual(0, len(get_case_ids_in_domain(self.domain))) def testBlankRows(self): # don't create new cases for rows left blank config = self._config(self.default_headers, create_new_cases=True) file = MockExcelFile(header_columns=self.default_headers, num_rows=5, row_generator=blank_row_generator) res = do_import(file, config, self.domain) # no matching and no create new set - should do nothing self.assertEqual(0, res['created_count']) self.assertEqual(0, res['match_count']) self.assertEqual(0, len(get_case_ids_in_domain(self.domain))) def testBasicChunking(self): config = self._config(self.default_headers) file = MockExcelFile(header_columns=self.default_headers, num_rows=5) res = do_import(file, config, self.domain, chunksize=2) # 5 cases in chunks of 2 = 3 chunks self.assertEqual(3, res['num_chunks']) self.assertEqual(5, res['created_count']) self.assertEqual(5, len(get_case_ids_in_domain(self.domain))) @run_with_all_backends def testExternalIdChunking(self): # bootstrap a stub case external_id = 'importer-test-external-id' headers = ['external_id', 'age', 'sex', 'location'] config = self._config(headers, search_field='external_id') file = MockExcelFile(header_columns=headers, num_rows=3, row_generator=id_match_generator(external_id)) # the first one should create the case, and the remaining two should update it res = do_import(file, config, self.domain) self.assertEqual(1, res['created_count']) self.assertEqual(2, res['match_count']) self.assertFalse(res['errors']) self.assertEqual(2, res['num_chunks']) # the lookup causes an extra chunk # should just create the one case case_ids = self.accessor.get_case_ids_in_domain() self.assertEqual(1, len(case_ids)) [case] = self.accessor.get_cases(case_ids) self.assertEqual(external_id, case.external_id) for prop in self.default_headers[1:]: self.assertTrue(prop in case.get_case_property(prop)) @run_with_all_backends def testParentCase(self): headers = ['parent_id', 'name', 'case_id'] config = self._config(headers, create_new_cases=True, search_column='case_id') rows = 3 [parent_case] = self.factory.create_or_update_case( CaseStructure(attrs={'create': True})) self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) file = MockExcelFile(header_columns=headers, num_rows=rows, row_generator=id_match_generator( parent_case.case_id)) file_missing = MockExcelFile(header_columns=headers, num_rows=rows) # Should successfully match on `rows` cases res = do_import(file, config, self.domain) self.assertEqual(rows, res['created_count']) # Should be unable to find parent case on `rows` cases res = do_import(file_missing, config, self.domain) error_column_name = 'parent_id' self.assertEqual( rows, len(res['errors'][ImportErrors.InvalidParentId][error_column_name] ['rows']), "All cases should have missing parent") def import_mock_file(self, rows): config = self._config(rows[0]) case_rows = rows[1:] num_rows = len(case_rows) xls_file = MockExcelFile( header_columns=rows[0], num_rows=num_rows, row_generator=lambda _, i: case_rows[i], ) return do_import(xls_file, config, self.domain) @run_with_all_backends def testLocationOwner(self): # This is actually testing several different things, but I figure it's # worth it, as each of these tests takes a non-trivial amount of time. non_case_sharing = LocationType.objects.create(domain=self.domain, name='lt1', shares_cases=False) case_sharing = LocationType.objects.create(domain=self.domain, name='lt2', shares_cases=True) location = make_loc('loc-1', 'Loc 1', self.domain, case_sharing) make_loc('loc-2', 'Loc 2', self.domain, case_sharing) duplicate_loc = make_loc('loc-3', 'Loc 2', self.domain, case_sharing) improper_loc = make_loc('loc-4', 'Loc 4', self.domain, non_case_sharing) res = self.import_mock_file([ ['case_id', 'name', 'owner_id', 'owner_name'], ['', 'location-owner-id', location.group_id, ''], ['', 'location-owner-code', '', location.site_code], ['', 'location-owner-name', '', location.name], ['', 'duplicate-location-name', '', duplicate_loc.name], ['', 'non-case-owning-name', '', improper_loc.name], ]) case_ids = self.accessor.get_case_ids_in_domain() cases = {c.name: c for c in list(self.accessor.get_cases(case_ids))} self.assertEqual(cases['location-owner-id'].owner_id, location.group_id) self.assertEqual(cases['location-owner-code'].owner_id, location.group_id) self.assertEqual(cases['location-owner-name'].owner_id, location.group_id) error_message = ImportErrors.DuplicateLocationName error_column_name = None self.assertIn(error_message, res['errors']) self.assertEqual( res['errors'][error_message][error_column_name]['rows'], [4]) error_message = ImportErrors.InvalidOwnerId self.assertIn(error_message, res['errors']) error_column_name = 'owner_id' self.assertEqual( res['errors'][error_message][error_column_name]['rows'], [5]) def _typeTest(self, type_fields, error_message): config = self._config(self.default_headers, type_fields=type_fields) file = MockExcelFile(header_columns=self.default_headers, num_rows=3) res = do_import(file, config, self.domain) self.assertIn(self.default_headers[1], res['errors'][error_message]) self.assertEqual( res['errors'][error_message][self.default_headers[1]]['rows'], [1, 2, 3]) def testDateError(self): self._typeTest(['plain', 'date', 'plain', 'plain'], ImportErrors.InvalidDate) def testIntegerError(self): self._typeTest(['plain', 'integer', 'plain', 'plain'], ImportErrors.InvalidInteger)
class TestFormArchiving(TestCase, TestFileMixin): file_path = ('data', 'sample_xforms') root = os.path.dirname(__file__) def setUp(self): super(TestFormArchiving, self).setUp() self.casedb = CaseAccessors('test-domain') self.formdb = FormAccessors('test-domain') def tearDown(self): FormProcessorTestUtils.delete_all_xforms() FormProcessorTestUtils.delete_all_cases() super(TestFormArchiving, self).tearDown() def testArchive(self): case_id = 'ddb8e2b3-7ce0-43e4-ad45-d7a2eebe9169' xml_data = self.get_xml('basic') result = submit_form_locally( xml_data, 'test-domain', ) xform = result.xform self.assertTrue(xform.is_normal) self.assertEqual(0, len(xform.history)) lower_bound = datetime.utcnow() - timedelta(seconds=1) xform.archive(user_id='mr. librarian') upper_bound = datetime.utcnow() + timedelta(seconds=1) xform = self.formdb.get_form(xform.form_id) self.assertTrue(xform.is_archived) case = self.casedb.get_case(case_id) self.assertTrue(case.is_deleted) self.assertEqual(case.xform_ids, []) [archival] = xform.history self.assertTrue(lower_bound <= archival.date <= upper_bound) self.assertEqual('archive', archival.operation) self.assertEqual('mr. librarian', archival.user) lower_bound = datetime.utcnow() - timedelta(seconds=1) xform.unarchive(user_id='mr. researcher') upper_bound = datetime.utcnow() + timedelta(seconds=1) xform = self.formdb.get_form(xform.form_id) self.assertTrue(xform.is_normal) case = self.casedb.get_case(case_id) self.assertFalse(case.is_deleted) self.assertEqual(case.xform_ids, [xform.form_id]) [archival, restoration] = xform.history self.assertTrue(lower_bound <= restoration.date <= upper_bound) self.assertEqual('unarchive', restoration.operation) self.assertEqual('mr. researcher', restoration.user) def testSignal(self): global archive_counter, restore_counter archive_counter = 0 restore_counter = 0 def count_archive(**kwargs): global archive_counter archive_counter += 1 def count_unarchive(**kwargs): global restore_counter restore_counter += 1 xform_archived.connect(count_archive) xform_unarchived.connect(count_unarchive) xml_data = self.get_xml('basic') result = submit_form_locally( xml_data, 'test-domain', ) self.assertEqual(0, archive_counter) self.assertEqual(0, restore_counter) result.xform.archive() self.assertEqual(1, archive_counter) self.assertEqual(0, restore_counter) xform = self.formdb.get_form(result.xform.form_id) xform.unarchive() self.assertEqual(1, archive_counter) self.assertEqual(1, restore_counter)
def _get_case_ids(self): return CaseAccessors(domain=self.domain_name).get_case_ids_in_domain()
def setUp(self): super(EditFormTest, self).setUp() self.interface = FormProcessorInterface(self.domain) self.casedb = CaseAccessors(self.domain) self.formdb = FormAccessors(self.domain)
class ENikshay2BMigrator(object): def __init__(self, domain, commit): self.domain = domain self.commit = commit self.accessor = CaseAccessors(self.domain) self.factory = CaseFactory(self.domain) self.total_persons = 0 self.total_occurrences = 0 self.total_episodes = 0 self.total_tests = 0 self.total_referrals = 0 self.total_trails = 0 self.total_secondary_owners = 0 self.total_drtb_hiv = 0 @property @memoized def locations(self): return {loc.location_id: loc for loc in SQLLocation.objects.filter(domain=self.domain).prefetch_related('location_type')} @property @memoized def location_ids_by_pk(self): return {loc.pk: loc.location_id for loc in self.locations.values()} def get_ancestors_by_type(self, location): """Get all direct ancestors found in self.locations""" ancestors_by_type = {location.location_type.code: location} loc = location while loc.parent_id and loc.parent_id in self.location_ids_by_pk: parent = self.locations[self.location_ids_by_pk[loc.parent_id]] ancestors_by_type[parent.location_type.code] = parent loc = parent return ancestors_by_type def migrate(self): person_ids = self.get_relevant_person_case_ids() persons = self.get_relevant_person_case_sets(person_ids) for person in with_progress_bar(persons, len(person_ids)): self.migrate_person_case_set(person) def get_relevant_person_case_ids(self): return self.accessor.get_case_ids_in_domain(CASE_TYPE_PERSON) def get_relevant_person_case_sets(self, person_ids): """ Generator returning all relevant cases for the migration, grouped by person. This is a pretty nasty method, but it was the only way I could figure out how to group the queries together, rather than performing multiple queries per person case. """ for person_chunk in chunked(person_ids, 100): person_chunk = list(filter(None, person_chunk)) all_persons = {} # case_id: PersonCaseSet for person in self.accessor.get_cases(person_chunk): # enrolled_in_private is blank/not set AND case_version is blank/not set # AND owner_id is within the location set being migrated if (person.get_case_property(ENROLLED_IN_PRIVATE) != 'true' and not person.get_case_property(CASE_VERSION)): all_persons[person.case_id] = PersonCaseSet(person) referrals_and_occurrences_to_person = {} type_to_bucket = {CASE_TYPE_OCCURRENCE: 'occurrences', CASE_TYPE_REFERRAL: 'referrals', CASE_TYPE_TRAIL: 'trails'} for case in self.accessor.get_reverse_indexed_cases( [person_id for person_id in all_persons]): bucket = type_to_bucket.get(case.type, None) if bucket: for index in case.indices: if index.referenced_id in all_persons: getattr(all_persons[index.referenced_id], bucket).append(case) if bucket != 'trails': referrals_and_occurrences_to_person[case.case_id] = index.referenced_id break type_to_bucket = {CASE_TYPE_EPISODE: 'episodes', CASE_TYPE_TEST: 'tests', CASE_TYPE_TRAIL: 'trails'} episodes_to_person = {} for case in self.accessor.get_reverse_indexed_cases(list(referrals_and_occurrences_to_person)): bucket = type_to_bucket.get(case.type, None) if bucket: for index in case.indices: person_id = referrals_and_occurrences_to_person.get(index.referenced_id) if person_id: getattr(all_persons[person_id], bucket).append(case) if case.type == CASE_TYPE_EPISODE: episodes_to_person[case.case_id] = person_id break for case in self.accessor.get_reverse_indexed_cases(list(episodes_to_person)): if case.type == CASE_TYPE_DRTB_HIV_REFERRAL: for index in case.indices: person_id = episodes_to_person.get(index.referenced_id) if person_id: all_persons[person_id].drtb_hiv.append(case) break for person in all_persons.values(): if person.occurrences: person.latest_occurrence = max((case.opened_on, case) for case in person.occurrences)[1] yield person
class EditFormTest(TestCase, TestFileMixin): ID = '7H46J37FGH3' domain = 'test-form-edits' file_path = ('data', 'deprecation') root = os.path.dirname(__file__) def setUp(self): super(EditFormTest, self).setUp() self.interface = FormProcessorInterface(self.domain) self.casedb = CaseAccessors(self.domain) self.formdb = FormAccessors(self.domain) def tearDown(self): FormProcessorTestUtils.delete_all_xforms(self.domain) FormProcessorTestUtils.delete_all_cases(self.domain) UnfinishedSubmissionStub.objects.all().delete() super(EditFormTest, self).tearDown() def test_basic_edit(self): original_xml = self.get_xml('original') edit_xml = self.get_xml('edit') xform = submit_form_locally(original_xml, self.domain).xform self.assertEqual(self.ID, xform.form_id) self.assertTrue(xform.is_normal) self.assertEqual("", xform.form_data['vitals']['height']) self.assertEqual("other", xform.form_data['assessment']['categories']) xform = submit_form_locally(edit_xml, self.domain).xform self.assertEqual(self.ID, xform.form_id) self.assertTrue(xform.is_normal) self.assertEqual("100", xform.form_data['vitals']['height']) self.assertEqual("Edited Baby!", xform.form_data['assessment']['categories']) self.assertEqual(1, len(xform.history)) self.assertEqual('edit', xform.history[0].operation) deprecated_xform = self.formdb.get_form(xform.deprecated_form_id) self.assertEqual(self.ID, deprecated_xform.orig_id) self.assertNotEqual(self.ID, deprecated_xform.form_id) self.assertTrue(deprecated_xform.is_deprecated) self.assertEqual("", deprecated_xform.form_data['vitals']['height']) self.assertEqual( "other", deprecated_xform.form_data['assessment']['categories']) self.assertEqual(xform.received_on, deprecated_xform.received_on) self.assertEqual(xform.deprecated_form_id, deprecated_xform.form_id) self.assertTrue(xform.edited_on > deprecated_xform.received_on) self.assertEqual(deprecated_xform.get_xml(), original_xml) self.assertEqual(xform.get_xml(), edit_xml) def test_edit_form_with_attachments(self): attachment_source = './corehq/ex-submodules/casexml/apps/case/tests/data/attachments/fruity.jpg' attachment_file = open(attachment_source, 'rb') attachments = { 'fruity_file': UploadedFile(attachment_file, 'fruity_file', content_type='image/jpeg') } def _get_xml(date, form_id): return """<?xml version='1.0' ?> <data uiVersion="1" version="1" name="" xmlns="http://openrosa.org/formdesigner/123"> <name>fgg</name> <date>2011-06-07</date> <n1:meta xmlns:n1="http://openrosa.org/jr/xforms"> <n1:deviceID>354957031935664</n1:deviceID> <n1:timeStart>{date}</n1:timeStart> <n1:timeEnd>{date}</n1:timeEnd> <n1:username>bcdemo</n1:username> <n1:userID>user-abc</n1:userID> <n1:instanceID>{form_id}</n1:instanceID> </n1:meta> </data>""".format(date=date, attachment_source=attachment_source, form_id=form_id) form_id = uuid.uuid4().hex original_xml = _get_xml('2016-03-01T12:04:16Z', form_id) submit_form_locally( original_xml, self.domain, attachments=attachments, ) form = self.formdb.get_form(form_id) self.assertIn('fruity_file', form.attachments) self.assertIn(original_xml, form.get_xml()) # edit form edit_xml = _get_xml('2016-04-01T12:04:16Z', form_id) submit_form_locally( edit_xml, self.domain, ) form = self.formdb.get_form(form_id) self.assertIsNotNone(form.edited_on) self.assertIsNotNone(form.deprecated_form_id) self.assertIn('fruity_file', form.attachments) self.assertIn(edit_xml, form.get_xml()) def test_edit_an_error(self): form_id = uuid.uuid4().hex case_block = CaseBlock( create=True, case_id='', # this should cause the submission to error case_type='person', owner_id='some-owner', ) form, _ = submit_case_blocks(case_block.as_string(), domain=self.domain, form_id=form_id) self.assertTrue(form.is_error) self.assertTrue('IllegalCaseId' in form.problem) case_block.case_id = uuid.uuid4().hex form, _ = submit_case_blocks(case_block.as_string(), domain=self.domain, form_id=form_id) self.assertFalse(form.is_error) self.assertEqual(None, getattr(form, 'problem', None)) def test_broken_save(self): """ Test that if the second form submission terminates unexpectedly and the main form isn't saved, then there are no side effects such as the original having been marked as deprecated. """ original_xml = self.get_xml('original') edit_xml = self.get_xml('edit') result = submit_form_locally(original_xml, self.domain) xform = result.xform self.assertEqual(self.ID, xform.form_id) self.assertTrue(xform.is_normal) self.assertEqual(self.domain, xform.domain) self.assertEqual( UnfinishedSubmissionStub.objects.filter(xform_id=self.ID).count(), 0) with patch.object(self.interface.processor, 'save_processed_models', side_effect=HTTPError): with self.assertRaises(HTTPError): submit_form_locally(edit_xml, self.domain) # it didn't go through, so make sure there are no edits still self.assertIsNone(getattr(xform, 'deprecated_form_id', None)) xform = self.formdb.get_form(self.ID) self.assertIsNotNone(xform) self.assertEqual( UnfinishedSubmissionStub.objects.filter(xform_id=self.ID).count(), 0) def test_case_management(self): form_id = uuid.uuid4().hex case_id = uuid.uuid4().hex owner_id = uuid.uuid4().hex case_block = CaseBlock(create=True, case_id=case_id, case_type='person', owner_id=owner_id, update={ 'property': 'original value' }).as_string() submit_case_blocks(case_block, domain=self.domain, form_id=form_id) # validate some assumptions case = self.casedb.get_case(case_id) self.assertEqual(case.type, 'person') self.assertEqual(case.dynamic_case_properties()['property'], 'original value') self.assertEqual([form_id], case.xform_ids) if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False): self.assertEqual(2, len(case.actions)) for a in case.actions: self.assertEqual(form_id, a.xform_id) # submit a new form with a different case update case_block = CaseBlock(create=True, case_id=case_id, case_type='newtype', owner_id=owner_id, update={ 'property': 'edited value' }).as_string() xform, _ = submit_case_blocks(case_block, domain=self.domain, form_id=form_id) case = self.casedb.get_case(case_id) self.assertEqual(case.type, 'newtype') self.assertEqual(case.dynamic_case_properties()['property'], 'edited value') self.assertEqual([form_id], case.xform_ids) self.assertEqual(case.server_modified_on, xform.edited_on) if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False): self.assertEqual(2, len(case.actions)) for a in case.actions: self.assertEqual(form_id, a.xform_id) def test_second_edit_fails(self): form_id = uuid.uuid4().hex case_id = uuid.uuid4().hex case_block = CaseBlock( create=True, case_id=case_id, case_type='person', ).as_string() submit_case_blocks(case_block, domain=self.domain, form_id=form_id) # submit an edit form with a bad case update (for example a bad ID) case_block = CaseBlock( create=True, case_id='', case_type='person', ).as_string() submit_case_blocks(case_block, domain=self.domain, form_id=form_id) xform = self.formdb.get_form(form_id) self.assertTrue(xform.is_error) deprecated_xform = self.formdb.get_form(xform.deprecated_form_id) self.assertTrue(deprecated_xform.is_deprecated) def test_case_management_ordering(self): case_id = uuid.uuid4().hex owner_id = uuid.uuid4().hex # create a case case_block = CaseBlock( create=True, case_id=case_id, case_type='person', owner_id=owner_id, ).as_string() create_form_id = submit_case_blocks(case_block, domain=self.domain)[0].form_id # validate that worked case = self.casedb.get_case(case_id) self.assertEqual([create_form_id], case.xform_ids) if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False): self.assertTrue( create_form_id in [a.xform_id for a in case.actions]) for a in case.actions: self.assertEqual(create_form_id, a.xform_id) edit_date = datetime.utcnow() # set some property value case_block = CaseBlock(create=False, case_id=case_id, date_modified=edit_date, update={ 'property': 'first value', }).as_string() edit_form_id = submit_case_blocks(case_block, domain=self.domain)[0].form_id # validate that worked case = self.casedb.get_case(case_id) self.assertEqual(case.dynamic_case_properties()['property'], 'first value') self.assertEqual([create_form_id, edit_form_id], case.xform_ids) if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False): self.assertTrue( all(form_id in [a.xform_id for a in case.actions] for form_id in [create_form_id, edit_form_id])) # submit a second (new) form updating the value case_block = CaseBlock(create=False, case_id=case_id, update={ 'property': 'final value', }).as_string() second_edit_form_id = submit_case_blocks(case_block, domain=self.domain)[0].form_id # validate that worked case = self.casedb.get_case(case_id) self.assertEqual(case.dynamic_case_properties()['property'], 'final value') self.assertEqual([create_form_id, edit_form_id, second_edit_form_id], case.xform_ids) if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False): self.assertTrue( all(form_id in [a.xform_id for a in case.actions] for form_id in [create_form_id, edit_form_id, second_edit_form_id])) # deprecate the middle edit case_block = CaseBlock( create=False, case_id=case_id, date_modified= edit_date, # need to use the previous edit date for action sort comparisons update={ 'property': 'edited value', 'added_property': 'added value', }).as_string() submit_case_blocks(case_block, domain=self.domain, form_id=edit_form_id) # ensure that the middle edit stays in the right place and is applied # before the final one case = self.casedb.get_case(case_id) self.assertEqual(case.dynamic_case_properties()['property'], 'final value') self.assertEqual(case.dynamic_case_properties()['added_property'], 'added value') self.assertEqual([create_form_id, edit_form_id, second_edit_form_id], case.xform_ids) if not getattr(settings, 'TESTS_SHOULD_USE_SQL_BACKEND', False): self.assertEqual([ create_form_id, create_form_id, edit_form_id, second_edit_form_id ], [a.xform_id for a in case.actions]) @softer_assert() def test_edit_different_xmlns(self): form_id = uuid.uuid4().hex case1_id = uuid.uuid4().hex case2_id = uuid.uuid4().hex xmlns1 = 'http://commcarehq.org/xmlns1' xmlns2 = 'http://commcarehq.org/xmlns2' case_block = CaseBlock( create=True, case_id=case1_id, case_type='person', owner_id='owner1', ).as_string() xform, cases = submit_case_blocks(case_block, domain=self.domain, xmlns=xmlns1, form_id=form_id) self.assertTrue(xform.is_normal) self.assertEqual(form_id, xform.form_id) case_block = CaseBlock( create=True, case_id=case2_id, case_type='goat', owner_id='owner1', ).as_string() # submit new form with same form ID but different XMLNS xform, cases = submit_case_blocks(case_block, domain=self.domain, xmlns=xmlns2, form_id=form_id) self.assertTrue(xform.is_normal) self.assertNotEqual(form_id, xform.form_id) # form should have a different ID def test_copy_operations(self): original_xml = self.get_xml('original') edit_xml = self.get_xml('edit') xform = submit_form_locally(original_xml, self.domain).xform xform.archive(user_id='user1') xform.unarchive(user_id='user2') xform = submit_form_locally(edit_xml, self.domain).xform self.assertEqual(3, len(xform.history)) self.assertEqual('archive', xform.history[0].operation) self.assertEqual('unarchive', xform.history[1].operation) self.assertEqual('edit', xform.history[2].operation)
def process_change(self, change): try: case = CaseAccessors(change.metadata.domain).get_case(change.id)
class TestFormArchiving(TestCase, TestFileMixin): file_path = ('data', 'sample_xforms') root = os.path.dirname(__file__) def setUp(self): super(TestFormArchiving, self).setUp() self.casedb = CaseAccessors('test-domain') self.formdb = FormAccessors('test-domain') def tearDown(self): FormProcessorTestUtils.delete_all_xforms() FormProcessorTestUtils.delete_all_cases() super(TestFormArchiving, self).tearDown() def testArchive(self): case_id = 'ddb8e2b3-7ce0-43e4-ad45-d7a2eebe9169' xml_data = self.get_xml('basic') result = submit_form_locally( xml_data, 'test-domain', ) xform = result.xform self.assertTrue(xform.is_normal) self.assertEqual(0, len(xform.history)) lower_bound = datetime.utcnow() - timedelta(seconds=1) xform.archive(user_id='mr. librarian') upper_bound = datetime.utcnow() + timedelta(seconds=1) xform = self.formdb.get_form(xform.form_id) self.assertTrue(xform.is_archived) case = self.casedb.get_case(case_id) self.assertTrue(case.is_deleted) self.assertEqual(case.xform_ids, []) [archival] = xform.history self.assertTrue(lower_bound <= archival.date <= upper_bound) self.assertEqual('archive', archival.operation) self.assertEqual('mr. librarian', archival.user) lower_bound = datetime.utcnow() - timedelta(seconds=1) xform.unarchive(user_id='mr. researcher') upper_bound = datetime.utcnow() + timedelta(seconds=1) xform = self.formdb.get_form(xform.form_id) self.assertTrue(xform.is_normal) case = self.casedb.get_case(case_id) self.assertFalse(case.is_deleted) self.assertEqual(case.xform_ids, [xform.form_id]) [archival, restoration] = xform.history self.assertTrue(lower_bound <= restoration.date <= upper_bound) self.assertEqual('unarchive', restoration.operation) self.assertEqual('mr. researcher', restoration.user) def testUnfinishedArchiveStub(self): # Test running the celery task reprocess_archive_stubs on an existing archive stub case_id = 'ddb8e2b3-7ce0-43e4-ad45-d7a2eebe9169' xml_data = self.get_xml('basic') result = submit_form_locally( xml_data, 'test-domain', ) xform = result.xform self.assertTrue(xform.is_normal) self.assertEqual(0, len(xform.history)) # Mock the archive function throwing an error with mock.patch('couchforms.signals.xform_archived.send') as mock_send: try: mock_send.side_effect = Exception xform.archive(user_id='librarian') except Exception: pass # Get the form with the updated history, it should be archived xform = self.formdb.get_form(xform.form_id) self.assertEqual(1, len(xform.history)) self.assertTrue(xform.is_archived) [archival] = xform.history self.assertEqual('archive', archival.operation) self.assertEqual('librarian', archival.user) # The case associated with the form should still exist, it was not rebuilt because of the exception case = self.casedb.get_case(case_id) self.assertFalse(case.is_deleted) # There should be a stub for the unfinished archive unfinished_archive_stubs = UnfinishedArchiveStub.objects.filter() self.assertEqual(len(unfinished_archive_stubs), 1) self.assertEqual(unfinished_archive_stubs[0].history_updated, True) self.assertEqual(unfinished_archive_stubs[0].user_id, 'librarian') self.assertEqual(unfinished_archive_stubs[0].domain, 'test-domain') self.assertEqual(unfinished_archive_stubs[0].archive, True) # Manually call the periodic celery task that reruns archiving/unarchiving actions reprocess_archive_stubs() # The case and stub should both be deleted now case = self.casedb.get_case(case_id) self.assertTrue(case.is_deleted) unfinished_archive_stubs_after_reprocessing = UnfinishedArchiveStub.objects.filter() self.assertEqual(len(unfinished_archive_stubs_after_reprocessing), 0) def testUnfinishedUnarchiveStub(self): # Test running the celery task reprocess_archive_stubs on an existing unarchive stub case_id = 'ddb8e2b3-7ce0-43e4-ad45-d7a2eebe9169' xml_data = self.get_xml('basic') result = submit_form_locally( xml_data, 'test-domain', ) xform = result.xform self.assertTrue(xform.is_normal) self.assertEqual(0, len(xform.history)) # Archive the form successfully xform.archive(user_id='librarian') # Mock the unarchive function throwing an error with mock.patch('couchforms.signals.xform_unarchived.send') as mock_send: try: mock_send.side_effect = Exception xform.unarchive(user_id='librarian') except Exception: pass # Make sure the history only has an archive and an unarchive xform = self.formdb.get_form(xform.form_id) self.assertEqual(2, len(xform.history)) self.assertFalse(xform.is_archived) self.assertEqual('archive', xform.history[0].operation) self.assertEqual('librarian', xform.history[0].user) self.assertEqual('unarchive', xform.history[1].operation) self.assertEqual('librarian', xform.history[1].user) # The case should not exist because the unarchived form was not rebuilt case = self.casedb.get_case(case_id) self.assertTrue(case.is_deleted) # There should be a stub for the unfinished unarchive unfinished_archive_stubs = UnfinishedArchiveStub.objects.filter() self.assertEqual(len(unfinished_archive_stubs), 1) self.assertEqual(unfinished_archive_stubs[0].history_updated, True) self.assertEqual(unfinished_archive_stubs[0].user_id, 'librarian') self.assertEqual(unfinished_archive_stubs[0].domain, 'test-domain') self.assertEqual(unfinished_archive_stubs[0].archive, False) # Manually call the periodic celery task that reruns archiving/unarchiving actions reprocess_archive_stubs() # The case should be back, and the stub should be deleted now case = self.casedb.get_case(case_id) self.assertFalse(case.is_deleted) unfinished_archive_stubs_after_reprocessing = UnfinishedArchiveStub.objects.filter() self.assertEqual(len(unfinished_archive_stubs_after_reprocessing), 0) def testUnarchivingWithArchiveStub(self): # Test a user-initiated unarchive with an existing archive stub case_id = 'ddb8e2b3-7ce0-43e4-ad45-d7a2eebe9169' xml_data = self.get_xml('basic') result = submit_form_locally( xml_data, 'test-domain', ) xform = result.xform self.assertTrue(xform.is_normal) self.assertEqual(0, len(xform.history)) # Mock the archive function throwing an error with mock.patch('couchforms.signals.xform_archived.send') as mock_send: try: mock_send.side_effect = Exception xform.archive(user_id='librarian') except Exception: pass # There should be a stub for the unfinished archive unfinished_archive_stubs = UnfinishedArchiveStub.objects.filter() self.assertEqual(len(unfinished_archive_stubs), 1) self.assertEqual(unfinished_archive_stubs[0].history_updated, True) self.assertEqual(unfinished_archive_stubs[0].user_id, 'librarian') self.assertEqual(unfinished_archive_stubs[0].domain, 'test-domain') self.assertEqual(unfinished_archive_stubs[0].archive, True) # Call an unarchive xform.unarchive(user_id='librarian') # The unfinished archive stub should be deleted unfinished_archive_stubs = UnfinishedArchiveStub.objects.filter() self.assertEqual(len(unfinished_archive_stubs), 0) # The case should exist because the case close was unarchived case = self.casedb.get_case(case_id) self.assertFalse(case.is_deleted) # Manually call the periodic celery task that reruns archiving/unarchiving actions reprocess_archive_stubs() # Make sure the case still exists (to double check that the archive stub was deleted) case = self.casedb.get_case(case_id) self.assertFalse(case.is_deleted) def testArchivingWithUnarchiveStub(self): # Test a user-initiated archive with an existing unarchive stub case_id = 'ddb8e2b3-7ce0-43e4-ad45-d7a2eebe9169' xml_data = self.get_xml('basic') result = submit_form_locally( xml_data, 'test-domain', ) xform = result.xform self.assertTrue(xform.is_normal) self.assertEqual(0, len(xform.history)) # Archive the form successfully xform.archive(user_id='librarian') # Mock the unarchive function throwing an error with mock.patch('couchforms.signals.xform_unarchived.send') as mock_send: try: mock_send.side_effect = Exception xform.unarchive(user_id='librarian') except Exception: pass # There should be a stub for the unfinished unarchive unfinished_archive_stubs = UnfinishedArchiveStub.objects.filter() self.assertEqual(len(unfinished_archive_stubs), 1) self.assertEqual(unfinished_archive_stubs[0].history_updated, True) self.assertEqual(unfinished_archive_stubs[0].user_id, 'librarian') self.assertEqual(unfinished_archive_stubs[0].domain, 'test-domain') self.assertEqual(unfinished_archive_stubs[0].archive, False) # Call an archive xform.archive(user_id='librarian') # The unfinished archive stub should be deleted unfinished_archive_stubs = UnfinishedArchiveStub.objects.filter() self.assertEqual(len(unfinished_archive_stubs), 0) # The case should not exist because the case close was archived case = self.casedb.get_case(case_id) self.assertTrue(case.is_deleted) # Manually call the periodic celery task that reruns archiving/unarchiving actions reprocess_archive_stubs() # The history should not have been added to, make sure that it still only has one entry # Make sure the case still does not exist (to double check that the unarchive stub was deleted) case = self.casedb.get_case(case_id) self.assertTrue(case.is_deleted) def testUnfinishedArchiveStubErrorAddingHistory(self): # Test running the celery task reprocess_archive_stubs on an existing archive stub where the archive # initially failed on updating the history case_id = 'ddb8e2b3-7ce0-43e4-ad45-d7a2eebe9169' xml_data = self.get_xml('basic') result = submit_form_locally( xml_data, 'test-domain', ) xform = result.xform self.assertTrue(xform.is_normal) self.assertEqual(0, len(xform.history)) # Mock the couch and sql archive function throwing an error (so that this test works for both) tmp = 'corehq.form_processor.backends.%s.dbaccessors.%s' with mock.patch(tmp % ('sql', 'FormAccessorSQL.set_archived_state')) as mock_operation_sql: with mock.patch(tmp % ('couch', 'XFormOperation')) as mock_operation_couch: try: mock_operation_sql.side_effect = Exception mock_operation_couch.side_effect = Exception xform.archive(user_id='librarian') except Exception: pass # Get the form with the updated history, make sure it has not been archived yet xform = self.formdb.get_form(xform.form_id) self.assertEqual(0, len(xform.history)) self.assertFalse(xform.is_archived) # The case associated with the form should still exist, it was not rebuilt because of the exception case = self.casedb.get_case(case_id) self.assertFalse(case.is_deleted) # There should be a stub for the unfinished archive, and the history should not be updated yet unfinished_archive_stubs = UnfinishedArchiveStub.objects.filter() self.assertEqual(len(unfinished_archive_stubs), 1) self.assertEqual(unfinished_archive_stubs[0].history_updated, False) self.assertEqual(unfinished_archive_stubs[0].user_id, 'librarian') self.assertEqual(unfinished_archive_stubs[0].domain, 'test-domain') self.assertEqual(unfinished_archive_stubs[0].archive, True) # Manually call the periodic celery task that reruns archiving/unarchiving actions reprocess_archive_stubs() # Make sure the history shows an archive now xform = self.formdb.get_form(xform.form_id) self.assertEqual(1, len(xform.history)) self.assertTrue(xform.is_archived) [archival] = xform.history self.assertEqual('archive', archival.operation) self.assertEqual('librarian', archival.user) # The case and stub should both be deleted now case = self.casedb.get_case(case_id) self.assertTrue(case.is_deleted) unfinished_archive_stubs_after_reprocessing = UnfinishedArchiveStub.objects.filter() self.assertEqual(len(unfinished_archive_stubs_after_reprocessing), 0) def testUnfinishedUnarchiveStubErrorAddingHistory(self): # Test running the celery task reprocess_archive_stubs on an existing archive stub where the archive # initially failed on updating the history case_id = 'ddb8e2b3-7ce0-43e4-ad45-d7a2eebe9169' xml_data = self.get_xml('basic') result = submit_form_locally( xml_data, 'test-domain', ) xform = result.xform self.assertTrue(xform.is_normal) self.assertEqual(0, len(xform.history)) # Archive the form successfully xform.archive(user_id='librarian') # Mock the couch and sql archive function throwing an error (so that this test works for both) tmp = 'corehq.form_processor.backends.%s.dbaccessors.%s' with mock.patch(tmp % ('sql', 'FormAccessorSQL.set_archived_state')) as mock_operation_sql: with mock.patch(tmp % ('couch', 'XFormOperation')) as mock_operation_couch: try: mock_operation_sql.side_effect = Exception mock_operation_couch.side_effect = Exception xform.unarchive(user_id='librarian') except Exception: pass # Get the form with the updated history, make sure it only has one entry (the archive) xform = self.formdb.get_form(xform.form_id) self.assertEqual(1, len(xform.history)) self.assertTrue(xform.is_archived) [archival] = xform.history self.assertEqual('archive', archival.operation) self.assertEqual('librarian', archival.user) # The case associated with the form should not exist, it was not rebuilt because of the exception case = self.casedb.get_case(case_id) self.assertTrue(case.is_deleted) # There should be a stub for the unfinished archive, and the history should not be updated yet unfinished_archive_stubs = UnfinishedArchiveStub.objects.filter() self.assertEqual(len(unfinished_archive_stubs), 1) self.assertEqual(unfinished_archive_stubs[0].history_updated, False) self.assertEqual(unfinished_archive_stubs[0].user_id, 'librarian') self.assertEqual(unfinished_archive_stubs[0].domain, 'test-domain') self.assertEqual(unfinished_archive_stubs[0].archive, False) # Manually call the periodic celery task that reruns archiving/unarchiving actions reprocess_archive_stubs() # Make sure the history shows an archive and an unarchive now xform = self.formdb.get_form(xform.form_id) self.assertEqual(2, len(xform.history)) self.assertFalse(xform.is_archived) self.assertEqual('archive', xform.history[0].operation) self.assertEqual('librarian', xform.history[0].user) self.assertEqual('unarchive', xform.history[1].operation) self.assertEqual('librarian', xform.history[1].user) # The case should be back, and the stub should be deleted now case = self.casedb.get_case(case_id) self.assertFalse(case.is_deleted) unfinished_archive_stubs_after_reprocessing = UnfinishedArchiveStub.objects.filter() self.assertEqual(len(unfinished_archive_stubs_after_reprocessing), 0) def testSignal(self): global archive_counter, restore_counter archive_counter = 0 restore_counter = 0 def count_archive(**kwargs): global archive_counter archive_counter += 1 def count_unarchive(**kwargs): global restore_counter restore_counter += 1 xform_archived.connect(count_archive) xform_unarchived.connect(count_unarchive) xml_data = self.get_xml('basic') result = submit_form_locally( xml_data, 'test-domain', ) self.assertEqual(0, archive_counter) self.assertEqual(0, restore_counter) result.xform.archive() self.assertEqual(1, archive_counter) self.assertEqual(0, restore_counter) xform = self.formdb.get_form(result.xform.form_id) xform.unarchive() self.assertEqual(1, archive_counter) self.assertEqual(1, restore_counter)
class CleanOwnerSyncPayload(object): def __init__(self, timing_context, case_ids_to_sync, restore_state): self.restore_state = restore_state self.case_accessor = CaseAccessors(self.restore_state.domain) self.case_ids_to_sync = case_ids_to_sync self.all_maybe_syncing = copy(case_ids_to_sync) self.checked_cases = set() self.child_indices = defaultdict(set) self.extension_indices = defaultdict(set) self.all_dependencies_syncing = set() self.closed_cases = set() self.potential_elements_to_sync = {} self.timing_context = timing_context self._track_load = case_load_counter("cleanowner_restore", restore_state.domain) def extend_response(self, response): with self.timing_context('process_case_batches'): while self.case_ids_to_sync: self.process_case_batch(self._get_next_case_batch()) with self.timing_context('update_index_trees'): self.update_index_trees() with self.timing_context('update_case_ids_on_phone'): self.update_case_ids_on_phone() with self.timing_context( 'move_no_longer_owned_cases_to_dependent_list_if_necessary'): self.move_no_longer_owned_cases_to_dependent_list_if_necessary() with self.timing_context('purge_and_get_irrelevant_cases'): irrelevant_cases = self.purge_and_get_irrelevant_cases() with self.timing_context('compile_response'): self.compile_response(irrelevant_cases, response) def _get_next_case_batch(self): ids = pop_ids(self.case_ids_to_sync, chunk_size) self._track_load(len(ids)) return [ case for case in self.case_accessor.get_cases(ids) if not case.is_deleted and case_needs_to_sync( case, last_sync_log=self.restore_state.last_sync_log) ] def process_case_batch(self, case_batch): updates = get_case_sync_updates(self.restore_state.domain, case_batch, self.restore_state.last_sync_log) for update in updates: case = update.case self.potential_elements_to_sync[ case.case_id] = PotentialSyncElement( case_stub=CaseStub(case.case_id, case.type), sync_xml_items=get_xml_for_response( update, self.restore_state)) self._process_case_update(case) self._mark_case_as_checked(case)
def get(self, request, domain, episode_id, *args, **kwargs): try: episode = CaseAccessors(domain).get_case(episode_id) return JsonResponse(EpisodeAdherenceUpdate(domain, episode).update_json()) except EnikshayTaskException, e: return HttpResponse(e)
class CleanOwnerCaseSyncOperation(object): def __init__(self, timing_context, restore_state, async_task=None): self.timing_context = timing_context self.restore_state = restore_state self.case_accessor = CaseAccessors(self.restore_state.domain) self.async_task = async_task @property @memoized def cleanliness_flags(self): return dict( OwnershipCleanlinessFlag.objects.filter( domain=self.restore_state.domain, owner_id__in=self.restore_state.owner_ids).values_list( 'owner_id', 'is_clean')) @property def payload_class(self): if self.async_task is not None: return partial(AsyncCleanOwnerPayload, current_task=self.async_task) return CleanOwnerSyncPayload def is_clean(self, owner_id): return self.cleanliness_flags.get(owner_id, False) def is_new_owner(self, owner_id): return (self.restore_state.is_initial or owner_id not in self.restore_state.last_sync_log.owner_ids_on_phone) def extend_response(self, response): with self.timing_context('get_case_ids_to_sync'): case_ids_to_sync = self.get_case_ids_to_sync() sync_payload = self.payload_class(self.timing_context, case_ids_to_sync, self.restore_state) return sync_payload.extend_response(response) def get_case_ids_to_sync(self): case_ids_to_sync = set() for owner_id in self.restore_state.owner_ids: case_ids_to_sync = case_ids_to_sync | set( self.get_case_ids_for_owner(owner_id)) if (not self.restore_state.is_initial and any([ not self.is_clean(owner_id) for owner_id in self.restore_state.owner_ids ])): # if it's a steady state sync and we have any dirty owners, then we also need to # include ALL cases on the phone that have been modified since the last sync as # possible candidates to sync (since they may have been closed or reassigned by someone else) # don't bother checking ones we've already decided to check other_ids_to_check = self.restore_state.last_sync_log.case_ids_on_phone - case_ids_to_sync case_ids_to_sync = case_ids_to_sync | set( filter_cases_modified_since( self.case_accessor, list(other_ids_to_check), self.restore_state.last_sync_log.date)) return case_ids_to_sync def get_case_ids_for_owner(self, owner_id): if EXTENSION_CASES_SYNC_ENABLED.enabled(self.restore_state.domain): return self._get_case_ids_for_owners_with_extensions(owner_id) else: return self._get_case_ids_for_owners_without_extensions(owner_id) def _get_case_ids_for_owners_without_extensions(self, owner_id): if self.is_clean(owner_id): if self.is_new_owner(owner_id): # for a clean owner's initial sync the base set is just the open ids return set( self.case_accessor.get_open_case_ids_for_owner(owner_id)) else: # for a clean owner's steady state sync, the base set is anything modified since last sync return set( self.case_accessor.get_case_ids_modified_with_owner_since( owner_id, self.restore_state.last_sync_log.date)) else: # TODO: we may want to be smarter than this # right now just return the whole footprint and do any filtering later # Note: This will also return extensions if they exist. return get_case_footprint_info(self.restore_state.domain, owner_id).all_ids def _get_case_ids_for_owners_with_extensions(self, owner_id): """Fetches base and extra cases when extensions are enabled""" if not self.is_clean( owner_id) or self.restore_state.is_first_extension_sync: # If this is the first time a user with extensions has synced after # the extension flag is toggled, pull all the cases so that the # extension parameters get set correctly return get_case_footprint_info(self.restore_state.domain, owner_id).all_ids else: if self.is_new_owner(owner_id): # for a clean owner's initial sync the base set is just the open ids and their extensions all_case_ids = set( self.case_accessor.get_open_case_ids_for_owner(owner_id)) new_case_ids = set(all_case_ids) while new_case_ids: all_case_ids = all_case_ids | new_case_ids extension_case_ids = set( self.case_accessor.get_extension_case_ids( new_case_ids)) new_case_ids = extension_case_ids - all_case_ids return all_case_ids else: # for a clean owner's steady state sync, the base set is anything modified since last sync modified_non_extension_cases = set( self.case_accessor.get_case_ids_modified_with_owner_since( owner_id, self.restore_state.last_sync_log.date)) # we also need to fetch unowned extension cases that have been modified extension_case_ids = list(self.restore_state.last_sync_log. extension_index_tree.indices.keys()) modified_extension_cases = set( filter_cases_modified_since( self.case_accessor, extension_case_ids, self.restore_state.last_sync_log.date)) return modified_non_extension_cases | modified_extension_cases
def recipient(self): if self.recipient_type == self.RECIPIENT_TYPE_CASE: try: case = CaseAccessors(self.domain).get_case(self.recipient_id) except CaseNotFound: return None
def sync_user_case(commcare_user, case_type, owner_id, case=None): """ Each time a CommCareUser is saved this method gets called and creates or updates a case associated with the user with the user's details. This is also called to create user cases when the usercase is used for the first time. """ with CriticalSection( ['user_case_%s_for_%s' % (case_type, commcare_user._id)]): domain = commcare_user.project fields = _get_user_case_fields(commcare_user, case_type, owner_id) case = case or CaseAccessors( domain.name).get_case_by_domain_hq_user_id(commcare_user._id, case_type) close = commcare_user.to_be_deleted() or not commcare_user.is_active user_case_helper = _UserCaseHelper(domain, owner_id) def case_should_be_reopened(case, user_case_should_be_closed): return case and case.closed and not user_case_should_be_closed if not case: user_case_helper.create_user_case(commcare_user, fields) else: if case_should_be_reopened(case, close): user_case_helper.re_open_case(case) changed_fields = _get_changed_fields(case, fields) close_case = close and not case.closed if changed_fields or close_case:
def handle(self, domain, repeater_id, filename, **options): accessor = CaseAccessors(domain) records = iter_repeat_records_by_domain(domain, repeater_id=repeater_id) record_count = get_repeat_record_count(domain, repeater_id=repeater_id) row_names = [ 'VoucherID', 'EventOccurDate', 'EventID', 'BeneficiaryUUID', 'BeneficiaryType', 'Location', 'Amount', 'DTOLocation', 'InvestigationType', 'PersonId', 'AgencyId', 'EnikshayApprover', 'EnikshayRole', 'EnikshayApprovalDate', 'Succeeded', # Some records did succeed when we sent them. # Include this so they don't re-pay people. ] seen_voucher_ids = set() duplicate_voucher_ids = set() errors = [] with open(filename, 'w') as f: writer = csv.writer(f) writer.writerow(row_names) for record in with_progress_bar(records, length=record_count): try: payload = json.loads( record.get_payload())['voucher_details'][0] voucher_id = record.payload_id payload['Succeeded'] = record.succeeded except Exception as e: errors.append([record.payload_id, six.text_type(e)]) continue if voucher_id in seen_voucher_ids: duplicate_voucher_ids.add(voucher_id) else: seen_voucher_ids.add(voucher_id) row = [ payload.get(name) if payload.get(name) is not None else "" for name in row_names ] writer.writerow(row) print("{} duplicates found".format(len(duplicate_voucher_ids))) if duplicate_voucher_ids: with open('duplicates_{}'.format(filename), 'w') as f: writer = csv.writer(f) for duplicate_id in duplicate_voucher_ids: writer.writerow([duplicate_id]) print("{} errors".format(len(errors))) if errors: with open('errors_{}'.format(filename), 'w') as f: writer = csv.writer(f) writer.writerow(['episode_id', 'error']) for error in errors: writer.writerow(errors)