def _get_direct_dependencies(domain, cases):
case_accessor = CaseAccessors(domain)
extension_cases = set(case_accessor.get_extension_case_ids(cases))
indexed_cases = set(case_accessor.get_indexed_case_ids(cases))
return DirectDependencies(
all=extension_cases | indexed_cases, indexed_cases=indexed_cases, extension_cases=extension_cases
)
def close_cases(self, all_cases, occurrence_case_id, retain_case):
# remove duplicates in case ids to remove so that we dont retain and close
# the same case by mistake
all_case_ids = set([case.case_id for case in all_cases])
retain_case_id = retain_case.case_id
case_ids_to_close = all_case_ids.copy()
case_ids_to_close.remove(retain_case_id)
case_accessor = CaseAccessors(DOMAIN)
closing_extension_case_ids = case_accessor.get_extension_case_ids(
case_ids_to_close)
self.writerow({
"occurrence_case_id":
occurrence_case_id,
"retain_case_id":
retain_case_id,
"closed_case_ids":
','.join(map(str, case_ids_to_close)),
"closed_extension_case_ids":
','.join(map(str, closing_extension_case_ids)),
"person_case_version":
self.person_case.get_case_property('case_version'),
"person_case_dataset":
self.person_case.get_case_property('dataset')
})
if self.commit:
updates = [(case_id, {
'referral_closed_reason':
"duplicate_reconciliation",
'referral_closed_date':
datetime.datetime.now(pytz.timezone(ENIKSHAY_TIMEZONE)).date()
}, True) for case_id in case_ids_to_close]
bulk_update_cases(DOMAIN, updates, self.__module__)
def _get_direct_dependencies(domain, case_ids):
assert not isinstance(case_ids, basestring)
case_accessor = CaseAccessors(domain)
extension_cases = set(case_accessor.get_extension_case_ids(case_ids))
indexed_cases = set(case_accessor.get_indexed_case_ids(case_ids))
return DirectDependencies(all=extension_cases | indexed_cases,
indexed_cases=indexed_cases,
extension_cases=extension_cases)
def _get_direct_dependencies(domain, case_ids):
assert not isinstance(case_ids, (six.text_type, bytes))
case_accessor = CaseAccessors(domain)
extension_cases = set(case_accessor.get_extension_case_ids(case_ids))
indexed_cases = set(case_accessor.get_indexed_case_ids(case_ids))
return DirectDependencies(
all=extension_cases | indexed_cases,
indexed_cases=indexed_cases,
extension_cases=extension_cases
)
def close_cases(self, all_cases, retain_case, associated_case_id,
reconciling_case_type):
# remove duplicates in case ids to remove so that we don't retain and close
# the same case by mistake
all_case_ids = set([case.case_id for case in all_cases])
retain_case_id = retain_case.case_id
case_ids_to_close = all_case_ids.copy()
case_ids_to_close.remove(retain_case_id)
case_accessor = CaseAccessors(DOMAIN)
closing_extension_case_ids = case_accessor.get_extension_case_ids(
case_ids_to_close)
self.writerow({
"case_type":
reconciling_case_type,
"associated_case_id":
associated_case_id,
"retain_case_id":
retain_case_id,
"closed_case_ids":
','.join(map(str, case_ids_to_close)),
"closed_extension_case_ids":
','.join(map(str, closing_extension_case_ids)),
"retained_case_date_opened":
str(retain_case.opened_on),
"retained_case_episode_type":
retain_case.get_case_property("episode_type"),
"retained_case_is_active":
retain_case.get_case_property("is_active"),
"closed_cases_details": ({
a_case.case_id: {
"last_modified_at(utc)": str(last_user_edit_at(a_case)),
"episode_type": a_case.get_case_property("episode_type"),
"is_active": a_case.get_case_property("is_active")
}
for a_case in all_cases if a_case.case_id != retain_case_id
})
})
if self.commit:
updates = [(case_id, {
'close_reason': "duplicate_reconciliation"
}, True) for case_id in case_ids_to_close]
bulk_update_cases(DOMAIN, updates, self.__module__)
class CleanOwnerCaseSyncOperation(object):
def __init__(self, timing_context, restore_state, async_task=None):
self.timing_context = timing_context
self.restore_state = restore_state
self.case_accessor = CaseAccessors(self.restore_state.domain)
self.async_task = async_task
@property
@memoized
def cleanliness_flags(self):
return dict(
OwnershipCleanlinessFlag.objects.filter(
domain=self.restore_state.domain,
owner_id__in=self.restore_state.owner_ids).values_list(
'owner_id', 'is_clean'))
@property
def payload_class(self):
if self.async_task is not None:
return partial(AsyncCleanOwnerPayload,
current_task=self.async_task)
return CleanOwnerSyncPayload
def is_clean(self, owner_id):
return self.cleanliness_flags.get(owner_id, False)
def is_new_owner(self, owner_id):
return (self.restore_state.is_initial or owner_id
not in self.restore_state.last_sync_log.owner_ids_on_phone)
def extend_response(self, response):
with self.timing_context('get_case_ids_to_sync'):
case_ids_to_sync = self.get_case_ids_to_sync()
sync_payload = self.payload_class(self.timing_context,
case_ids_to_sync, self.restore_state)
return sync_payload.extend_response(response)
def get_case_ids_to_sync(self):
case_ids_to_sync = set()
for owner_id in self.restore_state.owner_ids:
case_ids_to_sync = case_ids_to_sync | set(
self.get_case_ids_for_owner(owner_id))
if (not self.restore_state.is_initial and any([
not self.is_clean(owner_id)
for owner_id in self.restore_state.owner_ids
])):
# if it's a steady state sync and we have any dirty owners, then we also need to
# include ALL cases on the phone that have been modified since the last sync as
# possible candidates to sync (since they may have been closed or reassigned by someone else)
# don't bother checking ones we've already decided to check
other_ids_to_check = self.restore_state.last_sync_log.case_ids_on_phone - case_ids_to_sync
case_ids_to_sync = case_ids_to_sync | set(
filter_cases_modified_since(
self.case_accessor, list(other_ids_to_check),
self.restore_state.last_sync_log.date))
return case_ids_to_sync
def get_case_ids_for_owner(self, owner_id):
if EXTENSION_CASES_SYNC_ENABLED.enabled(self.restore_state.domain):
return self._get_case_ids_for_owners_with_extensions(owner_id)
else:
return self._get_case_ids_for_owners_without_extensions(owner_id)
def _get_case_ids_for_owners_without_extensions(self, owner_id):
if self.is_clean(owner_id):
if self.is_new_owner(owner_id):
# for a clean owner's initial sync the base set is just the open ids
return set(
self.case_accessor.get_open_case_ids_for_owner(owner_id))
else:
# for a clean owner's steady state sync, the base set is anything modified since last sync
return set(
self.case_accessor.get_case_ids_modified_with_owner_since(
owner_id, self.restore_state.last_sync_log.date))
else:
# TODO: we may want to be smarter than this
# right now just return the whole footprint and do any filtering later
# Note: This will also return extensions if they exist.
return get_case_footprint_info(self.restore_state.domain,
owner_id).all_ids
def _get_case_ids_for_owners_with_extensions(self, owner_id):
"""Fetches base and extra cases when extensions are enabled"""
if not self.is_clean(
owner_id) or self.restore_state.is_first_extension_sync:
# If this is the first time a user with extensions has synced after
# the extension flag is toggled, pull all the cases so that the
# extension parameters get set correctly
return get_case_footprint_info(self.restore_state.domain,
owner_id).all_ids
else:
if self.is_new_owner(owner_id):
# for a clean owner's initial sync the base set is just the open ids and their extensions
all_case_ids = set(
self.case_accessor.get_open_case_ids_for_owner(owner_id))
new_case_ids = set(all_case_ids)
while new_case_ids:
all_case_ids = all_case_ids | new_case_ids
extension_case_ids = set(
self.case_accessor.get_extension_case_ids(
new_case_ids))
new_case_ids = extension_case_ids - all_case_ids
return all_case_ids
else:
# for a clean owner's steady state sync, the base set is anything modified since last sync
modified_non_extension_cases = set(
self.case_accessor.get_case_ids_modified_with_owner_since(
owner_id, self.restore_state.last_sync_log.date))
# we also need to fetch unowned extension cases that have been modified
extension_case_ids = list(self.restore_state.last_sync_log.
extension_index_tree.indices.keys())
modified_extension_cases = set(
filter_cases_modified_since(
self.case_accessor, extension_case_ids,
self.restore_state.last_sync_log.date))
return modified_non_extension_cases | modified_extension_cases
print("processing %d docs from db %s" % (num_case_ids, db))
for i, case_id in enumerate(case_ids):
yield case_id
if i % 1000 == 0 and self.log_progress:
print("processed %d / %d docs from db %s" % (i, num_case_ids, db))
def close_cases(self, all_cases, occurrence_case_id, drug_id, retain_case, retain_reason):
# remove duplicates in case ids to remove so that we dont retain and close
# the same case by mistake
all_case_ids = {case.case_id for case in all_cases}
retain_case_id = retain_case.case_id
case_ids_to_close = all_case_ids.copy()
case_ids_to_close.remove(retain_case_id)
case_accessor = CaseAccessors(DOMAIN)
closing_extension_case_ids = case_accessor.get_extension_case_ids(case_ids_to_close)
self.writerow({
"occurrence_case_id": occurrence_case_id,
"person_case_id": self.person_case_id,
"drug_id": drug_id,
"retain_case_id": retain_case_id,
"retain_reason": retain_reason,
"closed_case_ids": ','.join(map(str, case_ids_to_close)),
"closed_extension_case_ids": ','.join(map(str, closing_extension_case_ids))
})
if self.commit:
updates = [(case_id, {'close_reason': "duplicate_reconciliation"}, True)
for case_id in case_ids_to_close]
bulk_update_cases(DOMAIN, updates, self.__module__)
class CleanOwnerCaseSyncOperation(object):
def __init__(self, timing_context, restore_state, async_task=None):
self.timing_context = timing_context
self.restore_state = restore_state
self.case_accessor = CaseAccessors(self.restore_state.domain)
self.async_task = async_task
@property
@memoized
def cleanliness_flags(self):
return dict(
OwnershipCleanlinessFlag.objects.filter(
domain=self.restore_state.domain,
owner_id__in=self.restore_state.owner_ids
).values_list('owner_id', 'is_clean')
)
@property
def payload_class(self):
if self.async_task is not None:
return partial(AsyncCleanOwnerPayload, current_task=self.async_task)
return CleanOwnerSyncPayload
def is_clean(self, owner_id):
return self.cleanliness_flags.get(owner_id, False)
def is_new_owner(self, owner_id):
return (
self.restore_state.is_initial or
owner_id not in self.restore_state.last_sync_log.owner_ids_on_phone
)
def get_payload(self):
self.restore_state.mark_as_new_format()
with self.timing_context('get_case_ids_to_sync'):
case_ids_to_sync = self.get_case_ids_to_sync()
sync_payload = self.payload_class(self.timing_context, case_ids_to_sync, self.restore_state)
return sync_payload.get_payload()
def get_case_ids_to_sync(self):
case_ids_to_sync = set()
for owner_id in self.restore_state.owner_ids:
case_ids_to_sync = case_ids_to_sync | set(self.get_case_ids_for_owner(owner_id))
if (not self.restore_state.is_initial and
any([not self.is_clean(owner_id) for owner_id in self.restore_state.owner_ids])):
# if it's a steady state sync and we have any dirty owners, then we also need to
# include ALL cases on the phone that have been modified since the last sync as
# possible candidates to sync (since they may have been closed or reassigned by someone else)
# don't bother checking ones we've already decided to check
other_ids_to_check = self.restore_state.last_sync_log.case_ids_on_phone - case_ids_to_sync
case_ids_to_sync = case_ids_to_sync | set(filter_cases_modified_since(
self.case_accessor, list(other_ids_to_check), self.restore_state.last_sync_log.date
))
return case_ids_to_sync
def get_case_ids_for_owner(self, owner_id):
if EXTENSION_CASES_SYNC_ENABLED.enabled(self.restore_state.domain):
return self._get_case_ids_for_owners_with_extensions(owner_id)
else:
return self._get_case_ids_for_owners_without_extensions(owner_id)
def _get_case_ids_for_owners_without_extensions(self, owner_id):
if self.is_clean(owner_id):
if self.is_new_owner(owner_id):
# for a clean owner's initial sync the base set is just the open ids
return set(self.case_accessor.get_open_case_ids_for_owner(owner_id))
else:
# for a clean owner's steady state sync, the base set is anything modified since last sync
return set(self.case_accessor.get_case_ids_modified_with_owner_since(
owner_id, self.restore_state.last_sync_log.date
))
else:
# TODO: we may want to be smarter than this
# right now just return the whole footprint and do any filtering later
# Note: This will also return extensions if they exist.
return get_case_footprint_info(self.restore_state.domain, owner_id).all_ids
def _get_case_ids_for_owners_with_extensions(self, owner_id):
"""Fetches base and extra cases when extensions are enabled"""
if not self.is_clean(owner_id) or self.restore_state.is_first_extension_sync:
# If this is the first time a user with extensions has synced after
# the extension flag is toggled, pull all the cases so that the
# extension parameters get set correctly
return get_case_footprint_info(self.restore_state.domain, owner_id).all_ids
else:
if self.is_new_owner(owner_id):
# for a clean owner's initial sync the base set is just the open ids and their extensions
all_case_ids = set(self.case_accessor.get_open_case_ids_for_owner(owner_id))
new_case_ids = set(all_case_ids)
while new_case_ids:
all_case_ids = all_case_ids | new_case_ids
extension_case_ids = set(self.case_accessor.get_extension_case_ids(new_case_ids))
new_case_ids = extension_case_ids - all_case_ids
return all_case_ids
else:
# for a clean owner's steady state sync, the base set is anything modified since last sync
modified_non_extension_cases = set(self.case_accessor.get_case_ids_modified_with_owner_since(
owner_id, self.restore_state.last_sync_log.date
))
# we also need to fetch unowned extension cases that have been modified
extension_case_ids = self.restore_state.last_sync_log.extension_index_tree.indices.keys()
modified_extension_cases = set(filter_cases_modified_since(
self.case_accessor, extension_case_ids, self.restore_state.last_sync_log.date
))
return modified_non_extension_cases | modified_extension_cases
class ImporterTest(TestCase):
def setUp(self):
super(ImporterTest, self).setUp()
self.domain_obj = create_domain("importer-test")
self.domain = self.domain_obj.name
self.default_case_type = 'importer-test-casetype'
self.couch_user = WebUser.create(None, "test", "foobar", None, None)
self.couch_user.add_domain_membership(self.domain, is_admin=True)
self.couch_user.save()
self.accessor = CaseAccessors(self.domain)
self.factory = CaseFactory(domain=self.domain,
case_defaults={
'case_type': self.default_case_type,
})
delete_all_cases()
def tearDown(self):
self.couch_user.delete(deleted_by=None)
self.domain_obj.delete()
super(ImporterTest, self).tearDown()
def _config(self,
col_names,
search_column=None,
case_type=None,
search_field='case_id',
create_new_cases=True):
return ImporterConfig(
couch_user_id=self.couch_user._id,
case_type=case_type or self.default_case_type,
excel_fields=col_names,
case_fields=[''] * len(col_names),
custom_fields=col_names,
search_column=search_column or col_names[0],
search_field=search_field,
create_new_cases=create_new_cases,
)
@run_with_all_backends
@patch('corehq.apps.case_importer.tasks.bulk_import_async.update_state')
def testImportFileMissing(self, update_state):
# by using a made up upload_id, we ensure it's not referencing any real file
case_upload = CaseUploadRecord(upload_id=str(uuid.uuid4()),
task_id=str(uuid.uuid4()))
case_upload.save()
res = bulk_import_async.delay(self._config(['anything']), self.domain,
case_upload.upload_id)
self.assertIsInstance(res.result, Ignore)
update_state.assert_called_with(
state=states.FAILURE,
meta=get_interned_exception(
'Sorry, your session has expired. Please start over and try again.'
))
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
@run_with_all_backends
def testImportBasic(self):
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
['case_id-0', 'age-0', 'sex-0', 'location-0'],
['case_id-1', 'age-1', 'sex-1', 'location-1'],
['case_id-2', 'age-2', 'sex-2', 'location-2'],
['case_id-3', 'age-3', 'sex-3', 'location-3'],
['case_id-4', 'age-4', 'sex-4', 'location-4'],
)
res = do_import(file, config, self.domain)
self.assertEqual(5, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertFalse(res['errors'])
self.assertEqual(1, res['num_chunks'])
case_ids = self.accessor.get_case_ids_in_domain()
cases = list(self.accessor.get_cases(case_ids))
self.assertEqual(5, len(cases))
properties_seen = set()
for case in cases:
self.assertEqual(self.couch_user._id, case.user_id)
self.assertEqual(self.couch_user._id, case.owner_id)
self.assertEqual(self.default_case_type, case.type)
for prop in ['age', 'sex', 'location']:
self.assertTrue(prop in case.get_case_property(prop))
self.assertFalse(
case.get_case_property(prop) in properties_seen)
properties_seen.add(case.get_case_property(prop))
@run_with_all_backends
def testCreateCasesWithDuplicateExternalIds(self):
config = self._config(
['case_id', 'age', 'sex', 'location', 'external_id'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location', 'external_id'],
['case_id-0', 'age-0', 'sex-0', 'location-0', 'external_id-0'],
['case_id-1', 'age-1', 'sex-1', 'location-1', 'external_id-0'],
['case_id-2', 'age-2', 'sex-2', 'location-2', 'external_id-1'],
)
res = do_import(file, config, self.domain)
self.assertEqual(3, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertFalse(res['errors'])
case_ids = self.accessor.get_case_ids_in_domain()
self.assertItemsEqual(
[case.external_id for case in self.accessor.get_cases(case_ids)],
['external_id-0', 'external_id-0', 'external_id-1'])
@run_with_all_backends
def testImportNamedColumns(self):
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
['case_id-0', 'age-0', 'sex-0', 'location-0'],
['case_id-1', 'age-1', 'sex-1', 'location-1'],
['case_id-2', 'age-2', 'sex-2', 'location-2'],
['case_id-3', 'age-3', 'sex-3', 'location-3'],
)
res = do_import(file, config, self.domain)
self.assertEqual(4, res['created_count'])
self.assertEqual(4, len(self.accessor.get_case_ids_in_domain()))
@run_with_all_backends
def testImportTrailingWhitespace(self):
cols = ['case_id', 'age', 'sex\xa0', 'location']
config = self._config(cols)
file = make_worksheet_wrapper(
['case_id', 'age', 'sex\xa0', 'location'],
['case_id-0', 'age-0', 'sex\xa0-0', 'location-0'],
)
res = do_import(file, config, self.domain)
self.assertEqual(1, res['created_count'])
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
case = self.accessor.get_case(case_ids[0])
self.assertTrue(bool(case.get_case_property(
'sex'))) # make sure the value also got properly set
@run_with_all_backends
def testCaseIdMatching(self):
# bootstrap a stub case
[case] = self.factory.create_or_update_case(
CaseStructure(attrs={
'create': True,
'update': {
'importer_test_prop': 'foo'
},
}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
[case.case_id, 'age-0', 'sex-0', 'location-0'],
[case.case_id, 'age-1', 'sex-1', 'location-1'],
[case.case_id, 'age-2', 'sex-2', 'location-2'],
)
res = do_import(file, config, self.domain)
self.assertEqual(0, res['created_count'])
self.assertEqual(3, res['match_count'])
self.assertFalse(res['errors'])
# shouldn't create any more cases, just the one
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
[case] = self.accessor.get_cases(case_ids)
for prop in ['age', 'sex', 'location']:
self.assertTrue(prop in case.get_case_property(prop))
# shouldn't touch existing properties
self.assertEqual('foo', case.get_case_property('importer_test_prop'))
@run_with_all_backends
def testCaseLookupTypeCheck(self):
[case] = self.factory.create_or_update_case(
CaseStructure(attrs={
'create': True,
'case_type': 'nonmatch-type',
}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
[case.case_id, 'age-0', 'sex-0', 'location-0'],
[case.case_id, 'age-1', 'sex-1', 'location-1'],
[case.case_id, 'age-2', 'sex-2', 'location-2'],
)
res = do_import(file, config, self.domain)
# because the type is wrong these shouldn't match
self.assertEqual(3, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(4, len(self.accessor.get_case_ids_in_domain()))
@run_with_all_backends
def testCaseLookupDomainCheck(self):
self.factory.domain = 'wrong-domain'
[case] = self.factory.create_or_update_case(
CaseStructure(attrs={
'create': True,
}))
self.assertEqual(0, len(self.accessor.get_case_ids_in_domain()))
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
[case.case_id, 'age-0', 'sex-0', 'location-0'],
[case.case_id, 'age-1', 'sex-1', 'location-1'],
[case.case_id, 'age-2', 'sex-2', 'location-2'],
)
res = do_import(file, config, self.domain)
# because the domain is wrong these shouldn't match
self.assertEqual(3, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(3, len(self.accessor.get_case_ids_in_domain()))
@run_with_all_backends
def testExternalIdMatching(self):
# bootstrap a stub case
external_id = 'importer-test-external-id'
[case] = self.factory.create_or_update_case(
CaseStructure(attrs={
'create': True,
'external_id': external_id,
}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
headers = ['external_id', 'age', 'sex', 'location']
config = self._config(headers, search_field='external_id')
file = make_worksheet_wrapper(
['external_id', 'age', 'sex', 'location'],
['importer-test-external-id', 'age-0', 'sex-0', 'location-0'],
['importer-test-external-id', 'age-1', 'sex-1', 'location-1'],
['importer-test-external-id', 'age-2', 'sex-2', 'location-2'],
)
res = do_import(file, config, self.domain)
self.assertEqual(0, res['created_count'])
self.assertEqual(3, res['match_count'])
self.assertFalse(res['errors'])
# shouldn't create any more cases, just the one
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
@run_with_all_backends
def test_external_id_matching_on_create_with_custom_column_name(self):
headers = ['id_column', 'age', 'sex', 'location']
external_id = 'external-id-test'
config = self._config(headers[1:],
search_column='id_column',
search_field='external_id')
file = make_worksheet_wrapper(
['id_column', 'age', 'sex', 'location'],
['external-id-test', 'age-0', 'sex-0', 'location-0'],
['external-id-test', 'age-1', 'sex-1', 'location-1'],
)
res = do_import(file, config, self.domain)
self.assertFalse(res['errors'])
self.assertEqual(1, res['created_count'])
self.assertEqual(1, res['match_count'])
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
case = self.accessor.get_case(case_ids[0])
self.assertEqual(external_id, case.external_id)
def testNoCreateNew(self):
config = self._config(['case_id', 'age', 'sex', 'location'],
create_new_cases=False)
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
['case_id-0', 'age-0', 'sex-0', 'location-0'],
['case_id-1', 'age-1', 'sex-1', 'location-1'],
['case_id-2', 'age-2', 'sex-2', 'location-2'],
['case_id-3', 'age-3', 'sex-3', 'location-3'],
['case_id-4', 'age-4', 'sex-4', 'location-4'],
)
res = do_import(file, config, self.domain)
# no matching and no create new set - should do nothing
self.assertEqual(0, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
def testBlankRows(self):
# don't create new cases for rows left blank
config = self._config(['case_id', 'age', 'sex', 'location'],
create_new_cases=True)
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
[None, None, None, None],
['', '', '', ''],
)
res = do_import(file, config, self.domain)
# no matching and no create new set - should do nothing
self.assertEqual(0, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
@patch('corehq.apps.case_importer.do_import.CASEBLOCK_CHUNKSIZE', 2)
def testBasicChunking(self):
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
['case_id-0', 'age-0', 'sex-0', 'location-0'],
['case_id-1', 'age-1', 'sex-1', 'location-1'],
['case_id-2', 'age-2', 'sex-2', 'location-2'],
['case_id-3', 'age-3', 'sex-3', 'location-3'],
['case_id-4', 'age-4', 'sex-4', 'location-4'],
)
res = do_import(file, config, self.domain)
# 5 cases in chunks of 2 = 3 chunks
self.assertEqual(3, res['num_chunks'])
self.assertEqual(5, res['created_count'])
self.assertEqual(5, len(get_case_ids_in_domain(self.domain)))
@run_with_all_backends
def testExternalIdChunking(self):
# bootstrap a stub case
external_id = 'importer-test-external-id'
headers = ['external_id', 'age', 'sex', 'location']
config = self._config(headers, search_field='external_id')
file = make_worksheet_wrapper(
['external_id', 'age', 'sex', 'location'],
['importer-test-external-id', 'age-0', 'sex-0', 'location-0'],
['importer-test-external-id', 'age-1', 'sex-1', 'location-1'],
['importer-test-external-id', 'age-2', 'sex-2', 'location-2'],
)
# the first one should create the case, and the remaining two should update it
res = do_import(file, config, self.domain)
self.assertEqual(1, res['created_count'])
self.assertEqual(2, res['match_count'])
self.assertFalse(res['errors'])
self.assertEqual(2,
res['num_chunks']) # the lookup causes an extra chunk
# should just create the one case
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
[case] = self.accessor.get_cases(case_ids)
self.assertEqual(external_id, case.external_id)
for prop in ['age', 'sex', 'location']:
self.assertTrue(prop in case.get_case_property(prop))
@run_with_all_backends
def testParentCase(self):
headers = ['parent_id', 'name', 'case_id']
config = self._config(headers,
create_new_cases=True,
search_column='case_id')
rows = 3
[parent_case] = self.factory.create_or_update_case(
CaseStructure(attrs={'create': True}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
file = make_worksheet_wrapper(
['parent_id', 'name', 'case_id'],
[parent_case.case_id, 'name-0', 'case_id-0'],
[parent_case.case_id, 'name-1', 'case_id-1'],
[parent_case.case_id, 'name-2', 'case_id-2'],
)
# Should successfully match on `rows` cases
res = do_import(file, config, self.domain)
self.assertEqual(rows, res['created_count'])
# Should create child cases
self.assertEqual(
len(self.accessor.get_reverse_indexed_cases([parent_case.case_id
])), 3)
self.assertEqual(
self.accessor.get_extension_case_ids([parent_case.case_id]), [])
file_missing = make_worksheet_wrapper(
['parent_id', 'name', 'case_id'],
['parent_id-0', 'name-0', 'case_id-0'],
['parent_id-1', 'name-1', 'case_id-1'],
['parent_id-2', 'name-2', 'case_id-2'],
)
# Should be unable to find parent case on `rows` cases
res = do_import(file_missing, config, self.domain)
error_column_name = 'parent_id'
self.assertEqual(
rows,
len(res['errors'][exceptions.InvalidParentId.title]
[error_column_name]['rows']),
"All cases should have missing parent")
@run_with_all_backends
def testExtensionCase(self):
headers = [
'parent_id', 'name', 'case_id', 'parent_relationship_type',
'parent_identifier'
]
config = self._config(headers,
create_new_cases=True,
search_column='case_id')
[parent_case] = self.factory.create_or_update_case(
CaseStructure(attrs={'create': True}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
file = make_worksheet_wrapper(
headers,
[parent_case.case_id, 'name-0', 'case_id-0', 'extension', 'host'],
[
parent_case.case_id, 'name-1', 'case_id-1', 'extension',
'mother'
],
[parent_case.case_id, 'name-2', 'case_id-2', 'child', 'parent'],
)
# Should successfully match on `rows` cases
res = do_import(file, config, self.domain)
self.assertEqual(res['created_count'], 3)
# Of the 3, 2 should be extension cases
extension_case_ids = self.accessor.get_extension_case_ids(
[parent_case.case_id])
self.assertEqual(len(extension_case_ids), 2)
extension_cases = self.accessor.get_cases(extension_case_ids)
# Check that identifier is set correctly
self.assertEqual({'host', 'mother'},
{c.indices[0].identifier
for c in extension_cases})
# This test will only run on SQL backend because of a bug in couch backend
# that overrides current domain with the 'domain' column value from excel
@override_settings(TESTS_SHOULD_USE_SQL_BACKEND=True)
@flag_enabled('DOMAIN_PERMISSIONS_MIRROR')
def test_multiple_domain_case_import(self):
mirror_domain1 = DomainPermissionsMirror(source=self.domain,
mirror='mirrordomain1')
mirror_domain2 = DomainPermissionsMirror(source=self.domain,
mirror='mirrordomain2')
mirror_domain1.save()
mirror_domain2.save()
headers_with_domain = ['case_id', 'name', 'artist', 'domain']
config_1 = self._config(headers_with_domain,
create_new_cases=True,
search_column='case_id')
case_with_domain_file = make_worksheet_wrapper(
['case_id', 'name', 'artist', 'domain'],
['', 'name-0', 'artist-0', self.domain],
['', 'name-1', 'artist-1', mirror_domain1.mirror],
['', 'name-2', 'artist-2', mirror_domain2.mirror],
['', 'name-3', 'artist-3', self.domain],
['', 'name-4', 'artist-4', self.domain],
['', 'name-5', 'artist-5', 'not-existing-domain'])
res = do_import(case_with_domain_file, config_1, self.domain)
self.assertEqual(5, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(1, res['failed_count'])
# Asserting current domain
cur_case_ids = self.accessor.get_case_ids_in_domain()
cur_cases = list(self.accessor.get_cases(cur_case_ids))
self.assertEqual(3, len(cur_cases))
#Asserting current domain case property
cases = {c.name: c for c in cur_cases}
self.assertEqual(cases['name-0'].get_case_property('artist'),
'artist-0')
# Asserting mirror domain 1
md1_case_ids = CaseAccessors(
mirror_domain1.mirror).get_case_ids_in_domain()
md1_cases = list(self.accessor.get_cases(md1_case_ids))
self.assertEqual(1, len(md1_cases))
# Asserting mirror domain 1 case property
md1_cases_pro = {c.name: c for c in md1_cases}
self.assertEqual(md1_cases_pro['name-1'].get_case_property('artist'),
'artist-1')
# Asserting mirror domain 2
md2_case_ids = CaseAccessors(
mirror_domain2.mirror).get_case_ids_in_domain()
md2_cases = list(self.accessor.get_cases(md2_case_ids))
self.assertEqual(1, len(md2_cases))
# Asserting mirror domain 2 case propperty
md2_cases_pro = {c.name: c for c in md2_cases}
self.assertEqual(md2_cases_pro['name-2'].get_case_property('artist'),
'artist-2')
# This test will only run on SQL backend because of a bug in couch backend
# that overrides current domain with the 'domain' column value from excel
@override_settings(TESTS_SHOULD_USE_SQL_BACKEND=True)
@flag_disabled('DOMAIN_PERMISSIONS_MIRROR')
def test_multiple_domain_case_import_mirror_domain_disabled(self):
headers_with_domain = ['case_id', 'name', 'artist', 'domain']
config_1 = self._config(headers_with_domain,
create_new_cases=True,
search_column='case_id')
case_with_domain_file = make_worksheet_wrapper(
['case_id', 'name', 'artist', 'domain'],
['', 'name-0', 'artist-0', self.domain],
['', 'name-1', 'artist-1', 'domain-1'],
['', 'name-2', 'artist-2', 'domain-2'],
['', 'name-3', 'artist-3', self.domain],
['', 'name-4', 'artist-4', self.domain],
['', 'name-5', 'artist-5', 'not-existing-domain'])
res = do_import(case_with_domain_file, config_1, self.domain)
self.assertEqual(6, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(0, res['failed_count'])
case_ids = self.accessor.get_case_ids_in_domain()
# Asserting current domain
cur_cases = list(self.accessor.get_cases(case_ids))
self.assertEqual(6, len(cur_cases))
#Asserting domain case property
cases = {c.name: c for c in cur_cases}
self.assertEqual(cases['name-0'].get_case_property('domain'),
self.domain)
def import_mock_file(self, rows):
config = self._config(rows[0])
xls_file = make_worksheet_wrapper(*rows)
return do_import(xls_file, config, self.domain)
@run_with_all_backends
def testLocationOwner(self):
# This is actually testing several different things, but I figure it's
# worth it, as each of these tests takes a non-trivial amount of time.
non_case_sharing = LocationType.objects.create(domain=self.domain,
name='lt1',
shares_cases=False)
case_sharing = LocationType.objects.create(domain=self.domain,
name='lt2',
shares_cases=True)
location = make_loc('loc-1', 'Loc 1', self.domain, case_sharing.code)
make_loc('loc-2', 'Loc 2', self.domain, case_sharing.code)
duplicate_loc = make_loc('loc-3', 'Loc 2', self.domain,
case_sharing.code)
improper_loc = make_loc('loc-4', 'Loc 4', self.domain,
non_case_sharing.code)
res = self.import_mock_file([
['case_id', 'name', 'owner_id', 'owner_name'],
['', 'location-owner-id', location.location_id, ''],
['', 'location-owner-code', '', location.site_code],
['', 'location-owner-name', '', location.name],
['', 'duplicate-location-name', '', duplicate_loc.name],
['', 'non-case-owning-name', '', improper_loc.name],
])
case_ids = self.accessor.get_case_ids_in_domain()
cases = {c.name: c for c in list(self.accessor.get_cases(case_ids))}
self.assertEqual(cases['location-owner-id'].owner_id,
location.location_id)
self.assertEqual(cases['location-owner-code'].owner_id,
location.location_id)
self.assertEqual(cases['location-owner-name'].owner_id,
location.location_id)
error_message = exceptions.DuplicateLocationName.title
error_column_name = None
self.assertIn(error_message, res['errors'])
self.assertEqual(
res['errors'][error_message][error_column_name]['rows'], [5])
error_message = exceptions.InvalidOwner.title
self.assertIn(error_message, res['errors'])
error_column_name = 'owner_name'
self.assertEqual(
res['errors'][error_message][error_column_name]['rows'], [6])
@run_with_all_backends
def test_opened_on(self):
case = self.factory.create_case()
new_date = '2015-04-30T14:41:53.000000Z'
with flag_enabled('BULK_UPLOAD_DATE_OPENED'):
self.import_mock_file([['case_id', 'date_opened'],
[case.case_id, new_date]])
case = CaseAccessors(self.domain).get_case(case.case_id)
self.assertEqual(case.opened_on,
PhoneTime(parse_datetime(new_date)).done())
class CleanOwnerCaseSyncOperation(object):
def __init__(self, restore_state):
self.restore_state = restore_state
self.case_accessor = CaseAccessors(self.restore_state.domain)
@property
@memoized
def cleanliness_flags(self):
return dict(
OwnershipCleanlinessFlag.objects.filter(
domain=self.restore_state.domain,
owner_id__in=self.restore_state.owner_ids
).values_list('owner_id', 'is_clean')
)
def is_clean(self, owner_id):
return self.cleanliness_flags.get(owner_id, False)
def get_payload(self):
response = self.restore_state.restore_class()
case_ids_to_sync = set()
for owner_id in self.restore_state.owner_ids:
case_ids_to_sync = case_ids_to_sync | set(self.get_case_ids_for_owner(owner_id))
if (not self.restore_state.is_initial and
any([not self.is_clean(owner_id) for owner_id in self.restore_state.owner_ids])):
# if it's a steady state sync and we have any dirty owners, then we also need to
# include ALL cases on the phone that have been modified since the last sync as
# possible candidates to sync (since they may have been closed or reassigned by someone else)
# don't bother checking ones we've already decided to check
other_ids_to_check = self.restore_state.last_sync_log.case_ids_on_phone - case_ids_to_sync
case_ids_to_sync = case_ids_to_sync | set(filter_cases_modified_since(
self.case_accessor, list(other_ids_to_check), self.restore_state.last_sync_log.date
))
all_maybe_syncing = copy(case_ids_to_sync)
all_synced = set()
child_indices = defaultdict(set)
extension_indices = defaultdict(set)
all_dependencies_syncing = set()
closed_cases = set()
potential_updates_to_sync = []
while case_ids_to_sync:
ids = pop_ids(case_ids_to_sync, chunk_size)
# todo: see if we can avoid wrapping - serialization depends on it heavily for now
case_batch = filter(
partial(case_needs_to_sync, last_sync_log=self.restore_state.last_sync_log),
[case for case in self.case_accessor.get_cases(ids)
if not case.is_deleted]
)
updates = get_case_sync_updates(
self.restore_state.domain, case_batch, self.restore_state.last_sync_log
)
for update in updates:
case = update.case
all_synced.add(case.case_id)
potential_updates_to_sync.append(update)
# update the indices in the new sync log
if case.indices:
# and double check footprint for non-live cases
extension_indices[case.case_id] = {
index.identifier: index.referenced_id
for index in case.indices
if index.relationship == CASE_INDEX_EXTENSION
}
child_indices[case.case_id] = {index.identifier: index.referenced_id for index in case.indices
if index.relationship == CASE_INDEX_CHILD}
for index in case.indices:
if index.referenced_id not in all_maybe_syncing:
case_ids_to_sync.add(index.referenced_id)
if not _is_live(case, self.restore_state):
all_dependencies_syncing.add(case.case_id)
if case.closed:
closed_cases.add(case.case_id)
# commtrack ledger sections for this batch
commtrack_elements = get_stock_payload(
self.restore_state.project, self.restore_state.stock_settings,
[CaseStub(update.case.case_id, update.case.type) for update in updates]
)
response.extend(commtrack_elements)
# add any new values to all_syncing
all_maybe_syncing = all_maybe_syncing | case_ids_to_sync
# update sync token - marking it as the new format
self.restore_state.current_sync_log = SimplifiedSyncLog.wrap(
self.restore_state.current_sync_log.to_json()
)
self.restore_state.current_sync_log.log_format = LOG_FORMAT_SIMPLIFIED
self.restore_state.current_sync_log.extensions_checked = True
index_tree = IndexTree(indices=child_indices)
extension_index_tree = IndexTree(indices=extension_indices)
case_ids_on_phone = all_synced
primary_cases_syncing = all_synced - all_dependencies_syncing
if not self.restore_state.is_initial:
case_ids_on_phone = case_ids_on_phone | self.restore_state.last_sync_log.case_ids_on_phone
# subtract primary cases from dependencies since they must be newly primary
all_dependencies_syncing = all_dependencies_syncing | (
self.restore_state.last_sync_log.dependent_case_ids_on_phone -
primary_cases_syncing
)
index_tree = self.restore_state.last_sync_log.index_tree.apply_updates(index_tree)
self.restore_state.current_sync_log.case_ids_on_phone = case_ids_on_phone
self.restore_state.current_sync_log.dependent_case_ids_on_phone = all_dependencies_syncing
self.restore_state.current_sync_log.index_tree = index_tree
self.restore_state.current_sync_log.extension_index_tree = extension_index_tree
self.restore_state.current_sync_log.closed_cases = closed_cases
_move_no_longer_owned_cases_to_dependent_list_if_necessary(self.restore_state, self.case_accessor)
self.restore_state.current_sync_log.purge_dependent_cases()
purged_cases = case_ids_on_phone - self.restore_state.current_sync_log.case_ids_on_phone
# don't sync purged cases that were never on the phone
if self.restore_state.is_initial:
irrelevant_cases = purged_cases
else:
irrelevant_cases = purged_cases - self.restore_state.last_sync_log.case_ids_on_phone
for update in potential_updates_to_sync:
if update.case.case_id not in irrelevant_cases:
append_update_to_response(response, update, self.restore_state)
return response
def get_case_ids_for_owner(self, owner_id):
if EXTENSION_CASES_SYNC_ENABLED.enabled(self.restore_state.domain):
return self._get_case_ids_for_owners_with_extensions(owner_id)
else:
return self._get_case_ids_for_owners_without_extensions(owner_id)
def _get_case_ids_for_owners_without_extensions(self, owner_id):
if self.is_clean(owner_id):
if self.restore_state.is_initial:
# for a clean owner's initial sync the base set is just the open ids
return set(self.case_accessor.get_open_case_ids(owner_id))
else:
# for a clean owner's steady state sync, the base set is anything modified since last sync
return set(self.case_accessor.get_case_ids_modified_with_owner_since(
owner_id, self.restore_state.last_sync_log.date
))
else:
# TODO: we may want to be smarter than this
# right now just return the whole footprint and do any filtering later
# Note: This will also return extensions if they exist.
return get_case_footprint_info(self.restore_state.domain, owner_id).all_ids
def _get_case_ids_for_owners_with_extensions(self, owner_id):
"""Fetches base and extra cases when extensions are enabled"""
if not self.is_clean(owner_id) or self.restore_state.is_first_extension_sync:
# If this is the first time a user with extensions has synced after
# the extension flag is toggled, pull all the cases so that the
# extension parameters get set correctly
return get_case_footprint_info(self.restore_state.domain, owner_id).all_ids
else:
if self.restore_state.is_initial:
# for a clean owner's initial sync the base set is just the open ids and their extensions
all_case_ids = set(self.case_accessor.get_open_case_ids(owner_id))
new_case_ids = set(all_case_ids)
while new_case_ids:
all_case_ids = all_case_ids | new_case_ids
extension_case_ids = set(self.case_accessor.get_extension_case_ids(new_case_ids))
new_case_ids = extension_case_ids - all_case_ids
return all_case_ids
else:
# for a clean owner's steady state sync, the base set is anything modified since last sync
modified_non_extension_cases = set(self.case_accessor.get_case_ids_modified_with_owner_since(
owner_id, self.restore_state.last_sync_log.date
))
# we also need to fetch unowned extension cases that have been modified
extension_case_ids = self.restore_state.last_sync_log.extension_index_tree.indices.keys()
modified_extension_cases = set(filter_cases_modified_since(
self.case_accessor, extension_case_ids, self.restore_state.last_sync_log.date
))
return modified_non_extension_cases | modified_extension_cases
