class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('infile')
parser.add_argument('outfile')
def handle(self, infile, outfile, *args, **options):
self.case_accessor = CaseAccessors('icds-cas')
with open(infile, 'r', encoding='utf-8') as old, open(outfile, 'w', encoding='utf-8') as new:
reader = csv.reader(old)
writer = csv.writer(new)
headers = next(reader)
writer.writerow(headers)
for row in reader:
case_id = row[4]
hh_id = row[10]
if hh_id:
person, hh = self.case_accessor.get_cases([case_id, hh_id], ordered=True)
else:
person = self.case_accessor.get_case(case_id)
hh = None
if hh:
row[18] = hh.get_case_property('name')
row[19] = hh.get_case_property('hh_num')
row[20] = person.get_case_property('name')
writer.writerow(row)
class Command(BaseCommand):
"""https://manage.dimagi.com/default.asp?265190
Some ccs_record cases have evidence of a delivery occurring, but do not
have an associated delivery form. The implications on the case are that an
add property is not set and the schedule phase is 2.
"""
def add_arguments(self, parser):
parser.add_argument(
'csv_file',
help="File path for csv file",
)
def handle(self, csv_file, **options):
self.domain = 'icds-cas'
self.case_accessor = CaseAccessors(self.domain)
with open(csv_file, "w", encoding='utf-8') as csv_file:
field_names = [
'case_id', 'owner_id', 'modified_on', 'server_modified_on',
'add', 'edd', 'ccs_phase', 'num_pnc_visits',
'current_schedule_phase'
]
csv_writer = csv.DictWriter(csv_file,
field_names,
extrasaction='ignore')
csv_writer.writeheader()
for ccs_case in self._get_cases():
properties = copy.deepcopy(ccs_case.case_json)
if 'add' in properties:
continue
if properties.get('current_schedule_phase') != '2':
continue
properties.update({
'case_id':
ccs_case.case_id,
'owner_id':
ccs_case.owner_id,
'modified_on':
ccs_case.modified_on,
'server_modified_on':
ccs_case.server_modified_on
})
csv_writer.writerow(properties)
def _get_cases(self):
dbs = get_db_aliases_for_partitioned_query()
for db in dbs:
ccs_record_case_ids = (CommCareCaseSQL.objects.using(db).filter(
domain=self.domain, type='ccs_record',
closed=False).values_list('case_id', flat=True))
for case_ids in chunked(ccs_record_case_ids, 100):
cases = self.case_accessor.get_cases(list(case_ids))
for case in cases:
yield case
def get_parent_of_case(domain, case_id, parent_case_type):
case_accessor = CaseAccessors(domain)
try:
if not isinstance(case_id, basestring):
case_id = case_id.case_id
child_case = case_accessor.get_case(case_id)
except CaseNotFound:
raise ENikshayCaseNotFound("Couldn't find case: {}".format(case_id))
parent_case_ids = [
indexed_case.referenced_id for indexed_case in child_case.indices
if indexed_case.referenced_type == parent_case_type
]
parent_cases = case_accessor.get_cases(parent_case_ids)
open_parent_cases = [
occurrence_case for occurrence_case in parent_cases
if not occurrence_case.closed
]
if not open_parent_cases:
raise ENikshayCaseNotFound(
"Couldn't find any open {} cases for id: {}".format(
parent_case_type, case_id))
return open_parent_cases[0]
def test_update_adherence_confidence(self):
self.create_case_structure()
case_accessor = CaseAccessors(self.domain)
adherence_dates = [
datetime(2005, 7, 10),
datetime(2016, 8, 10),
datetime(2016, 8, 11),
]
adherence_cases = self.create_adherence_cases(adherence_dates)
update_adherence_confidence_level(
self.domain,
self.person_id,
datetime(2016, 8, 10, tzinfo=pytz.UTC),
datetime(2016, 8, 11, tzinfo=pytz.UTC),
"new_confidence_level",
)
adherence_case_ids = [adherence_date.strftime("%Y-%m-%d") for adherence_date in adherence_dates]
adherence_cases = {case.case_id: case for case in case_accessor.get_cases(adherence_case_ids)}
self.assertEqual(
adherence_cases[adherence_case_ids[0]].dynamic_case_properties()['adherence_confidence'],
'medium',
)
self.assertEqual(
adherence_cases[adherence_case_ids[1]].dynamic_case_properties()['adherence_confidence'],
'new_confidence_level',
)
self.assertEqual(
adherence_cases[adherence_case_ids[2]].dynamic_case_properties()['adherence_confidence'],
'new_confidence_level',
)
def get_parent_of_case(domain, case_id, parent_case_type):
case_accessor = CaseAccessors(domain)
try:
if not isinstance(case_id, basestring):
case_id = case_id.case_id
child_case = case_accessor.get_case(case_id)
except CaseNotFound:
raise ENikshayCaseNotFound(
"Couldn't find case: {}".format(case_id)
)
parent_case_ids = [
indexed_case.referenced_id for indexed_case in child_case.indices
if indexed_case.referenced_type == parent_case_type
]
parent_cases = case_accessor.get_cases(parent_case_ids)
open_parent_cases = [
occurrence_case for occurrence_case in parent_cases
if not occurrence_case.closed
]
if not open_parent_cases:
raise ENikshayCaseNotFound(
"Couldn't find any open {} cases for id: {}".format(parent_case_type, case_id)
)
return open_parent_cases[0]
def test_update_adherence_confidence(self):
self.create_case_structure()
case_accessor = CaseAccessors(self.domain)
adherence_dates = [
datetime(2005, 7, 10),
datetime(2016, 8, 10),
datetime(2016, 8, 11),
]
adherence_cases = self.create_adherence_cases(adherence_dates)
update_adherence_confidence_level(
self.domain,
self.person_id,
datetime(2016, 8, 10, tzinfo=pytz.UTC),
datetime(2016, 8, 11, tzinfo=pytz.UTC),
"new_confidence_level",
)
adherence_case_ids = [
adherence_date.strftime("%Y-%m-%d-%H-%M")
for adherence_date in adherence_dates
]
adherence_cases = {
case.case_id: case
for case in case_accessor.get_cases(adherence_case_ids)
}
def recalculate_stagnant_cases():
domain = 'icds-cas'
config_ids = [
'static-icds-cas-static-ccs_record_cases_monthly_v2',
'static-icds-cas-static-ccs_record_cases_monthly_tableau_v2',
'static-icds-cas-static-child_cases_monthly_v2',
'static-icds-cas-static-child_cases_monthly_tableau_v2',
]
stagnant_cases = set()
for config_id in config_ids:
config, is_static = get_datasource_config(config_id, domain)
adapter = get_indicator_adapter(config)
case_ids = _find_stagnant_cases(adapter)
celery_task_logger.info("Found {} stagnant cases in config {}".format(
len(case_ids), config_id))
stagnant_cases = stagnant_cases.union(set(case_ids))
celery_task_logger.info(
"Total number of stagant cases is now {}".format(
len(stagnant_cases)))
case_accessor = CaseAccessors(domain)
num_stagnant_cases = len(stagnant_cases)
current_case_num = 0
for case_ids in chunked(stagnant_cases, 1000):
current_case_num += len(case_ids)
cases = case_accessor.get_cases(list(case_ids))
for case in cases:
publish_case_saved(case, send_post_save_signal=False)
celery_task_logger.info("Resaved {} / {} cases".format(
current_case_num, num_stagnant_cases))
def recalculate_stagnant_cases():
domain = 'icds-cas'
config_ids = [
'static-icds-cas-static-ccs_record_cases_monthly_v2',
'static-icds-cas-static-ccs_record_cases_monthly_tableau_v2',
'static-icds-cas-static-child_cases_monthly_v2',
]
stagnant_cases = set()
for config_id in config_ids:
config, is_static = get_datasource_config(config_id, domain)
adapter = get_indicator_adapter(config)
case_ids = _find_stagnant_cases(adapter)
celery_task_logger.info(
"Found {} stagnant cases in config {}".format(len(case_ids), config_id)
)
stagnant_cases = stagnant_cases.union(set(case_ids))
celery_task_logger.info(
"Total number of stagant cases is now {}".format(len(stagnant_cases))
)
case_accessor = CaseAccessors(domain)
num_stagnant_cases = len(stagnant_cases)
current_case_num = 0
for case_ids in chunked(stagnant_cases, 1000):
current_case_num += len(case_ids)
cases = case_accessor.get_cases(list(case_ids))
for case in cases:
publish_case_saved(case, send_post_save_signal=False)
celery_task_logger.info(
"Resaved {} / {} cases".format(current_case_num, num_stagnant_cases)
)
class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('infile')
parser.add_argument('outfile')
def handle(self, infile, outfile, *args, **options):
self.case_accessor = CaseAccessors('icds-cas')
with open(infile, 'r') as old, open(outfile, 'w') as new:
reader = csv.reader(old)
writer = csv.writer(new)
headers = next(reader)
writer.writerow(headers)
for row in reader:
owner = row[2]
case_id = row[3]
hh_id = row[4]
if hh_id:
person, hh = self.case_accessor.get_cases([case_id, hh_id],
ordered=True)
else:
person = self.case_accessor.get_case(case_id)
hh = None
row[15] = SQLLocation.objects.get(location_id=owner).name
if hh:
row[16] = hh.get_case_property('name')
row[17] = hh.get_case_property('hh_num')
row[18] = person.get_case_property('name')
row[19] = person.get_case_property('dob')
row[20] = person.get_case_property('sex')
writer.writerow(row)
def _owner_from_extension(self, item, context, case_id):
if item['owner_id'] == UNOWNED_EXTENSION_OWNER_ID:
accessor = CaseAccessors(context.root_doc['domain'])
indices = {case_id}
last_indices = set()
loop_counter = 0
# only allow 10 iterations at most in case there are loops
while indices != last_indices and loop_counter < 10:
last_indices |= indices
indices |= set(accessor.get_indexed_case_ids(indices))
loop_counter += 1
cases = accessor.get_cases(list(indices))
cases_with_owners = [
case for case in cases if case.owner_id
and case.owner_id != UNOWNED_EXTENSION_OWNER_ID
]
if len(cases_with_owners) != 0:
# This shouldn't happen in this world, but will feasibly
# occur depending on our migration path from parent/child ->
# extension cases. Once a migration path is decided revisit
# alerting in this case
return cases_with_owners[0].owner_id
household_cases = [
case for case in cases if case.type == 'household'
]
assert len(household_cases) == 1
class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('infile')
parser.add_argument('outfile')
def handle(self, infile, outfile, *args, **options):
self.case_accessor = CaseAccessors('icds-cas')
with open(infile, 'r', encoding='utf-8') as old, open(outfile, 'w', encoding='utf-8') as new:
reader = csv.reader(old)
writer = csv.writer(new)
headers = next(reader)
writer.writerow(headers)
for row in reader:
case_id = row[4]
hh_id = row[10]
if hh_id:
person, hh = self.case_accessor.get_cases([case_id, hh_id], ordered=True)
else:
person = self.case_accessor.get_case(case_id)
hh = None
if hh:
row[18] = hh.get_case_property('name')
row[19] = hh.get_case_property('hh_num')
row[20] = person.get_case_property('name')
writer.writerow(row)
def get_cleanliness_flag_from_scratch(domain, owner_id):
casedb = CaseAccessors(domain)
footprint_info = get_case_footprint_info(domain, owner_id)
owned_cases = footprint_info.base_ids
cases_to_check = footprint_info.all_ids - owned_cases
if cases_to_check:
closed_owned_case_ids = set(casedb.get_closed_case_ids_for_owner(owner_id))
cases_to_check = cases_to_check - closed_owned_case_ids - footprint_info.extension_ids
# check extension cases that are unowned or owned by others
extension_cases_to_check = footprint_info.extension_ids - closed_owned_case_ids - owned_cases
while extension_cases_to_check:
extension_case = extension_cases_to_check.pop()
dependent_cases = set(get_dependent_case_info(domain, [extension_case]).all_ids)
unowned_dependent_cases = dependent_cases - owned_cases
extension_cases_to_check = extension_cases_to_check - dependent_cases
dependent_cases_owned_by_other_owners = {
dependent_case.case_id
for dependent_case in casedb.get_cases(list(unowned_dependent_cases))
if dependent_case.owner_id != UNOWNED_EXTENSION_OWNER_ID
}
if dependent_cases_owned_by_other_owners:
hint_id = dependent_cases & owned_cases
# can't get back from extension case to owned case e.g. host is a child of owned case
if hint_id:
return CleanlinessFlag(False, hint_id.pop())
if cases_to_check:
# it wasn't in any of the open or closed IDs - it must be dirty
reverse_index_infos = casedb.get_all_reverse_indices_info(list(cases_to_check))
reverse_index_ids = set([r.case_id for r in reverse_index_infos])
indexed_with_right_owner = (reverse_index_ids & (owned_cases | closed_owned_case_ids))
found_deleted_cases = False
while indexed_with_right_owner:
hint_id = indexed_with_right_owner.pop()
infos_for_this_owner = _get_info_by_case_id(reverse_index_infos, hint_id)
for info in infos_for_this_owner:
try:
case = CaseAccessors(domain).get_case(info.referenced_id)
if not case.is_deleted:
return CleanlinessFlag(False, hint_id)
else:
found_deleted_cases = True
except ResourceNotFound:
# the case doesn't exist - don't use it as a dirty flag
found_deleted_cases = True
if found_deleted_cases:
# if we made it all the way to the end of the loop without returning anything
# then the owner was only flagged as dirty due to missing cases,
# This implies the owner is still clean.
return CleanlinessFlag(True, None)
else:
# I don't believe code can ever be hit, but if it is we should fail hard
# until we can better understand it.
raise IllegalCaseId('Owner {} in domain {} has an invalid index reference chain!!'.format(
owner_id, domain
))
return CleanlinessFlag(True, None)
def run(self, iteration_key):
case_ids = self._get_case_ids()
doc_processor = self.doc_processor(self.domain)
for chunk in chunked(case_ids, 100):
case_accessor = CaseAccessors(self.domain)
doc_processor.process_bulk_docs(
case_accessor.get_cases(list(chunk)))
return len(case_ids), 0
class CleanOwnerSyncPayload(object):
def __init__(self, timing_context, case_ids_to_sync, restore_state):
self.restore_state = restore_state
self.case_accessor = CaseAccessors(self.restore_state.domain)
self.case_ids_to_sync = case_ids_to_sync
self.all_maybe_syncing = copy(case_ids_to_sync)
self.checked_cases = set()
self.child_indices = defaultdict(set)
self.extension_indices = defaultdict(set)
self.all_dependencies_syncing = set()
self.closed_cases = set()
self.potential_elements_to_sync = {}
self.timing_context = timing_context
def extend_response(self, response):
with self.timing_context('process_case_batches'):
while self.case_ids_to_sync:
self.process_case_batch(self._get_next_case_batch())
with self.timing_context('update_index_trees'):
self.update_index_trees()
with self.timing_context('update_case_ids_on_phone'):
self.update_case_ids_on_phone()
with self.timing_context(
'move_no_longer_owned_cases_to_dependent_list_if_necessary'):
self.move_no_longer_owned_cases_to_dependent_list_if_necessary()
with self.timing_context('purge_and_get_irrelevant_cases'):
irrelevant_cases = self.purge_and_get_irrelevant_cases()
with self.timing_context('compile_response'):
self.compile_response(irrelevant_cases, response)
def _get_next_case_batch(self):
ids = pop_ids(self.case_ids_to_sync, chunk_size)
return [
case for case in self.case_accessor.get_cases(ids)
if not case.is_deleted and case_needs_to_sync(
case, last_sync_log=self.restore_state.last_sync_log)
]
def process_case_batch(self, case_batch):
updates = get_case_sync_updates(self.restore_state.domain, case_batch,
self.restore_state.last_sync_log)
for update in updates:
case = update.case
self.potential_elements_to_sync[
case.case_id] = PotentialSyncElement(
case_stub=CaseStub(case.case_id, case.type),
sync_xml_items=get_xml_for_response(
update, self.restore_state))
self._process_case_update(case)
self._mark_case_as_checked(case)
def get_cleanliness_flag_from_scratch(domain, owner_id):
casedb = CaseAccessors(domain)
footprint_info = get_case_footprint_info(domain, owner_id)
owned_cases = footprint_info.base_ids
cases_to_check = footprint_info.all_ids - owned_cases
if cases_to_check:
closed_owned_case_ids = set(casedb.get_closed_case_ids_for_owner(owner_id))
cases_to_check = cases_to_check - closed_owned_case_ids - footprint_info.extension_ids
# check extension cases that are unowned or owned by others
extension_cases_to_check = footprint_info.extension_ids - closed_owned_case_ids - owned_cases
while extension_cases_to_check:
extension_case = extension_cases_to_check.pop()
dependent_cases = set(get_dependent_case_info(domain, [extension_case]).all_ids)
unowned_dependent_cases = dependent_cases - owned_cases
extension_cases_to_check = extension_cases_to_check - dependent_cases
dependent_cases_owned_by_other_owners = {
dependent_case.case_id
for dependent_case in casedb.get_cases(list(unowned_dependent_cases))
if dependent_case.owner_id != UNOWNED_EXTENSION_OWNER_ID
}
if dependent_cases_owned_by_other_owners:
hint_id = dependent_cases & owned_cases
# can't get back from extension case to owned case e.g. host is a child of owned case
if hint_id:
return CleanlinessFlag(False, hint_id.pop())
if cases_to_check:
# it wasn't in any of the open or closed IDs - it must be dirty
reverse_index_infos = casedb.get_all_reverse_indices_info(list(cases_to_check))
reverse_index_ids = set([r.case_id for r in reverse_index_infos])
indexed_with_right_owner = (reverse_index_ids & (owned_cases | closed_owned_case_ids))
found_deleted_cases = False
while indexed_with_right_owner:
hint_id = indexed_with_right_owner.pop()
infos_for_this_owner = _get_info_by_case_id(reverse_index_infos, hint_id)
for info in infos_for_this_owner:
try:
case = CaseAccessors(domain).get_case(info.referenced_id)
if not case.is_deleted:
return CleanlinessFlag(False, hint_id)
else:
found_deleted_cases = True
except ResourceNotFound:
# the case doesn't exist - don't use it as a dirty flag
found_deleted_cases = True
if found_deleted_cases:
# if we made it all the way to the end of the loop without returning anything
# then the owner was only flagged as dirty due to missing cases,
# This implies the owner is still clean.
return CleanlinessFlag(True, None)
else:
# I don't believe code can ever be hit, but if it is we should fail hard
# until we can better understand it.
raise IllegalCaseId('Owner {} in domain {} has an invalid index reference chain!!'.format(
owner_id, domain
))
class Command(BaseCommand):
"""https://manage.dimagi.com/default.asp?265190
Some ccs_record cases have evidence of a delivery occurring, but do not
have an associated delivery form. The implications on the case are that an
add property is not set and the schedule phase is 2.
"""
def add_arguments(self, parser):
parser.add_argument(
'csv_file',
help="File path for csv file",
)
def handle(self, csv_file, **options):
self.domain = 'icds-cas'
self.case_accessor = CaseAccessors(self.domain)
with open(csv_file, "w", encoding='utf-8') as csv_file:
field_names = [
'case_id', 'owner_id', 'modified_on', 'server_modified_on',
'add', 'edd', 'ccs_phase', 'num_pnc_visits', 'current_schedule_phase'
]
csv_writer = csv.DictWriter(csv_file, field_names, extrasaction='ignore')
csv_writer.writeheader()
for ccs_case in self._get_cases():
properties = copy.deepcopy(ccs_case.case_json)
if 'add' in properties:
continue
if properties.get('current_schedule_phase') != '2':
continue
properties.update({
'case_id': ccs_case.case_id,
'owner_id': ccs_case.owner_id,
'modified_on': ccs_case.modified_on,
'server_modified_on': ccs_case.server_modified_on
})
csv_writer.writerow(properties)
def _get_cases(self):
dbs = get_db_aliases_for_partitioned_query()
for db in dbs:
ccs_record_case_ids = (
CommCareCaseSQL.objects
.using(db)
.filter(domain=self.domain, type='ccs_record', closed=False)
.values_list('case_id', flat=True)
)
for case_ids in chunked(ccs_record_case_ids, 100):
cases = self.case_accessor.get_cases(list(case_ids))
for case in cases:
yield case
def test_cases_created(self):
accessor = CaseAccessors(DOMAIN)
case_ids = accessor.get_case_ids_in_domain()
cases = list(accessor.get_cases(case_ids))
self.assertEqual(len(cases), 18)
self.assertTrue(
all(case.parent.type == FOODRECALL_CASE_TYPE for case in cases
if case.type == FOOD_CASE_TYPE))
def cases_referenced_by_xform(xform):
"""
Returns a list of CommCareCase or CommCareCaseSQL given a JSON
representation of an XFormInstance
"""
assert xform.domain, "Form is missing 'domain'"
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors
case_ids = get_case_ids_from_form(xform)
case_accessor = CaseAccessors(xform.domain)
return list(case_accessor.get_cases(list(case_ids)))
def cases_referenced_by_xform(xform):
"""
Returns a list of CommCareCase or CommCareCaseSQL given a JSON
representation of an XFormInstance
"""
assert xform.domain, "Form is missing 'domain'"
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors
case_ids = get_case_ids_from_form(xform)
case_accessor = CaseAccessors(xform.domain)
return list(case_accessor.get_cases(list(case_ids)))
def cases_referenced_by_xform(xform):
"""
Returns a list of CommCareCase or CommCareCaseSQL given a JSON
representation of an XFormInstance
"""
from corehq.form_processor.backends.couch.dbaccessors import CaseAccessorCouch
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors
case_ids = get_case_ids_from_form(xform)
domain = get_and_check_xform_domain(xform)
case_accessor = CaseAccessors(domain)
if domain is None:
assert case_accessor.db_accessor == CaseAccessorCouch
return list(case_accessor.get_cases(list(case_ids)))
def cases_referenced_by_xform(xform):
"""
Returns a list of CommCareCase or CommCareCaseSQL given a JSON
representation of an XFormInstance
"""
from corehq.form_processor.backends.couch.dbaccessors import CaseAccessorCouch
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors
case_ids = get_case_ids_from_form(xform)
domain = get_and_check_xform_domain(xform)
case_accessor = CaseAccessors(domain)
if domain is None:
assert case_accessor.db_accessor == CaseAccessorCouch
return list(case_accessor.get_cases(list(case_ids)))
def hint_still_valid(domain, hint):
"""
For a given domain/owner/cleanliness hint check if it's still valid
"""
casedb = CaseAccessors(domain)
try:
hint_case = casedb.get_case(hint)
hint_owner = hint_case.owner_id
except CaseNotFound:
# hint was deleted
return False
dependent_case_ids = set(get_dependent_case_info(domain, [hint]).all_ids)
return any([c.owner_id != hint_owner and c.owner_id != UNOWNED_EXTENSION_OWNER_ID
for c in casedb.get_cases(list(dependent_case_ids))])
def hint_still_valid(domain, hint):
"""
For a given domain/owner/cleanliness hint check if it's still valid
"""
casedb = CaseAccessors(domain)
try:
hint_case = casedb.get_case(hint)
hint_owner = hint_case.owner_id
except CaseNotFound:
# hint was deleted
return False
dependent_case_ids = set(get_dependent_case_info(domain, [hint]).all_ids)
return any([c.owner_id != hint_owner and c.owner_id != UNOWNED_EXTENSION_OWNER_ID
for c in casedb.get_cases(list(dependent_case_ids))])
def handle(self, *args, **options):
domain = options["domain"]
case_id = options["case_id"]
case_accessor = CaseAccessors(domain=domain)
case = case_accessor.get_case(case_id)
if (
not case.is_deleted
and raw_input(
"\n".join(["Case {} is not already deleted. Are you sure you want to delete it? (y/N)".format(case_id)])
).lower()
!= "y"
):
sys.exit(0)
dependent_case_ids = get_entire_case_network(domain, [case_id])
cases_to_delete = filter(lambda case: not case.is_deleted, case_accessor.get_cases(dependent_case_ids))
if cases_to_delete:
with open(options["filename"], "w") as csvfile:
writer = csv.writer(csvfile)
headers = ["case id", "case type", "owner", "opened by", "app version"]
writer.writerow(headers)
print headers
for case in cases_to_delete:
form = FormAccessors(domain=domain).get_form(case.xform_ids[0])
app_version_info = get_app_version_info(
domain, form.build_id, form.form_data["@version"], form.metadata
)
row = [
case.case_id,
case.type,
cached_owner_id_to_display(case.owner_id) or case.owner_id,
cached_owner_id_to_display(case.opened_by),
app_version_info.build_version,
]
writer.writerow(row)
print row
if (
cases_to_delete
and raw_input("\n".join(["Delete these {} cases? (y/N)".format(len(cases_to_delete))])).lower() == "y"
):
case_accessor.soft_delete_cases([c.case_id for c in cases_to_delete])
print "deleted {} cases".format(len(cases_to_delete))
if cases_to_delete:
print "details here: {}".format(options["filename"])
else:
print "didn't find any cases to delete"
def get_all_parents_of_case(domain, case_id):
case_accessor = CaseAccessors(domain)
try:
if not isinstance(case_id, basestring):
case_id = case_id.case_id
child_case = case_accessor.get_case(case_id)
except CaseNotFound:
raise ENikshayCaseNotFound("Couldn't find case: {}".format(case_id))
parent_case_ids = [
indexed_case.referenced_id for indexed_case in child_case.indices
]
parent_cases = case_accessor.get_cases(parent_case_ids)
return parent_cases
def get_expected_rows(self):
# Swap out the external IDs in the test fixture for the real IDs
accessor = CaseAccessors(DOMAIN)
case_ids = accessor.get_case_ids_in_domain()
case_ids_by_external_id = {
c.external_id: c.case_id
for c in accessor.get_cases(case_ids)
}
def substitute_real_ids(row):
return {
key: case_ids_by_external_id[val]
if val in case_ids_by_external_id else val
for key, val in row.items()
}
return map(substitute_real_ids, get_expected_report())
def _overwrite_report(filename, actual_report):
"""For use when making changes - force overwrites test data"""
accessor = CaseAccessors(DOMAIN)
case_ids = accessor.get_case_ids_in_domain()
external_ids_by_case_id = {
c.case_id: c.external_id
for c in accessor.get_cases(case_ids)
}
rows = [[
external_ids_by_case_id[val] if val in external_ids_by_case_id else val
for val in row
] for row in actual_report.rows]
with open(os.path.join(os.path.dirname(__file__), 'data', filename),
'w') as f:
writer = csv.writer(f)
writer.writerow(actual_report.headers)
writer.writerows(rows)
def parse_person_case_ids(person_case_ids, domain):
"""
ensure case ids are open person cases
:param person_case_ids: comma separated case id
:return: parsed case ids or return None in case unable to
"""
person_case_ids = person_case_ids.split(',')
case_accessor = CaseAccessors(domain)
try:
person_cases = case_accessor.get_cases(person_case_ids)
except CaseNotFound:
return None
parsed_person_case_ids = []
for person_case in person_cases:
if not person_case or person_case.type != 'person' or person_case.closed:
return None
else:
parsed_person_case_ids.append(person_case.case_id)
return parsed_person_case_ids
def _update_case_id_properties(domain, user):
"""Some case properties store the ID of related cases. This updates those IDs"""
accessor = CaseAccessors(domain)
case_ids = accessor.get_case_ids_in_domain()
cases = list(accessor.get_cases(case_ids))
case_ids_by_external_id = {c.external_id: c.case_id for c in cases}
case_blocks = []
for case in cases:
update = {}
for k, v in case.dynamic_case_properties().items():
if v in case_ids_by_external_id:
update[k] = case_ids_by_external_id[v]
if update:
case_blocks.append(
CaseBlock(
case_id=case.case_id,
user_id=user._id,
update=update,
).as_xml())
def perform_resave_on_cases(domain, start_date, end_date, no_input):
_, _, case_ids_missing_in_es, _ = compare_cases(domain, 'CommCareCase', start_date, end_date)
print("%s Ids found for cases missing in ES." % len(case_ids_missing_in_es))
if len(case_ids_missing_in_es) < 1000:
print(case_ids_missing_in_es)
if no_input is not True:
ok = input("Type 'ok' to continue: ")
if ok != "ok":
print("No changes made")
return
case_accessor = CaseAccessors(domain)
for case_ids in chunked(with_progress_bar(case_ids_missing_in_es), 100):
cases = case_accessor.get_cases(list(case_ids))
found_case_ids = set()
for case in cases:
resave_case(domain, case, send_post_save_signal=False)
found_case_ids.add(case.case_id)
for case_id in set(case_ids) - found_case_ids:
print("case not found %s" % case_id)
def perform_resave_on_cases(domain, start_date, end_date, no_input):
_, _, case_ids_missing_in_es, _ = compare_cases(domain, 'CommCareCase',
start_date, end_date)
print("%s Ids found for cases missing in ES." %
len(case_ids_missing_in_es))
if len(case_ids_missing_in_es) < 1000:
print(case_ids_missing_in_es)
if no_input is not True:
ok = input("Type 'ok' to continue: ")
if ok != "ok":
print("No changes made")
return
case_accessor = CaseAccessors(domain)
for case_ids in chunked(with_progress_bar(case_ids_missing_in_es), 100):
cases = case_accessor.get_cases(list(case_ids))
found_case_ids = set()
for case in cases:
resave_case(domain, case, send_post_save_signal=False)
found_case_ids.add(case.case_id)
for case_id in set(case_ids) - found_case_ids:
print("case not found %s" % case_id)
class CleanOwnerSyncPayload(object):
def __init__(self, timing_context, case_ids_to_sync, restore_state):
self.restore_state = restore_state
self.case_accessor = CaseAccessors(self.restore_state.domain)
self.response = self.restore_state.restore_class()
self.case_ids_to_sync = case_ids_to_sync
self.all_maybe_syncing = copy(case_ids_to_sync)
self.checked_cases = set()
self.child_indices = defaultdict(set)
self.extension_indices = defaultdict(set)
self.all_dependencies_syncing = set()
self.closed_cases = set()
self.potential_elements_to_sync = {}
self.timing_context = timing_context
def get_payload(self):
with self.timing_context('process_case_batches'):
while self.case_ids_to_sync:
self.process_case_batch(self._get_next_case_batch())
with self.timing_context('update_index_trees'):
self.update_index_trees()
with self.timing_context('update_case_ids_on_phone'):
self.update_case_ids_on_phone()
with self.timing_context('move_no_longer_owned_cases_to_dependent_list_if_necessary'):
self.move_no_longer_owned_cases_to_dependent_list_if_necessary()
with self.timing_context('purge_and_get_irrelevant_cases'):
irrelevant_cases = self.purge_and_get_irrelevant_cases()
with self.timing_context('compile_response'):
self.compile_response(irrelevant_cases)
return self.response
def _get_next_case_batch(self):
ids = pop_ids(self.case_ids_to_sync, chunk_size)
return [
case for case in self.case_accessor.get_cases(ids)
if not case.is_deleted and case_needs_to_sync(case, last_sync_log=self.restore_state.last_sync_log)
]
def process_case_batch(self, case_batch):
updates = get_case_sync_updates(
self.restore_state.domain, case_batch, self.restore_state.last_sync_log
)
for update in updates:
case = update.case
self.potential_elements_to_sync[case.case_id] = PotentialSyncElement(
case_stub=CaseStub(case.case_id, case.type),
sync_xml_items=get_xml_for_response(update, self.restore_state)
)
self._process_case_update(case)
self._mark_case_as_checked(case)
def _process_case_update(self, case):
if case.indices:
self._update_indices_in_new_synclog(case)
self._add_unchecked_indices_to_be_checked(case)
if not _is_live(case, self.restore_state):
self.all_dependencies_syncing.add(case.case_id)
if case.closed:
self.closed_cases.add(case.case_id)
def _mark_case_as_checked(self, case):
self.checked_cases.add(case.case_id)
def _update_indices_in_new_synclog(self, case):
self.extension_indices[case.case_id] = {
index.identifier: index.referenced_id
for index in case.indices
if index.relationship == CASE_INDEX_EXTENSION
}
self.child_indices[case.case_id] = {
index.identifier: index.referenced_id
for index in case.indices
if index.relationship == CASE_INDEX_CHILD
}
def _add_unchecked_indices_to_be_checked(self, case):
for index in case.indices:
if index.referenced_id not in self.all_maybe_syncing:
self.case_ids_to_sync.add(index.referenced_id)
self.all_maybe_syncing |= self.case_ids_to_sync
def move_no_longer_owned_cases_to_dependent_list_if_necessary(self):
if not self.restore_state.is_initial:
removed_owners = (
set(self.restore_state.last_sync_log.owner_ids_on_phone) - set(self.restore_state.owner_ids)
)
if removed_owners:
# if we removed any owner ids, then any cases that belonged to those owners need
# to be moved to the dependent list
case_ids_to_try_purging = self.case_accessor.get_case_ids_by_owners(list(removed_owners))
for to_purge in case_ids_to_try_purging:
if to_purge in self.restore_state.current_sync_log.case_ids_on_phone:
self.restore_state.current_sync_log.dependent_case_ids_on_phone.add(to_purge)
def update_index_trees(self):
index_tree = IndexTree(indices=self.child_indices)
extension_index_tree = IndexTree(indices=self.extension_indices)
if not self.restore_state.is_initial:
index_tree = self.restore_state.last_sync_log.index_tree.apply_updates(index_tree)
extension_index_tree = self.restore_state.last_sync_log.extension_index_tree.apply_updates(
extension_index_tree
)
self.restore_state.current_sync_log.index_tree = index_tree
self.restore_state.current_sync_log.extension_index_tree = extension_index_tree
def update_case_ids_on_phone(self):
case_ids_on_phone = self.checked_cases
primary_cases_syncing = self.checked_cases - self.all_dependencies_syncing
if not self.restore_state.is_initial:
case_ids_on_phone |= self.restore_state.last_sync_log.case_ids_on_phone
# subtract primary cases from dependencies since they must be newly primary
self.all_dependencies_syncing |= (
self.restore_state.last_sync_log.dependent_case_ids_on_phone -
primary_cases_syncing
)
self.restore_state.current_sync_log.case_ids_on_phone = case_ids_on_phone
self.restore_state.current_sync_log.dependent_case_ids_on_phone = self.all_dependencies_syncing
self.restore_state.current_sync_log.closed_cases = self.closed_cases
def purge_and_get_irrelevant_cases(self):
original_case_ids_on_phone = self.restore_state.current_sync_log.case_ids_on_phone.copy()
self.restore_state.current_sync_log.purge_dependent_cases()
purged_cases = original_case_ids_on_phone - self.restore_state.current_sync_log.case_ids_on_phone
# don't sync purged cases that were never on the phone
if self.restore_state.is_initial:
irrelevant_cases = purged_cases
else:
irrelevant_cases = purged_cases - self.restore_state.last_sync_log.case_ids_on_phone
return irrelevant_cases
def compile_response(self, irrelevant_cases):
relevant_sync_elements = [
potential_sync_element
for syncable_case_id, potential_sync_element in self.potential_elements_to_sync.iteritems()
if syncable_case_id not in irrelevant_cases
]
with self.timing_context('add_commtrack_elements_to_response'):
self._add_commtrack_elements_to_response(relevant_sync_elements)
self._add_case_elements_to_response(relevant_sync_elements)
def _add_commtrack_elements_to_response(self, relevant_sync_elements):
commtrack_elements = get_stock_payload(
self.restore_state.project, self.restore_state.stock_settings,
[
potential_sync_element.case_stub
for potential_sync_element in relevant_sync_elements
]
)
self.response.extend(commtrack_elements)
def _add_case_elements_to_response(self, relevant_sync_elements):
for relevant_case in relevant_sync_elements:
for xml_item in relevant_case.sync_xml_items:
self.response.append(xml_item)
class ImporterTest(TestCase):
def setUp(self):
super(ImporterTest, self).setUp()
self.domain = create_domain("importer-test").name
self.default_case_type = 'importer-test-casetype'
self.default_headers = ['case_id', 'age', 'sex', 'location']
self.couch_user = WebUser.create(None, "test", "foobar")
self.couch_user.add_domain_membership(self.domain, is_admin=True)
self.couch_user.save()
self.accessor = CaseAccessors(self.domain)
self.factory = CaseFactory(domain=self.domain,
case_defaults={
'case_type': self.default_case_type,
})
delete_all_cases()
def tearDown(self):
self.couch_user.delete()
delete_all_locations()
LocationType.objects.all().delete()
super(ImporterTest, self).tearDown()
def _config(self,
col_names=None,
search_column=None,
case_type=None,
search_field='case_id',
named_columns=False,
create_new_cases=True,
type_fields=None):
col_names = col_names or self.default_headers
case_type = case_type or self.default_case_type
search_column = search_column or col_names[0]
type_fields = type_fields if type_fields is not None else [
'plain'
] * len(col_names)
return ImporterConfig(
couch_user_id=self.couch_user._id,
case_type=case_type,
excel_fields=col_names,
case_fields=[''] * len(col_names),
custom_fields=col_names,
type_fields=type_fields,
search_column=search_column,
search_field=search_field,
named_columns=named_columns,
create_new_cases=create_new_cases,
key_column='',
value_column='',
)
@run_with_all_backends
def testImportNone(self):
res = bulk_import_async(self._config(), self.domain, None)
self.assertEqual(
'Sorry, your session has expired. Please start over and try again.',
unicode(res['errors']))
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
@run_with_all_backends
def testImporterErrors(self):
with mock.patch(
'corehq.apps.importer.tasks.importer_util.get_spreadsheet',
side_effect=ImporterError()):
res = bulk_import_async(self._config(), self.domain, None)
self.assertEqual(
'The session containing the file you uploaded has expired - please upload a new one.',
unicode(res['errors']))
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
@run_with_all_backends
def testImportBasic(self):
config = self._config(self.default_headers)
file = MockExcelFile(header_columns=self.default_headers, num_rows=5)
res = do_import(file, config, self.domain)
self.assertEqual(5, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertFalse(res['errors'])
self.assertEqual(1, res['num_chunks'])
case_ids = self.accessor.get_case_ids_in_domain()
cases = list(self.accessor.get_cases(case_ids))
self.assertEqual(5, len(cases))
properties_seen = set()
for case in cases:
self.assertEqual(self.couch_user._id, case.user_id)
self.assertEqual(self.couch_user._id, case.owner_id)
self.assertEqual(self.default_case_type, case.type)
for prop in self.default_headers[1:]:
self.assertTrue(prop in case.get_case_property(prop))
self.assertFalse(
case.get_case_property(prop) in properties_seen)
properties_seen.add(case.get_case_property(prop))
@run_with_all_backends
def testImportNamedColumns(self):
config = self._config(self.default_headers, named_columns=True)
file = MockExcelFile(header_columns=self.default_headers, num_rows=5)
res = do_import(file, config, self.domain)
# we create 1 less since we knock off the header column
self.assertEqual(4, res['created_count'])
self.assertEqual(4, len(self.accessor.get_case_ids_in_domain()))
@run_with_all_backends
def testImportTrailingWhitespace(self):
cols = ['case_id', 'age', u'sex\xa0', 'location']
config = self._config(cols, named_columns=True)
file = MockExcelFile(header_columns=cols, num_rows=2)
res = do_import(file, config, self.domain)
# we create 1 less since we knock off the header column
self.assertEqual(1, res['created_count'])
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
case = self.accessor.get_case(case_ids[0])
self.assertTrue(bool(case.get_case_property(
'sex'))) # make sure the value also got properly set
@run_with_all_backends
def testCaseIdMatching(self):
# bootstrap a stub case
[case] = self.factory.create_or_update_case(
CaseStructure(attrs={
'create': True,
'update': {
'importer_test_prop': 'foo'
},
}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
config = self._config(self.default_headers)
file = MockExcelFile(header_columns=self.default_headers,
num_rows=3,
row_generator=id_match_generator(case.case_id))
res = do_import(file, config, self.domain)
self.assertEqual(0, res['created_count'])
self.assertEqual(3, res['match_count'])
self.assertFalse(res['errors'])
# shouldn't create any more cases, just the one
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
[case] = self.accessor.get_cases(case_ids)
for prop in self.default_headers[1:]:
self.assertTrue(prop in case.get_case_property(prop))
# shouldn't touch existing properties
self.assertEqual('foo', case.get_case_property('importer_test_prop'))
@run_with_all_backends
def testCaseLookupTypeCheck(self):
[case] = self.factory.create_or_update_case(
CaseStructure(attrs={
'create': True,
'case_type': 'nonmatch-type',
}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
config = self._config(self.default_headers)
file = MockExcelFile(header_columns=self.default_headers,
num_rows=3,
row_generator=id_match_generator(case.case_id))
res = do_import(file, config, self.domain)
# because the type is wrong these shouldn't match
self.assertEqual(3, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(4, len(self.accessor.get_case_ids_in_domain()))
@run_with_all_backends
def testCaseLookupDomainCheck(self):
self.factory.domain = 'wrong-domain'
[case] = self.factory.create_or_update_case(
CaseStructure(attrs={
'create': True,
}))
self.assertEqual(0, len(self.accessor.get_case_ids_in_domain()))
config = self._config(self.default_headers)
file = MockExcelFile(header_columns=self.default_headers,
num_rows=3,
row_generator=id_match_generator(case.case_id))
res = do_import(file, config, self.domain)
# because the domain is wrong these shouldn't match
self.assertEqual(3, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(3, len(self.accessor.get_case_ids_in_domain()))
@run_with_all_backends
def testExternalIdMatching(self):
# bootstrap a stub case
external_id = 'importer-test-external-id'
[case] = self.factory.create_or_update_case(
CaseStructure(attrs={
'create': True,
'external_id': external_id,
}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
headers = ['external_id', 'age', 'sex', 'location']
config = self._config(headers, search_field='external_id')
file = MockExcelFile(header_columns=headers,
num_rows=3,
row_generator=id_match_generator(external_id))
res = do_import(file, config, self.domain)
self.assertEqual(0, res['created_count'])
self.assertEqual(3, res['match_count'])
self.assertFalse(res['errors'])
# shouldn't create any more cases, just the one
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
@run_with_all_backends
def test_external_id_matching_on_create_with_custom_column_name(self):
headers = ['id_column', 'age', 'sex', 'location']
external_id = 'external-id-test'
config = self._config(headers[1:],
search_column='id_column',
search_field='external_id')
file = MockExcelFile(header_columns=headers,
num_rows=2,
row_generator=id_match_generator(external_id))
res = do_import(file, config, self.domain)
self.assertEqual(1, res['created_count'])
self.assertEqual(1, res['match_count'])
self.assertFalse(res['errors'])
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
case = self.accessor.get_case(case_ids[0])
self.assertEqual(external_id, case.external_id)
def testNoCreateNew(self):
config = self._config(self.default_headers, create_new_cases=False)
file = MockExcelFile(header_columns=self.default_headers, num_rows=5)
res = do_import(file, config, self.domain)
# no matching and no create new set - should do nothing
self.assertEqual(0, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
def testBlankRows(self):
# don't create new cases for rows left blank
config = self._config(self.default_headers, create_new_cases=True)
file = MockExcelFile(header_columns=self.default_headers,
num_rows=5,
row_generator=blank_row_generator)
res = do_import(file, config, self.domain)
# no matching and no create new set - should do nothing
self.assertEqual(0, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
def testBasicChunking(self):
config = self._config(self.default_headers)
file = MockExcelFile(header_columns=self.default_headers, num_rows=5)
res = do_import(file, config, self.domain, chunksize=2)
# 5 cases in chunks of 2 = 3 chunks
self.assertEqual(3, res['num_chunks'])
self.assertEqual(5, res['created_count'])
self.assertEqual(5, len(get_case_ids_in_domain(self.domain)))
@run_with_all_backends
def testExternalIdChunking(self):
# bootstrap a stub case
external_id = 'importer-test-external-id'
headers = ['external_id', 'age', 'sex', 'location']
config = self._config(headers, search_field='external_id')
file = MockExcelFile(header_columns=headers,
num_rows=3,
row_generator=id_match_generator(external_id))
# the first one should create the case, and the remaining two should update it
res = do_import(file, config, self.domain)
self.assertEqual(1, res['created_count'])
self.assertEqual(2, res['match_count'])
self.assertFalse(res['errors'])
self.assertEqual(2,
res['num_chunks']) # the lookup causes an extra chunk
# should just create the one case
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
[case] = self.accessor.get_cases(case_ids)
self.assertEqual(external_id, case.external_id)
for prop in self.default_headers[1:]:
self.assertTrue(prop in case.get_case_property(prop))
@run_with_all_backends
def testParentCase(self):
headers = ['parent_id', 'name', 'case_id']
config = self._config(headers,
create_new_cases=True,
search_column='case_id')
rows = 3
[parent_case] = self.factory.create_or_update_case(
CaseStructure(attrs={'create': True}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
file = MockExcelFile(header_columns=headers,
num_rows=rows,
row_generator=id_match_generator(
parent_case.case_id))
file_missing = MockExcelFile(header_columns=headers, num_rows=rows)
# Should successfully match on `rows` cases
res = do_import(file, config, self.domain)
self.assertEqual(rows, res['created_count'])
# Should be unable to find parent case on `rows` cases
res = do_import(file_missing, config, self.domain)
error_column_name = 'parent_id'
self.assertEqual(
rows,
len(res['errors'][ImportErrors.InvalidParentId][error_column_name]
['rows']), "All cases should have missing parent")
def import_mock_file(self, rows):
config = self._config(rows[0])
case_rows = rows[1:]
num_rows = len(case_rows)
xls_file = MockExcelFile(
header_columns=rows[0],
num_rows=num_rows,
row_generator=lambda _, i: case_rows[i],
)
return do_import(xls_file, config, self.domain)
@run_with_all_backends
def testLocationOwner(self):
# This is actually testing several different things, but I figure it's
# worth it, as each of these tests takes a non-trivial amount of time.
non_case_sharing = LocationType.objects.create(domain=self.domain,
name='lt1',
shares_cases=False)
case_sharing = LocationType.objects.create(domain=self.domain,
name='lt2',
shares_cases=True)
location = make_loc('loc-1', 'Loc 1', self.domain, case_sharing)
make_loc('loc-2', 'Loc 2', self.domain, case_sharing)
duplicate_loc = make_loc('loc-3', 'Loc 2', self.domain, case_sharing)
improper_loc = make_loc('loc-4', 'Loc 4', self.domain,
non_case_sharing)
res = self.import_mock_file([
['case_id', 'name', 'owner_id', 'owner_name'],
['', 'location-owner-id', location.group_id, ''],
['', 'location-owner-code', '', location.site_code],
['', 'location-owner-name', '', location.name],
['', 'duplicate-location-name', '', duplicate_loc.name],
['', 'non-case-owning-name', '', improper_loc.name],
])
case_ids = self.accessor.get_case_ids_in_domain()
cases = {c.name: c for c in list(self.accessor.get_cases(case_ids))}
self.assertEqual(cases['location-owner-id'].owner_id,
location.group_id)
self.assertEqual(cases['location-owner-code'].owner_id,
location.group_id)
self.assertEqual(cases['location-owner-name'].owner_id,
location.group_id)
error_message = ImportErrors.DuplicateLocationName
error_column_name = None
self.assertIn(error_message, res['errors'])
self.assertEqual(
res['errors'][error_message][error_column_name]['rows'], [4])
error_message = ImportErrors.InvalidOwnerId
self.assertIn(error_message, res['errors'])
error_column_name = 'owner_id'
self.assertEqual(
res['errors'][error_message][error_column_name]['rows'], [5])
def _typeTest(self, type_fields, error_message):
config = self._config(self.default_headers, type_fields=type_fields)
file = MockExcelFile(header_columns=self.default_headers, num_rows=3)
res = do_import(file, config, self.domain)
self.assertIn(self.default_headers[1], res['errors'][error_message])
self.assertEqual(
res['errors'][error_message][self.default_headers[1]]['rows'],
[1, 2, 3])
def testDateError(self):
self._typeTest(['plain', 'date', 'plain', 'plain'],
ImportErrors.InvalidDate)
def testIntegerError(self):
self._typeTest(['plain', 'integer', 'plain', 'plain'],
ImportErrors.InvalidInteger)
def _get_case_ids_by_external_id():
accessor = CaseAccessors(DOMAIN)
case_ids = accessor.get_case_ids_in_domain()
return {c.external_id: c.case_id for c in accessor.get_cases(case_ids)}
class CleanOwnerCaseSyncOperation(object):
def __init__(self, restore_state):
self.restore_state = restore_state
self.case_accessor = CaseAccessors(self.restore_state.domain)
@property
@memoized
def cleanliness_flags(self):
return dict(
OwnershipCleanlinessFlag.objects.filter(
domain=self.restore_state.domain,
owner_id__in=self.restore_state.owner_ids
).values_list('owner_id', 'is_clean')
)
def is_clean(self, owner_id):
return self.cleanliness_flags.get(owner_id, False)
def get_payload(self):
response = self.restore_state.restore_class()
case_ids_to_sync = set()
for owner_id in self.restore_state.owner_ids:
case_ids_to_sync = case_ids_to_sync | set(self.get_case_ids_for_owner(owner_id))
if (not self.restore_state.is_initial and
any([not self.is_clean(owner_id) for owner_id in self.restore_state.owner_ids])):
# if it's a steady state sync and we have any dirty owners, then we also need to
# include ALL cases on the phone that have been modified since the last sync as
# possible candidates to sync (since they may have been closed or reassigned by someone else)
# don't bother checking ones we've already decided to check
other_ids_to_check = self.restore_state.last_sync_log.case_ids_on_phone - case_ids_to_sync
case_ids_to_sync = case_ids_to_sync | set(filter_cases_modified_since(
self.case_accessor, list(other_ids_to_check), self.restore_state.last_sync_log.date
))
all_maybe_syncing = copy(case_ids_to_sync)
all_synced = set()
child_indices = defaultdict(set)
extension_indices = defaultdict(set)
all_dependencies_syncing = set()
closed_cases = set()
potential_updates_to_sync = []
while case_ids_to_sync:
ids = pop_ids(case_ids_to_sync, chunk_size)
# todo: see if we can avoid wrapping - serialization depends on it heavily for now
case_batch = filter(
partial(case_needs_to_sync, last_sync_log=self.restore_state.last_sync_log),
[case for case in self.case_accessor.get_cases(ids)
if not case.is_deleted]
)
updates = get_case_sync_updates(
self.restore_state.domain, case_batch, self.restore_state.last_sync_log
)
for update in updates:
case = update.case
all_synced.add(case.case_id)
potential_updates_to_sync.append(update)
# update the indices in the new sync log
if case.indices:
# and double check footprint for non-live cases
extension_indices[case.case_id] = {
index.identifier: index.referenced_id
for index in case.indices
if index.relationship == CASE_INDEX_EXTENSION
}
child_indices[case.case_id] = {index.identifier: index.referenced_id for index in case.indices
if index.relationship == CASE_INDEX_CHILD}
for index in case.indices:
if index.referenced_id not in all_maybe_syncing:
case_ids_to_sync.add(index.referenced_id)
if not _is_live(case, self.restore_state):
all_dependencies_syncing.add(case.case_id)
if case.closed:
closed_cases.add(case.case_id)
# commtrack ledger sections for this batch
commtrack_elements = get_stock_payload(
self.restore_state.project, self.restore_state.stock_settings,
[CaseStub(update.case.case_id, update.case.type) for update in updates]
)
response.extend(commtrack_elements)
# add any new values to all_syncing
all_maybe_syncing = all_maybe_syncing | case_ids_to_sync
# update sync token - marking it as the new format
self.restore_state.current_sync_log = SimplifiedSyncLog.wrap(
self.restore_state.current_sync_log.to_json()
)
self.restore_state.current_sync_log.log_format = LOG_FORMAT_SIMPLIFIED
self.restore_state.current_sync_log.extensions_checked = True
index_tree = IndexTree(indices=child_indices)
extension_index_tree = IndexTree(indices=extension_indices)
case_ids_on_phone = all_synced
primary_cases_syncing = all_synced - all_dependencies_syncing
if not self.restore_state.is_initial:
case_ids_on_phone = case_ids_on_phone | self.restore_state.last_sync_log.case_ids_on_phone
# subtract primary cases from dependencies since they must be newly primary
all_dependencies_syncing = all_dependencies_syncing | (
self.restore_state.last_sync_log.dependent_case_ids_on_phone -
primary_cases_syncing
)
index_tree = self.restore_state.last_sync_log.index_tree.apply_updates(index_tree)
self.restore_state.current_sync_log.case_ids_on_phone = case_ids_on_phone
self.restore_state.current_sync_log.dependent_case_ids_on_phone = all_dependencies_syncing
self.restore_state.current_sync_log.index_tree = index_tree
self.restore_state.current_sync_log.extension_index_tree = extension_index_tree
self.restore_state.current_sync_log.closed_cases = closed_cases
_move_no_longer_owned_cases_to_dependent_list_if_necessary(self.restore_state, self.case_accessor)
self.restore_state.current_sync_log.purge_dependent_cases()
purged_cases = case_ids_on_phone - self.restore_state.current_sync_log.case_ids_on_phone
# don't sync purged cases that were never on the phone
if self.restore_state.is_initial:
irrelevant_cases = purged_cases
else:
irrelevant_cases = purged_cases - self.restore_state.last_sync_log.case_ids_on_phone
for update in potential_updates_to_sync:
if update.case.case_id not in irrelevant_cases:
append_update_to_response(response, update, self.restore_state)
return response
def get_case_ids_for_owner(self, owner_id):
if EXTENSION_CASES_SYNC_ENABLED.enabled(self.restore_state.domain):
return self._get_case_ids_for_owners_with_extensions(owner_id)
else:
return self._get_case_ids_for_owners_without_extensions(owner_id)
def _get_case_ids_for_owners_without_extensions(self, owner_id):
if self.is_clean(owner_id):
if self.restore_state.is_initial:
# for a clean owner's initial sync the base set is just the open ids
return set(self.case_accessor.get_open_case_ids(owner_id))
else:
# for a clean owner's steady state sync, the base set is anything modified since last sync
return set(self.case_accessor.get_case_ids_modified_with_owner_since(
owner_id, self.restore_state.last_sync_log.date
))
else:
# TODO: we may want to be smarter than this
# right now just return the whole footprint and do any filtering later
# Note: This will also return extensions if they exist.
return get_case_footprint_info(self.restore_state.domain, owner_id).all_ids
def _get_case_ids_for_owners_with_extensions(self, owner_id):
"""Fetches base and extra cases when extensions are enabled"""
if not self.is_clean(owner_id) or self.restore_state.is_first_extension_sync:
# If this is the first time a user with extensions has synced after
# the extension flag is toggled, pull all the cases so that the
# extension parameters get set correctly
return get_case_footprint_info(self.restore_state.domain, owner_id).all_ids
else:
if self.restore_state.is_initial:
# for a clean owner's initial sync the base set is just the open ids and their extensions
all_case_ids = set(self.case_accessor.get_open_case_ids(owner_id))
new_case_ids = set(all_case_ids)
while new_case_ids:
all_case_ids = all_case_ids | new_case_ids
extension_case_ids = set(self.case_accessor.get_extension_case_ids(new_case_ids))
new_case_ids = extension_case_ids - all_case_ids
return all_case_ids
else:
# for a clean owner's steady state sync, the base set is anything modified since last sync
modified_non_extension_cases = set(self.case_accessor.get_case_ids_modified_with_owner_since(
owner_id, self.restore_state.last_sync_log.date
))
# we also need to fetch unowned extension cases that have been modified
extension_case_ids = self.restore_state.last_sync_log.extension_index_tree.indices.keys()
modified_extension_cases = set(filter_cases_modified_since(
self.case_accessor, extension_case_ids, self.restore_state.last_sync_log.date
))
return modified_non_extension_cases | modified_extension_cases
def filter_cases(request, domain, app_id, module_id, parent_id=None):
app = Application.get(app_id)
module = app.get_module(module_id)
auth_cookie = request.COOKIES.get('sessionid')
requires_parent_cases = string_to_boolean(
request.GET.get('requires_parent_cases', 'false'))
xpath = EntriesHelper.get_filter_xpath(module)
instances = get_instances_for_module(app,
module,
additional_xpaths=[xpath])
extra_instances = [{'id': inst.id, 'src': inst.src} for inst in instances]
use_formplayer = toggles.USE_FORMPLAYER.enabled(domain)
accessor = CaseAccessors(domain)
# touchforms doesn't like this to be escaped
xpath = HTMLParser.HTMLParser().unescape(xpath)
case_type = module.case_type
if xpath or should_use_sql_backend(domain):
# if we need to do a custom filter, send it to touchforms for processing
additional_filters = {
"properties/case_type": case_type,
"footprint": True
}
helper = BaseSessionDataHelper(domain, request.couch_user)
result = helper.filter_cases(xpath,
additional_filters,
DjangoAuth(auth_cookie),
extra_instances=extra_instances,
use_formplayer=use_formplayer)
if result.get('status', None) == 'error':
code = result.get('code', 500)
message = result.get(
'message', _("Something went wrong filtering your cases."))
if code == 500:
notify_exception(None, message=message)
return json_response(message, status_code=code)
case_ids = result.get("cases", [])
else:
# otherwise just use our built in api with the defaults
case_ids = [
res.id for res in get_filtered_cases(
domain,
status=CASE_STATUS_OPEN,
case_type=case_type,
user_id=request.couch_user._id,
footprint=True,
ids_only=True,
)
]
cases = accessor.get_cases(case_ids)
if parent_id:
cases = filter(lambda c: c.parent and c.parent.case_id == parent_id,
cases)
# refilter these because we might have accidentally included footprint cases
# in the results from touchforms. this is a little hacky but the easiest
# (quick) workaround. should be revisted when we optimize the case list.
cases = filter(lambda c: c.type == case_type, cases)
cases = [c.to_api_json(lite=True) for c in cases if c]
response = {'cases': cases}
if requires_parent_cases:
# Subtract already fetched cases from parent list
parent_ids = set(map(lambda c: c['indices']['parent']['case_id'], cases)) - \
set(map(lambda c: c['case_id'], cases))
parents = accessor.get_cases(list(parent_ids))
parents = [c.to_api_json(lite=True) for c in parents]
response.update({'parents': parents})
return json_response(response)
class EnikshayCaseFactory(object):
domain = None
patient_detail = None
def __init__(self, domain, patient_detail, nikshay_codes_to_location):
self.domain = domain
self.patient_detail = patient_detail
self.case_accessor = CaseAccessors(domain)
self.nikshay_codes_to_location = nikshay_codes_to_location
@property
def nikshay_id(self):
return self.patient_detail.PregId
@property
@memoized
def existing_person_case(self):
"""
Get the existing person case for this nikshay ID, or None if no person case exists
"""
matching_external_ids = self.case_accessor.get_cases_by_external_id(self.nikshay_id, case_type='person')
if matching_external_ids:
assert len(matching_external_ids) == 1
return matching_external_ids[0]
return None
@property
def creating_person_case(self):
return self.existing_person_case is not None
@property
@memoized
def existing_occurrence_case(self):
"""
Get the existing occurrence case for this nikshay ID, or None if no occurrence case exists
"""
if self.existing_person_case:
try:
return get_open_occurrence_case_from_person(
self.domain, self.existing_person_case.case_id
)
except ENikshayCaseNotFound:
return None
@property
@memoized
def existing_episode_case(self):
"""
Get the existing episode case for this nikshay ID, or None if no episode case exists
"""
if self.existing_occurrence_case:
try:
return get_open_episode_case_from_occurrence(
self.domain, self.existing_occurrence_case.case_id
)
except ENikshayCaseNotFound:
return None
def get_case_structures_to_create(self):
person_structure = self.get_person_case_structure()
ocurrence_structure = self.get_occurrence_case_structure(person_structure)
episode_structure = self.get_episode_case_structure(ocurrence_structure)
test_structures = [
self.get_test_case_structure(followup, ocurrence_structure) for followup in self._followups
]
return [episode_structure] + test_structures
def get_person_case_structure(self):
kwargs = {
'attrs': {
'case_type': PERSON_CASE_TYPE,
'external_id': self.nikshay_id,
'update': {
'age': self.patient_detail.page,
'age_entered': self.patient_detail.page,
'contact_phone_number': validate_phone_number(self.patient_detail.pmob),
'current_address': self.patient_detail.paddress,
'current_address_district_choice': self.district.location_id,
'current_address_state_choice': self.state.location_id,
'dob': date(date.today().year - self.patient_detail.page, 7, 1),
'dob_known': 'no',
'first_name': self.patient_detail.first_name,
'last_name': self.patient_detail.last_name,
'name': self.patient_detail.pname,
'nikshay_id': self.nikshay_id,
'person_id': 'FROM_NIKSHAY_' + self.nikshay_id,
'secondary_contact_name_address': (
(self.patient_detail.cname or '')
+ ', '
+ (self.patient_detail.caddress or '')
),
'secondary_contact_phone_number': validate_phone_number(self.patient_detail.cmob),
'sex': self.patient_detail.sex,
'tu_choice': self.tu.name,
'migration_created_case': 'true',
},
},
}
if self.phi:
if self.phi.location_type.code == 'phi':
kwargs['attrs']['owner_id'] = self.phi.location_id
kwargs['attrs']['update']['phi'] = self.phi.name
else:
kwargs['attrs']['owner_id'] = ARCHIVED_CASE_OWNER_ID
kwargs['attrs']['update']['archive_reason'] = 'migration_not_phi_location'
kwargs['attrs']['update']['migration_error'] = 'not_phi_location'
kwargs['attrs']['update']['migration_error_details'] = self._nikshay_code
else:
kwargs['attrs']['owner_id'] = ARCHIVED_CASE_OWNER_ID
kwargs['attrs']['update']['archive_reason'] = 'migration_location_not_found'
kwargs['attrs']['update']['migration_error'] = 'location_not_found'
kwargs['attrs']['update']['migration_error_details'] = self._nikshay_code
if self.patient_detail.paadharno is not None:
kwargs['attrs']['update']['aadhaar_number'] = self.patient_detail.paadharno
if self.existing_person_case is not None:
kwargs['case_id'] = self.existing_person_case.case_id
kwargs['attrs']['create'] = False
else:
kwargs['attrs']['create'] = True
return CaseStructure(**kwargs)
def get_occurrence_case_structure(self, person_structure):
"""
This gets the occurrence case structure with a nested person case structure.
"""
kwargs = {
'attrs': {
'case_type': OCCURRENCE_CASE_TYPE,
'owner_id': '-',
'update': {
'current_episode_type': 'confirmed_tb',
'ihv_date': self.patient_detail.ihv_date,
'initial_home_visit_status': self.patient_detail.initial_home_visit_status,
'name': 'Occurrence #1',
'occurrence_episode_count': 1,
'occurrence_id': datetime.utcnow().strftime('%Y%m%d%H%M%S%f')[:-3],
'migration_created_case': 'true',
},
},
'indices': [CaseIndex(
person_structure,
identifier='host',
relationship=CASE_INDEX_EXTENSION,
related_type=PERSON_CASE_TYPE,
)],
}
if self._outcome:
# TODO - store with correct value
kwargs['attrs']['update']['hiv_status'] = self._outcome.HIVStatus
if self.existing_occurrence_case:
kwargs['case_id'] = self.existing_occurrence_case.case_id
kwargs['attrs']['create'] = False
else:
kwargs['attrs']['create'] = True
return CaseStructure(**kwargs)
def get_episode_case_structure(self, occurrence_structure):
"""
This gets the episode case structure with a nested occurrence and person case structures
inside of it.
"""
kwargs = {
'attrs': {
'case_type': EPISODE_CASE_TYPE,
'date_opened': self.patient_detail.pregdate1,
'owner_id': '-',
'update': {
'date_of_mo_signature': self.patient_detail.date_of_mo_signature,
'disease_classification': self.patient_detail.disease_classification,
'dots_99_enabled': 'false',
'episode_pending_registration': 'no',
'episode_type': 'confirmed_tb',
'name': 'Episode #1: Confirmed TB (Patient)',
'occupation': self.patient_detail.occupation,
'patient_type_choice': self.patient_detail.patient_type_choice,
'treatment_initiation_date': self.patient_detail.treatment_initiation_date,
'treatment_supporter_designation': self.patient_detail.treatment_supporter_designation,
'treatment_supporter_first_name': self.patient_detail.treatment_supporter_first_name,
'treatment_supporter_last_name': self.patient_detail.treatment_supporter_last_name,
'treatment_supporter_mobile_number': validate_phone_number(self.patient_detail.dotmob),
'migration_created_case': 'true',
},
},
'indices': [CaseIndex(
occurrence_structure,
identifier='host',
relationship=CASE_INDEX_EXTENSION,
related_type=OCCURRENCE_CASE_TYPE,
)],
}
if self.existing_episode_case:
kwargs['case_id'] = self.existing_episode_case.case_id
kwargs['attrs']['create'] = False
else:
kwargs['attrs']['create'] = True
return CaseStructure(**kwargs)
def get_test_case_structure(self, followup, occurrence_structure):
kwargs = {
'attrs': {
'create': True,
'case_type': TEST_CASE_TYPE,
'owner_id': '-',
'update': {
'date_tested': followup.TestDate,
'migration_created_case': 'true',
'migration_followup_id': followup.id,
},
},
'indices': [CaseIndex(
occurrence_structure,
identifier='host',
relationship=CASE_INDEX_EXTENSION,
related_type=occurrence_structure.attrs['case_type'],
)],
# this prevents creating duplicate occurrence data on creation of the test cases
'walk_related': False,
}
if self.existing_occurrence_case:
matching_test_case = next((
extension_case for extension_case in self.case_accessor.get_cases([
index.referenced_id for index in
self.existing_occurrence_case.reverse_indices
])
if (
extension_case.type == TEST_CASE_TYPE
and followup.id == int(extension_case.dynamic_case_properties().get('migration_followup_id', -1))
)
), None)
if matching_test_case:
kwargs['case_id'] = matching_test_case.case_id
kwargs['attrs']['create'] = False
else:
kwargs['attrs']['create'] = True
return CaseStructure(**kwargs)
@property
@memoized
def _outcome(self):
zero_or_one_outcomes = list(Outcome.objects.filter(PatientId=self.patient_detail))
if zero_or_one_outcomes:
return zero_or_one_outcomes[0]
else:
return None
@property
@memoized
def _followups(self):
return Followup.objects.filter(PatientID=self.patient_detail)
@property
def state(self):
return self.nikshay_codes_to_location.get(self.patient_detail.PregId.split('-')[0])
@property
def district(self):
return self.nikshay_codes_to_location.get('-'.join(self.patient_detail.PregId.split('-')[:2]))
@property
def tu(self):
return self.nikshay_codes_to_location.get('-'.join(self.patient_detail.PregId.split('-')[:3]))
@property
def phi(self):
return self.nikshay_codes_to_location.get(self._nikshay_code)
@property
def _nikshay_code(self):
return '-'.join(self.patient_detail.PregId.split('-')[:4])
class ImporterTest(TestCase):
def setUp(self):
super(ImporterTest, self).setUp()
self.domain_obj = create_domain("importer-test")
self.domain = self.domain_obj.name
self.default_case_type = 'importer-test-casetype'
self.couch_user = WebUser.create(None, "test", "foobar")
self.couch_user.add_domain_membership(self.domain, is_admin=True)
self.couch_user.save()
self.accessor = CaseAccessors(self.domain)
self.factory = CaseFactory(domain=self.domain, case_defaults={
'case_type': self.default_case_type,
})
delete_all_cases()
def tearDown(self):
self.couch_user.delete()
self.domain_obj.delete()
super(ImporterTest, self).tearDown()
def _config(self, col_names, search_column=None, case_type=None,
search_field='case_id', create_new_cases=True):
return ImporterConfig(
couch_user_id=self.couch_user._id,
case_type=case_type or self.default_case_type,
excel_fields=col_names,
case_fields=[''] * len(col_names),
custom_fields=col_names,
search_column=search_column or col_names[0],
search_field=search_field,
create_new_cases=create_new_cases,
)
@run_with_all_backends
@patch('corehq.apps.case_importer.tasks.bulk_import_async.update_state')
def testImportNone(self, update_state):
res = bulk_import_async.delay(self._config(['anything']), self.domain, None)
self.assertIsInstance(res.result, Ignore)
update_state.assert_called_with(
state=states.FAILURE,
meta={'errors': 'Sorry, your session has expired. Please start over and try again.'})
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
@run_with_all_backends
def testImportBasic(self):
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
['case_id-0', 'age-0', 'sex-0', 'location-0'],
['case_id-1', 'age-1', 'sex-1', 'location-1'],
['case_id-2', 'age-2', 'sex-2', 'location-2'],
['case_id-3', 'age-3', 'sex-3', 'location-3'],
['case_id-4', 'age-4', 'sex-4', 'location-4'],
)
res = do_import(file, config, self.domain)
self.assertEqual(5, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertFalse(res['errors'])
self.assertEqual(1, res['num_chunks'])
case_ids = self.accessor.get_case_ids_in_domain()
cases = list(self.accessor.get_cases(case_ids))
self.assertEqual(5, len(cases))
properties_seen = set()
for case in cases:
self.assertEqual(self.couch_user._id, case.user_id)
self.assertEqual(self.couch_user._id, case.owner_id)
self.assertEqual(self.default_case_type, case.type)
for prop in ['age', 'sex', 'location']:
self.assertTrue(prop in case.get_case_property(prop))
self.assertFalse(case.get_case_property(prop) in properties_seen)
properties_seen.add(case.get_case_property(prop))
@run_with_all_backends
def testImportNamedColumns(self):
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
['case_id-0', 'age-0', 'sex-0', 'location-0'],
['case_id-1', 'age-1', 'sex-1', 'location-1'],
['case_id-2', 'age-2', 'sex-2', 'location-2'],
['case_id-3', 'age-3', 'sex-3', 'location-3'],
)
res = do_import(file, config, self.domain)
self.assertEqual(4, res['created_count'])
self.assertEqual(4, len(self.accessor.get_case_ids_in_domain()))
@run_with_all_backends
def testImportTrailingWhitespace(self):
cols = ['case_id', 'age', 'sex\xa0', 'location']
config = self._config(cols)
file = make_worksheet_wrapper(
['case_id', 'age', 'sex\xa0', 'location'],
['case_id-0', 'age-0', 'sex\xa0-0', 'location-0'],
)
res = do_import(file, config, self.domain)
self.assertEqual(1, res['created_count'])
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
case = self.accessor.get_case(case_ids[0])
self.assertTrue(bool(case.get_case_property('sex'))) # make sure the value also got properly set
@run_with_all_backends
def testCaseIdMatching(self):
# bootstrap a stub case
[case] = self.factory.create_or_update_case(CaseStructure(attrs={
'create': True,
'update': {'importer_test_prop': 'foo'},
}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
[case.case_id, 'age-0', 'sex-0', 'location-0'],
[case.case_id, 'age-1', 'sex-1', 'location-1'],
[case.case_id, 'age-2', 'sex-2', 'location-2'],
)
res = do_import(file, config, self.domain)
self.assertEqual(0, res['created_count'])
self.assertEqual(3, res['match_count'])
self.assertFalse(res['errors'])
# shouldn't create any more cases, just the one
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
[case] = self.accessor.get_cases(case_ids)
for prop in ['age', 'sex', 'location']:
self.assertTrue(prop in case.get_case_property(prop))
# shouldn't touch existing properties
self.assertEqual('foo', case.get_case_property('importer_test_prop'))
@run_with_all_backends
def testCaseLookupTypeCheck(self):
[case] = self.factory.create_or_update_case(CaseStructure(attrs={
'create': True,
'case_type': 'nonmatch-type',
}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
[case.case_id, 'age-0', 'sex-0', 'location-0'],
[case.case_id, 'age-1', 'sex-1', 'location-1'],
[case.case_id, 'age-2', 'sex-2', 'location-2'],
)
res = do_import(file, config, self.domain)
# because the type is wrong these shouldn't match
self.assertEqual(3, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(4, len(self.accessor.get_case_ids_in_domain()))
@run_with_all_backends
def testCaseLookupDomainCheck(self):
self.factory.domain = 'wrong-domain'
[case] = self.factory.create_or_update_case(CaseStructure(attrs={
'create': True,
}))
self.assertEqual(0, len(self.accessor.get_case_ids_in_domain()))
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
[case.case_id, 'age-0', 'sex-0', 'location-0'],
[case.case_id, 'age-1', 'sex-1', 'location-1'],
[case.case_id, 'age-2', 'sex-2', 'location-2'],
)
res = do_import(file, config, self.domain)
# because the domain is wrong these shouldn't match
self.assertEqual(3, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(3, len(self.accessor.get_case_ids_in_domain()))
@run_with_all_backends
def testExternalIdMatching(self):
# bootstrap a stub case
external_id = 'importer-test-external-id'
[case] = self.factory.create_or_update_case(CaseStructure(
attrs={
'create': True,
'external_id': external_id,
}
))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
headers = ['external_id', 'age', 'sex', 'location']
config = self._config(headers, search_field='external_id')
file = make_worksheet_wrapper(
['external_id', 'age', 'sex', 'location'],
['importer-test-external-id', 'age-0', 'sex-0', 'location-0'],
['importer-test-external-id', 'age-1', 'sex-1', 'location-1'],
['importer-test-external-id', 'age-2', 'sex-2', 'location-2'],
)
res = do_import(file, config, self.domain)
self.assertEqual(0, res['created_count'])
self.assertEqual(3, res['match_count'])
self.assertFalse(res['errors'])
# shouldn't create any more cases, just the one
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
@run_with_all_backends
def test_external_id_matching_on_create_with_custom_column_name(self):
headers = ['id_column', 'age', 'sex', 'location']
external_id = 'external-id-test'
config = self._config(headers[1:], search_column='id_column', search_field='external_id')
file = make_worksheet_wrapper(
['id_column', 'age', 'sex', 'location'],
['external-id-test', 'age-0', 'sex-0', 'location-0'],
['external-id-test', 'age-1', 'sex-1', 'location-1'],
)
res = do_import(file, config, self.domain)
self.assertFalse(res['errors'])
self.assertEqual(1, res['created_count'])
self.assertEqual(1, res['match_count'])
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
case = self.accessor.get_case(case_ids[0])
self.assertEqual(external_id, case.external_id)
def testNoCreateNew(self):
config = self._config(['case_id', 'age', 'sex', 'location'], create_new_cases=False)
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
['case_id-0', 'age-0', 'sex-0', 'location-0'],
['case_id-1', 'age-1', 'sex-1', 'location-1'],
['case_id-2', 'age-2', 'sex-2', 'location-2'],
['case_id-3', 'age-3', 'sex-3', 'location-3'],
['case_id-4', 'age-4', 'sex-4', 'location-4'],
)
res = do_import(file, config, self.domain)
# no matching and no create new set - should do nothing
self.assertEqual(0, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
def testBlankRows(self):
# don't create new cases for rows left blank
config = self._config(['case_id', 'age', 'sex', 'location'], create_new_cases=True)
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
[None, None, None, None],
['', '', '', ''],
)
res = do_import(file, config, self.domain)
# no matching and no create new set - should do nothing
self.assertEqual(0, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
def testBasicChunking(self):
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
['case_id-0', 'age-0', 'sex-0', 'location-0'],
['case_id-1', 'age-1', 'sex-1', 'location-1'],
['case_id-2', 'age-2', 'sex-2', 'location-2'],
['case_id-3', 'age-3', 'sex-3', 'location-3'],
['case_id-4', 'age-4', 'sex-4', 'location-4'],
)
res = do_import(file, config, self.domain, chunksize=2)
# 5 cases in chunks of 2 = 3 chunks
self.assertEqual(3, res['num_chunks'])
self.assertEqual(5, res['created_count'])
self.assertEqual(5, len(get_case_ids_in_domain(self.domain)))
@run_with_all_backends
def testExternalIdChunking(self):
# bootstrap a stub case
external_id = 'importer-test-external-id'
headers = ['external_id', 'age', 'sex', 'location']
config = self._config(headers, search_field='external_id')
file = make_worksheet_wrapper(
['external_id', 'age', 'sex', 'location'],
['importer-test-external-id', 'age-0', 'sex-0', 'location-0'],
['importer-test-external-id', 'age-1', 'sex-1', 'location-1'],
['importer-test-external-id', 'age-2', 'sex-2', 'location-2'],
)
# the first one should create the case, and the remaining two should update it
res = do_import(file, config, self.domain)
self.assertEqual(1, res['created_count'])
self.assertEqual(2, res['match_count'])
self.assertFalse(res['errors'])
self.assertEqual(2, res['num_chunks']) # the lookup causes an extra chunk
# should just create the one case
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
[case] = self.accessor.get_cases(case_ids)
self.assertEqual(external_id, case.external_id)
for prop in ['age', 'sex', 'location']:
self.assertTrue(prop in case.get_case_property(prop))
@run_with_all_backends
def testParentCase(self):
headers = ['parent_id', 'name', 'case_id']
config = self._config(headers, create_new_cases=True, search_column='case_id')
rows = 3
[parent_case] = self.factory.create_or_update_case(CaseStructure(attrs={'create': True}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
file = make_worksheet_wrapper(
['parent_id', 'name', 'case_id'],
[parent_case.case_id, 'name-0', 'case_id-0'],
[parent_case.case_id, 'name-1', 'case_id-1'],
[parent_case.case_id, 'name-2', 'case_id-2'],
)
file_missing = make_worksheet_wrapper(
['parent_id', 'name', 'case_id'],
['parent_id-0', 'name-0', 'case_id-0'],
['parent_id-1', 'name-1', 'case_id-1'],
['parent_id-2', 'name-2', 'case_id-2'],
)
# Should successfully match on `rows` cases
res = do_import(file, config, self.domain)
self.assertEqual(rows, res['created_count'])
# Should be unable to find parent case on `rows` cases
res = do_import(file_missing, config, self.domain)
error_column_name = 'parent_id'
self.assertEqual(rows,
len(res['errors'][ImportErrors.InvalidParentId][error_column_name]['rows']),
"All cases should have missing parent")
def import_mock_file(self, rows):
config = self._config(rows[0])
xls_file = make_worksheet_wrapper(*rows)
return do_import(xls_file, config, self.domain)
@run_with_all_backends
def testLocationOwner(self):
# This is actually testing several different things, but I figure it's
# worth it, as each of these tests takes a non-trivial amount of time.
non_case_sharing = LocationType.objects.create(
domain=self.domain, name='lt1', shares_cases=False
)
case_sharing = LocationType.objects.create(
domain=self.domain, name='lt2', shares_cases=True
)
location = make_loc('loc-1', 'Loc 1', self.domain, case_sharing.code)
make_loc('loc-2', 'Loc 2', self.domain, case_sharing.code)
duplicate_loc = make_loc('loc-3', 'Loc 2', self.domain, case_sharing.code)
improper_loc = make_loc('loc-4', 'Loc 4', self.domain, non_case_sharing.code)
res = self.import_mock_file([
['case_id', 'name', 'owner_id', 'owner_name'],
['', 'location-owner-id', location.group_id, ''],
['', 'location-owner-code', '', location.site_code],
['', 'location-owner-name', '', location.name],
['', 'duplicate-location-name', '', duplicate_loc.name],
['', 'non-case-owning-name', '', improper_loc.name],
])
case_ids = self.accessor.get_case_ids_in_domain()
cases = {c.name: c for c in list(self.accessor.get_cases(case_ids))}
self.assertEqual(cases['location-owner-id'].owner_id, location.group_id)
self.assertEqual(cases['location-owner-code'].owner_id, location.group_id)
self.assertEqual(cases['location-owner-name'].owner_id, location.group_id)
error_message = ImportErrors.DuplicateLocationName
error_column_name = None
self.assertIn(error_message, res['errors'])
self.assertEqual(res['errors'][error_message][error_column_name]['rows'], [5])
error_message = ImportErrors.InvalidOwnerId
self.assertIn(error_message, res['errors'])
error_column_name = 'owner_id'
self.assertEqual(res['errors'][error_message][error_column_name]['rows'], [6])
@run_with_all_backends
def test_opened_on(self):
case = self.factory.create_case()
new_date = '2015-04-30T14:41:53.000000Z'
with flag_enabled('BULK_UPLOAD_DATE_OPENED'):
self.import_mock_file([
['case_id', 'date_opened'],
[case.case_id, new_date]
])
case = CaseAccessors(self.domain).get_case(case.case_id)
self.assertEqual(case.opened_on, PhoneTime(parse_datetime(new_date)).done())
@run_with_all_backends
def test_columns_and_rows_align(self):
case_owner = CommCareUser.create(self.domain, 'username', 'pw')
res = self.import_mock_file([
['case_id', 'name', '', 'favorite_color', 'owner_id'],
['', 'Jeff', '', 'blue', case_owner._id],
['', 'Caroline', '', 'yellow', case_owner._id],
])
self.assertEqual(res['errors'], {})
case_ids = self.accessor.get_case_ids_in_domain()
cases = {c.name: c for c in list(self.accessor.get_cases(case_ids))}
self.assertEqual(cases['Jeff'].owner_id, case_owner._id)
self.assertEqual(cases['Jeff'].get_case_property('favorite_color'), 'blue')
self.assertEqual(cases['Caroline'].owner_id, case_owner._id)
self.assertEqual(cases['Caroline'].get_case_property('favorite_color'), 'yellow')
class ENikshay2BMigrator(object):
def __init__(self, domain, commit):
self.domain = domain
self.commit = commit
self.accessor = CaseAccessors(self.domain)
self.factory = CaseFactory(self.domain)
self.total_persons = 0
self.total_occurrences = 0
self.total_episodes = 0
self.total_tests = 0
self.total_referrals = 0
self.total_trails = 0
self.total_secondary_owners = 0
self.total_drtb_hiv = 0
@property
@memoized
def locations(self):
return {
loc.location_id: loc
for loc in SQLLocation.objects.filter(
domain=self.domain).prefetch_related('location_type')
}
@property
@memoized
def location_ids_by_pk(self):
return {loc.pk: loc.location_id for loc in self.locations.values()}
def get_ancestors_by_type(self, location):
"""Get all direct ancestors found in self.locations"""
ancestors_by_type = {location.location_type.code: location}
loc = location
while loc.parent_id and loc.parent_id in self.location_ids_by_pk:
parent = self.locations[self.location_ids_by_pk[loc.parent_id]]
ancestors_by_type[parent.location_type.code] = parent
loc = parent
return ancestors_by_type
def migrate(self):
person_ids = self.get_relevant_person_case_ids()
persons = self.get_relevant_person_case_sets(person_ids)
for person in with_progress_bar(persons, len(person_ids)):
self.migrate_person_case_set(person)
def get_relevant_person_case_ids(self):
return self.accessor.get_case_ids_in_domain(CASE_TYPE_PERSON)
def get_relevant_person_case_sets(self, person_ids):
"""
Generator returning all relevant cases for the migration, grouped by person.
This is a pretty nasty method, but it was the only way I could figure
out how to group the queries together, rather than performing multiple
queries per person case.
"""
for person_chunk in chunked(person_ids, 100):
person_chunk = list(filter(None, person_chunk))
all_persons = {} # case_id: PersonCaseSet
for person in self.accessor.get_cases(person_chunk):
# enrolled_in_private is blank/not set AND case_version is blank/not set
# AND owner_id is within the location set being migrated
if (person.get_case_property(ENROLLED_IN_PRIVATE) != 'true'
and not person.get_case_property(CASE_VERSION)):
all_persons[person.case_id] = PersonCaseSet(person)
referrals_and_occurrences_to_person = {}
type_to_bucket = {
CASE_TYPE_OCCURRENCE: 'occurrences',
CASE_TYPE_REFERRAL: 'referrals',
CASE_TYPE_TRAIL: 'trails'
}
for case in self.accessor.get_reverse_indexed_cases(
[person_id for person_id in all_persons]):
bucket = type_to_bucket.get(case.type, None)
if bucket:
for index in case.indices:
if index.referenced_id in all_persons:
getattr(all_persons[index.referenced_id],
bucket).append(case)
if bucket != 'trails':
referrals_and_occurrences_to_person[
case.case_id] = index.referenced_id
break
type_to_bucket = {
CASE_TYPE_EPISODE: 'episodes',
CASE_TYPE_TEST: 'tests',
CASE_TYPE_TRAIL: 'trails'
}
episodes_to_person = {}
for case in self.accessor.get_reverse_indexed_cases(
referrals_and_occurrences_to_person.keys()):
bucket = type_to_bucket.get(case.type, None)
if bucket:
for index in case.indices:
person_id = referrals_and_occurrences_to_person.get(
index.referenced_id)
if person_id:
getattr(all_persons[person_id],
bucket).append(case)
if case.type == CASE_TYPE_EPISODE:
episodes_to_person[case.case_id] = person_id
break
for case in self.accessor.get_reverse_indexed_cases(
episodes_to_person.keys()):
if case.type == CASE_TYPE_DRTB_HIV_REFERRAL:
for index in case.indices:
person_id = episodes_to_person.get(index.referenced_id)
if person_id:
all_persons[person_id].drtb_hiv.append(case)
break
for person in all_persons.values():
if person.occurrences:
person.latest_occurrence = max(
(case.opened_on, case)
for case in person.occurrences)[1]
yield person
def filter_cases(request, domain, app_id, module_id, parent_id=None):
app = Application.get(app_id)
module = app.get_module(module_id)
auth_cookie = request.COOKIES.get('sessionid')
requires_parent_cases = string_to_boolean(request.GET.get('requires_parent_cases', 'false'))
xpath = EntriesHelper.get_filter_xpath(module)
instances = get_instances_for_module(app, module, additional_xpaths=[xpath])
extra_instances = [{'id': inst.id, 'src': inst.src} for inst in instances]
accessor = CaseAccessors(domain)
# touchforms doesn't like this to be escaped
xpath = HTMLParser.HTMLParser().unescape(xpath)
case_type = module.case_type
if xpath or should_use_sql_backend(domain):
# if we need to do a custom filter, send it to touchforms for processing
additional_filters = {
"properties/case_type": case_type,
"footprint": True
}
helper = BaseSessionDataHelper(domain, request.couch_user)
result = helper.filter_cases(xpath, additional_filters, DjangoAuth(auth_cookie),
extra_instances=extra_instances)
if result.get('status', None) == 'error':
code = result.get('code', 500)
message = result.get('message', _("Something went wrong filtering your cases."))
if code == 500:
notify_exception(None, message=message)
return json_response(message, status_code=code)
case_ids = result.get("cases", [])
else:
# otherwise just use our built in api with the defaults
case_ids = [res.id for res in get_filtered_cases(
domain,
status=CASE_STATUS_OPEN,
case_type=case_type,
user_id=request.couch_user._id,
footprint=True,
ids_only=True,
)]
cases = accessor.get_cases(case_ids)
if parent_id:
cases = filter(lambda c: c.parent and c.parent.case_id == parent_id, cases)
# refilter these because we might have accidentally included footprint cases
# in the results from touchforms. this is a little hacky but the easiest
# (quick) workaround. should be revisted when we optimize the case list.
cases = filter(lambda c: c.type == case_type, cases)
cases = [c.to_api_json(lite=True) for c in cases if c]
response = {'cases': cases}
if requires_parent_cases:
# Subtract already fetched cases from parent list
parent_ids = set(map(lambda c: c['indices']['parent']['case_id'], cases)) - \
set(map(lambda c: c['case_id'], cases))
parents = accessor.get_cases(list(parent_ids))
parents = [c.to_api_json(lite=True) for c in parents]
response.update({'parents': parents})
return json_response(response)
class ImporterTest(TestCase):
def setUp(self):
super(ImporterTest, self).setUp()
self.domain_obj = create_domain("importer-test")
self.domain = self.domain_obj.name
self.default_case_type = 'importer-test-casetype'
self.couch_user = WebUser.create(None, "test", "foobar", None, None)
self.couch_user.add_domain_membership(self.domain, is_admin=True)
self.couch_user.save()
self.accessor = CaseAccessors(self.domain)
self.factory = CaseFactory(domain=self.domain,
case_defaults={
'case_type': self.default_case_type,
})
delete_all_cases()
def tearDown(self):
self.couch_user.delete()
self.domain_obj.delete()
super(ImporterTest, self).tearDown()
def _config(self,
col_names,
search_column=None,
case_type=None,
search_field='case_id',
create_new_cases=True):
return ImporterConfig(
couch_user_id=self.couch_user._id,
case_type=case_type or self.default_case_type,
excel_fields=col_names,
case_fields=[''] * len(col_names),
custom_fields=col_names,
search_column=search_column or col_names[0],
search_field=search_field,
create_new_cases=create_new_cases,
)
@run_with_all_backends
@patch('corehq.apps.case_importer.tasks.bulk_import_async.update_state')
def testImportFileMissing(self, update_state):
# by using a made up upload_id, we ensure it's not referencing any real file
case_upload = CaseUploadRecord(upload_id=str(uuid.uuid4()),
task_id=str(uuid.uuid4()))
case_upload.save()
res = bulk_import_async.delay(self._config(['anything']), self.domain,
case_upload.upload_id)
self.assertIsInstance(res.result, Ignore)
update_state.assert_called_with(
state=states.FAILURE,
meta=get_interned_exception(
'Sorry, your session has expired. Please start over and try again.'
))
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
@run_with_all_backends
def testImportBasic(self):
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
['case_id-0', 'age-0', 'sex-0', 'location-0'],
['case_id-1', 'age-1', 'sex-1', 'location-1'],
['case_id-2', 'age-2', 'sex-2', 'location-2'],
['case_id-3', 'age-3', 'sex-3', 'location-3'],
['case_id-4', 'age-4', 'sex-4', 'location-4'],
)
res = do_import(file, config, self.domain)
self.assertEqual(5, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertFalse(res['errors'])
self.assertEqual(1, res['num_chunks'])
case_ids = self.accessor.get_case_ids_in_domain()
cases = list(self.accessor.get_cases(case_ids))
self.assertEqual(5, len(cases))
properties_seen = set()
for case in cases:
self.assertEqual(self.couch_user._id, case.user_id)
self.assertEqual(self.couch_user._id, case.owner_id)
self.assertEqual(self.default_case_type, case.type)
for prop in ['age', 'sex', 'location']:
self.assertTrue(prop in case.get_case_property(prop))
self.assertFalse(
case.get_case_property(prop) in properties_seen)
properties_seen.add(case.get_case_property(prop))
@run_with_all_backends
def testImportNamedColumns(self):
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
['case_id-0', 'age-0', 'sex-0', 'location-0'],
['case_id-1', 'age-1', 'sex-1', 'location-1'],
['case_id-2', 'age-2', 'sex-2', 'location-2'],
['case_id-3', 'age-3', 'sex-3', 'location-3'],
)
res = do_import(file, config, self.domain)
self.assertEqual(4, res['created_count'])
self.assertEqual(4, len(self.accessor.get_case_ids_in_domain()))
@run_with_all_backends
def testImportTrailingWhitespace(self):
cols = ['case_id', 'age', 'sex\xa0', 'location']
config = self._config(cols)
file = make_worksheet_wrapper(
['case_id', 'age', 'sex\xa0', 'location'],
['case_id-0', 'age-0', 'sex\xa0-0', 'location-0'],
)
res = do_import(file, config, self.domain)
self.assertEqual(1, res['created_count'])
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
case = self.accessor.get_case(case_ids[0])
self.assertTrue(bool(case.get_case_property(
'sex'))) # make sure the value also got properly set
@run_with_all_backends
def testCaseIdMatching(self):
# bootstrap a stub case
[case] = self.factory.create_or_update_case(
CaseStructure(attrs={
'create': True,
'update': {
'importer_test_prop': 'foo'
},
}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
[case.case_id, 'age-0', 'sex-0', 'location-0'],
[case.case_id, 'age-1', 'sex-1', 'location-1'],
[case.case_id, 'age-2', 'sex-2', 'location-2'],
)
res = do_import(file, config, self.domain)
self.assertEqual(0, res['created_count'])
self.assertEqual(3, res['match_count'])
self.assertFalse(res['errors'])
# shouldn't create any more cases, just the one
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
[case] = self.accessor.get_cases(case_ids)
for prop in ['age', 'sex', 'location']:
self.assertTrue(prop in case.get_case_property(prop))
# shouldn't touch existing properties
self.assertEqual('foo', case.get_case_property('importer_test_prop'))
@run_with_all_backends
def testCaseLookupTypeCheck(self):
[case] = self.factory.create_or_update_case(
CaseStructure(attrs={
'create': True,
'case_type': 'nonmatch-type',
}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
[case.case_id, 'age-0', 'sex-0', 'location-0'],
[case.case_id, 'age-1', 'sex-1', 'location-1'],
[case.case_id, 'age-2', 'sex-2', 'location-2'],
)
res = do_import(file, config, self.domain)
# because the type is wrong these shouldn't match
self.assertEqual(3, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(4, len(self.accessor.get_case_ids_in_domain()))
@run_with_all_backends
def testCaseLookupDomainCheck(self):
self.factory.domain = 'wrong-domain'
[case] = self.factory.create_or_update_case(
CaseStructure(attrs={
'create': True,
}))
self.assertEqual(0, len(self.accessor.get_case_ids_in_domain()))
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
[case.case_id, 'age-0', 'sex-0', 'location-0'],
[case.case_id, 'age-1', 'sex-1', 'location-1'],
[case.case_id, 'age-2', 'sex-2', 'location-2'],
)
res = do_import(file, config, self.domain)
# because the domain is wrong these shouldn't match
self.assertEqual(3, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(3, len(self.accessor.get_case_ids_in_domain()))
@run_with_all_backends
def testExternalIdMatching(self):
# bootstrap a stub case
external_id = 'importer-test-external-id'
[case] = self.factory.create_or_update_case(
CaseStructure(attrs={
'create': True,
'external_id': external_id,
}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
headers = ['external_id', 'age', 'sex', 'location']
config = self._config(headers, search_field='external_id')
file = make_worksheet_wrapper(
['external_id', 'age', 'sex', 'location'],
['importer-test-external-id', 'age-0', 'sex-0', 'location-0'],
['importer-test-external-id', 'age-1', 'sex-1', 'location-1'],
['importer-test-external-id', 'age-2', 'sex-2', 'location-2'],
)
res = do_import(file, config, self.domain)
self.assertEqual(0, res['created_count'])
self.assertEqual(3, res['match_count'])
self.assertFalse(res['errors'])
# shouldn't create any more cases, just the one
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
@run_with_all_backends
def test_external_id_matching_on_create_with_custom_column_name(self):
headers = ['id_column', 'age', 'sex', 'location']
external_id = 'external-id-test'
config = self._config(headers[1:],
search_column='id_column',
search_field='external_id')
file = make_worksheet_wrapper(
['id_column', 'age', 'sex', 'location'],
['external-id-test', 'age-0', 'sex-0', 'location-0'],
['external-id-test', 'age-1', 'sex-1', 'location-1'],
)
res = do_import(file, config, self.domain)
self.assertFalse(res['errors'])
self.assertEqual(1, res['created_count'])
self.assertEqual(1, res['match_count'])
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
case = self.accessor.get_case(case_ids[0])
self.assertEqual(external_id, case.external_id)
def testNoCreateNew(self):
config = self._config(['case_id', 'age', 'sex', 'location'],
create_new_cases=False)
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
['case_id-0', 'age-0', 'sex-0', 'location-0'],
['case_id-1', 'age-1', 'sex-1', 'location-1'],
['case_id-2', 'age-2', 'sex-2', 'location-2'],
['case_id-3', 'age-3', 'sex-3', 'location-3'],
['case_id-4', 'age-4', 'sex-4', 'location-4'],
)
res = do_import(file, config, self.domain)
# no matching and no create new set - should do nothing
self.assertEqual(0, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
def testBlankRows(self):
# don't create new cases for rows left blank
config = self._config(['case_id', 'age', 'sex', 'location'],
create_new_cases=True)
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
[None, None, None, None],
['', '', '', ''],
)
res = do_import(file, config, self.domain)
# no matching and no create new set - should do nothing
self.assertEqual(0, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
@patch('corehq.apps.case_importer.do_import.CASEBLOCK_CHUNKSIZE', 2)
def testBasicChunking(self):
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
['case_id-0', 'age-0', 'sex-0', 'location-0'],
['case_id-1', 'age-1', 'sex-1', 'location-1'],
['case_id-2', 'age-2', 'sex-2', 'location-2'],
['case_id-3', 'age-3', 'sex-3', 'location-3'],
['case_id-4', 'age-4', 'sex-4', 'location-4'],
)
res = do_import(file, config, self.domain)
# 5 cases in chunks of 2 = 3 chunks
self.assertEqual(3, res['num_chunks'])
self.assertEqual(5, res['created_count'])
self.assertEqual(5, len(get_case_ids_in_domain(self.domain)))
@run_with_all_backends
def testExternalIdChunking(self):
# bootstrap a stub case
external_id = 'importer-test-external-id'
headers = ['external_id', 'age', 'sex', 'location']
config = self._config(headers, search_field='external_id')
file = make_worksheet_wrapper(
['external_id', 'age', 'sex', 'location'],
['importer-test-external-id', 'age-0', 'sex-0', 'location-0'],
['importer-test-external-id', 'age-1', 'sex-1', 'location-1'],
['importer-test-external-id', 'age-2', 'sex-2', 'location-2'],
)
# the first one should create the case, and the remaining two should update it
res = do_import(file, config, self.domain)
self.assertEqual(1, res['created_count'])
self.assertEqual(2, res['match_count'])
self.assertFalse(res['errors'])
self.assertEqual(2,
res['num_chunks']) # the lookup causes an extra chunk
# should just create the one case
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
[case] = self.accessor.get_cases(case_ids)
self.assertEqual(external_id, case.external_id)
for prop in ['age', 'sex', 'location']:
self.assertTrue(prop in case.get_case_property(prop))
@run_with_all_backends
def testParentCase(self):
headers = ['parent_id', 'name', 'case_id']
config = self._config(headers,
create_new_cases=True,
search_column='case_id')
rows = 3
[parent_case] = self.factory.create_or_update_case(
CaseStructure(attrs={'create': True}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
file = make_worksheet_wrapper(
['parent_id', 'name', 'case_id'],
[parent_case.case_id, 'name-0', 'case_id-0'],
[parent_case.case_id, 'name-1', 'case_id-1'],
[parent_case.case_id, 'name-2', 'case_id-2'],
)
file_missing = make_worksheet_wrapper(
['parent_id', 'name', 'case_id'],
['parent_id-0', 'name-0', 'case_id-0'],
['parent_id-1', 'name-1', 'case_id-1'],
['parent_id-2', 'name-2', 'case_id-2'],
)
# Should successfully match on `rows` cases
res = do_import(file, config, self.domain)
self.assertEqual(rows, res['created_count'])
# Should be unable to find parent case on `rows` cases
res = do_import(file_missing, config, self.domain)
error_column_name = 'parent_id'
self.assertEqual(
rows,
len(res['errors'][exceptions.InvalidParentId.title]
[error_column_name]['rows']),
"All cases should have missing parent")
def import_mock_file(self, rows):
config = self._config(rows[0])
xls_file = make_worksheet_wrapper(*rows)
return do_import(xls_file, config, self.domain)
@run_with_all_backends
def testLocationOwner(self):
# This is actually testing several different things, but I figure it's
# worth it, as each of these tests takes a non-trivial amount of time.
non_case_sharing = LocationType.objects.create(domain=self.domain,
name='lt1',
shares_cases=False)
case_sharing = LocationType.objects.create(domain=self.domain,
name='lt2',
shares_cases=True)
location = make_loc('loc-1', 'Loc 1', self.domain, case_sharing.code)
make_loc('loc-2', 'Loc 2', self.domain, case_sharing.code)
duplicate_loc = make_loc('loc-3', 'Loc 2', self.domain,
case_sharing.code)
improper_loc = make_loc('loc-4', 'Loc 4', self.domain,
non_case_sharing.code)
res = self.import_mock_file([
['case_id', 'name', 'owner_id', 'owner_name'],
['', 'location-owner-id', location.group_id, ''],
['', 'location-owner-code', '', location.site_code],
['', 'location-owner-name', '', location.name],
['', 'duplicate-location-name', '', duplicate_loc.name],
['', 'non-case-owning-name', '', improper_loc.name],
])
case_ids = self.accessor.get_case_ids_in_domain()
cases = {c.name: c for c in list(self.accessor.get_cases(case_ids))}
self.assertEqual(cases['location-owner-id'].owner_id,
location.group_id)
self.assertEqual(cases['location-owner-code'].owner_id,
location.group_id)
self.assertEqual(cases['location-owner-name'].owner_id,
location.group_id)
error_message = exceptions.DuplicateLocationName.title
error_column_name = None
self.assertIn(error_message, res['errors'])
self.assertEqual(
res['errors'][error_message][error_column_name]['rows'], [5])
error_message = exceptions.InvalidOwner.title
self.assertIn(error_message, res['errors'])
error_column_name = 'owner_name'
self.assertEqual(
res['errors'][error_message][error_column_name]['rows'], [6])
@run_with_all_backends
def test_opened_on(self):
case = self.factory.create_case()
new_date = '2015-04-30T14:41:53.000000Z'
with flag_enabled('BULK_UPLOAD_DATE_OPENED'):
self.import_mock_file([['case_id', 'date_opened'],
[case.case_id, new_date]])
case = CaseAccessors(self.domain).get_case(case.case_id)
self.assertEqual(case.opened_on,
PhoneTime(parse_datetime(new_date)).done())
