def testParentCase(self):
headers = ['parent_id', 'name', 'case_id']
config = self._config(headers,
create_new_cases=True,
search_column='case_id')
rows = 3
[parent_case] = self.factory.create_or_update_case(
CaseStructure(attrs={'create': True}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
file = MockExcelFile(header_columns=headers,
num_rows=rows,
row_generator=id_match_generator(
parent_case.case_id))
file_missing = MockExcelFile(header_columns=headers, num_rows=rows)
# Should successfully match on `rows` cases
res = do_import(file, config, self.domain)
self.assertEqual(rows, res['created_count'])
# Should be unable to find parent case on `rows` cases
res = do_import(file_missing, config, self.domain)
error_column_name = 'parent_id'
self.assertEqual(
rows,
len(res['errors'][ImportErrors.InvalidParentId][error_column_name]
['rows']), "All cases should have missing parent")
def testParentCase(self):
headers = ['parent_id', 'name', 'case_id']
config = self._config(headers,
create_new_cases=True,
search_column='case_id')
rows = 3
parent_case = CommCareCase(domain=self.domain,
type=self.default_case_type)
parent_case.save()
file = MockExcelFile(header_columns=headers,
num_rows=rows,
row_generator=id_match_generator(
parent_case['_id']))
file_missing = MockExcelFile(header_columns=headers, num_rows=rows)
# Should successfully match on `rows` cases
res = do_import(file, config, self.domain)
self.assertEqual(rows, res['created_count'])
# Should be unable to find parent case on `rows` cases
res = do_import(file_missing, config, self.domain)
self.assertEqual(
rows, len(res['errors'][ImportErrors.InvalidParentId]['rows']),
"All cases should have missing parent")
def testExternalIdChunking(self):
# bootstrap a stub case
external_id = 'importer-test-external-id'
headers = ['external_id', 'age', 'sex', 'location']
config = self._config(headers, search_field='external_id')
file = MockExcelFile(header_columns=headers,
num_rows=3,
row_generator=id_match_generator(external_id))
# the first one should create the case, and the remaining two should update it
res = do_import(file, config, self.domain)
self.assertEqual(1, res['created_count'])
self.assertEqual(2, res['match_count'])
self.assertFalse(res['errors'])
self.assertEqual(2,
res['num_chunks']) # the lookup causes an extra chunk
# should just create the one case
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
[case] = self.accessor.get_cases(case_ids)
self.assertEqual(external_id, case.external_id)
for prop in self.default_headers[1:]:
self.assertTrue(prop in case.get_case_property(prop))
def testImportNamedColumns(self):
config = self._config(self.default_headers, named_columns=True)
file = MockExcelFile(header_columns=self.default_headers, num_rows=5)
res = do_import(file, config, self.domain)
# we create 1 less since we knock off the header column
self.assertEqual(4, res['created_count'])
self.assertEqual(4, len(get_case_ids_in_domain(self.domain)))
def testImportNamedColumns(self):
config = self._config(self.default_headers, named_columns=True)
file = MockExcelFile(header_columns=self.default_headers, num_rows=5)
res = do_import(file, config, self.domain)
# we create 1 less since we knock off the header column
self.assertEqual(4, res['created_count'])
self.assertEqual(4, len(get_case_ids_in_domain(self.domain)))
def testCaseIdMatching(self):
# bootstrap a stub case
[case] = self.factory.create_or_update_case(
CaseStructure(attrs={
'create': True,
'update': {
'importer_test_prop': 'foo'
},
}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
config = self._config(self.default_headers)
file = MockExcelFile(header_columns=self.default_headers,
num_rows=3,
row_generator=id_match_generator(case.case_id))
res = do_import(file, config, self.domain)
self.assertEqual(0, res['created_count'])
self.assertEqual(3, res['match_count'])
self.assertFalse(res['errors'])
# shouldn't create any more cases, just the one
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
[case] = self.accessor.get_cases(case_ids)
for prop in self.default_headers[1:]:
self.assertTrue(prop in case.get_case_property(prop))
# shouldn't touch existing properties
self.assertEqual('foo', case.get_case_property('importer_test_prop'))
def _typeTest(self, type_fields, error_message):
config = self._config(self.default_headers, type_fields=type_fields)
file = MockExcelFile(header_columns=self.default_headers, num_rows=3)
res = do_import(file, config, self.domain)
self.assertIn(self.default_headers[1], res['errors'][error_message])
self.assertEqual(
res['errors'][error_message][self.default_headers[1]]['rows'],
[1, 2, 3])
def testBasicChunking(self):
config = self._config(self.default_headers)
file = MockExcelFile(header_columns=self.default_headers, num_rows=5)
res = do_import(file, config, self.domain, chunksize=2)
# 5 cases in chunks of 2 = 3 chunks
self.assertEqual(3, res['num_chunks'])
self.assertEqual(5, res['created_count'])
self.assertEqual(5, len(get_case_ids_in_domain(self.domain)))
def testBasicChunking(self):
config = self._config(self.default_headers)
file = MockExcelFile(header_columns=self.default_headers, num_rows=5)
res = do_import(file, config, self.domain, chunksize=2)
# 5 cases in chunks of 2 = 3 chunks
self.assertEqual(3, res['num_chunks'])
self.assertEqual(5, res['created_count'])
self.assertEqual(5, len(get_case_ids_in_domain(self.domain)))
def testNoCreateNew(self):
config = self._config(self.default_headers, create_new_cases=False)
file = MockExcelFile(header_columns=self.default_headers, num_rows=5)
res = do_import(file, config, self.domain)
# no matching and no create new set - should do nothing
self.assertEqual(0, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
def testNoCreateNew(self):
config = self._config(self.default_headers, create_new_cases=False)
file = MockExcelFile(header_columns=self.default_headers, num_rows=5)
res = do_import(file, config, self.domain)
# no matching and no create new set - should do nothing
self.assertEqual(0, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
def import_mock_file(self, rows):
config = self._config(rows[0])
case_rows = rows[1:]
num_rows = len(case_rows)
xls_file = MockExcelFile(
header_columns=rows[0],
num_rows=num_rows,
row_generator=lambda _, i: case_rows[i],
)
return do_import(xls_file, config, self.domain)
def import_mock_file(self, rows):
config = self._config(rows[0])
case_rows = rows[1:]
num_rows = len(case_rows)
xls_file = MockExcelFile(
header_columns=rows[0],
num_rows=num_rows,
row_generator=lambda _, i: case_rows[i],
)
return do_import(xls_file, config, self.domain)
def testImportTrailingWhitespace(self):
cols = ['case_id', 'age', u'sex\xa0', 'location']
config = self._config(cols, named_columns=True)
file = MockExcelFile(header_columns=cols, num_rows=2)
res = do_import(file, config, self.domain)
# we create 1 less since we knock off the header column
self.assertEqual(1, res['created_count'])
case_ids = get_case_ids_in_domain(self.domain)
self.assertEqual(1, len(case_ids))
case = CommCareCase.get(case_ids[0])
self.assertTrue(bool(case.sex)) # make sure the value also got properly set
def testImportTrailingWhitespace(self):
cols = ['case_id', 'age', u'sex\xa0', 'location']
config = self._config(cols, named_columns=True)
file = MockExcelFile(header_columns=cols, num_rows=2)
res = do_import(file, config, self.domain)
# we create 1 less since we knock off the header column
self.assertEqual(1, res['created_count'])
case_ids = get_case_ids_in_domain(self.domain)
self.assertEqual(1, len(case_ids))
case = CommCareCase.get(case_ids[0])
self.assertTrue(bool(
case.sex)) # make sure the value also got properly set
def testBlankRows(self):
# don't create new cases for rows left blank
config = self._config(self.default_headers, create_new_cases=True)
file = MockExcelFile(header_columns=self.default_headers,
num_rows=5,
row_generator=blank_row_generator)
res = do_import(file, config, self.domain)
# no matching and no create new set - should do nothing
self.assertEqual(0, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
def testCaseLookupTypeCheck(self):
case = CommCareCase(domain=self.domain, type='nonmatch-type')
case.save()
self.assertEqual(1, len(get_case_ids_in_domain(self.domain)))
config = self._config(self.default_headers)
file = MockExcelFile(header_columns=self.default_headers, num_rows=3,
row_generator=id_match_generator(case._id))
res = do_import(file, config, self.domain)
# because the type is wrong these shouldn't match
self.assertEqual(3, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(4, len(get_case_ids_in_domain(self.domain)))
def testParentCase(self):
headers = ['parent_id', 'name', 'case_id']
config = self._config(headers, create_new_cases=True, search_column='case_id')
rows = 3
parent_case = CommCareCase(domain=self.domain, type=self.default_case_type)
parent_case.save()
file = MockExcelFile(header_columns=headers,
num_rows=rows,
row_generator=id_match_generator(parent_case['_id']))
file_missing = MockExcelFile(header_columns=headers,
num_rows=rows)
# Should successfully match on `rows` cases
res = do_import(file, config, self.domain)
self.assertEqual(rows, res['created_count'])
# Should be unable to find parent case on `rows` cases
res = do_import(file_missing, config, self.domain)
self.assertEqual(rows,
len(res['errors'][ImportErrors.InvalidParentId]['rows']),
"All cases should have missing parent")
def testBlankRows(self):
# don't create new cases for rows left blank
config = self._config(self.default_headers, create_new_cases=True)
file = MockExcelFile(
header_columns=self.default_headers,
num_rows=5,
row_generator=blank_row_generator
)
res = do_import(file, config, self.domain)
# no matching and no create new set - should do nothing
self.assertEqual(0, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
def testCaseLookupTypeCheck(self):
[case] = self.factory.create_or_update_case(
CaseStructure(attrs={
'create': True,
'case_type': 'nonmatch-type',
}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
config = self._config(self.default_headers)
file = MockExcelFile(header_columns=self.default_headers,
num_rows=3,
row_generator=id_match_generator(case.case_id))
res = do_import(file, config, self.domain)
# because the type is wrong these shouldn't match
self.assertEqual(3, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(4, len(self.accessor.get_case_ids_in_domain()))
def test_external_id_matching_on_create_with_custom_column_name(self):
headers = ['id_column', 'age', 'sex', 'location']
external_id = 'external-id-test'
config = self._config(headers[1:],
search_column='id_column',
search_field='external_id')
file = MockExcelFile(header_columns=headers,
num_rows=2,
row_generator=id_match_generator(external_id))
res = do_import(file, config, self.domain)
self.assertEqual(1, res['created_count'])
self.assertEqual(1, res['match_count'])
self.assertFalse(res['errors'])
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
case = self.accessor.get_case(case_ids[0])
self.assertEqual(external_id, case.external_id)
def testImportBasic(self):
config = self._config(self.default_headers)
file = MockExcelFile(header_columns=self.default_headers, num_rows=5)
res = do_import(file, config, self.domain)
self.assertEqual(5, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertFalse(res['errors'])
self.assertEqual(1, res['num_chunks'])
cases = list(get_cases_in_domain(self.domain))
self.assertEqual(5, len(cases))
properties_seen = set()
for case in cases:
self.assertEqual(self.couch_user._id, case.user_id)
self.assertEqual(self.couch_user._id, case.owner_id)
self.assertEqual(self.default_case_type, case.type)
for prop in self.default_headers[1:]:
self.assertTrue(prop in case.get_case_property(prop))
self.assertFalse(case.get_case_property(prop) in properties_seen)
properties_seen.add(case.get_case_property(prop))
def testExternalIdMatching(self):
# bootstrap a stub case
case = CommCareCase(domain=self.domain, type=self.default_case_type)
external_id = 'importer-test-external-id'
case.external_id = external_id
case.save()
self.assertEqual(1, len(get_case_ids_in_domain(self.domain)))
headers = ['external_id', 'age', 'sex', 'location']
config = self._config(headers, search_field='external_id')
file = MockExcelFile(header_columns=headers, num_rows=3,
row_generator=id_match_generator(external_id))
res = do_import(file, config, self.domain)
self.assertEqual(0, res['created_count'])
self.assertEqual(3, res['match_count'])
self.assertFalse(res['errors'])
# shouldn't create any more cases, just the one
self.assertEqual(1, len(get_case_ids_in_domain(self.domain)))
def handle(self, *args, **options):
if len(args) != 4:
raise CommandError('Usage is import_cases %s' % self.args)
start = datetime.now()
export_file, config_file, domain, user_id = args
if '@' in user_id:
user = WebUser.get_by_username(user_id)
else:
user = WebUser.get(user_id)
if not user.is_member_of(domain):
raise CommandError("%s can't access %s" % (user, domain))
with open(config_file, 'r') as f:
config = ImporterConfig.from_json(f.read())
config.couch_user_id = user._id
spreadsheet = ExcelFile(export_file, True)
print json.dumps(do_import(spreadsheet, config, domain))
print 'finished in %s seconds' % (datetime.now() - start).seconds
def testImportBasic(self):
config = self._config(self.default_headers)
file = MockExcelFile(header_columns=self.default_headers, num_rows=5)
res = do_import(file, config, self.domain)
self.assertEqual(5, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertFalse(res['errors'])
self.assertEqual(1, res['num_chunks'])
cases = list(get_cases_in_domain(self.domain))
self.assertEqual(5, len(cases))
properties_seen = set()
for case in cases:
self.assertEqual(self.couch_user._id, case.user_id)
self.assertEqual(self.couch_user._id, case.owner_id)
self.assertEqual(self.default_case_type, case.type)
for prop in self.default_headers[1:]:
self.assertTrue(prop in case.get_case_property(prop))
self.assertFalse(
case.get_case_property(prop) in properties_seen)
properties_seen.add(case.get_case_property(prop))
def testCaseIdMatching(self):
# bootstrap a stub case
case = CommCareCase(domain=self.domain, type=self.default_case_type)
case.importer_test_prop = 'foo'
case.save()
self.assertEqual(1, len(get_case_ids_in_domain(self.domain)))
config = self._config(self.default_headers)
file = MockExcelFile(header_columns=self.default_headers, num_rows=3, row_generator=id_match_generator(case._id))
res = do_import(file, config, self.domain)
self.assertEqual(0, res['created_count'])
self.assertEqual(3, res['match_count'])
self.assertFalse(res['errors'])
# shouldn't create any more cases, just the one
self.assertEqual(1, len(get_case_ids_in_domain(self.domain)))
[case] = get_cases_in_domain(self.domain)
for prop in self.default_headers[1:]:
self.assertTrue(prop in case.get_case_property(prop))
# shouldn't touch existing properties
self.assertEqual('foo', case.importer_test_prop)
def testExternalIdMatching(self):
# bootstrap a stub case
external_id = 'importer-test-external-id'
[case] = self.factory.create_or_update_case(
CaseStructure(attrs={
'create': True,
'external_id': external_id,
}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
headers = ['external_id', 'age', 'sex', 'location']
config = self._config(headers, search_field='external_id')
file = MockExcelFile(header_columns=headers,
num_rows=3,
row_generator=id_match_generator(external_id))
res = do_import(file, config, self.domain)
self.assertEqual(0, res['created_count'])
self.assertEqual(3, res['match_count'])
self.assertFalse(res['errors'])
# shouldn't create any more cases, just the one
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
def testExternalIdChunking(self):
# bootstrap a stub case
external_id = 'importer-test-external-id'
headers = ['external_id', 'age', 'sex', 'location']
config = self._config(headers, search_field='external_id')
file = MockExcelFile(header_columns=headers, num_rows=3,
row_generator=id_match_generator(external_id))
# the first one should create the case, and the remaining two should update it
res = do_import(file, config, self.domain)
self.assertEqual(1, res['created_count'])
self.assertEqual(2, res['match_count'])
self.assertFalse(res['errors'])
self.assertEqual(2, res['num_chunks']) # the lookup causes an extra chunk
# should just create the one case
self.assertEqual(1, len(get_case_ids_in_domain(self.domain)))
[case] = get_cases_in_domain(self.domain)
self.assertEqual(external_id, case.external_id)
for prop in self.default_headers[1:]:
self.assertTrue(prop in case.get_case_property(prop))
def _typeTest(self, type_fields, error_message):
config = self._config(self.default_headers, type_fields=type_fields)
file = MockExcelFile(header_columns=self.default_headers, num_rows=3)
res = do_import(file, config, self.domain)
self.assertIn(self.default_headers[1], res['errors'][error_message])
self.assertEqual(res['errors'][error_message][self.default_headers[1]]['rows'], [1, 2, 3])
[case] = get_cases_in_domain(self.domain)
for prop in self.default_headers[1:]:
self.assertTrue(prop in case.get_case_property(prop))
# shouldn't touch existing properties
self.assertEqual('foo', case.importer_test_prop)
def testCaseLookupTypeCheck(self):
case = CommCareCase(domain=self.domain, type='nonmatch-type')
case.save()
self.assertEqual(1, len(get_case_ids_in_domain(self.domain)))
config = self._config(self.default_headers)
file = MockExcelFile(header_columns=self.default_headers,
num_rows=3,
row_generator=id_match_generator(case._id))
res = do_import(file, config, self.domain)
# because the type is wrong these shouldn't match
self.assertEqual(3, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(4, len(get_case_ids_in_domain(self.domain)))
def testCaseLookupDomainCheck(self):
case = CommCareCase(domain='not-right-domain',
type=self.default_case_type)
case.save()
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
config = self._config(self.default_headers)
file = MockExcelFile(header_columns=self.default_headers,
num_rows=3,
row_generator=id_match_generator(case._id))
res = do_import(file, config, self.domain)
def testImporterErrors(self):
res = do_import(MockExcelFile(has_errors=True), self._config(), self.domain)
self.assertEqual('HAS_ERRORS', res['error'])
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
def testImportNone(self):
res = do_import(None, self._config(), self.domain)
self.assertEqual('EXPIRED', res['error'])
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
def testImportNone(self):
res = do_import(None, self._config(), self.domain)
self.assertEqual('EXPIRED', res['error'])
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
def testImporterErrors(self):
res = do_import(MockExcelFile(has_errors=True), self._config(),
self.domain)
self.assertEqual('HAS_ERRORS', res['error'])
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
class ImporterTest(TestCase):
def setUp(self):
self.domain = create_domain("importer-test").name
self.default_case_type = 'importer-test-casetype'
self.default_headers = ['case_id', 'age', 'sex', 'location']
self.couch_user = WebUser.create(None, "test", "foobar")
self.couch_user.add_domain_membership(self.domain, is_admin=True)
self.couch_user.save()
delete_all_cases()
def tearDown(self):
self.couch_user.delete()
def _config(self,
col_names=None,
search_column=None,
case_type=None,
search_field='case_id',
named_columns=False,
create_new_cases=True):
col_names = col_names or self.default_headers
case_type = case_type or self.default_case_type
search_column = search_column or col_names[0]
return ImporterConfig(
couch_user_id=self.couch_user._id,
case_type=case_type,
excel_fields=col_names,
case_fields=[''] * len(col_names),
custom_fields=col_names,
type_fields=['plain'] * len(col_names),
search_column=search_column,
search_field=search_field,
named_columns=named_columns,
create_new_cases=create_new_cases,
key_column='',
value_column='',
)
def testImportNone(self):
res = do_import(None, self._config(), self.domain)
self.assertEqual('EXPIRED', res['error'])
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
def testImporterErrors(self):
res = do_import(MockExcelFile(has_errors=True), self._config(),
self.domain)
self.assertEqual('HAS_ERRORS', res['error'])
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
def testImportBasic(self):
config = self._config(self.default_headers)
file = MockExcelFile(header_columns=self.default_headers, num_rows=5)
res = do_import(file, config, self.domain)
self.assertEqual(5, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertFalse(res['errors'])
self.assertEqual(1, res['num_chunks'])
cases = list(get_cases_in_domain(self.domain))
self.assertEqual(5, len(cases))
properties_seen = set()
for case in cases:
self.assertEqual(self.couch_user._id, case.user_id)
self.assertEqual(self.couch_user._id, case.owner_id)
self.assertEqual(self.default_case_type, case.type)
for prop in self.default_headers[1:]:
self.assertTrue(prop in case.get_case_property(prop))
self.assertFalse(
case.get_case_property(prop) in properties_seen)
properties_seen.add(case.get_case_property(prop))
def testImportNamedColumns(self):
config = self._config(self.default_headers, named_columns=True)
file = MockExcelFile(header_columns=self.default_headers, num_rows=5)
res = do_import(file, config, self.domain)
# we create 1 less since we knock off the header column
self.assertEqual(4, res['created_count'])
self.assertEqual(4, len(get_case_ids_in_domain(self.domain)))
def testImportTrailingWhitespace(self):
cols = ['case_id', 'age', u'sex\xa0', 'location']
config = self._config(cols, named_columns=True)
file = MockExcelFile(header_columns=cols, num_rows=2)
res = do_import(file, config, self.domain)
# we create 1 less since we knock off the header column
self.assertEqual(1, res['created_count'])
case_ids = get_case_ids_in_domain(self.domain)
self.assertEqual(1, len(case_ids))
case = CommCareCase.get(case_ids[0])
self.assertTrue(bool(
case.sex)) # make sure the value also got properly set
def testCaseIdMatching(self):
# bootstrap a stub case
case = CommCareCase(domain=self.domain, type=self.default_case_type)
case.importer_test_prop = 'foo'
case.save()
self.assertEqual(1, len(get_case_ids_in_domain(self.domain)))
config = self._config(self.default_headers)
file = MockExcelFile(header_columns=self.default_headers,
num_rows=3,
row_generator=id_match_generator(case._id))
res = do_import(file, config, self.domain)
self.assertEqual(0, res['created_count'])
self.assertEqual(3, res['match_count'])
self.assertFalse(res['errors'])
# shouldn't create any more cases, just the one
self.assertEqual(1, len(get_case_ids_in_domain(self.domain)))
[case] = get_cases_in_domain(self.domain)
for prop in self.default_headers[1:]:
self.assertTrue(prop in case.get_case_property(prop))
# shouldn't touch existing properties
self.assertEqual('foo', case.importer_test_prop)
def testImporterErrors(self):
res = do_import(ImporterError(), self._config(), self.domain)
self.assertEqual('HAS_ERRORS', res['errors'])
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
