def testParentCase(self):
headers = ['parent_id', 'name', 'case_id']
config = self._config(headers,
create_new_cases=True,
search_column='case_id')
rows = 3
[parent_case] = self.factory.create_or_update_case(
CaseStructure(attrs={'create': True}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
file = make_worksheet_wrapper(
['parent_id', 'name', 'case_id'],
[parent_case.case_id, 'name-0', 'case_id-0'],
[parent_case.case_id, 'name-1', 'case_id-1'],
[parent_case.case_id, 'name-2', 'case_id-2'],
)
file_missing = make_worksheet_wrapper(
['parent_id', 'name', 'case_id'],
[u'parent_id-0', u'name-0', u'case_id-0'],
[u'parent_id-1', u'name-1', u'case_id-1'],
[u'parent_id-2', u'name-2', u'case_id-2'],
)
# Should successfully match on `rows` cases
res = do_import(file, config, self.domain)
self.assertEqual(rows, res['created_count'])
# Should be unable to find parent case on `rows` cases
res = do_import(file_missing, config, self.domain)
error_column_name = 'parent_id'
self.assertEqual(
rows,
len(res['errors'][ImportErrors.InvalidParentId][error_column_name]
['rows']), "All cases should have missing parent")
def testParentCase(self):
headers = ['parent_id', 'name', 'case_id']
config = self._config(headers, create_new_cases=True, search_column='case_id')
rows = 3
[parent_case] = self.factory.create_or_update_case(CaseStructure(attrs={'create': True}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
file = make_worksheet_wrapper(
['parent_id', 'name', 'case_id'],
[parent_case.case_id, 'name-0', 'case_id-0'],
[parent_case.case_id, 'name-1', 'case_id-1'],
[parent_case.case_id, 'name-2', 'case_id-2'],
)
file_missing = make_worksheet_wrapper(
['parent_id', 'name', 'case_id'],
['parent_id-0', 'name-0', 'case_id-0'],
['parent_id-1', 'name-1', 'case_id-1'],
['parent_id-2', 'name-2', 'case_id-2'],
)
# Should successfully match on `rows` cases
res = do_import(file, config, self.domain)
self.assertEqual(rows, res['created_count'])
# Should be unable to find parent case on `rows` cases
res = do_import(file_missing, config, self.domain)
error_column_name = 'parent_id'
self.assertEqual(rows,
len(res['errors'][ImportErrors.InvalidParentId][error_column_name]['rows']),
"All cases should have missing parent")
def testExternalIdChunking(self):
# bootstrap a stub case
external_id = 'importer-test-external-id'
headers = ['external_id', 'age', 'sex', 'location']
config = self._config(headers, search_field='external_id')
file = make_worksheet_wrapper(
['external_id', 'age', 'sex', 'location'],
['importer-test-external-id', 'age-0', 'sex-0', 'location-0'],
['importer-test-external-id', 'age-1', 'sex-1', 'location-1'],
['importer-test-external-id', 'age-2', 'sex-2', 'location-2'],
)
# the first one should create the case, and the remaining two should update it
res = do_import(file, config, self.domain)
self.assertEqual(1, res['created_count'])
self.assertEqual(2, res['match_count'])
self.assertFalse(res['errors'])
self.assertEqual(2,
res['num_chunks']) # the lookup causes an extra chunk
# should just create the one case
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
[case] = self.accessor.get_cases(case_ids)
self.assertEqual(external_id, case.external_id)
for prop in ['age', 'sex', 'location']:
self.assertTrue(prop in case.get_case_property(prop))
def testExternalIdChunking(self):
# bootstrap a stub case
external_id = 'importer-test-external-id'
headers = ['external_id', 'age', 'sex', 'location']
config = self._config(headers, search_field='external_id')
file = make_worksheet_wrapper(
['external_id', 'age', 'sex', 'location'],
['importer-test-external-id', 'age-0', 'sex-0', 'location-0'],
['importer-test-external-id', 'age-1', 'sex-1', 'location-1'],
['importer-test-external-id', 'age-2', 'sex-2', 'location-2'],
)
# the first one should create the case, and the remaining two should update it
res = do_import(file, config, self.domain)
self.assertEqual(1, res['created_count'])
self.assertEqual(2, res['match_count'])
self.assertFalse(res['errors'])
self.assertEqual(2, res['num_chunks']) # the lookup causes an extra chunk
# should just create the one case
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
[case] = self.accessor.get_cases(case_ids)
self.assertEqual(external_id, case.external_id)
for prop in ['age', 'sex', 'location']:
self.assertTrue(prop in case.get_case_property(prop))
def testImportBasic(self):
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
[u'case_id-0', u'age-0', u'sex-0', u'location-0'],
[u'case_id-1', u'age-1', u'sex-1', u'location-1'],
[u'case_id-2', u'age-2', u'sex-2', u'location-2'],
[u'case_id-3', u'age-3', u'sex-3', u'location-3'],
[u'case_id-4', u'age-4', u'sex-4', u'location-4'],
)
res = do_import(file, config, self.domain)
self.assertEqual(5, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertFalse(res['errors'])
self.assertEqual(1, res['num_chunks'])
case_ids = self.accessor.get_case_ids_in_domain()
cases = list(self.accessor.get_cases(case_ids))
self.assertEqual(5, len(cases))
properties_seen = set()
for case in cases:
self.assertEqual(self.couch_user._id, case.user_id)
self.assertEqual(self.couch_user._id, case.owner_id)
self.assertEqual(self.default_case_type, case.type)
for prop in ['age', 'sex', 'location']:
self.assertTrue(prop in case.get_case_property(prop))
self.assertFalse(
case.get_case_property(prop) in properties_seen)
properties_seen.add(case.get_case_property(prop))
def testCaseIdMatching(self):
# bootstrap a stub case
[case] = self.factory.create_or_update_case(
CaseStructure(attrs={
'create': True,
'update': {
'importer_test_prop': 'foo'
},
}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
[case.case_id, 'age-0', 'sex-0', 'location-0'],
[case.case_id, 'age-1', 'sex-1', 'location-1'],
[case.case_id, 'age-2', 'sex-2', 'location-2'],
)
res = do_import(file, config, self.domain)
self.assertEqual(0, res['created_count'])
self.assertEqual(3, res['match_count'])
self.assertFalse(res['errors'])
# shouldn't create any more cases, just the one
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
[case] = self.accessor.get_cases(case_ids)
for prop in ['age', 'sex', 'location']:
self.assertTrue(prop in case.get_case_property(prop))
# shouldn't touch existing properties
self.assertEqual('foo', case.get_case_property('importer_test_prop'))
def testExternalIdMatching(self):
# bootstrap a stub case
external_id = 'importer-test-external-id'
[case] = self.factory.create_or_update_case(CaseStructure(
attrs={
'create': True,
'external_id': external_id,
}
))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
headers = ['external_id', 'age', 'sex', 'location']
config = self._config(headers, search_field='external_id')
file = make_worksheet_wrapper(
['external_id', 'age', 'sex', 'location'],
['importer-test-external-id', 'age-0', 'sex-0', 'location-0'],
['importer-test-external-id', 'age-1', 'sex-1', 'location-1'],
['importer-test-external-id', 'age-2', 'sex-2', 'location-2'],
)
res = do_import(file, config, self.domain)
self.assertEqual(0, res['created_count'])
self.assertEqual(3, res['match_count'])
self.assertFalse(res['errors'])
# shouldn't create any more cases, just the one
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
def testExternalIdMatching(self):
# bootstrap a stub case
external_id = 'importer-test-external-id'
[case] = self.factory.create_or_update_case(
CaseStructure(attrs={
'create': True,
'external_id': external_id,
}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
headers = ['external_id', 'age', 'sex', 'location']
config = self._config(headers, search_field='external_id')
file = make_worksheet_wrapper(
['external_id', 'age', 'sex', 'location'],
['importer-test-external-id', 'age-0', 'sex-0', 'location-0'],
['importer-test-external-id', 'age-1', 'sex-1', 'location-1'],
['importer-test-external-id', 'age-2', 'sex-2', 'location-2'],
)
res = do_import(file, config, self.domain)
self.assertEqual(0, res['created_count'])
self.assertEqual(3, res['match_count'])
self.assertFalse(res['errors'])
# shouldn't create any more cases, just the one
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
def testCaseIdMatching(self):
# bootstrap a stub case
[case] = self.factory.create_or_update_case(CaseStructure(attrs={
'create': True,
'update': {'importer_test_prop': 'foo'},
}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
[case.case_id, 'age-0', 'sex-0', 'location-0'],
[case.case_id, 'age-1', 'sex-1', 'location-1'],
[case.case_id, 'age-2', 'sex-2', 'location-2'],
)
res = do_import(file, config, self.domain)
self.assertEqual(0, res['created_count'])
self.assertEqual(3, res['match_count'])
self.assertFalse(res['errors'])
# shouldn't create any more cases, just the one
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
[case] = self.accessor.get_cases(case_ids)
for prop in ['age', 'sex', 'location']:
self.assertTrue(prop in case.get_case_property(prop))
# shouldn't touch existing properties
self.assertEqual('foo', case.get_case_property('importer_test_prop'))
def testImportBasic(self):
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
['case_id-0', 'age-0', 'sex-0', 'location-0'],
['case_id-1', 'age-1', 'sex-1', 'location-1'],
['case_id-2', 'age-2', 'sex-2', 'location-2'],
['case_id-3', 'age-3', 'sex-3', 'location-3'],
['case_id-4', 'age-4', 'sex-4', 'location-4'],
)
res = do_import(file, config, self.domain)
self.assertEqual(5, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertFalse(res['errors'])
self.assertEqual(1, res['num_chunks'])
case_ids = self.accessor.get_case_ids_in_domain()
cases = list(self.accessor.get_cases(case_ids))
self.assertEqual(5, len(cases))
properties_seen = set()
for case in cases:
self.assertEqual(self.couch_user._id, case.user_id)
self.assertEqual(self.couch_user._id, case.owner_id)
self.assertEqual(self.default_case_type, case.type)
for prop in ['age', 'sex', 'location']:
self.assertTrue(prop in case.get_case_property(prop))
self.assertFalse(case.get_case_property(prop) in properties_seen)
properties_seen.add(case.get_case_property(prop))
def _import_rows(self, rows, get_by_user_id):
get_by_user_id.return_value = WebUser(
_id=self.couch_user_id, domain=self.domain, username='******')
case_upload_record = CaseUploadRecord(
upload_id=uuid.uuid4(),
task_id=uuid.uuid4(),
domain=self.domain,
)
case_upload_record.save()
self.addCleanup(case_upload_record.delete)
tracker = CaseUpload(case_upload_record.upload_id)
# mock internals to have record_cases use our case_upload_record
tracker.__dict__['_case_upload_record'] = case_upload_record
config = self._get_config(rows[0])
xls_file = make_worksheet_wrapper(*rows)
do_import(xls_file, config, self.domain,
record_form_callback=tracker.record_form)
return case_upload_record
def _import_rows(self, rows, get_by_user_id):
get_by_user_id.return_value = WebUser(
_id=self.couch_user_id, domain=self.domain, username='******')
case_upload_record = CaseUploadRecord(
upload_id=uuid.uuid4(),
task_id=uuid.uuid4(),
domain=self.domain,
)
case_upload_record.save()
self.addCleanup(case_upload_record.delete)
tracker = CaseUpload(case_upload_record.upload_id)
# mock internals to have record_cases use our case_upload_record
tracker.__dict__['_case_upload_record'] = case_upload_record
config = self._get_config(rows[0])
xls_file = make_worksheet_wrapper(*rows)
do_import(xls_file, config, self.domain,
record_form_callback=tracker.record_form)
return case_upload_record
def testImportNamedColumns(self):
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
[u'case_id-0', u'age-0', u'sex-0', u'location-0'],
[u'case_id-1', u'age-1', u'sex-1', u'location-1'],
[u'case_id-2', u'age-2', u'sex-2', u'location-2'],
[u'case_id-3', u'age-3', u'sex-3', u'location-3'],
)
res = do_import(file, config, self.domain)
self.assertEqual(4, res['created_count'])
self.assertEqual(4, len(self.accessor.get_case_ids_in_domain()))
def testImportNamedColumns(self):
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
['case_id-0', 'age-0', 'sex-0', 'location-0'],
['case_id-1', 'age-1', 'sex-1', 'location-1'],
['case_id-2', 'age-2', 'sex-2', 'location-2'],
['case_id-3', 'age-3', 'sex-3', 'location-3'],
)
res = do_import(file, config, self.domain)
self.assertEqual(4, res['created_count'])
self.assertEqual(4, len(self.accessor.get_case_ids_in_domain()))
def testImportTrailingWhitespace(self):
cols = ['case_id', 'age', 'sex\xa0', 'location']
config = self._config(cols)
file = make_worksheet_wrapper(
['case_id', 'age', 'sex\xa0', 'location'],
['case_id-0', 'age-0', 'sex\xa0-0', 'location-0'],
)
res = do_import(file, config, self.domain)
self.assertEqual(1, res['created_count'])
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
case = self.accessor.get_case(case_ids[0])
self.assertTrue(bool(case.get_case_property('sex'))) # make sure the value also got properly set
def testBlankRows(self):
# don't create new cases for rows left blank
config = self._config(['case_id', 'age', 'sex', 'location'], create_new_cases=True)
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
[None, None, None, None],
['', '', '', ''],
)
res = do_import(file, config, self.domain)
# no matching and no create new set - should do nothing
self.assertEqual(0, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
def testBasicChunking(self):
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
['case_id-0', 'age-0', 'sex-0', 'location-0'],
['case_id-1', 'age-1', 'sex-1', 'location-1'],
['case_id-2', 'age-2', 'sex-2', 'location-2'],
['case_id-3', 'age-3', 'sex-3', 'location-3'],
['case_id-4', 'age-4', 'sex-4', 'location-4'],
)
res = do_import(file, config, self.domain, chunksize=2)
# 5 cases in chunks of 2 = 3 chunks
self.assertEqual(3, res['num_chunks'])
self.assertEqual(5, res['created_count'])
self.assertEqual(5, len(get_case_ids_in_domain(self.domain)))
def testBasicChunking(self):
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
[u'case_id-0', u'age-0', u'sex-0', u'location-0'],
[u'case_id-1', u'age-1', u'sex-1', u'location-1'],
[u'case_id-2', u'age-2', u'sex-2', u'location-2'],
[u'case_id-3', u'age-3', u'sex-3', u'location-3'],
[u'case_id-4', u'age-4', u'sex-4', u'location-4'],
)
res = do_import(file, config, self.domain, chunksize=2)
# 5 cases in chunks of 2 = 3 chunks
self.assertEqual(3, res['num_chunks'])
self.assertEqual(5, res['created_count'])
self.assertEqual(5, len(get_case_ids_in_domain(self.domain)))
def testNoCreateNew(self):
config = self._config(['case_id', 'age', 'sex', 'location'], create_new_cases=False)
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
['case_id-0', 'age-0', 'sex-0', 'location-0'],
['case_id-1', 'age-1', 'sex-1', 'location-1'],
['case_id-2', 'age-2', 'sex-2', 'location-2'],
['case_id-3', 'age-3', 'sex-3', 'location-3'],
['case_id-4', 'age-4', 'sex-4', 'location-4'],
)
res = do_import(file, config, self.domain)
# no matching and no create new set - should do nothing
self.assertEqual(0, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
def testCaseLookupTypeCheck(self):
[case] = self.factory.create_or_update_case(CaseStructure(attrs={
'create': True,
'case_type': 'nonmatch-type',
}))
self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
config = self._config(['case_id', 'age', 'sex', 'location'])
file = make_worksheet_wrapper(
['case_id', 'age', 'sex', 'location'],
[case.case_id, 'age-0', 'sex-0', 'location-0'],
[case.case_id, 'age-1', 'sex-1', 'location-1'],
[case.case_id, 'age-2', 'sex-2', 'location-2'],
)
res = do_import(file, config, self.domain)
# because the type is wrong these shouldn't match
self.assertEqual(3, res['created_count'])
self.assertEqual(0, res['match_count'])
self.assertEqual(4, len(self.accessor.get_case_ids_in_domain()))
def test_external_id_matching_on_create_with_custom_column_name(self):
headers = ['id_column', 'age', 'sex', 'location']
external_id = 'external-id-test'
config = self._config(headers[1:], search_column='id_column', search_field='external_id')
file = make_worksheet_wrapper(
['id_column', 'age', 'sex', 'location'],
['external-id-test', 'age-0', 'sex-0', 'location-0'],
['external-id-test', 'age-1', 'sex-1', 'location-1'],
)
res = do_import(file, config, self.domain)
self.assertFalse(res['errors'])
self.assertEqual(1, res['created_count'])
self.assertEqual(1, res['match_count'])
case_ids = self.accessor.get_case_ids_in_domain()
self.assertEqual(1, len(case_ids))
case = self.accessor.get_case(case_ids[0])
self.assertEqual(external_id, case.external_id)
def handle(self, export_file, config_file, domain, user_id, **options):
start = datetime.utcnow()
if '@' in user_id:
user = WebUser.get_by_username(user_id)
else:
user = WebUser.get(user_id)
if not user.is_member_of(domain):
raise CommandError("%s can't access %s" % (user, domain))
with open(config_file, 'r', encoding='utf-8') as f:
config = ImporterConfig.from_json(f.read())
config.couch_user_id = user._id
with get_spreadsheet(export_file) as spreadsheet:
print(json.dumps(do_import(spreadsheet, config, domain),
default=json_handler))
print('finished in %s seconds' % (datetime.utcnow() - start).seconds)
def handle(self, *args, **options):
if len(args) != 4:
raise CommandError('Usage is import_cases %s' % self.args)
start = datetime.utcnow()
export_file, config_file, domain, user_id = args
if '@' in user_id:
user = WebUser.get_by_username(user_id)
else:
user = WebUser.get(user_id)
if not user.is_member_of(domain):
raise CommandError("%s can't access %s" % (user, domain))
with open(config_file, 'r') as f:
config = ImporterConfig.from_json(f.read())
config.couch_user_id = user._id
with get_spreadsheet(export_file) as spreadsheet:
print json.dumps(do_import(spreadsheet, config, domain),
default=json_handler)
print 'finished in %s seconds' % (datetime.utcnow() - start).seconds
def handle(self, *args, **options):
if len(args) != 4:
raise CommandError('Usage is import_cases %s' % self.args)
start = datetime.utcnow()
export_file, config_file, domain, user_id = args
if '@' in user_id:
user = WebUser.get_by_username(user_id)
else:
user = WebUser.get(user_id)
if not user.is_member_of(domain):
raise CommandError("%s can't access %s" % (user, domain))
with open(config_file, 'r') as f:
config = ImporterConfig.from_json(f.read())
config.couch_user_id = user._id
with get_spreadsheet(export_file) as spreadsheet:
print json.dumps(do_import(spreadsheet, config, domain),
default=json_handler)
print 'finished in %s seconds' % (datetime.utcnow() -
start).seconds
def import_mock_file(self, rows):
config = self._config(rows[0])
xls_file = make_worksheet_wrapper(*rows)
return do_import(xls_file, config, self.domain)
def import_mock_file(self, rows):
config = self._config(rows[0])
xls_file = make_worksheet_wrapper(*rows)
return do_import(xls_file, config, self.domain)
