def testParentCase(self): headers = ['parent_id', 'name', 'case_id'] config = self._config(headers, create_new_cases=True, search_column='case_id') rows = 3 [parent_case] = self.factory.create_or_update_case( CaseStructure(attrs={'create': True})) self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) file = make_worksheet_wrapper( ['parent_id', 'name', 'case_id'], [parent_case.case_id, 'name-0', 'case_id-0'], [parent_case.case_id, 'name-1', 'case_id-1'], [parent_case.case_id, 'name-2', 'case_id-2'], ) file_missing = make_worksheet_wrapper( ['parent_id', 'name', 'case_id'], [u'parent_id-0', u'name-0', u'case_id-0'], [u'parent_id-1', u'name-1', u'case_id-1'], [u'parent_id-2', u'name-2', u'case_id-2'], ) # Should successfully match on `rows` cases res = do_import(file, config, self.domain) self.assertEqual(rows, res['created_count']) # Should be unable to find parent case on `rows` cases res = do_import(file_missing, config, self.domain) error_column_name = 'parent_id' self.assertEqual( rows, len(res['errors'][ImportErrors.InvalidParentId][error_column_name] ['rows']), "All cases should have missing parent")
def testParentCase(self): headers = ['parent_id', 'name', 'case_id'] config = self._config(headers, create_new_cases=True, search_column='case_id') rows = 3 [parent_case] = self.factory.create_or_update_case(CaseStructure(attrs={'create': True})) self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) file = make_worksheet_wrapper( ['parent_id', 'name', 'case_id'], [parent_case.case_id, 'name-0', 'case_id-0'], [parent_case.case_id, 'name-1', 'case_id-1'], [parent_case.case_id, 'name-2', 'case_id-2'], ) file_missing = make_worksheet_wrapper( ['parent_id', 'name', 'case_id'], ['parent_id-0', 'name-0', 'case_id-0'], ['parent_id-1', 'name-1', 'case_id-1'], ['parent_id-2', 'name-2', 'case_id-2'], ) # Should successfully match on `rows` cases res = do_import(file, config, self.domain) self.assertEqual(rows, res['created_count']) # Should be unable to find parent case on `rows` cases res = do_import(file_missing, config, self.domain) error_column_name = 'parent_id' self.assertEqual(rows, len(res['errors'][ImportErrors.InvalidParentId][error_column_name]['rows']), "All cases should have missing parent")
def testExternalIdChunking(self): # bootstrap a stub case external_id = 'importer-test-external-id' headers = ['external_id', 'age', 'sex', 'location'] config = self._config(headers, search_field='external_id') file = make_worksheet_wrapper( ['external_id', 'age', 'sex', 'location'], ['importer-test-external-id', 'age-0', 'sex-0', 'location-0'], ['importer-test-external-id', 'age-1', 'sex-1', 'location-1'], ['importer-test-external-id', 'age-2', 'sex-2', 'location-2'], ) # the first one should create the case, and the remaining two should update it res = do_import(file, config, self.domain) self.assertEqual(1, res['created_count']) self.assertEqual(2, res['match_count']) self.assertFalse(res['errors']) self.assertEqual(2, res['num_chunks']) # the lookup causes an extra chunk # should just create the one case case_ids = self.accessor.get_case_ids_in_domain() self.assertEqual(1, len(case_ids)) [case] = self.accessor.get_cases(case_ids) self.assertEqual(external_id, case.external_id) for prop in ['age', 'sex', 'location']: self.assertTrue(prop in case.get_case_property(prop))
def testImportBasic(self): config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], [u'case_id-0', u'age-0', u'sex-0', u'location-0'], [u'case_id-1', u'age-1', u'sex-1', u'location-1'], [u'case_id-2', u'age-2', u'sex-2', u'location-2'], [u'case_id-3', u'age-3', u'sex-3', u'location-3'], [u'case_id-4', u'age-4', u'sex-4', u'location-4'], ) res = do_import(file, config, self.domain) self.assertEqual(5, res['created_count']) self.assertEqual(0, res['match_count']) self.assertFalse(res['errors']) self.assertEqual(1, res['num_chunks']) case_ids = self.accessor.get_case_ids_in_domain() cases = list(self.accessor.get_cases(case_ids)) self.assertEqual(5, len(cases)) properties_seen = set() for case in cases: self.assertEqual(self.couch_user._id, case.user_id) self.assertEqual(self.couch_user._id, case.owner_id) self.assertEqual(self.default_case_type, case.type) for prop in ['age', 'sex', 'location']: self.assertTrue(prop in case.get_case_property(prop)) self.assertFalse( case.get_case_property(prop) in properties_seen) properties_seen.add(case.get_case_property(prop))
def testCaseIdMatching(self): # bootstrap a stub case [case] = self.factory.create_or_update_case( CaseStructure(attrs={ 'create': True, 'update': { 'importer_test_prop': 'foo' }, })) self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], [case.case_id, 'age-0', 'sex-0', 'location-0'], [case.case_id, 'age-1', 'sex-1', 'location-1'], [case.case_id, 'age-2', 'sex-2', 'location-2'], ) res = do_import(file, config, self.domain) self.assertEqual(0, res['created_count']) self.assertEqual(3, res['match_count']) self.assertFalse(res['errors']) # shouldn't create any more cases, just the one case_ids = self.accessor.get_case_ids_in_domain() self.assertEqual(1, len(case_ids)) [case] = self.accessor.get_cases(case_ids) for prop in ['age', 'sex', 'location']: self.assertTrue(prop in case.get_case_property(prop)) # shouldn't touch existing properties self.assertEqual('foo', case.get_case_property('importer_test_prop'))
def testExternalIdMatching(self): # bootstrap a stub case external_id = 'importer-test-external-id' [case] = self.factory.create_or_update_case(CaseStructure( attrs={ 'create': True, 'external_id': external_id, } )) self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) headers = ['external_id', 'age', 'sex', 'location'] config = self._config(headers, search_field='external_id') file = make_worksheet_wrapper( ['external_id', 'age', 'sex', 'location'], ['importer-test-external-id', 'age-0', 'sex-0', 'location-0'], ['importer-test-external-id', 'age-1', 'sex-1', 'location-1'], ['importer-test-external-id', 'age-2', 'sex-2', 'location-2'], ) res = do_import(file, config, self.domain) self.assertEqual(0, res['created_count']) self.assertEqual(3, res['match_count']) self.assertFalse(res['errors']) # shouldn't create any more cases, just the one self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
def testExternalIdMatching(self): # bootstrap a stub case external_id = 'importer-test-external-id' [case] = self.factory.create_or_update_case( CaseStructure(attrs={ 'create': True, 'external_id': external_id, })) self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) headers = ['external_id', 'age', 'sex', 'location'] config = self._config(headers, search_field='external_id') file = make_worksheet_wrapper( ['external_id', 'age', 'sex', 'location'], ['importer-test-external-id', 'age-0', 'sex-0', 'location-0'], ['importer-test-external-id', 'age-1', 'sex-1', 'location-1'], ['importer-test-external-id', 'age-2', 'sex-2', 'location-2'], ) res = do_import(file, config, self.domain) self.assertEqual(0, res['created_count']) self.assertEqual(3, res['match_count']) self.assertFalse(res['errors']) # shouldn't create any more cases, just the one self.assertEqual(1, len(self.accessor.get_case_ids_in_domain()))
def testCaseIdMatching(self): # bootstrap a stub case [case] = self.factory.create_or_update_case(CaseStructure(attrs={ 'create': True, 'update': {'importer_test_prop': 'foo'}, })) self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], [case.case_id, 'age-0', 'sex-0', 'location-0'], [case.case_id, 'age-1', 'sex-1', 'location-1'], [case.case_id, 'age-2', 'sex-2', 'location-2'], ) res = do_import(file, config, self.domain) self.assertEqual(0, res['created_count']) self.assertEqual(3, res['match_count']) self.assertFalse(res['errors']) # shouldn't create any more cases, just the one case_ids = self.accessor.get_case_ids_in_domain() self.assertEqual(1, len(case_ids)) [case] = self.accessor.get_cases(case_ids) for prop in ['age', 'sex', 'location']: self.assertTrue(prop in case.get_case_property(prop)) # shouldn't touch existing properties self.assertEqual('foo', case.get_case_property('importer_test_prop'))
def testImportBasic(self): config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], ['case_id-0', 'age-0', 'sex-0', 'location-0'], ['case_id-1', 'age-1', 'sex-1', 'location-1'], ['case_id-2', 'age-2', 'sex-2', 'location-2'], ['case_id-3', 'age-3', 'sex-3', 'location-3'], ['case_id-4', 'age-4', 'sex-4', 'location-4'], ) res = do_import(file, config, self.domain) self.assertEqual(5, res['created_count']) self.assertEqual(0, res['match_count']) self.assertFalse(res['errors']) self.assertEqual(1, res['num_chunks']) case_ids = self.accessor.get_case_ids_in_domain() cases = list(self.accessor.get_cases(case_ids)) self.assertEqual(5, len(cases)) properties_seen = set() for case in cases: self.assertEqual(self.couch_user._id, case.user_id) self.assertEqual(self.couch_user._id, case.owner_id) self.assertEqual(self.default_case_type, case.type) for prop in ['age', 'sex', 'location']: self.assertTrue(prop in case.get_case_property(prop)) self.assertFalse(case.get_case_property(prop) in properties_seen) properties_seen.add(case.get_case_property(prop))
def _import_rows(self, rows, get_by_user_id): get_by_user_id.return_value = WebUser( _id=self.couch_user_id, domain=self.domain, username='******') case_upload_record = CaseUploadRecord( upload_id=uuid.uuid4(), task_id=uuid.uuid4(), domain=self.domain, ) case_upload_record.save() self.addCleanup(case_upload_record.delete) tracker = CaseUpload(case_upload_record.upload_id) # mock internals to have record_cases use our case_upload_record tracker.__dict__['_case_upload_record'] = case_upload_record config = self._get_config(rows[0]) xls_file = make_worksheet_wrapper(*rows) do_import(xls_file, config, self.domain, record_form_callback=tracker.record_form) return case_upload_record
def testImportNamedColumns(self): config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], [u'case_id-0', u'age-0', u'sex-0', u'location-0'], [u'case_id-1', u'age-1', u'sex-1', u'location-1'], [u'case_id-2', u'age-2', u'sex-2', u'location-2'], [u'case_id-3', u'age-3', u'sex-3', u'location-3'], ) res = do_import(file, config, self.domain) self.assertEqual(4, res['created_count']) self.assertEqual(4, len(self.accessor.get_case_ids_in_domain()))
def testImportNamedColumns(self): config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], ['case_id-0', 'age-0', 'sex-0', 'location-0'], ['case_id-1', 'age-1', 'sex-1', 'location-1'], ['case_id-2', 'age-2', 'sex-2', 'location-2'], ['case_id-3', 'age-3', 'sex-3', 'location-3'], ) res = do_import(file, config, self.domain) self.assertEqual(4, res['created_count']) self.assertEqual(4, len(self.accessor.get_case_ids_in_domain()))
def testImportTrailingWhitespace(self): cols = ['case_id', 'age', 'sex\xa0', 'location'] config = self._config(cols) file = make_worksheet_wrapper( ['case_id', 'age', 'sex\xa0', 'location'], ['case_id-0', 'age-0', 'sex\xa0-0', 'location-0'], ) res = do_import(file, config, self.domain) self.assertEqual(1, res['created_count']) case_ids = self.accessor.get_case_ids_in_domain() self.assertEqual(1, len(case_ids)) case = self.accessor.get_case(case_ids[0]) self.assertTrue(bool(case.get_case_property('sex'))) # make sure the value also got properly set
def testBlankRows(self): # don't create new cases for rows left blank config = self._config(['case_id', 'age', 'sex', 'location'], create_new_cases=True) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], [None, None, None, None], ['', '', '', ''], ) res = do_import(file, config, self.domain) # no matching and no create new set - should do nothing self.assertEqual(0, res['created_count']) self.assertEqual(0, res['match_count']) self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
def testBasicChunking(self): config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], ['case_id-0', 'age-0', 'sex-0', 'location-0'], ['case_id-1', 'age-1', 'sex-1', 'location-1'], ['case_id-2', 'age-2', 'sex-2', 'location-2'], ['case_id-3', 'age-3', 'sex-3', 'location-3'], ['case_id-4', 'age-4', 'sex-4', 'location-4'], ) res = do_import(file, config, self.domain, chunksize=2) # 5 cases in chunks of 2 = 3 chunks self.assertEqual(3, res['num_chunks']) self.assertEqual(5, res['created_count']) self.assertEqual(5, len(get_case_ids_in_domain(self.domain)))
def testBasicChunking(self): config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], [u'case_id-0', u'age-0', u'sex-0', u'location-0'], [u'case_id-1', u'age-1', u'sex-1', u'location-1'], [u'case_id-2', u'age-2', u'sex-2', u'location-2'], [u'case_id-3', u'age-3', u'sex-3', u'location-3'], [u'case_id-4', u'age-4', u'sex-4', u'location-4'], ) res = do_import(file, config, self.domain, chunksize=2) # 5 cases in chunks of 2 = 3 chunks self.assertEqual(3, res['num_chunks']) self.assertEqual(5, res['created_count']) self.assertEqual(5, len(get_case_ids_in_domain(self.domain)))
def testNoCreateNew(self): config = self._config(['case_id', 'age', 'sex', 'location'], create_new_cases=False) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], ['case_id-0', 'age-0', 'sex-0', 'location-0'], ['case_id-1', 'age-1', 'sex-1', 'location-1'], ['case_id-2', 'age-2', 'sex-2', 'location-2'], ['case_id-3', 'age-3', 'sex-3', 'location-3'], ['case_id-4', 'age-4', 'sex-4', 'location-4'], ) res = do_import(file, config, self.domain) # no matching and no create new set - should do nothing self.assertEqual(0, res['created_count']) self.assertEqual(0, res['match_count']) self.assertEqual(0, len(get_case_ids_in_domain(self.domain)))
def testCaseLookupTypeCheck(self): [case] = self.factory.create_or_update_case(CaseStructure(attrs={ 'create': True, 'case_type': 'nonmatch-type', })) self.assertEqual(1, len(self.accessor.get_case_ids_in_domain())) config = self._config(['case_id', 'age', 'sex', 'location']) file = make_worksheet_wrapper( ['case_id', 'age', 'sex', 'location'], [case.case_id, 'age-0', 'sex-0', 'location-0'], [case.case_id, 'age-1', 'sex-1', 'location-1'], [case.case_id, 'age-2', 'sex-2', 'location-2'], ) res = do_import(file, config, self.domain) # because the type is wrong these shouldn't match self.assertEqual(3, res['created_count']) self.assertEqual(0, res['match_count']) self.assertEqual(4, len(self.accessor.get_case_ids_in_domain()))
def test_external_id_matching_on_create_with_custom_column_name(self): headers = ['id_column', 'age', 'sex', 'location'] external_id = 'external-id-test' config = self._config(headers[1:], search_column='id_column', search_field='external_id') file = make_worksheet_wrapper( ['id_column', 'age', 'sex', 'location'], ['external-id-test', 'age-0', 'sex-0', 'location-0'], ['external-id-test', 'age-1', 'sex-1', 'location-1'], ) res = do_import(file, config, self.domain) self.assertFalse(res['errors']) self.assertEqual(1, res['created_count']) self.assertEqual(1, res['match_count']) case_ids = self.accessor.get_case_ids_in_domain() self.assertEqual(1, len(case_ids)) case = self.accessor.get_case(case_ids[0]) self.assertEqual(external_id, case.external_id)
def handle(self, export_file, config_file, domain, user_id, **options): start = datetime.utcnow() if '@' in user_id: user = WebUser.get_by_username(user_id) else: user = WebUser.get(user_id) if not user.is_member_of(domain): raise CommandError("%s can't access %s" % (user, domain)) with open(config_file, 'r', encoding='utf-8') as f: config = ImporterConfig.from_json(f.read()) config.couch_user_id = user._id with get_spreadsheet(export_file) as spreadsheet: print(json.dumps(do_import(spreadsheet, config, domain), default=json_handler)) print('finished in %s seconds' % (datetime.utcnow() - start).seconds)
def handle(self, *args, **options): if len(args) != 4: raise CommandError('Usage is import_cases %s' % self.args) start = datetime.utcnow() export_file, config_file, domain, user_id = args if '@' in user_id: user = WebUser.get_by_username(user_id) else: user = WebUser.get(user_id) if not user.is_member_of(domain): raise CommandError("%s can't access %s" % (user, domain)) with open(config_file, 'r') as f: config = ImporterConfig.from_json(f.read()) config.couch_user_id = user._id with get_spreadsheet(export_file) as spreadsheet: print json.dumps(do_import(spreadsheet, config, domain), default=json_handler) print 'finished in %s seconds' % (datetime.utcnow() - start).seconds
def import_mock_file(self, rows): config = self._config(rows[0]) xls_file = make_worksheet_wrapper(*rows) return do_import(xls_file, config, self.domain)