def test_import_named_update(self): cred = testcommon.DummyCredentials() rules = xmlload(StringIO(import_named_xml)) rules.add(ImportSource('local_case_id', 'Id')) src = datasrc.DataImpSrc('foo', StringIO(data)) imp = dataimp.DataImp(cred, 1, src, rules) self.failIf(imp.errors, imp.errors) self.assertEqual(imp.errors.count(), 0) self.assertEqual(imp.new_cnt, 3) self.assertEqual(imp.update_cnt, 0) self.assertListEq(self._fetch_rows(), [ ('BLOGS', dt('2001-11-24'), 0, 'confirmed', 'NSW', None, '100', 1.0, 'Unknown', None, True, False, False), ('SMITH', age3y, 366, 'confirmed', 'NSW', None, '101', 2.0, 'Unknown', None, False, True, True), ('JONES', age4m, 31, 'preliminary', 'NSW', None, '102', None, 'Unknown', None, False, False, False), ]) src = datasrc.DataImpSrc('foo', StringIO(data + data_update)) imp = dataimp.DataImp(cred, 1, src, rules) self.assertEqual(imp.new_cnt, 1) self.assertEqual(imp.update_cnt, 4) self.failIf(imp.errors, imp.errors) self.assertEqual(imp.errors.count(), 0) self.assertListEq(self._fetch_rows(), [ ('BLOGS', dt('2001-11-24'), 0, 'confirmed', 'NSW', None, '100', 1.0, 'Unknown', None, True, False, False), ('SMITH', dt('2000-01-20'), 0, 'confirmed', 'NSW', None, '101', 2.0, 'Unknown', None, False, False, False), ('JONES', age4m, 31, 'preliminary', 'NSW', None, '102', None, 'Unknown', None, False, False, False), ('WILLIAMS', dt('1940-12-02'), 0, 'excluded', 'NSW', None, '104', 3.0, 'Unknown', None, False, True, False), ])
def test_preview_errors(self): cred = testcommon.DummyCredentials() rules = xmlload(StringIO(error_named_xml)) data = '''\ Surname,Status,DoB,Age,Duration,Contact,Date,Id,Likes blogs,XXX,24/11/2001,,,False,,100, smith,confirmed,,10000,2,True,,101,Apples jones,suspected,2008-1-30,,,Unknown,,102,Tomato ,,,,,,,, williams,,,,a,XX,XX,103, ,,,,,, ''' src = datasrc.DataImpSrc('foo', StringIO(data)) imp = dataimp.PreviewImport(cred, 1, src, rules) self.assertListEq(list(imp.errors), [ 'foo: record 5 (line 7): Column count is not constant: has 7 columns, expected 9', "record 1 (line 2): Status: 'XXX' not a valid choice", "record 2 (line 3): Date of birth/Age: date/time '10000' does not match format 'DD/MM/YYYY'", "record 3 (line 4): Date of birth/Age: date/time '2008-1-30' does not match format 'DD/MM/YYYY'", 'record 3 (line 4): Favourite foods: tomato not valid choice(s)', 'record 4 (line 5): Either Surname or Local ID must be specified', 'record 4 (line 5): Exposure History (SARS): Contact with case: this field must be answered', "record 5 (line 6): Contact with case: 'XX' not a valid choice", 'record 5 (line 6): Exposure History (SARS): Contact with case: this field must be answered', 'record 5 (line 6): Exposure History (SARS): Contact duration (hours): value must be a number', 'record 5 (line 6): Exposure History (SARS): Date of first contact: could not parse date "XX"', ]) self.assertEqual(imp.errors.count(), 11) self.failUnless(0 not in imp.errors) self.failUnless(1 in imp.errors) self.failUnless(5 in imp.errors) self.failUnless(6 not in imp.errors) self.assertEqual(imp.errors.get(4), [ 'record 4 (line 5): Either Surname or Local ID must be specified', 'record 4 (line 5): Exposure History (SARS): Contact with case: this field must be answered', ]) # Check "too many errors" handling lines = data.splitlines() data = '\n'.join([lines[0]] + [lines[1]] * 101) src = datasrc.DataImpSrc('foo', StringIO(data)) imp = dataimp.PreviewImport(cred, 1, src, rules) self.assertEqual(list(imp.errors)[0], 'More than %s errors, giving up' % imp.errors.MAX_ERRORS) self.assertEqual(imp.errors.count(), 100)
def test_import_named(self): cred = testcommon.DummyCredentials() rules = xmlload(StringIO(import_named_xml)) src = datasrc.DataImpSrc('foo', StringIO(data)) imp = dataimp.DataImp(cred, 1, src, rules) self.failIf(imp.errors, imp.errors) self.assertEqual(imp.errors.count(), 0) self.assertEqual(imp.new_cnt, 3) self.assertEqual(imp.update_cnt, 0) self.assertListEq(self._fetch_rows(), [ ('BLOGS', dt('2001-11-24'), 0, 'confirmed', 'NSW', None, None, 1.0, 'Unknown', None, True, False, False), ('SMITH', age3y, 366, 'confirmed', 'NSW', None, None, 2.0, 'Unknown', None, False, True, True), ('JONES', age4m, 31, 'preliminary', 'NSW', None, None, None, 'Unknown', None, False, False, False), ])
def test_datasrc_named(self): rules = xmlload(StringIO(named_rules)) f = StringIO(data) src = datasrc.DataImpSrc('foo', f) self.assertEqual(src.size, len(data)) # Preview src.update_preview(rules) self.assertEqual(src.preview.n_cols, 2) self.assertEqual(src.preview.n_rows, 3) self.assertEqual(src.preview.col_names, ['surname', 'case_status']) self.assertEqual(len(src.preview.rows), 3) self.assertEqual(src.preview.rows, [['blogs', 'confirmed'], ['smith', 'confirmed'], ['jones', 'suspected']]) self.assertEqual(src.preview.colvalues('case_status'), ['confirmed', 'suspected']) self.assertEqual(src.preview.colpreview('surname'), ['blogs', 'smith', 'jones']) self.assertEqual(src.preview.colpreview('case_status'), ['confirmed', 'confirmed', 'suspected']) src.release()
def _test_preview(self, rules_xml, data): cred = testcommon.DummyCredentials() rules = xmlload(StringIO(rules_xml)) src = datasrc.DataImpSrc('foo', StringIO(data)) now = DateTime.DateTime(2010,7,20,17,23,1) imp = testcommon.freeze_time(now, dataimp.PreviewImport, cred, 1, src, rules) self.failIf(imp.errors, imp.errors) self.assertEqual(imp.group_header, [('Demographics', 4), ('Exposure History (SARS)', 3)]) self.assertEqual(imp.header, [ 'Status', 'Surname', 'Date of birth/Age', 'Locality/Suburb', 'Contact with case', 'Contact duration (hours)', 'Favourite foods', ]) self.assertEqual(imp.rows, [ ['Confirmed', 'BLOGS', '24/11/2001 (8y)', 'NSW', 'Unknown', '1', 'Icecream'], ['Confirmed', 'SMITH', '3 years', 'NSW', 'Unknown', '2', 'Lamb Chops/Apples'], ['Preliminary', 'JONES', '4 months', 'NSW', 'Unknown', None, None] ])
def test_null_src(self): self.failIf(bool(datasrc.NullDataImpSrc)) self.assertEqual(datasrc.NullDataImpSrc.preview.colvalues('x'), None) self.assertEqual(datasrc.NullDataImpSrc.preview.colpreview('x'), []) datasrc.NullDataImpSrc.release() self.failIf(datasrc.DataImpSrc('foo', StringIO()))