def test_import_named_update(self):
cred = testcommon.DummyCredentials()
rules = xmlload(StringIO(import_named_xml))
rules.add(ImportSource('local_case_id', 'Id'))
src = datasrc.DataImpSrc('foo', StringIO(data))
imp = dataimp.DataImp(cred, 1, src, rules)
self.failIf(imp.errors, imp.errors)
self.assertEqual(imp.errors.count(), 0)
self.assertEqual(imp.new_cnt, 3)
self.assertEqual(imp.update_cnt, 0)
self.assertListEq(self._fetch_rows(), [
('BLOGS', dt('2001-11-24'), 0, 'confirmed', 'NSW', None, '100',
1.0, 'Unknown', None, True, False, False),
('SMITH', age3y, 366, 'confirmed', 'NSW', None, '101',
2.0, 'Unknown', None, False, True, True),
('JONES', age4m, 31, 'preliminary', 'NSW', None, '102',
None, 'Unknown', None, False, False, False),
])
src = datasrc.DataImpSrc('foo', StringIO(data + data_update))
imp = dataimp.DataImp(cred, 1, src, rules)
self.assertEqual(imp.new_cnt, 1)
self.assertEqual(imp.update_cnt, 4)
self.failIf(imp.errors, imp.errors)
self.assertEqual(imp.errors.count(), 0)
self.assertListEq(self._fetch_rows(), [
('BLOGS', dt('2001-11-24'), 0, 'confirmed', 'NSW', None, '100',
1.0, 'Unknown', None, True, False, False),
('SMITH', dt('2000-01-20'), 0, 'confirmed', 'NSW', None, '101',
2.0, 'Unknown', None, False, False, False),
('JONES', age4m, 31, 'preliminary', 'NSW', None, '102',
None, 'Unknown', None, False, False, False),
('WILLIAMS', dt('1940-12-02'), 0, 'excluded', 'NSW', None, '104',
3.0, 'Unknown', None, False, True, False),
])
def test_preview_errors(self):
cred = testcommon.DummyCredentials()
rules = xmlload(StringIO(error_named_xml))
data = '''\
Surname,Status,DoB,Age,Duration,Contact,Date,Id,Likes
blogs,XXX,24/11/2001,,,False,,100,
smith,confirmed,,10000,2,True,,101,Apples
jones,suspected,2008-1-30,,,Unknown,,102,Tomato
,,,,,,,,
williams,,,,a,XX,XX,103,
,,,,,,
'''
src = datasrc.DataImpSrc('foo', StringIO(data))
imp = dataimp.PreviewImport(cred, 1, src, rules)
self.assertListEq(list(imp.errors), [
'foo: record 5 (line 7): Column count is not constant: has 7 columns, expected 9',
"record 1 (line 2): Status: 'XXX' not a valid choice",
"record 2 (line 3): Date of birth/Age: date/time '10000' does not match format 'DD/MM/YYYY'",
"record 3 (line 4): Date of birth/Age: date/time '2008-1-30' does not match format 'DD/MM/YYYY'",
'record 3 (line 4): Favourite foods: tomato not valid choice(s)',
'record 4 (line 5): Either Surname or Local ID must be specified',
'record 4 (line 5): Exposure History (SARS): Contact with case: this field must be answered',
"record 5 (line 6): Contact with case: 'XX' not a valid choice",
'record 5 (line 6): Exposure History (SARS): Contact with case: this field must be answered',
'record 5 (line 6): Exposure History (SARS): Contact duration (hours): value must be a number',
'record 5 (line 6): Exposure History (SARS): Date of first contact: could not parse date "XX"',
])
self.assertEqual(imp.errors.count(), 11)
self.failUnless(0 not in imp.errors)
self.failUnless(1 in imp.errors)
self.failUnless(5 in imp.errors)
self.failUnless(6 not in imp.errors)
self.assertEqual(imp.errors.get(4), [
'record 4 (line 5): Either Surname or Local ID must be specified',
'record 4 (line 5): Exposure History (SARS): Contact with case: this field must be answered',
])
# Check "too many errors" handling
lines = data.splitlines()
data = '\n'.join([lines[0]] + [lines[1]] * 101)
src = datasrc.DataImpSrc('foo', StringIO(data))
imp = dataimp.PreviewImport(cred, 1, src, rules)
self.assertEqual(list(imp.errors)[0],
'More than %s errors, giving up' % imp.errors.MAX_ERRORS)
self.assertEqual(imp.errors.count(), 100)
def test_import_named(self):
cred = testcommon.DummyCredentials()
rules = xmlload(StringIO(import_named_xml))
src = datasrc.DataImpSrc('foo', StringIO(data))
imp = dataimp.DataImp(cred, 1, src, rules)
self.failIf(imp.errors, imp.errors)
self.assertEqual(imp.errors.count(), 0)
self.assertEqual(imp.new_cnt, 3)
self.assertEqual(imp.update_cnt, 0)
self.assertListEq(self._fetch_rows(), [
('BLOGS', dt('2001-11-24'), 0, 'confirmed', 'NSW', None, None,
1.0, 'Unknown', None, True, False, False),
('SMITH', age3y, 366, 'confirmed', 'NSW', None, None,
2.0, 'Unknown', None, False, True, True),
('JONES', age4m, 31, 'preliminary', 'NSW', None, None,
None, 'Unknown', None, False, False, False),
])
def test_datasrc_named(self):
rules = xmlload(StringIO(named_rules))
f = StringIO(data)
src = datasrc.DataImpSrc('foo', f)
self.assertEqual(src.size, len(data))
# Preview
src.update_preview(rules)
self.assertEqual(src.preview.n_cols, 2)
self.assertEqual(src.preview.n_rows, 3)
self.assertEqual(src.preview.col_names, ['surname', 'case_status'])
self.assertEqual(len(src.preview.rows), 3)
self.assertEqual(src.preview.rows,
[['blogs', 'confirmed'], ['smith', 'confirmed'],
['jones', 'suspected']])
self.assertEqual(src.preview.colvalues('case_status'),
['confirmed', 'suspected'])
self.assertEqual(src.preview.colpreview('surname'),
['blogs', 'smith', 'jones'])
self.assertEqual(src.preview.colpreview('case_status'),
['confirmed', 'confirmed', 'suspected'])
src.release()
def _test_preview(self, rules_xml, data):
cred = testcommon.DummyCredentials()
rules = xmlload(StringIO(rules_xml))
src = datasrc.DataImpSrc('foo', StringIO(data))
now = DateTime.DateTime(2010,7,20,17,23,1)
imp = testcommon.freeze_time(now, dataimp.PreviewImport,
cred, 1, src, rules)
self.failIf(imp.errors, imp.errors)
self.assertEqual(imp.group_header, [('Demographics', 4),
('Exposure History (SARS)', 3)])
self.assertEqual(imp.header, [
'Status', 'Surname', 'Date of birth/Age', 'Locality/Suburb',
'Contact with case',
'Contact duration (hours)',
'Favourite foods',
])
self.assertEqual(imp.rows, [
['Confirmed', 'BLOGS', '24/11/2001 (8y)', 'NSW',
'Unknown', '1', 'Icecream'],
['Confirmed', 'SMITH', '3 years', 'NSW',
'Unknown', '2', 'Lamb Chops/Apples'],
['Preliminary', 'JONES', '4 months', 'NSW',
'Unknown', None, None]
])
def test_null_src(self):
self.failIf(bool(datasrc.NullDataImpSrc))
self.assertEqual(datasrc.NullDataImpSrc.preview.colvalues('x'), None)
self.assertEqual(datasrc.NullDataImpSrc.preview.colpreview('x'), [])
datasrc.NullDataImpSrc.release()
self.failIf(datasrc.DataImpSrc('foo', StringIO()))