def test_encoding_with_file(self):
path = 'sample_text_iso88591.csv'
load_csv(self.cursor, 'testtable', path, encoding='latin-1')
expected = [
('iso88591', chr(0xe6)), # chr(0xe6) -> æ
]
self.cursor.execute('SELECT col1, col2 FROM testtable')
self.assertEqual(list(self.cursor), expected)
def test_fallback_with_file(self):
with warnings.catch_warnings(record=True) as warning_list:
warnings.simplefilter('always')
path = 'sample_text_iso88591.csv'
load_csv(self.cursor, 'testtable', path) # <- No encoding arg.
self.assertEqual(len(warning_list), 1)
expected = "using fallback 'latin-1'"
self.assertIn(expected, str(warning_list[0].message))
expected = [
('iso88591', chr(0xe6)), # chr(0xe6) -> æ
]
self.cursor.execute('SELECT col1, col2 FROM testtable')
self.assertEqual(list(self.cursor), expected)
def test_encoding_with_stream(self):
csvfile = self.get_stream((
b'col1,col2\n'
b'1,\xe6\n' # '\xe6' -> æ (ash)
b'2,\xf0\n' # '\xf0' -> ð (eth)
b'3,\xfe\n' # '\xfe' -> þ (thorn)
), encoding='latin-1')
load_csv(self.cursor, 'testtable1', csvfile, encoding='latin-1')
expected = [
('1', chr(0xe6)), # chr(0xe6) -> æ
('2', chr(0xf0)), # chr(0xf0) -> ð
('3', chr(0xfe)), # chr(0xfe) -> þ
]
self.cursor.execute('SELECT col1, col2 FROM testtable1')
self.assertEqual(list(self.cursor), expected)
def test_fallback_with_stream(self):
with warnings.catch_warnings(record=True): # Catch warnings issued
csvfile = self.get_stream(( # when running Python 2.
b'col1,col2\n'
b'1,\xe6\n' # '\xe6' -> æ (ash)
b'2,\xf0\n' # '\xf0' -> ð (eth)
b'3,\xfe\n' # '\xfe' -> þ (thorn)
), encoding='latin-1')
load_csv(self.cursor, 'testtable1', csvfile) # <- No encoding arg.
expected = [
('1', chr(0xe6)), # chr(0xe6) -> æ
('2', chr(0xf0)), # chr(0xf0) -> ð
('3', chr(0xfe)), # chr(0xfe) -> þ
]
self.cursor.execute('SELECT col1, col2 FROM testtable1')
self.assertEqual(list(self.cursor), expected)
def from_csv(cls, file, encoding=None, **fmtparams):
if isinstance(file, string_types) or isinstance(file, file_types):
data_list = [file]
else:
data_list = file
new_cls = cls.__new__(cls)
new_cls._connection = DEFAULT_CONNECTION
cursor = new_cls._connection.cursor()
with savepoint(cursor):
table = new_table_name(cursor)
for obj in data_list:
load_csv(cursor, table, obj, encoding=encoding, **fmtparams)
new_cls._table = table if table_exists(cursor, table) else None
new_cls._data = file
new_cls._args = (encoding, )
new_cls._kwds = fmtparams
new_cls._update_list = []
return new_cls
def test_fallback_with_StringIO(self):
if not StringIO: # <- Python 2.x only.
return
csvfile = StringIO(
b'col1,col2\n'
b'1,\xe6\n' # '\xe6' -> æ (ash)
b'2,\xf0\n' # '\xf0' -> ð (eth)
b'3,\xfe\n' # '\xfe' -> þ (thorn)
)
with warnings.catch_warnings(record=True):
load_csv(self.cursor, 'testtable1', csvfile)
expected = [
('1', chr(0xe6)), # chr(0xe6) -> æ
('2', chr(0xf0)), # chr(0xf0) -> ð
('3', chr(0xfe)), # chr(0xfe) -> þ
]
self.cursor.execute('SELECT col1, col2 FROM testtable1')
self.assertEqual(list(self.cursor), expected)
def test_fallback_with_exhaustible_object(self):
"""Exhaustible iterators and unseekable file-like objects
can only be iterated over once. This means that the usual
fallback behavior can not be applied and the function must
raise an exception.
"""
if not sys.version_info[0] == 2:
return
csvfile = self.get_stream((
b'col1,col2\n'
b'1,\xe6\n' # '\xe6' -> æ (ash)
b'2,\xf0\n' # '\xf0' -> ð (eth)
b'3,\xfe\n' # '\xfe' -> þ (thorn)
), encoding='latin-1')
generator = (x for x in csvfile) # <- Make stream unseekable.
with self.assertRaises(UnicodeDecodeError) as cm:
load_csv(self.cursor, 'testtable', generator)
error_message = str(cm.exception)
self.assertIn('cannot attempt fallback', error_message.lower())
def test_encoding_mismatch(self):
path = 'sample_text_iso88591.csv'
wrong_encoding = 'utf-8' # <- Doesn't match file.
with self.assertRaises(UnicodeDecodeError):
load_csv(self.cursor, 'testtable', path, wrong_encoding)
