def test_encoding_with_file(self): path = 'sample_text_iso88591.csv' load_csv(self.cursor, 'testtable', path, encoding='latin-1') expected = [ ('iso88591', chr(0xe6)), # chr(0xe6) -> æ ] self.cursor.execute('SELECT col1, col2 FROM testtable') self.assertEqual(list(self.cursor), expected)
def test_fallback_with_file(self): with warnings.catch_warnings(record=True) as warning_list: warnings.simplefilter('always') path = 'sample_text_iso88591.csv' load_csv(self.cursor, 'testtable', path) # <- No encoding arg. self.assertEqual(len(warning_list), 1) expected = "using fallback 'latin-1'" self.assertIn(expected, str(warning_list[0].message)) expected = [ ('iso88591', chr(0xe6)), # chr(0xe6) -> æ ] self.cursor.execute('SELECT col1, col2 FROM testtable') self.assertEqual(list(self.cursor), expected)
def test_encoding_with_stream(self): csvfile = self.get_stream(( b'col1,col2\n' b'1,\xe6\n' # '\xe6' -> æ (ash) b'2,\xf0\n' # '\xf0' -> ð (eth) b'3,\xfe\n' # '\xfe' -> þ (thorn) ), encoding='latin-1') load_csv(self.cursor, 'testtable1', csvfile, encoding='latin-1') expected = [ ('1', chr(0xe6)), # chr(0xe6) -> æ ('2', chr(0xf0)), # chr(0xf0) -> ð ('3', chr(0xfe)), # chr(0xfe) -> þ ] self.cursor.execute('SELECT col1, col2 FROM testtable1') self.assertEqual(list(self.cursor), expected)
def test_fallback_with_stream(self): with warnings.catch_warnings(record=True): # Catch warnings issued csvfile = self.get_stream(( # when running Python 2. b'col1,col2\n' b'1,\xe6\n' # '\xe6' -> æ (ash) b'2,\xf0\n' # '\xf0' -> ð (eth) b'3,\xfe\n' # '\xfe' -> þ (thorn) ), encoding='latin-1') load_csv(self.cursor, 'testtable1', csvfile) # <- No encoding arg. expected = [ ('1', chr(0xe6)), # chr(0xe6) -> æ ('2', chr(0xf0)), # chr(0xf0) -> ð ('3', chr(0xfe)), # chr(0xfe) -> þ ] self.cursor.execute('SELECT col1, col2 FROM testtable1') self.assertEqual(list(self.cursor), expected)
def from_csv(cls, file, encoding=None, **fmtparams): if isinstance(file, string_types) or isinstance(file, file_types): data_list = [file] else: data_list = file new_cls = cls.__new__(cls) new_cls._connection = DEFAULT_CONNECTION cursor = new_cls._connection.cursor() with savepoint(cursor): table = new_table_name(cursor) for obj in data_list: load_csv(cursor, table, obj, encoding=encoding, **fmtparams) new_cls._table = table if table_exists(cursor, table) else None new_cls._data = file new_cls._args = (encoding, ) new_cls._kwds = fmtparams new_cls._update_list = [] return new_cls
def test_fallback_with_StringIO(self): if not StringIO: # <- Python 2.x only. return csvfile = StringIO( b'col1,col2\n' b'1,\xe6\n' # '\xe6' -> æ (ash) b'2,\xf0\n' # '\xf0' -> ð (eth) b'3,\xfe\n' # '\xfe' -> þ (thorn) ) with warnings.catch_warnings(record=True): load_csv(self.cursor, 'testtable1', csvfile) expected = [ ('1', chr(0xe6)), # chr(0xe6) -> æ ('2', chr(0xf0)), # chr(0xf0) -> ð ('3', chr(0xfe)), # chr(0xfe) -> þ ] self.cursor.execute('SELECT col1, col2 FROM testtable1') self.assertEqual(list(self.cursor), expected)
def test_fallback_with_exhaustible_object(self): """Exhaustible iterators and unseekable file-like objects can only be iterated over once. This means that the usual fallback behavior can not be applied and the function must raise an exception. """ if not sys.version_info[0] == 2: return csvfile = self.get_stream(( b'col1,col2\n' b'1,\xe6\n' # '\xe6' -> æ (ash) b'2,\xf0\n' # '\xf0' -> ð (eth) b'3,\xfe\n' # '\xfe' -> þ (thorn) ), encoding='latin-1') generator = (x for x in csvfile) # <- Make stream unseekable. with self.assertRaises(UnicodeDecodeError) as cm: load_csv(self.cursor, 'testtable', generator) error_message = str(cm.exception) self.assertIn('cannot attempt fallback', error_message.lower())
def test_encoding_mismatch(self): path = 'sample_text_iso88591.csv' wrong_encoding = 'utf-8' # <- Doesn't match file. with self.assertRaises(UnicodeDecodeError): load_csv(self.cursor, 'testtable', path, wrong_encoding)