def test_unhandled_types(self): """Should raise error, not return a generator.""" with self.assertRaises(TypeError): get_reader(object()) with self.assertRaises(TypeError): get_reader([object(), object()])
def test_csv(self): reader = get_reader('sample_text_utf8.csv', encoding='utf-8') expected = [ { 'col1': 'utf8', 'col2': chr(0x003b1) }, # chr(0x003b1) -> α ] self.assertEqual(list(reader), expected) reader = get_reader('sample_text_iso88591.csv', encoding='iso8859-1') expected = [ { 'col1': 'iso88591', 'col2': chr(0xe6) }, # chr(0xe6) -> æ ] self.assertEqual(list(reader), expected) path = 'sample_text_utf8.csv' encoding = 'utf-8' if PY2: fh = io.open(path, 'rb') else: fh = open(path, 'rt', encoding=encoding, newline='') with fh: reader = get_reader(fh, encoding=encoding) expected = [ { 'col1': 'utf8', 'col2': chr(0x003b1) }, # chr(0x003b1) -> α ] self.assertEqual(list(reader), expected)
def test_csv(self): reader = get_reader('sample_text_utf8.csv', encoding='utf-8') expected = [ ['col1', 'col2'], ['utf8', chr(0x003b1)], # chr(0x003b1) -> α ] self.assertEqual(list(reader), expected) reader = get_reader('sample_text_iso88591.csv', encoding='iso8859-1') expected = [ ['col1', 'col2'], ['iso88591', chr(0xe6)], # chr(0xe6) -> æ ] self.assertEqual(list(reader), expected) path = 'sample_text_utf8.csv' encoding = 'utf-8' def open_file(path, encoding): # <- Helper function. if PY2: return open(path, 'rb') return open(path, 'rt', encoding=encoding, newline='') with open_file(path, encoding) as fh: reader = get_reader(fh, encoding=encoding) expected = [ ['col1', 'col2'], ['utf8', chr(0x003b1)], # chr(0x003b1) -> α ] self.assertEqual(list(reader), expected)
def test_readerlike_wrapping(self): """Reader-like lists should simply be wrapped.""" readerlike = [['col1', 'col2'], [1, 'a'], [2, 'b']] reader = get_reader(readerlike) self.assertEqual(list(reader), readerlike) readerlike = [('col1', 'col2'), (1, 'a'), (2, 'b')] reader = get_reader(readerlike) self.assertEqual(list(reader), readerlike)
def test_datatest(self): select = datatest.Selector([['A', 'B'], ['x', 1], ['y', 2]]) query = select(('A', 'B')) reader = get_reader(query) # <- datatest.Query self.assertEqual(list(reader), [('A', 'B'), ('x', 1), ('y', 2)]) reader = get_reader(select) # <- datatest.Selector self.assertEqual(list(reader), [('A', 'B'), ('x', 1), ('y', 2)]) result = select({'A': 'B'}).execute() reader = get_reader(query) # <- datatest.Result self.assertEqual(list(reader), [('A', 'B'), ('x', 1), ('y', 2)])
def test_dicts(self): records = [ {'col1': 'first'}, {'col1': 'second'}, ] reader = get_reader(records) # <- No fieldnames arg. self.assertIsInstance(reader, Reader) expected = [['col1'], ['first'], ['second']] self.assertEqual(list(reader), expected) reader = get_reader(records, ['col1']) # <- Give fieldnames arg. self.assertIsInstance(reader, Reader) expected = [['col1'], ['first'], ['second']] self.assertEqual(list(reader), expected)
def test_excel(self): reader = get_reader('sample_excel2007.xlsx') expected = [ ['col1', 'col2'], ['excel2007', 1], ] self.assertEqual(list(reader), expected) reader = get_reader('sample_excel1997.xls') expected = [ ['col1', 'col2'], ['excel1997', 1], ] self.assertEqual(list(reader), expected)
def test_pandas(self): df = pandas.DataFrame({ 'col1': (1, 2, 3), 'col2': ('a', 'b', 'c'), }) reader = get_reader(df, index=False) expected = [ ['col1', 'col2'], [1, 'a'], [2, 'b'], [3, 'c'], ] self.assertEqual(list(reader), expected) self.assertIsInstance(reader, Reader) s = pandas.Series(['a', 'b', 'c'], name='mydata') reader = get_reader(s, index=False) expected = [ ['mydata'], ['a'], ['b'], ['c'], ] self.assertEqual(list(reader), expected) self.assertIsInstance(reader, Reader) index = pandas.Index(['x', 'y', 'z'], name='myindex') reader = get_reader(index) expected = [ ['myindex'], ['x'], ['y'], ['z'], ] self.assertEqual(list(reader), expected) multiindex = pandas.MultiIndex.from_tuples( tuples=[('x', 'one'), ('x', 'two'), ('y', 'three')], names=['A', 'B'], ) reader = get_reader(multiindex) expected = [ ['A', 'B'], ['x', 'one'], ['x', 'two'], ['y', 'three'], ] self.assertEqual(list(reader), expected)
def test_squint(self): select = squint.Select([['A', 'B'], ['x', 1], ['y', 2]]) query = select(('A', 'B')) reader = get_reader(query) # <- squint.Query self.assertEqual(list(reader), [('A', 'B'), ('x', 1), ('y', 2)]) self.assertIsInstance(reader, Reader) reader = get_reader(select) # <- squint.Select self.assertEqual(list(reader), [('A', 'B'), ('x', 1), ('y', 2)]) self.assertIsInstance(reader, Reader) result = select({'A': 'B'}).execute() reader = get_reader(query) # <- squint.Result self.assertEqual(list(reader), [('A', 'B'), ('x', 1), ('y', 2)]) self.assertIsInstance(reader, Reader)
def test_dbf(self): reader = get_reader('sample_dbase.dbf') expected = [ ['COL1', 'COL2'], ['dBASE', 1], ] self.assertEqual(list(reader), expected)
def test_namedtuples(self): ntup = collections.namedtuple('ntup', ['col1', 'col2']) records = [ntup(1, 'a'), ntup(2, 'b')] reader = get_reader(records) expected = [('col1', 'col2'), (1, 'a'), (2, 'b')] self.assertEqual(list(reader), expected)
def test_csv(self): reader = get_reader('sample_text_utf8.csv', encoding='utf-8') expected = [ ['col1', 'col2'], ['utf8', unicode_alpha], ] self.assertEqual(list(reader), expected) self.assertIsInstance(reader, Reader) reader = get_reader('sample_text_utf8_bom.csv', encoding='utf-8') expected = [ ['col1', 'col2'], ['utf8', unicode_alpha], ] self.assertEqual(list(reader), expected) self.assertIsInstance(reader, Reader) reader = get_reader('sample_text_iso88591.csv', encoding='iso8859-1') expected = [ ['col1', 'col2'], ['iso88591', unicode_ash], ] self.assertEqual(list(reader), expected) self.assertIsInstance(reader, Reader) path = 'sample_text_utf8.csv' encoding = 'utf-8' def open_file(path, encoding): # <- Helper function. if PY2: return open(path, 'rb') return open(path, 'rt', encoding=encoding, newline='') with open_file(path, encoding) as fh: reader = get_reader(fh, encoding=encoding) expected = [ ['col1', 'col2'], ['utf8', unicode_alpha], ] self.assertEqual(list(reader), expected) self.assertIsInstance(reader, Reader)
def test_pandas(self): df = pandas.DataFrame({ 'col1': (1, 2, 3), 'col2': ('a', 'b', 'c'), }) reader = get_reader(df, index=False) expected = [ ['col1', 'col2'], [1, 'a'], [2, 'b'], [3, 'c'], ] self.assertEqual(list(reader), expected)
def load_data(self, objs, *args, **kwds): """Load data from one or more objects into the Select. The given *objs*, *\\*args*, and *\\*\\*kwds*, can be any values supported by the :class:`Select` class initialization. Load a single file into an empty Select:: select = datatest.Select() # <- Empty Select. select.load_data('myfile.csv') Add a single file to an already-populated Select:: select = datatest.Select('myfile1.csv') select.load_data('myfile2.xlsx', worksheet='Sheet2') Add multiple files to an already-populated Select:: select = datatest.Select('myfile1.csv') select.load_data(['myfile2.csv', 'myfile3.csv']) """ if isinstance(objs, string_types): obj_list = glob(objs) # Get shell-style wildcard matches. if not obj_list: __tracebackhide__ = True raise FileNotFoundError('no files matching {0!r}'.format(objs)) elif not isinstance(objs, list) \ or isinstance(objs[0], (list, tuple, dict)): # Not a list or is a obj_list = [objs] # reader-like list. else: obj_list = objs cursor = self._connection.cursor() with savepoint(cursor): table = self._table or new_table_name(cursor) for obj in obj_list: if ((isinstance(obj, string_types) and obj.lower().endswith('.csv')) or (isinstance(obj, file_types) and getattr(obj, 'name', '').lower().endswith('.csv'))): load_csv(cursor, table, obj, *args, **kwds) else: reader = get_reader(obj, *args, **kwds) load_data(cursor, table, reader) self._append_obj_string(obj) if not self._table and table_exists(cursor, table): self._table = table
def test_pandas(self): df = pandas.DataFrame({ 'col1': (1, 2, 3), 'col2': ('a', 'b', 'c'), }) reader = get_reader(df, index=False) expected = [ { 'col1': 1, 'col2': 'a' }, { 'col1': 2, 'col2': 'b' }, { 'col1': 3, 'col2': 'c' }, ] self.assertEqual(list(reader), expected)
def test_sql(self): connection = sqlite3.connect(':memory:') connection.executescript(""" CREATE TABLE mytable ( foo TEXT, bar REAL ); INSERT INTO mytable VALUES ('a', 0.8), ('a', 1.2), ('b', 2.5), ('b', 3.0); """) reader = get_reader(connection, 'mytable') expected = [ ('foo', 'bar'), ('a', 0.8), ('a', 1.2), ('b', 2.5), ('b', 3.0), ] self.assertEqual(list(reader), expected)
def test_unidentifiable_type(self): with self.assertRaises(TypeError): obj = object() reader = get_reader(obj)