def test_unhandled_types(self):
"""Should raise error, not return a generator."""
with self.assertRaises(TypeError):
get_reader(object())
with self.assertRaises(TypeError):
get_reader([object(), object()])
def test_csv(self):
reader = get_reader('sample_text_utf8.csv', encoding='utf-8')
expected = [
{
'col1': 'utf8',
'col2': chr(0x003b1)
}, # chr(0x003b1) -> α
]
self.assertEqual(list(reader), expected)
reader = get_reader('sample_text_iso88591.csv', encoding='iso8859-1')
expected = [
{
'col1': 'iso88591',
'col2': chr(0xe6)
}, # chr(0xe6) -> æ
]
self.assertEqual(list(reader), expected)
path = 'sample_text_utf8.csv'
encoding = 'utf-8'
if PY2:
fh = io.open(path, 'rb')
else:
fh = open(path, 'rt', encoding=encoding, newline='')
with fh:
reader = get_reader(fh, encoding=encoding)
expected = [
{
'col1': 'utf8',
'col2': chr(0x003b1)
}, # chr(0x003b1) -> α
]
self.assertEqual(list(reader), expected)
def test_csv(self):
reader = get_reader('sample_text_utf8.csv', encoding='utf-8')
expected = [
['col1', 'col2'],
['utf8', chr(0x003b1)], # chr(0x003b1) -> α
]
self.assertEqual(list(reader), expected)
reader = get_reader('sample_text_iso88591.csv', encoding='iso8859-1')
expected = [
['col1', 'col2'],
['iso88591', chr(0xe6)], # chr(0xe6) -> æ
]
self.assertEqual(list(reader), expected)
path = 'sample_text_utf8.csv'
encoding = 'utf-8'
def open_file(path, encoding): # <- Helper function.
if PY2:
return open(path, 'rb')
return open(path, 'rt', encoding=encoding, newline='')
with open_file(path, encoding) as fh:
reader = get_reader(fh, encoding=encoding)
expected = [
['col1', 'col2'],
['utf8', chr(0x003b1)], # chr(0x003b1) -> α
]
self.assertEqual(list(reader), expected)
def test_readerlike_wrapping(self):
"""Reader-like lists should simply be wrapped."""
readerlike = [['col1', 'col2'], [1, 'a'], [2, 'b']]
reader = get_reader(readerlike)
self.assertEqual(list(reader), readerlike)
readerlike = [('col1', 'col2'), (1, 'a'), (2, 'b')]
reader = get_reader(readerlike)
self.assertEqual(list(reader), readerlike)
def test_datatest(self):
select = datatest.Selector([['A', 'B'], ['x', 1], ['y', 2]])
query = select(('A', 'B'))
reader = get_reader(query) # <- datatest.Query
self.assertEqual(list(reader), [('A', 'B'), ('x', 1), ('y', 2)])
reader = get_reader(select) # <- datatest.Selector
self.assertEqual(list(reader), [('A', 'B'), ('x', 1), ('y', 2)])
result = select({'A': 'B'}).execute()
reader = get_reader(query) # <- datatest.Result
self.assertEqual(list(reader), [('A', 'B'), ('x', 1), ('y', 2)])
def test_dicts(self):
records = [
{'col1': 'first'},
{'col1': 'second'},
]
reader = get_reader(records) # <- No fieldnames arg.
self.assertIsInstance(reader, Reader)
expected = [['col1'], ['first'], ['second']]
self.assertEqual(list(reader), expected)
reader = get_reader(records, ['col1']) # <- Give fieldnames arg.
self.assertIsInstance(reader, Reader)
expected = [['col1'], ['first'], ['second']]
self.assertEqual(list(reader), expected)
def test_excel(self):
reader = get_reader('sample_excel2007.xlsx')
expected = [
['col1', 'col2'],
['excel2007', 1],
]
self.assertEqual(list(reader), expected)
reader = get_reader('sample_excel1997.xls')
expected = [
['col1', 'col2'],
['excel1997', 1],
]
self.assertEqual(list(reader), expected)
def test_pandas(self):
df = pandas.DataFrame({
'col1': (1, 2, 3),
'col2': ('a', 'b', 'c'),
})
reader = get_reader(df, index=False)
expected = [
['col1', 'col2'],
[1, 'a'],
[2, 'b'],
[3, 'c'],
]
self.assertEqual(list(reader), expected)
self.assertIsInstance(reader, Reader)
s = pandas.Series(['a', 'b', 'c'], name='mydata')
reader = get_reader(s, index=False)
expected = [
['mydata'],
['a'],
['b'],
['c'],
]
self.assertEqual(list(reader), expected)
self.assertIsInstance(reader, Reader)
index = pandas.Index(['x', 'y', 'z'], name='myindex')
reader = get_reader(index)
expected = [
['myindex'],
['x'],
['y'],
['z'],
]
self.assertEqual(list(reader), expected)
multiindex = pandas.MultiIndex.from_tuples(
tuples=[('x', 'one'), ('x', 'two'), ('y', 'three')],
names=['A', 'B'],
)
reader = get_reader(multiindex)
expected = [
['A', 'B'],
['x', 'one'],
['x', 'two'],
['y', 'three'],
]
self.assertEqual(list(reader), expected)
def test_squint(self):
select = squint.Select([['A', 'B'], ['x', 1], ['y', 2]])
query = select(('A', 'B'))
reader = get_reader(query) # <- squint.Query
self.assertEqual(list(reader), [('A', 'B'), ('x', 1), ('y', 2)])
self.assertIsInstance(reader, Reader)
reader = get_reader(select) # <- squint.Select
self.assertEqual(list(reader), [('A', 'B'), ('x', 1), ('y', 2)])
self.assertIsInstance(reader, Reader)
result = select({'A': 'B'}).execute()
reader = get_reader(query) # <- squint.Result
self.assertEqual(list(reader), [('A', 'B'), ('x', 1), ('y', 2)])
self.assertIsInstance(reader, Reader)
def test_dbf(self):
reader = get_reader('sample_dbase.dbf')
expected = [
['COL1', 'COL2'],
['dBASE', 1],
]
self.assertEqual(list(reader), expected)
def test_namedtuples(self):
ntup = collections.namedtuple('ntup', ['col1', 'col2'])
records = [ntup(1, 'a'), ntup(2, 'b')]
reader = get_reader(records)
expected = [('col1', 'col2'), (1, 'a'), (2, 'b')]
self.assertEqual(list(reader), expected)
def test_csv(self):
reader = get_reader('sample_text_utf8.csv', encoding='utf-8')
expected = [
['col1', 'col2'],
['utf8', unicode_alpha],
]
self.assertEqual(list(reader), expected)
self.assertIsInstance(reader, Reader)
reader = get_reader('sample_text_utf8_bom.csv', encoding='utf-8')
expected = [
['col1', 'col2'],
['utf8', unicode_alpha],
]
self.assertEqual(list(reader), expected)
self.assertIsInstance(reader, Reader)
reader = get_reader('sample_text_iso88591.csv', encoding='iso8859-1')
expected = [
['col1', 'col2'],
['iso88591', unicode_ash],
]
self.assertEqual(list(reader), expected)
self.assertIsInstance(reader, Reader)
path = 'sample_text_utf8.csv'
encoding = 'utf-8'
def open_file(path, encoding): # <- Helper function.
if PY2:
return open(path, 'rb')
return open(path, 'rt', encoding=encoding, newline='')
with open_file(path, encoding) as fh:
reader = get_reader(fh, encoding=encoding)
expected = [
['col1', 'col2'],
['utf8', unicode_alpha],
]
self.assertEqual(list(reader), expected)
self.assertIsInstance(reader, Reader)
def test_pandas(self):
df = pandas.DataFrame({
'col1': (1, 2, 3),
'col2': ('a', 'b', 'c'),
})
reader = get_reader(df, index=False)
expected = [
['col1', 'col2'],
[1, 'a'],
[2, 'b'],
[3, 'c'],
]
self.assertEqual(list(reader), expected)
def load_data(self, objs, *args, **kwds):
"""Load data from one or more objects into the Select. The
given *objs*, *\\*args*, and *\\*\\*kwds*, can be any values
supported by the :class:`Select` class initialization.
Load a single file into an empty Select::
select = datatest.Select() # <- Empty Select.
select.load_data('myfile.csv')
Add a single file to an already-populated Select::
select = datatest.Select('myfile1.csv')
select.load_data('myfile2.xlsx', worksheet='Sheet2')
Add multiple files to an already-populated Select::
select = datatest.Select('myfile1.csv')
select.load_data(['myfile2.csv', 'myfile3.csv'])
"""
if isinstance(objs, string_types):
obj_list = glob(objs) # Get shell-style wildcard matches.
if not obj_list:
__tracebackhide__ = True
raise FileNotFoundError('no files matching {0!r}'.format(objs))
elif not isinstance(objs, list) \
or isinstance(objs[0], (list, tuple, dict)): # Not a list or is a
obj_list = [objs] # reader-like list.
else:
obj_list = objs
cursor = self._connection.cursor()
with savepoint(cursor):
table = self._table or new_table_name(cursor)
for obj in obj_list:
if ((isinstance(obj, string_types)
and obj.lower().endswith('.csv')) or
(isinstance(obj, file_types)
and getattr(obj, 'name', '').lower().endswith('.csv'))):
load_csv(cursor, table, obj, *args, **kwds)
else:
reader = get_reader(obj, *args, **kwds)
load_data(cursor, table, reader)
self._append_obj_string(obj)
if not self._table and table_exists(cursor, table):
self._table = table
def test_pandas(self):
df = pandas.DataFrame({
'col1': (1, 2, 3),
'col2': ('a', 'b', 'c'),
})
reader = get_reader(df, index=False)
expected = [
{
'col1': 1,
'col2': 'a'
},
{
'col1': 2,
'col2': 'b'
},
{
'col1': 3,
'col2': 'c'
},
]
self.assertEqual(list(reader), expected)
def test_sql(self):
connection = sqlite3.connect(':memory:')
connection.executescript("""
CREATE TABLE mytable (
foo TEXT,
bar REAL
);
INSERT INTO mytable
VALUES ('a', 0.8),
('a', 1.2),
('b', 2.5),
('b', 3.0);
""")
reader = get_reader(connection, 'mytable')
expected = [
('foo', 'bar'),
('a', 0.8),
('a', 1.2),
('b', 2.5),
('b', 3.0),
]
self.assertEqual(list(reader), expected)
def test_unidentifiable_type(self):
with self.assertRaises(TypeError):
obj = object()
reader = get_reader(obj)