def test_filter(self):
query1 = Query(['col1'])
query2 = query1.filter(lambda x: x == 'a')
self.assertIsNot(query1, query2, 'should return new object')
source = Selector([('col1', 'col2'), ('a', '2'), ('b', '2')])
result = query2.execute(source)
self.assertEqual(result.fetch(), ['a'])
# No filter arg should default to bool()
source = Selector([('col1', ), (1, ), (2, ), (0, ), (3, )])
query = Query(set(['col1'])).filter() # <- No arg!
result = query.execute(source)
self.assertEqual(result.fetch(), set([1, 2, 3]))
def test_execute_query(self):
data = [['A', 'B'], ['x', 101], ['y', 202], ['z', 303]]
source = Selector(data)
# Test where-clause function.
def isodd(x):
return x % 2 == 1
result = source('A', B=isodd).fetch()
self.assertEqual(result, ['x', 'z'])
# Test replacing function.
def iseven(x):
return x % 2 == 0
result = source('A', B=iseven).fetch()
self.assertEqual(result, ['y'])
# Test callable-but-unhashable.
class IsEven(object):
__hash__ = None
def __call__(self, x):
return x % 2 == 0
unhashable_iseven = IsEven()
result = source('A', B=unhashable_iseven).fetch()
self.assertEqual(result, ['y'])
def test__copy__(self):
# Select-arg only.
query = Query(['B'])
copied = query.__copy__()
self.assertIs(copied.source, query.source)
self.assertEqual(copied.args, query.args)
self.assertEqual(copied.kwds, query.kwds)
self.assertEqual(copied._query_steps, query._query_steps)
self.assertIsNot(copied.kwds, query.kwds)
self.assertIsNot(copied._query_steps, query._query_steps)
# Select and keyword.
query = Query(['B'], C='x')
copied = query.__copy__()
self.assertIs(copied.source, query.source)
self.assertEqual(copied.args, query.args)
self.assertEqual(copied.kwds, query.kwds)
self.assertEqual(copied._query_steps, query._query_steps)
# Source, columns, and keyword.
source = Selector([('A', 'B'), (1, 2), (1, 2)])
query = Query(source, ['B'])
copied = query.__copy__()
self.assertIs(copied.source, query.source)
self.assertEqual(copied.args, query.args)
self.assertEqual(copied.kwds, query.kwds)
self.assertEqual(copied._query_steps, query._query_steps)
# Select and additional query methods.
query = Query(['B']).map(lambda x: str(x).upper())
copied = query.__copy__()
self.assertIs(copied.source, query.source)
self.assertEqual(copied.args, query.args)
self.assertEqual(copied.kwds, query.kwds)
self.assertEqual(copied._query_steps, query._query_steps)
def test_map(self):
query1 = Query(['col2'])
query2 = query1.map(int)
self.assertIsNot(query1, query2, 'should return new object')
source = Selector([('col1', 'col2'), ('a', '2'), ('b', '2')])
result = query2.execute(source)
self.assertEqual(result.fetch(), [2, 2])
def test_reduce(self):
query1 = Query(['col1'])
query2 = query1.reduce(lambda x, y: x + y)
self.assertIsNot(query1, query2, 'should return new object')
source = Selector([('col1', 'col2'), ('a', '2'), ('b', '2')])
result = query2.execute(source)
self.assertEqual(result, 'ab')
def test_fetch_datasource(self):
select = Selector([('A', 'B'), ('1', '2'), ('1', '2')])
query = Query(select, ['B'])
query._query_steps = [
('map', (int, ), {}),
('map', (lambda x: x * 2, ), {}),
('sum', (), {}),
]
result = query.fetch()
self.assertEqual(result, 8)
def test_init_with_selector(self):
source = Selector([('A', 'B'), (1, 2), (1, 2)])
query = Query(source, ['A'], B=2)
self.assertEqual(query.source, source)
self.assertEqual(query.args, (['A'], ))
self.assertEqual(query.kwds, {'B': 2})
self.assertEqual(query._query_steps, [])
with self.assertRaises(TypeError):
query = Query(None, ['foo'], bar='baz')
def test_init_with_invalid_args(self):
# Missing args.
with self.assertRaises(TypeError, msg='should require select args'):
Query()
# Bad "select" field.
source = Selector([('A', 'B'), (1, 2), (1, 2)])
with self.assertRaises(
LookupError,
msg=
'should fail immediately when fieldname conflicts with provided source'
):
query = Query(source, ['X'], B=2)
# Bad "where" field.
source = Selector([('A', 'B'), (1, 2), (1, 2)])
with self.assertRaises(
LookupError,
msg=
'should fail immediately when fieldname conflicts with provided "where" field'
):
query = Query(source, ['A'], Y=2)
def test_repr(self):
data = [['A', 'B'], ['x', 100], ['y', 200]]
# Empty selector.
select = Selector()
regex = ('^<datatest.Selector object at [^\n>]+>\n'
'Empty - contains no data\.$')
self.assertRegex(repr(select), regex)
# Data-only (no args)
source = Selector(data)
regex = ("^<datatest.Selector object at [^\n>]+>\n"
"Data from 1 source:\n"
" {0}$".format(re.escape(repr(data))))
self.assertRegex(repr(source), regex)
# Data with args and kwds.
iterable = iter(data)
source = Selector(iterable, 'foo',
bar='baz') # Args don't change repr.
regex = ('<datatest.Selector object at [^\n>]+>\n'
'Data from 1 source:\n'
' <[a-z_]+ object at [^\n>]+>')
self.assertRegex(repr(source), regex)
# Extended after instantiation.
data1 = [['A', 'B'], ['x', 100]]
data2 = [['A', 'B'], ['y', 200]]
data3 = [['A', 'B'], ['z', 300]]
source = Selector(data1)
source.load_data(data2)
source.load_data(data3)
actual_repr = repr(source)
self.assertTrue(actual_repr.startswith('<datatest.Selector object at'))
self.assertTrue(
actual_repr.endswith(">\n"
"Data from 3 sources:\n"
" [['A', 'B'], ['x', 100]]\n"
" [['A', 'B'], ['y', 200]]\n"
" [['A', 'B'], ['z', 300]]"))
def test_execute_datasource(self):
select = Selector([('A', 'B'), ('1', '2'), ('1', '2')])
query = Query(select, ['B'])
query._query_steps = [
('map', (int, ), {}),
('map', (lambda x: x * 2, ), {}),
('sum', (), {}),
]
result = query.execute()
self.assertEqual(result, 8)
query = Query(['A'])
regex = "expected 'Selector', got 'list'"
with self.assertRaisesRegex(TypeError, regex):
query.execute(['hello',
'world']) # <- Expects None or Query, not list!
def test_repr(self):
# Check "no selector" signature.
query = Query(['label1'])
regex = r"Query\(\[u?'label1'\]\)"
self.assertRegex(repr(query), regex)
# Check "no selector" with keyword string.
query = Query(['label1'], label2='x')
regex = r"Query\(\[u?'label1'\], label2='x'\)"
self.assertRegex(repr(query), regex)
# Check "no selector" with keyword list.
query = Query(['label1'], label2=['x', 'y'])
regex = r"Query\(\[u?'label1'\], label2=\[u?'x', u?'y'\]\)"
self.assertRegex(repr(query), regex)
# Check "selector-provided" signature.
select = Selector([('A', 'B'), ('x', 1), ('y', 2), ('z', 3)])
query = Query(select, ['B'])
short_repr = super(Selector, select).__repr__()
expected = "Query({0}, {1!r})".format(short_repr, ['B'])
#print(repr(query))
self.assertEqual(repr(query), expected)
# Check "from_object" signature.
query = Query.from_object([1, 2, 3])
expected = "Query.from_object([1, 2, 3])"
self.assertEqual(repr(query), expected)
# Check query steps.
query = Query(['label1']).distinct().count()
regex = r"Query\(\[u?'label1'\]\).distinct\(\).count\(\)"
self.assertRegex(repr(query), regex)
# Check query steps with function argument.
def upper(x):
return str(x.upper())
query = Query(['label1']).map(upper)
regex = r"Query\(\[u?'label1'\]\).map\(upper\)"
self.assertRegex(repr(query), regex)
# Check query steps with lambda argument.
lower = lambda x: str(x).lower()
query = Query(['label1']).map(lower)
regex = r"Query\(\[u?'label1'\]\).map\(<lambda>\)"
self.assertRegex(repr(query), regex)
def test_load_data(self):
select = Selector() # <- Empty selector.
self.assertEqual(select.fieldnames, [])
readerlike1 = [['col1', 'col2'], ['a', 1], ['b', 2]]
select.load_data(readerlike1)
self.assertEqual(select.fieldnames, ['col1', 'col2'])
readerlike2 = [['col1', 'col3'], ['c', 'x'], ['d', 'y']]
select.load_data(readerlike2)
self.assertEqual(select.fieldnames, ['col1', 'col2', 'col3'])
def test_query_objects(self):
source = Selector([('A', 'B'), ('1', '2'), ('1', '2')])
query_obj1 = source(['B'])
query_obj2 = source(['B'])
self.assertValid(query_obj1, query_obj2)
def setUp(self):
data = [['label1', 'label2', 'value'], ['a', 'x', '17'],
['a', 'x', '13'], ['a', 'y', '20'], ['a', 'z', '15'],
['b', 'z', '5'], ['b', 'y', '40'], ['b', 'x', '25']]
self.source = Selector(data)
def test_empty_selector(self):
select = Selector()
def test_fieldnames(self):
expected = ['label1', 'label2', 'value']
self.assertEqual(self.source.fieldnames, expected)
select = Selector() # <- Empty selector.
self.assertEqual(select.fieldnames, [], msg='should be empty list')
def test_repr(self):
data = [['A', 'B'], ['x', 100], ['y', 200]]
# Empty selector.
select = Selector()
self.assertEqual(repr(select), '<Selector (no data loaded)>')
# Data-only (no args)
select = Selector(data)
expected = "<Selector [['A', 'B'], ['x', 100], ['y', 200]]>"
self.assertEqual(repr(select), expected)
# Data with args (args don't affect repr)
iterable = iter(data)
select = Selector(iterable, 'foo', bar='baz')
regex = '<Selector <[a-z_]+ object at [^\n>]+>>'
self.assertRegex(repr(select), regex)
# Extended after instantiation.
select = Selector()
select.load_data([['A', 'B'], ['z', 300]])
select.load_data([['A', 'B'], ['y', 200]])
select.load_data([['A', 'B'], ['x', 100]])
expected = ("<Selector (3 sources):\n"
" [['A', 'B'], ['x', 100]]\n"
" [['A', 'B'], ['y', 200]]\n"
" [['A', 'B'], ['z', 300]]>")
self.assertEqual(repr(select), expected)
# Test long repr truncation.
select = Selector([
['xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'],
['yyyyyyyyyyyyyyyyyyyyyyyyyyyyyy'],
])
self.assertEqual(len(repr(select)), 72)
expected = "<Selector [['xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'], ['yyyyyyyyyyy...yyyyy']]>"
self.assertEqual(repr(select), expected)
def test_Query(self):
source = Selector([('A', 'B'), ('x', 1), ('y', 2)])
query = source({'A': 'B'}).apply(lambda x: next(x))
normalized = DictItems(query)
self.assertEqual(list(normalized), [('x', 1), ('y', 2)])
class TestSelector(unittest.TestCase):
def setUp(self):
data = [['label1', 'label2', 'value'], ['a', 'x', '17'],
['a', 'x', '13'], ['a', 'y', '20'], ['a', 'z', '15'],
['b', 'z', '5'], ['b', 'y', '40'], ['b', 'x', '25']]
self.source = Selector(data)
def test_empty_selector(self):
select = Selector()
def test_fieldnames(self):
expected = ['label1', 'label2', 'value']
self.assertEqual(self.source.fieldnames, expected)
select = Selector() # <- Empty selector.
self.assertEqual(select.fieldnames, [], msg='should be empty list')
def test_load_data(self):
select = Selector() # <- Empty selector.
self.assertEqual(select.fieldnames, [])
readerlike1 = [['col1', 'col2'], ['a', 1], ['b', 2]]
select.load_data(readerlike1)
self.assertEqual(select.fieldnames, ['col1', 'col2'])
readerlike2 = [['col1', 'col3'], ['c', 'x'], ['d', 'y']]
select.load_data(readerlike2)
self.assertEqual(select.fieldnames, ['col1', 'col2', 'col3'])
def test_repr(self):
data = [['A', 'B'], ['x', 100], ['y', 200]]
# Empty selector.
select = Selector()
self.assertEqual(repr(select), '<Selector (no data loaded)>')
# Data-only (no args)
select = Selector(data)
expected = "<Selector [['A', 'B'], ['x', 100], ['y', 200]]>"
self.assertEqual(repr(select), expected)
# Data with args (args don't affect repr)
iterable = iter(data)
select = Selector(iterable, 'foo', bar='baz')
regex = '<Selector <[a-z_]+ object at [^\n>]+>>'
self.assertRegex(repr(select), regex)
# Extended after instantiation.
select = Selector()
select.load_data([['A', 'B'], ['z', 300]])
select.load_data([['A', 'B'], ['y', 200]])
select.load_data([['A', 'B'], ['x', 100]])
expected = ("<Selector (3 sources):\n"
" [['A', 'B'], ['x', 100]]\n"
" [['A', 'B'], ['y', 200]]\n"
" [['A', 'B'], ['z', 300]]>")
self.assertEqual(repr(select), expected)
# Test long repr truncation.
select = Selector([
['xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'],
['yyyyyyyyyyyyyyyyyyyyyyyyyyyyyy'],
])
self.assertEqual(len(repr(select)), 72)
expected = "<Selector [['xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'], ['yyyyyyyyyyy...yyyyy']]>"
self.assertEqual(repr(select), expected)
def test_build_where_clause(self):
_build_where_clause = Selector._build_where_clause
result = _build_where_clause({'A': 'x'})
expected = ('A=?', ['x'])
self.assertEqual(result, expected)
result = _build_where_clause({'A': ['x', 'y']})
expected = ('A IN (?, ?)', ['x', 'y'])
self.assertEqual(result, expected)
userfunc = lambda x: len(x) == 1
result = _build_where_clause({'A': userfunc})
expected = ('FUNC{0}(A)'.format(id(userfunc)), [])
self.assertEqual(result, expected)
def test_execute_query(self):
data = [['A', 'B'], ['x', 101], ['y', 202], ['z', 303]]
source = Selector(data)
# Test where-clause function.
def isodd(x):
return x % 2 == 1
result = source('A', B=isodd).fetch()
self.assertEqual(result, ['x', 'z'])
# Test replacing function.
def iseven(x):
return x % 2 == 0
result = source('A', B=iseven).fetch()
self.assertEqual(result, ['y'])
# Test callable-but-unhashable.
class IsEven(object):
__hash__ = None
def __call__(self, x):
return x % 2 == 0
unhashable_iseven = IsEven()
result = source('A', B=unhashable_iseven).fetch()
self.assertEqual(result, ['y'])
def test_iter(self):
"""Test __iter__."""
result = [row for row in self.source]
expected = [
{
'label1': 'a',
'label2': 'x',
'value': '17'
},
{
'label1': 'a',
'label2': 'x',
'value': '13'
},
{
'label1': 'a',
'label2': 'y',
'value': '20'
},
{
'label1': 'a',
'label2': 'z',
'value': '15'
},
{
'label1': 'b',
'label2': 'z',
'value': '5'
},
{
'label1': 'b',
'label2': 'y',
'value': '40'
},
{
'label1': 'b',
'label2': 'x',
'value': '25'
},
]
self.assertEqual(expected, result)
def test_select_list_of_strings(self):
result = self.source._select(['label1'])
expected = ['a', 'a', 'a', 'a', 'b', 'b', 'b']
self.assertEqual(result.fetch(), expected)
def test_select_tuple_of_strings(self):
result = self.source._select(('label1', ))
expected = ('a', 'a', 'a', 'a', 'b', 'b', 'b')
self.assertEqual(result.fetch(), expected)
def test_select_set_of_strings(self):
result = self.source._select(set(['label1']))
expected = set(['a', 'b'])
self.assertEqual(result.fetch(), expected)
def test_select_field_not_found(self):
with self.assertRaises(LookupError):
result = self.source._select(['bad_field_name'])
def test_select_list_of_lists(self):
result = self.source._select([['label1']])
expected = [['a'], ['a'], ['a'], ['a'], ['b'], ['b'], ['b']]
self.assertEqual(result.fetch(), expected)
result = self.source._select([['label1', 'label2']])
expected = [['a', 'x'], ['a', 'x'], ['a', 'y'], ['a', 'z'], ['b', 'z'],
['b', 'y'], ['b', 'x']]
self.assertEqual(result.fetch(), expected)
def test_select_list_of_tuples(self):
result = self.source._select([('label1', )])
expected = [('a', ), ('a', ), ('a', ), ('a', ), ('b', ), ('b', ),
('b', )]
self.assertEqual(result.fetch(), expected)
def test_select_list_of_namedtuples(self):
namedtup = collections.namedtuple('namedtup', ['label1', 'label2'])
result = self.source._select([namedtup('label1', 'label2')])
expected = [
namedtup(label1='a', label2='x'),
namedtup(label1='a', label2='x'),
namedtup(label1='a', label2='y'),
namedtup(label1='a', label2='z'),
namedtup(label1='b', label2='z'),
namedtup(label1='b', label2='y'),
namedtup(label1='b', label2='x')
]
self.assertEqual(result.fetch(), expected)
def test_select_set_of_frozensets(self):
result = self.source._select(set([frozenset(['label1'])]))
expected = set([
frozenset(['a']),
frozenset(['a']),
frozenset(['a']),
frozenset(['a']),
frozenset(['b']),
frozenset(['b']),
frozenset(['b'])
])
self.assertEqual(result.fetch(), expected)
def test_select_dict(self):
result = self.source._select({'label1': ['value']})
expected = {
'a': ['17', '13', '20', '15'],
'b': ['5', '40', '25'],
}
self.assertEqual(result.fetch(), expected)
def test_select_dict2(self):
result = self.source._select({('label1', 'label2'): ['value']})
expected = {
('a', 'x'): ['17', '13'],
('a', 'y'): ['20'],
('a', 'z'): ['15'],
('b', 'x'): ['25'],
('b', 'y'): ['40'],
('b', 'z'): ['5'],
}
self.assertEqual(result.fetch(), expected)
def test_select_dict3(self):
result = self.source._select({('label1', 'label2'): [['value']]})
expected = {
('a', 'x'): [['17'], ['13']],
('a', 'y'): [['20']],
('a', 'z'): [['15']],
('b', 'x'): [['25']],
('b', 'y'): [['40']],
('b', 'z'): [['5']],
}
self.assertEqual(result.fetch(), expected)
def test_select_dict_with_namedtuple_keys(self):
namedtup = collections.namedtuple('namedtup', ['x', 'y'])
result = self.source._select({namedtup('label1', 'label2'): ['value']})
expected = {
namedtup(x='a', y='x'): ['17', '13'],
namedtup(x='a', y='y'): ['20'],
namedtup(x='a', y='z'): ['15'],
namedtup(x='b', y='x'): ['25'],
namedtup(x='b', y='y'): ['40'],
namedtup(x='b', y='z'): ['5'],
}
self.assertEqual(result.fetch(), expected)
def test_select_dict_with_values_container2(self):
result = self.source._select({'label1': [('label2', 'label2')]})
expected = {
'a': [('x', 'x'), ('x', 'x'), ('y', 'y'), ('z', 'z')],
'b': [('z', 'z'), ('y', 'y'), ('x', 'x')]
}
self.assertEqual(result.fetch(), expected)
result = self.source._select({'label1': [set(['label2', 'label2'])]})
expected = {
'a': [set(['x']), set(['x']),
set(['y']), set(['z'])],
'b': [set(['z']), set(['y']), set(['x'])],
}
self.assertEqual(result.fetch(), expected)
def test_select_alternate_mapping_type(self):
class CustomDict(dict):
pass
result = self.source._select(CustomDict({'label1': ['value']}))
result = result.fetch()
expected = {
'a': ['17', '13', '20', '15'],
'b': ['5', '40', '25'],
}
self.assertIsInstance(result, CustomDict)
self.assertEqual(result, expected)
def test_select_distinct(self):
result = self.source._select_distinct(['label1'])
expected = ['a', 'b']
self.assertEqual(list(result), expected)
result = self.source._select_distinct({'label1': ['label2']})
result = result.fetch()
expected = {'a': ['x', 'y', 'z'], 'b': ['z', 'y', 'x']}
self.assertIsInstance(result, dict)
# Sort values for SQLite versions earlier than 3.7.12
if (3, 7, 12) > sqlite3.sqlite_version_info:
sortvalues = lambda x: dict((k, sorted(v)) for k, v in x.items())
result = sortvalues(result)
expected = sortvalues(expected)
self.assertEqual(result, expected)
def test_select_aggregate(self):
# Not grouped, single result.
result = self.source._select_aggregate('COUNT', ['label2'])
self.assertEqual(result, 7)
# Not grouped, single result as set.
result = self.source._select_aggregate('COUNT', set(['label2']))
self.assertEqual(result, 3)
# Not grouped, multiple results.
result = self.source._select_aggregate('SUM', [['value', 'value']])
self.assertEqual(result, [135, 135])
# Simple group by (grouped by keys).
result = self.source._select_aggregate('SUM', {'label1': ['value']})
self.assertIsInstance(result, Result)
expected = {
'a': 65,
'b': 70,
}
self.assertEqual(result.fetch(), expected)
# Composite value.
result = self.source._select_aggregate(
'SUM', {'label1': [('value', 'value')]})
expected = {
'a': (65, 65),
'b': (70, 70),
}
self.assertEqual(dict(result), expected)
# Composite key and composite value.
result = self.source._select_aggregate(
'SUM', {('label1', 'label1'): [['value', 'value']]})
expected = {
('a', 'a'): [65, 65],
('b', 'b'): [70, 70],
}
self.assertEqual(dict(result), expected)
def test_call(self):
query = self.source(['label1'])
expected = ['a', 'a', 'a', 'a', 'b', 'b', 'b']
self.assertIsInstance(query, Query)
self.assertEqual(query.fetch(), expected)
query = self.source([('label1', 'label2')])
expected = [('a', 'x'), ('a', 'x'), ('a', 'y'), ('a', 'z'), ('b', 'z'),
('b', 'y'), ('b', 'x')]
self.assertIsInstance(query, Query)
self.assertEqual(query.fetch(), expected)
query = self.source([set(['label1', 'label2'])])
expected = [
set(['a', 'x']),
set(['a', 'x']),
set(['a', 'y']),
set(['a', 'z']),
set(['b', 'z']),
set(['b', 'y']),
set(['b', 'x'])
]
self.assertIsInstance(query, Query)
self.assertEqual(query.fetch(), expected)
query = self.source({'label1': ['label2']})
expected = {'a': ['x', 'x', 'y', 'z'], 'b': ['z', 'y', 'x']}
self.assertIsInstance(query, Query)
self.assertEqual(query.fetch(), expected)