def test_records_shortrows():
"""Test the records function on a table with short rows."""
table = (('foo', 'bar'), ('a', 1), ('b',))
actual = records(table)
expect = ({'foo': 'a', 'bar': 1}, {'foo': 'b', 'bar': None})
iassertequal(expect, actual)
def test_data():
"""Test the data function."""
table = (('foo', 'bar'), ('a', 1), ('b', 2))
actual = data(table)
expect = (('a', 1), ('b', 2))
iassertequal(expect, actual)
def test_unique():
table = (('foo', 'bar', 'baz'),
('A', 1, 2),
('B', '2', '3.4'),
('D', 'xyz', 9.0),
('B', u'3', u'7.8', True),
('B', '2', 42),
('E', None),
('D', 4, 12.3))
f1 = NamedTemporaryFile(delete=False)
f2 = NamedTemporaryFile(delete=False)
p = sort('foo')
q = p.pipe(unique('foo'))
q.pipe(topickle(f1.name))
q.pipe('remainder', topickle(f2.name))
p.push(table)
expectation = (('foo', 'bar', 'baz'),
('A', 1, 2),
('E', None))
iassertequal(expectation, frompickle(f1.name))
exremainder = (('foo', 'bar', 'baz'),
('B', '2', '3.4'),
('B', u'3', u'7.8', True),
('B', '2', 42),
('D', 'xyz', 9.0),
('D', 4, 12.3))
iassertequal(exremainder, frompickle(f2.name))
def test_records():
"""Test the records function."""
table = (('foo', 'bar'), ('a', 1), ('b', 2))
actual = records(table)
expect = ({'foo': 'a', 'bar': 1}, {'foo': 'b', 'bar': 2})
iassertequal(expect, actual)
def test_valuecounts():
"""Test the valuecounts function."""
table = (('foo', 'bar'), ('a', 1), ('b', 2), ('b', 7))
actual = valuecounts(table, 'foo')
expect = (('value', 'count', 'frequency'), ('b', 2, 2./3), ('a', 1, 1./3))
iassertequal(expect, actual)
def test_totsv():
t = [('fruit', 'city', 'sales'), ('orange', 'London', '12'),
('banana', 'London', '42'), ('orange', 'Paris', '31'),
('banana', 'Amsterdam', '74'), ('kiwi', 'Berlin', '55')]
f = NamedTemporaryFile(delete=False)
p = totsv(f.name)
p.push(t)
iassertequal(t, fromtsv(f.name))
def test_topickle():
t = [('fruit', 'city', 'sales'), ('orange', 'London', 12),
('banana', 'London', 42), ('orange', 'Paris', 31),
('banana', 'Amsterdam', 74), ('kiwi', 'Berlin', 55)]
f = NamedTemporaryFile(delete=False)
p = topickle(f.name)
p.push(t)
iassertequal(t, frompickle(f.name))
def test_parsecounts():
table = (('foo', 'bar', 'baz'),
('A', 'aaa', 2),
('B', u'2', '3.4'),
(u'B', u'3', u'7.8', True),
('D', '3.7', 9.0),
('E', 42))
actual = parsecounts(table, 'bar')
expect = (('type', 'count', 'errors'), ('float', 3, 1), ('int', 2, 2))
iassertequal(expect, actual)
def test_sort():
table = (('foo', 'bar'), ('C', '2'), ('A', '9'), ('A', '6'), ('F', '1'),
('D', '10'))
f = NamedTemporaryFile(delete=False)
p = sort('foo')
p.pipe(topickle(f.name))
p.push(table)
expectation = (('foo', 'bar'), ('A', '9'), ('A', '6'), ('C', '2'),
('D', '10'), ('F', '1'))
iassertequal(expectation, frompickle(f.name))
def test_rowlengths():
"""Test the rowlengths function."""
table = (('foo', 'bar', 'baz'),
('A', 1, 2),
('B', '2', '3.4'),
(u'B', u'3', u'7.8', True),
('D', 'xyz', 9.0),
('E', None),
('F', 9))
actual = rowlengths(table)
expect = (('length', 'count'), (3, 3), (2, 2), (4, 1))
iassertequal(expect, actual)
def test_topickle():
t = [('fruit', 'city', 'sales'),
('orange', 'London', 12),
('banana', 'London', 42),
('orange', 'Paris', 31),
('banana', 'Amsterdam', 74),
('kiwi', 'Berlin', 55)]
f = NamedTemporaryFile(delete=False)
p = topickle(f.name)
p.push(t)
iassertequal(t, frompickle(f.name))
def test_totsv():
t = [('fruit', 'city', 'sales'),
('orange', 'London', '12'),
('banana', 'London', '42'),
('orange', 'Paris', '31'),
('banana', 'Amsterdam', '74'),
('kiwi', 'Berlin', '55')]
f = NamedTemporaryFile(delete=False)
p = totsv(f.name)
p.push(t)
iassertequal(t, fromtsv(f.name))
def test_sort_buffered():
table = (('foo', 'bar'), ('C', '2'), ('A', '9'), ('A', '6'), ('F', '1'),
('D', '10'))
f = NamedTemporaryFile(delete=False)
p = sort('foo', buffersize=2)
p.pipe(topickle(f.name))
p.push(table)
expectation = (('foo', 'bar'), ('A', '9'), ('A', '6'), ('C', '2'),
('D', '10'), ('F', '1'))
actual = frompickle(f.name)
print list(actual)
iassertequal(expectation, actual)
def test_operator_overload():
table = (('foo', 'bar', 'baz'), ('A', 1, 2), ('B', '2', '3.4'),
('D', 'xyz', 9.0), ('B', u'3', u'7.8', True), ('B', '2', 42),
('E', None), ('D', 4, 12.3))
f1 = NamedTemporaryFile(delete=False)
p = sort('foo')
p | duplicates('foo') | topickle(f1.name)
p.push(table)
expectation = (('foo', 'bar', 'baz'), ('B', '2', '3.4'), ('B', u'3',
u'7.8', True),
('B', '2', 42), ('D', 'xyz', 9.0), ('D', 4, 12.3))
iassertequal(expectation, frompickle(f1.name))
def test_valuecounts_shortrows():
"""Test the valuecounts function with short rows."""
table = (('foo', 'bar'),
('a', True),
('x', True),
('b',),
('b', True),
('c', False),
('z', False))
actual = valuecounts(table, 'bar')
expect = (('value', 'count', 'frequency'),
(True, 3, 3./6),
(False, 2, 2./6),
(None, 1, 1./6))
iassertequal(expect, actual)
def test_basics():
t1 = (('foo', 'bar'), ('A', 1), ('B', 2))
w1 = FluentWrapper(t1)
eq_(('foo', 'bar'), w1.header())
eq_(petl.header(w1), w1.header())
iassertequal((('A', 1), ('B', 2)), w1.data())
iassertequal(petl.data(w1), w1.data())
w2 = w1.cut('bar', 'foo')
expect2 = (('bar', 'foo'), (1, 'A'), (2, 'B'))
iassertequal(expect2, w2)
iassertequal(petl.cut(w1, 'bar', 'foo'), w2)
w3 = w1.cut('bar', 'foo').cut('foo', 'bar')
iassertequal(t1, w3)
def test_unique():
table = (('foo', 'bar', 'baz'), ('A', 1, 2), ('B', '2', '3.4'),
('D', 'xyz', 9.0), ('B', u'3', u'7.8', True), ('B', '2', 42),
('E', None), ('D', 4, 12.3))
f1 = NamedTemporaryFile(delete=False)
f2 = NamedTemporaryFile(delete=False)
p = sort('foo')
q = p.pipe(unique('foo'))
q.pipe(topickle(f1.name))
q.pipe('remainder', topickle(f2.name))
p.push(table)
expectation = (('foo', 'bar', 'baz'), ('A', 1, 2), ('E', None))
iassertequal(expectation, frompickle(f1.name))
exremainder = (('foo', 'bar', 'baz'), ('B', '2', '3.4'), ('B', u'3',
u'7.8', True),
('B', '2', 42), ('D', 'xyz', 9.0), ('D', 4, 12.3))
iassertequal(exremainder, frompickle(f2.name))
def test_sort():
table = (('foo', 'bar'),
('C', '2'),
('A', '9'),
('A', '6'),
('F', '1'),
('D', '10'))
f = NamedTemporaryFile(delete=False)
p = sort('foo')
p.pipe(topickle(f.name))
p.push(table)
expectation = (('foo', 'bar'),
('A', '9'),
('A', '6'),
('C', '2'),
('D', '10'),
('F', '1'))
iassertequal(expectation, frompickle(f.name))
def test_basics():
t1 = (('foo', 'bar'),
('A', 1),
('B', 2))
w1 = FluentWrapper(t1)
eq_(('foo', 'bar'), w1.header())
eq_(petl.header(w1), w1.header())
iassertequal((('A', 1), ('B', 2)), w1.data())
iassertequal(petl.data(w1), w1.data())
w2 = w1.cut('bar', 'foo')
expect2 = (('bar', 'foo'),
(1, 'A'),
(2, 'B'))
iassertequal(expect2, w2)
iassertequal(petl.cut(w1, 'bar', 'foo'), w2)
w3 = w1.cut('bar', 'foo').cut('foo', 'bar')
iassertequal(t1, w3)
def test_diff():
tablea = (('foo', 'bar', 'baz'), ('A', 1, True), ('B', 2, False),
('C', 7, False), ('C', 9, True))
tableb = (('x', 'y', 'z'), ('A', 9, False), ('B', 2, False),
('B', 3, True), ('C', 9, True))
aminusb = (('foo', 'bar', 'baz'), ('A', 1, True), ('C', 7, False))
bminusa = (('foo', 'bar', 'baz'), ('A', 9, False), ('B', 3, True))
both = (('foo', 'bar', 'baz'), ('B', 2, False), ('C', 9, True))
f1 = NamedTemporaryFile(delete=False)
f2 = NamedTemporaryFile(delete=False)
f3 = NamedTemporaryFile(delete=False)
p = diff()
p.pipe('+', topickle(f1.name))
p.pipe('-', topickle(f2.name))
p.pipe(topickle(f3.name))
p.push(tablea, tableb)
added, subtracted, common = frompickle(f1.name), frompickle(
f2.name), frompickle(f3.name)
iassertequal(bminusa, added)
iassertequal(aminusb, subtracted)
iassertequal(both, common)
def test_sort_buffered():
table = (('foo', 'bar'),
('C', '2'),
('A', '9'),
('A', '6'),
('F', '1'),
('D', '10'))
f = NamedTemporaryFile(delete=False)
p = sort('foo', buffersize=2)
p.pipe(topickle(f.name))
p.push(table)
expectation = (('foo', 'bar'),
('A', '9'),
('A', '6'),
('C', '2'),
('D', '10'),
('F', '1'))
actual = frompickle(f.name)
print list(actual)
iassertequal(expectation, actual)
def test_stringpatterns():
table = (('foo', 'bar'),
('Mr. Foo', '123-1254'),
('Mrs. Bar', '234-1123'),
('Mr. Spo', '123-1254'),
(u'Mr. Baz', u'321 1434'),
(u'Mrs. Baz', u'321 1434'),
('Mr. Quux', '123-1254-XX'))
actual = stringpatterns(table, 'foo')
expect = (('pattern', 'count', 'frequency'),
('Aa. Aaa', 3, 3./6),
('Aaa. Aaa', 2, 2./6),
('Aa. Aaaa', 1, 1./6))
iassertequal(expect, actual)
actual = stringpatterns(table, 'bar')
expect = (('pattern', 'count', 'frequency'),
('999-9999', 3, 3./6),
('999 9999', 2, 2./6),
('999-9999-AA', 1, 1./6))
iassertequal(expect, actual)
def test_operator_overload():
table = (('foo', 'bar', 'baz'),
('A', 1, 2),
('B', '2', '3.4'),
('D', 'xyz', 9.0),
('B', u'3', u'7.8', True),
('B', '2', 42),
('E', None),
('D', 4, 12.3))
f1 = NamedTemporaryFile(delete=False)
p = sort('foo')
p | duplicates('foo') | topickle(f1.name)
p.push(table)
expectation = (('foo', 'bar', 'baz'),
('B', '2', '3.4'),
('B', u'3', u'7.8', True),
('B', '2', 42),
('D', 'xyz', 9.0),
('D', 4, 12.3))
iassertequal(expectation, frompickle(f1.name))
def test_itervalues():
"""Test the itervalues function."""
table = (('foo', 'bar', 'baz'),
('a', 1, True),
('b', 2),
('b', 7, False))
actual = itervalues(table, 'foo')
expect = ('a', 'b', 'b')
iassertequal(expect, actual)
actual = itervalues(table, 'bar')
expect = (1, 2, 7)
iassertequal(expect, actual)
actual = itervalues(table, ('foo', 'bar'))
expect = (('a', 1), ('b', 2), ('b', 7))
iassertequal(expect, actual)
actual = itervalues(table, 'baz')
expect = (True, None, False)
iassertequal(expect, actual)
def test_typecounts():
table = (('foo', 'bar', 'baz'),
('A', 1, 2.),
('B', u'2', 3.4),
(u'B', u'3', 7.8, True),
('D', u'xyz', 9.0),
('E', 42))
actual = typecounts(table, 'foo')
expect = (('type', 'count', 'frequency'), ('str', 4, 4./5), ('unicode', 1, 1./5))
iassertequal(expect, actual)
actual = typecounts(table, 'bar')
expect = (('type', 'count', 'frequency'), ('unicode', 3, 3./5), ('int', 2, 2./5))
iassertequal(expect, actual)
actual = typecounts(table, 'baz')
expect = (('type', 'count', 'frequency'), ('float', 4, 4./5), ('NoneType', 1, 1./5))
iassertequal(expect, actual)
def test_diff():
tablea = (('foo', 'bar', 'baz'),
('A', 1, True),
('B', 2, False),
('C', 7, False),
('C', 9, True))
tableb = (('x', 'y', 'z'),
('A', 9, False),
('B', 2, False),
('B', 3, True),
('C', 9, True))
aminusb = (('foo', 'bar', 'baz'),
('A', 1, True),
('C', 7, False))
bminusa = (('foo', 'bar', 'baz'),
('A', 9, False),
('B', 3, True))
both = (('foo', 'bar', 'baz'),
('B', 2, False),
('C', 9, True))
f1 = NamedTemporaryFile(delete=False)
f2 = NamedTemporaryFile(delete=False)
f3 = NamedTemporaryFile(delete=False)
p = diff()
p.pipe('+', topickle(f1.name))
p.pipe('-', topickle(f2.name))
p.pipe(topickle(f3.name))
p.push(tablea, tableb)
added, subtracted, common = frompickle(f1.name), frompickle(f2.name), frompickle(f3.name)
iassertequal(bminusa, added)
iassertequal(aminusb, subtracted)
iassertequal(both, common)
def test_partition():
t = [('fruit', 'city', 'sales'), ('orange', 'London', 12),
('banana', 'London', 42), ('orange', 'Paris', 31),
('banana', 'Amsterdam', 74), ('kiwi', 'Berlin', 55)]
p = partition('fruit')
p.pipe('orange', tocsv('oranges.csv'))
p.pipe('banana', tocsv('bananas.csv'))
p.push(t)
oranges_expected = [('fruit', 'city', 'sales'), ('orange', 'London', '12'),
('orange', 'Paris', '31')]
bananas_expected = [('fruit', 'city', 'sales'), ('banana', 'London', '42'),
('banana', 'Amsterdam', '74')]
oranges_actual = fromcsv('oranges.csv')
bananas_actual = fromcsv('bananas.csv')
iassertequal(oranges_expected, oranges_actual)
iassertequal(bananas_expected, bananas_actual)
# alternative syntax
p = partition('fruit')
p | ('orange', tocsv('oranges.csv'))
p | ('banana', tocsv('bananas.csv'))
p.push(t)
iassertequal(oranges_expected, oranges_actual)
iassertequal(bananas_expected, bananas_actual)
# test with callable discriminator
p = partition(lambda row: row['sales'] > 40)
p | (True, tocsv('high.csv'))
p | (False, tocsv('low.csv'))
p.push(t)
high_expected = [('fruit', 'city', 'sales'), ('banana', 'London', '42'),
('banana', 'Amsterdam', '74'), ('kiwi', 'Berlin', '55')]
low_expected = [('fruit', 'city', 'sales'), ('orange', 'London', '12'),
('orange', 'Paris', '31')]
high_actual = fromcsv('high.csv')
low_actual = fromcsv('low.csv')
iassertequal(high_expected, high_actual)
iassertequal(low_expected, low_actual)
def test_partition():
t = [('fruit', 'city', 'sales'),
('orange', 'London', 12),
('banana', 'London', 42),
('orange', 'Paris', 31),
('banana', 'Amsterdam', 74),
('kiwi', 'Berlin', 55)]
p = partition('fruit')
p.pipe('orange', tocsv('oranges.csv'))
p.pipe('banana', tocsv('bananas.csv'))
p.push(t)
oranges_expected = [('fruit', 'city', 'sales'),
('orange', 'London', '12'),
('orange', 'Paris', '31')]
bananas_expected = [('fruit', 'city', 'sales'),
('banana', 'London', '42'),
('banana', 'Amsterdam', '74')]
oranges_actual = fromcsv('oranges.csv')
bananas_actual = fromcsv('bananas.csv')
iassertequal(oranges_expected, oranges_actual)
iassertequal(bananas_expected, bananas_actual)
# alternative syntax
p = partition('fruit')
p | ('orange', tocsv('oranges.csv'))
p | ('banana', tocsv('bananas.csv'))
p.push(t)
iassertequal(oranges_expected, oranges_actual)
iassertequal(bananas_expected, bananas_actual)
# test with callable discriminator
p = partition(lambda row: row['sales'] > 40)
p | (True, tocsv('high.csv'))
p | (False, tocsv('low.csv'))
p.push(t)
high_expected = [('fruit', 'city', 'sales'),
('banana', 'London', '42'),
('banana', 'Amsterdam', '74'),
('kiwi', 'Berlin', '55')]
low_expected = [('fruit', 'city', 'sales'),
('orange', 'London', '12'),
('orange', 'Paris', '31')]
high_actual = fromcsv('high.csv')
low_actual = fromcsv('low.csv')
iassertequal(high_expected, high_actual)
iassertequal(low_expected, low_actual)