def test_convert():
table1 = (('foo', 'bar', 'baz'), ('A', 1, 2), ('B', '2', '3.4'),
(u'B', u'3', u'7.8', True), ('D', 'xyz', 9.0), ('E', None))
# test the simplest style - single field, lambda function
table2 = convert(table1, 'foo', lambda s: s.lower())
expect2 = (('foo', 'bar', 'baz'), ('a', 1, 2), ('b', '2', '3.4'),
(u'b', u'3', u'7.8', True), ('d', 'xyz', 9.0), ('e', None))
ieq(expect2, table2)
ieq(expect2, table2)
# test single field with method call
table3 = convert(table1, 'foo', 'lower')
expect3 = expect2
ieq(expect3, table3)
# test single field with method call with arguments
table4 = convert(table1, 'foo', 'replace', 'B', 'BB')
expect4 = (('foo', 'bar', 'baz'), ('A', 1, 2), ('BB', '2', '3.4'),
(u'BB', u'3', u'7.8', True), ('D', 'xyz', 9.0), ('E', None))
ieq(expect4, table4)
# test multiple fields with the same conversion
table5 = convert(table1, ('bar', 'baz'), str)
expect5 = (('foo', 'bar', 'baz'), ('A', '1', '2'), ('B', '2', '3.4'),
(u'B', u'3', u'7.8', True), ('D', 'xyz', '9.0'), ('E', 'None'))
ieq(expect5, table5)
# test convert with dictionary
table6 = convert(table1, 'foo', {'A': 'Z', 'B': 'Y'})
expect6 = (('foo', 'bar', 'baz'), ('Z', 1, 2), ('Y', '2', '3.4'),
(u'Y', u'3', u'7.8', True), ('D', 'xyz', 9.0), ('E', None))
ieq(expect6, table6)
def test_fieldconvert():
table1 = (('foo', 'bar', 'baz'), ('A', 1, 2), ('B', '2', '3.4'),
(u'B', u'3', u'7.8', True), ('D', 'xyz', 9.0), ('E', None))
# test the style where the converters functions are passed in as a
# dictionary
converters = {'foo': str, 'bar': int, 'baz': float}
table5 = convert(table1, converters, errorvalue='error')
expect5 = (
('foo', 'bar', 'baz'),
('A', 1, 2.0),
('B', 2, 3.4),
('B', 3, 7.8, True), # N.B., long rows are preserved
('D', 'error', 9.0),
('E', 'error')) # N.B., short rows are preserved
ieq(expect5, table5)
# test the style where the converters functions are added one at a time
table6 = convert(table1, errorvalue='err')
table6['foo'] = str
table6['bar'] = int
table6['baz'] = float
expect6 = (('foo', 'bar', 'baz'), ('A', 1, 2.0), ('B', 2, 3.4),
('B', 3, 7.8, True), ('D', 'err', 9.0), ('E', 'err'))
ieq(expect6, table6)
# test some different converters
table7 = convert(table1)
table7['foo'] = 'replace', 'B', 'BB'
expect7 = (('foo', 'bar', 'baz'), ('A', 1, 2), ('BB', '2', '3.4'),
(u'BB', u'3', u'7.8', True), ('D', 'xyz', 9.0), ('E', None))
ieq(expect7, table7)
# test the style where the converters functions are passed in as a list
converters = [str, int, float]
table8 = convert(table1, converters, errorvalue='error')
expect8 = (
('foo', 'bar', 'baz'),
('A', 1, 2.0),
('B', 2, 3.4),
('B', 3, 7.8, True), # N.B., long rows are preserved
('D', 'error', 9.0),
('E', 'error')) # N.B., short rows are preserved
ieq(expect8, table8)
# test the style where the converters functions are passed in as a list
converters = [str, None, float]
table9 = convert(table1, converters, errorvalue='error')
expect9 = (
('foo', 'bar', 'baz'),
('A', 1, 2.0),
('B', '2', 3.4),
('B', u'3', 7.8, True), # N.B., long rows are preserved
('D', 'xyz', 9.0),
('E', None)) # N.B., short rows are preserved
ieq(expect9, table9)
def test_convert():
table1 = (('foo', 'bar', 'baz'),
('A', 1, 2),
('B', '2', '3.4'),
(u'B', u'3', u'7.8', True),
('D', 'xyz', 9.0),
('E', None))
# test the simplest style - single field, lambda function
table2 = convert(table1, 'foo', lambda s: s.lower())
expect2 = (('foo', 'bar', 'baz'),
('a', 1, 2),
('b', '2', '3.4'),
(u'b', u'3', u'7.8', True),
('d', 'xyz', 9.0),
('e', None))
ieq(expect2, table2)
ieq(expect2, table2)
# test single field with method call
table3 = convert(table1, 'foo', 'lower')
expect3 = expect2
ieq(expect3, table3)
# test single field with method call with arguments
table4 = convert(table1, 'foo', 'replace', 'B', 'BB')
expect4 = (('foo', 'bar', 'baz'),
('A', 1, 2),
('BB', '2', '3.4'),
(u'BB', u'3', u'7.8', True),
('D', 'xyz', 9.0),
('E', None))
ieq(expect4, table4)
# test multiple fields with the same conversion
table5 = convert(table1, ('bar', 'baz'), str)
expect5 = (('foo', 'bar', 'baz'),
('A', '1', '2'),
('B', '2', '3.4'),
(u'B', u'3', u'7.8', True),
('D', 'xyz', '9.0'),
('E', 'None'))
ieq(expect5, table5)
# test convert with dictionary
table6 = convert(table1, 'foo', {'A': 'Z', 'B': 'Y'})
expect6 = (('foo', 'bar', 'baz'),
('Z', 1, 2),
('Y', '2', '3.4'),
(u'Y', u'3', u'7.8', True),
('D', 'xyz', 9.0),
('E', None))
ieq(expect6, table6)
def test_convert_where():
tbl1 = (('foo', 'bar'), ('a', 1), ('b', 2))
expect = (('foo', 'bar'), ('a', 1), ('b', 4))
actual = convert(tbl1,
'bar',
lambda v: v * 2,
where=lambda r: r.foo == 'b')
ieq(expect, actual)
ieq(expect, actual)
actual = convert(tbl1, 'bar', lambda v: v * 2, where="{foo} == 'b'")
ieq(expect, actual)
ieq(expect, actual)
def test_convert_where():
tbl1 = (('foo', 'bar'),
('a', 1),
('b', 2))
expect = (('foo', 'bar'),
('a', 1),
('b', 4))
actual = convert(tbl1, 'bar', lambda v: v*2, where=lambda r: r.foo == 'b')
ieq(expect, actual)
ieq(expect, actual)
actual = convert(tbl1, 'bar', lambda v: v*2, where="{foo} == 'b'")
ieq(expect, actual)
ieq(expect, actual)
def fetch_people_table():
planet_fetcher = CachedPlanetFetcher()
first_page_response = _fetch_people_page(1).json()
total_count = first_page_response['count']
fetched_results = first_page_response['results']
fetched_count = len(fetched_results)
remaining_count = total_count - fetched_count
remaining_pages = math.ceil(remaining_count / fetched_count)
table = fromdicts(fetched_results, header=PEOPLE_HEADER)
with ThreadPoolExecutor(max_workers=8) as executor:
response_futures = [
executor.submit(_fetch_people_page, page_number)
for page_number in range(2, 2 + remaining_pages)
]
for future in as_completed(response_futures):
page_response = future.result().json()
table = cat(
table, fromdicts(page_response['results'],
header=PEOPLE_HEADER))
table = addfields(table, [('date', lambda rec: datetime.fromisoformat(rec[
'edited'].replace('Z', '+00:00')).date().isoformat())])
table = cutout(table, 'edited')
table = convert(
table, 'homeworld', lambda homeworld_url: planet_fetcher.fetch(
homeworld_url).json()['name'])
return table
def test_convert_with_row_backwards_compat():
table = (('foo', 'bar'), (' a ', 1), (' b ', 2))
expect = (('foo', 'bar'), ('a', 1), ('b', 2))
actual = convert(table, 'foo', 'strip')
ieq(expect, actual)
def test_convert_translate():
table = (('foo', 'bar'), ('M', 12), ('F', 34), ('-', 56))
trans = {'M': 'male', 'F': 'female'}
result = convert(table, 'foo', trans)
expectation = (('foo', 'bar'), ('male', 12), ('female', 34), ('-', 56))
ieq(expectation, result)
def test_convert_with_row():
table = (('foo', 'bar'), ('a', 1), ('b', 2))
expect = (('foo', 'bar'), ('a', 'A'), ('b', 'B'))
actual = convert(table, 'bar', lambda v, row: row.foo.upper())
ieq(expect, actual)
def sub(table, field, pattern, repl, count=0, flags=0):
"""
Convenience function to convert values under the given field using a
regular expression substitution. See also :func:`re.sub`.
"""
prog = re.compile(pattern, flags)
conv = lambda v: prog.sub(repl, v, count=count)
return convert(table, field, conv)
def sub(table, field, pattern, repl, count=0, flags=0):
"""
Convenience function to convert values under the given field using a
regular expression substitution. See also :func:`re.sub`.
"""
prog = re.compile(pattern, flags)
conv = lambda v: prog.sub(repl, v, count=count)
return convert(table, field, conv)
def test_convert_with_row_backwards_compat():
table = (('foo', 'bar'),
(' a ', 1),
(' b ', 2))
expect = (('foo', 'bar'),
('a', 1),
('b', 2))
actual = convert(table, 'foo', 'strip')
ieq(expect, actual)
def test_convert_with_row():
table = (('foo', 'bar'),
('a', 1),
('b', 2))
expect = (('foo', 'bar'),
('a', 'A'),
('b', 'B'))
actual = convert(table, 'bar',
lambda v, row: row.foo.upper())
ieq(expect, actual)
def test_convert_translate():
table = (('foo', 'bar'),
('M', 12),
('F', 34),
('-', 56))
trans = {'M': 'male', 'F': 'female'}
result = convert(table, 'foo', trans)
expectation = (('foo', 'bar'),
('male', 12),
('female', 34),
('-', 56))
ieq(expectation, result)
def test_convert_empty():
table = (('foo', 'bar'),)
expect = (('foo', 'bar'),)
actual = convert(table, 'foo', int)
ieq(expect, actual)
def test_fieldconvert():
table1 = (('foo', 'bar', 'baz'),
('A', 1, 2),
('B', '2', '3.4'),
(u'B', u'3', u'7.8', True),
('D', 'xyz', 9.0),
('E', None))
# test the style where the converters functions are passed in as a
# dictionary
converters = {'foo': str, 'bar': int, 'baz': float}
table5 = convert(table1, converters, errorvalue='error')
expect5 = (('foo', 'bar', 'baz'),
('A', 1, 2.0),
('B', 2, 3.4),
('B', 3, 7.8, True), # N.B., long rows are preserved
('D', 'error', 9.0),
('E', 'error')) # N.B., short rows are preserved
ieq(expect5, table5)
# test the style where the converters functions are added one at a time
table6 = convert(table1, errorvalue='err')
table6['foo'] = str
table6['bar'] = int
table6['baz'] = float
expect6 = (('foo', 'bar', 'baz'),
('A', 1, 2.0),
('B', 2, 3.4),
('B', 3, 7.8, True),
('D', 'err', 9.0),
('E', 'err'))
ieq(expect6, table6)
# test some different converters
table7 = convert(table1)
table7['foo'] = 'replace', 'B', 'BB'
expect7 = (('foo', 'bar', 'baz'),
('A', 1, 2),
('BB', '2', '3.4'),
(u'BB', u'3', u'7.8', True),
('D', 'xyz', 9.0),
('E', None))
ieq(expect7, table7)
# test the style where the converters functions are passed in as a list
converters = [str, int, float]
table8 = convert(table1, converters, errorvalue='error')
expect8 = (('foo', 'bar', 'baz'),
('A', 1, 2.0),
('B', 2, 3.4),
('B', 3, 7.8, True), # N.B., long rows are preserved
('D', 'error', 9.0),
('E', 'error')) # N.B., short rows are preserved
ieq(expect8, table8)
# test the style where the converters functions are passed in as a list
converters = [str, None, float]
table9 = convert(table1, converters, errorvalue='error')
expect9 = (('foo', 'bar', 'baz'),
('A', 1, 2.0),
('B', '2', 3.4),
('B', u'3', 7.8, True), # N.B., long rows are preserved
('D', 'xyz', 9.0),
('E', None)) # N.B., short rows are preserved
ieq(expect9, table9)
