def test_convert(): table1 = (('foo', 'bar', 'baz'), ('A', 1, 2), ('B', '2', '3.4'), (u'B', u'3', u'7.8', True), ('D', 'xyz', 9.0), ('E', None)) # test the simplest style - single field, lambda function table2 = convert(table1, 'foo', lambda s: s.lower()) expect2 = (('foo', 'bar', 'baz'), ('a', 1, 2), ('b', '2', '3.4'), (u'b', u'3', u'7.8', True), ('d', 'xyz', 9.0), ('e', None)) ieq(expect2, table2) ieq(expect2, table2) # test single field with method call table3 = convert(table1, 'foo', 'lower') expect3 = expect2 ieq(expect3, table3) # test single field with method call with arguments table4 = convert(table1, 'foo', 'replace', 'B', 'BB') expect4 = (('foo', 'bar', 'baz'), ('A', 1, 2), ('BB', '2', '3.4'), (u'BB', u'3', u'7.8', True), ('D', 'xyz', 9.0), ('E', None)) ieq(expect4, table4) # test multiple fields with the same conversion table5 = convert(table1, ('bar', 'baz'), str) expect5 = (('foo', 'bar', 'baz'), ('A', '1', '2'), ('B', '2', '3.4'), (u'B', u'3', u'7.8', True), ('D', 'xyz', '9.0'), ('E', 'None')) ieq(expect5, table5) # test convert with dictionary table6 = convert(table1, 'foo', {'A': 'Z', 'B': 'Y'}) expect6 = (('foo', 'bar', 'baz'), ('Z', 1, 2), ('Y', '2', '3.4'), (u'Y', u'3', u'7.8', True), ('D', 'xyz', 9.0), ('E', None)) ieq(expect6, table6)
def test_fieldconvert(): table1 = (('foo', 'bar', 'baz'), ('A', 1, 2), ('B', '2', '3.4'), (u'B', u'3', u'7.8', True), ('D', 'xyz', 9.0), ('E', None)) # test the style where the converters functions are passed in as a # dictionary converters = {'foo': str, 'bar': int, 'baz': float} table5 = convert(table1, converters, errorvalue='error') expect5 = ( ('foo', 'bar', 'baz'), ('A', 1, 2.0), ('B', 2, 3.4), ('B', 3, 7.8, True), # N.B., long rows are preserved ('D', 'error', 9.0), ('E', 'error')) # N.B., short rows are preserved ieq(expect5, table5) # test the style where the converters functions are added one at a time table6 = convert(table1, errorvalue='err') table6['foo'] = str table6['bar'] = int table6['baz'] = float expect6 = (('foo', 'bar', 'baz'), ('A', 1, 2.0), ('B', 2, 3.4), ('B', 3, 7.8, True), ('D', 'err', 9.0), ('E', 'err')) ieq(expect6, table6) # test some different converters table7 = convert(table1) table7['foo'] = 'replace', 'B', 'BB' expect7 = (('foo', 'bar', 'baz'), ('A', 1, 2), ('BB', '2', '3.4'), (u'BB', u'3', u'7.8', True), ('D', 'xyz', 9.0), ('E', None)) ieq(expect7, table7) # test the style where the converters functions are passed in as a list converters = [str, int, float] table8 = convert(table1, converters, errorvalue='error') expect8 = ( ('foo', 'bar', 'baz'), ('A', 1, 2.0), ('B', 2, 3.4), ('B', 3, 7.8, True), # N.B., long rows are preserved ('D', 'error', 9.0), ('E', 'error')) # N.B., short rows are preserved ieq(expect8, table8) # test the style where the converters functions are passed in as a list converters = [str, None, float] table9 = convert(table1, converters, errorvalue='error') expect9 = ( ('foo', 'bar', 'baz'), ('A', 1, 2.0), ('B', '2', 3.4), ('B', u'3', 7.8, True), # N.B., long rows are preserved ('D', 'xyz', 9.0), ('E', None)) # N.B., short rows are preserved ieq(expect9, table9)
def test_convert_where(): tbl1 = (('foo', 'bar'), ('a', 1), ('b', 2)) expect = (('foo', 'bar'), ('a', 1), ('b', 4)) actual = convert(tbl1, 'bar', lambda v: v * 2, where=lambda r: r.foo == 'b') ieq(expect, actual) ieq(expect, actual) actual = convert(tbl1, 'bar', lambda v: v * 2, where="{foo} == 'b'") ieq(expect, actual) ieq(expect, actual)
def test_convert_where(): tbl1 = (('foo', 'bar'), ('a', 1), ('b', 2)) expect = (('foo', 'bar'), ('a', 1), ('b', 4)) actual = convert(tbl1, 'bar', lambda v: v*2, where=lambda r: r.foo == 'b') ieq(expect, actual) ieq(expect, actual) actual = convert(tbl1, 'bar', lambda v: v*2, where="{foo} == 'b'") ieq(expect, actual) ieq(expect, actual)
def fetch_people_table(): planet_fetcher = CachedPlanetFetcher() first_page_response = _fetch_people_page(1).json() total_count = first_page_response['count'] fetched_results = first_page_response['results'] fetched_count = len(fetched_results) remaining_count = total_count - fetched_count remaining_pages = math.ceil(remaining_count / fetched_count) table = fromdicts(fetched_results, header=PEOPLE_HEADER) with ThreadPoolExecutor(max_workers=8) as executor: response_futures = [ executor.submit(_fetch_people_page, page_number) for page_number in range(2, 2 + remaining_pages) ] for future in as_completed(response_futures): page_response = future.result().json() table = cat( table, fromdicts(page_response['results'], header=PEOPLE_HEADER)) table = addfields(table, [('date', lambda rec: datetime.fromisoformat(rec[ 'edited'].replace('Z', '+00:00')).date().isoformat())]) table = cutout(table, 'edited') table = convert( table, 'homeworld', lambda homeworld_url: planet_fetcher.fetch( homeworld_url).json()['name']) return table
def test_convert_with_row_backwards_compat(): table = (('foo', 'bar'), (' a ', 1), (' b ', 2)) expect = (('foo', 'bar'), ('a', 1), ('b', 2)) actual = convert(table, 'foo', 'strip') ieq(expect, actual)
def test_convert_translate(): table = (('foo', 'bar'), ('M', 12), ('F', 34), ('-', 56)) trans = {'M': 'male', 'F': 'female'} result = convert(table, 'foo', trans) expectation = (('foo', 'bar'), ('male', 12), ('female', 34), ('-', 56)) ieq(expectation, result)
def test_convert_with_row(): table = (('foo', 'bar'), ('a', 1), ('b', 2)) expect = (('foo', 'bar'), ('a', 'A'), ('b', 'B')) actual = convert(table, 'bar', lambda v, row: row.foo.upper()) ieq(expect, actual)
def sub(table, field, pattern, repl, count=0, flags=0): """ Convenience function to convert values under the given field using a regular expression substitution. See also :func:`re.sub`. """ prog = re.compile(pattern, flags) conv = lambda v: prog.sub(repl, v, count=count) return convert(table, field, conv)
def test_convert_empty(): table = (('foo', 'bar'),) expect = (('foo', 'bar'),) actual = convert(table, 'foo', int) ieq(expect, actual)
def test_fieldconvert(): table1 = (('foo', 'bar', 'baz'), ('A', 1, 2), ('B', '2', '3.4'), (u'B', u'3', u'7.8', True), ('D', 'xyz', 9.0), ('E', None)) # test the style where the converters functions are passed in as a # dictionary converters = {'foo': str, 'bar': int, 'baz': float} table5 = convert(table1, converters, errorvalue='error') expect5 = (('foo', 'bar', 'baz'), ('A', 1, 2.0), ('B', 2, 3.4), ('B', 3, 7.8, True), # N.B., long rows are preserved ('D', 'error', 9.0), ('E', 'error')) # N.B., short rows are preserved ieq(expect5, table5) # test the style where the converters functions are added one at a time table6 = convert(table1, errorvalue='err') table6['foo'] = str table6['bar'] = int table6['baz'] = float expect6 = (('foo', 'bar', 'baz'), ('A', 1, 2.0), ('B', 2, 3.4), ('B', 3, 7.8, True), ('D', 'err', 9.0), ('E', 'err')) ieq(expect6, table6) # test some different converters table7 = convert(table1) table7['foo'] = 'replace', 'B', 'BB' expect7 = (('foo', 'bar', 'baz'), ('A', 1, 2), ('BB', '2', '3.4'), (u'BB', u'3', u'7.8', True), ('D', 'xyz', 9.0), ('E', None)) ieq(expect7, table7) # test the style where the converters functions are passed in as a list converters = [str, int, float] table8 = convert(table1, converters, errorvalue='error') expect8 = (('foo', 'bar', 'baz'), ('A', 1, 2.0), ('B', 2, 3.4), ('B', 3, 7.8, True), # N.B., long rows are preserved ('D', 'error', 9.0), ('E', 'error')) # N.B., short rows are preserved ieq(expect8, table8) # test the style where the converters functions are passed in as a list converters = [str, None, float] table9 = convert(table1, converters, errorvalue='error') expect9 = (('foo', 'bar', 'baz'), ('A', 1, 2.0), ('B', '2', 3.4), ('B', u'3', 7.8, True), # N.B., long rows are preserved ('D', 'xyz', 9.0), ('E', None)) # N.B., short rows are preserved ieq(expect9, table9)