def fetch_people_table():
planet_fetcher = CachedPlanetFetcher()
first_page_response = _fetch_people_page(1).json()
total_count = first_page_response['count']
fetched_results = first_page_response['results']
fetched_count = len(fetched_results)
remaining_count = total_count - fetched_count
remaining_pages = math.ceil(remaining_count / fetched_count)
table = fromdicts(fetched_results, header=PEOPLE_HEADER)
with ThreadPoolExecutor(max_workers=8) as executor:
response_futures = [
executor.submit(_fetch_people_page, page_number)
for page_number in range(2, 2 + remaining_pages)
]
for future in as_completed(response_futures):
page_response = future.result().json()
table = cat(
table, fromdicts(page_response['results'],
header=PEOPLE_HEADER))
table = addfields(table, [('date', lambda rec: datetime.fromisoformat(rec[
'edited'].replace('Z', '+00:00')).date().isoformat())])
table = cutout(table, 'edited')
table = convert(
table, 'homeworld', lambda homeworld_url: planet_fetcher.fetch(
homeworld_url).json()['name'])
return table
def test_fromdicts_3():
data = [{'foo': 'a', 'bar': 1},
{'foo': 'b'},
{'foo': 'c', 'bar': 2, 'baz': True}]
actual = fromdicts(data, header=['foo', 'bar'])
expect = (('foo', 'bar'),
('a', 1),
('b', None),
('c', 2))
ieq(expect, actual)
ieq(expect, actual) # verify can iterate twice
def test_fromdicts_3():
data = [{"foo": "a", "bar": 1},
{"foo": "b"},
{"foo": "c", "bar": 2, "baz": True}]
actual = fromdicts(data, header=['foo', 'bar'])
expect = (('foo', 'bar'),
('a', 1),
('b', None),
('c', 2))
ieq(expect, actual)
ieq(expect, actual) # verify can iterate twice
def test_fromdicts_2():
data = [{"foo": "a", "bar": 1},
{"foo": "b"},
{"foo": "c", "bar": 2, "baz": True}]
actual = fromdicts(data)
expect = (('foo', 'bar', 'baz'),
('a', 1, None),
('b', None, None),
('c', 2, True))
ieq(expect, actual)
ieq(expect, actual) # verify can iterate twice
def test_fromdicts_1():
data = [{"foo": "a", "bar": 1},
{"foo": "b", "bar": 2},
{"foo": "c", "bar": 2}]
actual = fromdicts(data)
expect = (('foo', 'bar'),
('a', 1),
('b', 2),
('c', 2))
ieq(expect, actual)
ieq(expect, actual) # verify can iterate twice
def test_fromdicts_2():
data = [{'foo': 'a', 'bar': 1},
{'foo': 'b'},
{'foo': 'c', 'bar': 2, 'baz': True}]
actual = fromdicts(data)
# N.B., fields come out in sorted order
expect = (('bar', 'baz', 'foo'),
(1, None, 'a'),
(None, None, 'b'),
(2, True, 'c'))
ieq(expect, actual)
ieq(expect, actual) # verify can iterate twice
def test_fromdicts_3():
data = [{
"foo": "a",
"bar": 1
}, {
"foo": "b"
}, {
"foo": "c",
"bar": 2,
"baz": True
}]
actual = fromdicts(data, header=['foo', 'bar'])
expect = (('foo', 'bar'), ('a', 1), ('b', None), ('c', 2))
ieq(expect, actual)
ieq(expect, actual) # verify can iterate twice
def test_fromdicts_1():
data = [{
"foo": "a",
"bar": 1
}, {
"foo": "b",
"bar": 2
}, {
"foo": "c",
"bar": 2
}]
actual = fromdicts(data)
expect = (('foo', 'bar'), ('a', 1), ('b', 2), ('c', 2))
ieq(expect, actual)
ieq(expect, actual) # verify can iterate twice
def test_fromdicts_2():
data = [{
"foo": "a",
"bar": 1
}, {
"foo": "b"
}, {
"foo": "c",
"bar": 2,
"baz": True
}]
actual = fromdicts(data)
expect = (('foo', 'bar', 'baz'), ('a', 1, None), ('b', None, None),
('c', 2, True))
ieq(expect, actual)
ieq(expect, actual) # verify can iterate twice