Example #1
0
def fetch_people_table():
    planet_fetcher = CachedPlanetFetcher()

    first_page_response = _fetch_people_page(1).json()
    total_count = first_page_response['count']
    fetched_results = first_page_response['results']
    fetched_count = len(fetched_results)
    remaining_count = total_count - fetched_count
    remaining_pages = math.ceil(remaining_count / fetched_count)

    table = fromdicts(fetched_results, header=PEOPLE_HEADER)

    with ThreadPoolExecutor(max_workers=8) as executor:
        response_futures = [
            executor.submit(_fetch_people_page, page_number)
            for page_number in range(2, 2 + remaining_pages)
        ]
        for future in as_completed(response_futures):
            page_response = future.result().json()
            table = cat(
                table, fromdicts(page_response['results'],
                                 header=PEOPLE_HEADER))

    table = addfields(table, [('date', lambda rec: datetime.fromisoformat(rec[
        'edited'].replace('Z', '+00:00')).date().isoformat())])
    table = cutout(table, 'edited')
    table = convert(
        table, 'homeworld', lambda homeworld_url: planet_fetcher.fetch(
            homeworld_url).json()['name'])

    return table
Example #2
0
def test_fromdicts_3():

    data = [{'foo': 'a', 'bar': 1},
            {'foo': 'b'},
            {'foo': 'c', 'bar': 2, 'baz': True}]
    actual = fromdicts(data, header=['foo', 'bar'])
    expect = (('foo', 'bar'),
              ('a', 1),
              ('b', None),
              ('c', 2))
    ieq(expect, actual)
    ieq(expect, actual)  # verify can iterate twice
Example #3
0
def test_fromdicts_3():

    data = [{"foo": "a", "bar": 1},
            {"foo": "b"},
            {"foo": "c", "bar": 2, "baz": True}]
    actual = fromdicts(data, header=['foo', 'bar'])
    expect = (('foo', 'bar'),
              ('a', 1),
              ('b', None),
              ('c', 2))
    ieq(expect, actual)
    ieq(expect, actual)  # verify can iterate twice
Example #4
0
def test_fromdicts_2():

    data = [{"foo": "a", "bar": 1},
            {"foo": "b"},
            {"foo": "c", "bar": 2, "baz": True}]
    actual = fromdicts(data)
    expect = (('foo', 'bar', 'baz'),
              ('a', 1, None),
              ('b', None, None),
              ('c', 2, True))
    ieq(expect, actual)
    ieq(expect, actual)  # verify can iterate twice
Example #5
0
def test_fromdicts_1():

    data = [{"foo": "a", "bar": 1},
            {"foo": "b", "bar": 2},
            {"foo": "c", "bar": 2}]
    actual = fromdicts(data)
    expect = (('foo', 'bar'),
              ('a', 1),
              ('b', 2),
              ('c', 2))
    ieq(expect, actual)
    ieq(expect, actual)  # verify can iterate twice
Example #6
0
def test_fromdicts_2():

    data = [{'foo': 'a', 'bar': 1},
            {'foo': 'b'},
            {'foo': 'c', 'bar': 2, 'baz': True}]
    actual = fromdicts(data)
    # N.B., fields come out in sorted order
    expect = (('bar', 'baz', 'foo'),
              (1, None, 'a'),
              (None, None, 'b'),
              (2, True, 'c'))
    ieq(expect, actual)
    ieq(expect, actual)  # verify can iterate twice
Example #7
0
def test_fromdicts_3():

    data = [{
        "foo": "a",
        "bar": 1
    }, {
        "foo": "b"
    }, {
        "foo": "c",
        "bar": 2,
        "baz": True
    }]
    actual = fromdicts(data, header=['foo', 'bar'])
    expect = (('foo', 'bar'), ('a', 1), ('b', None), ('c', 2))
    ieq(expect, actual)
    ieq(expect, actual)  # verify can iterate twice
Example #8
0
def test_fromdicts_1():

    data = [{
        "foo": "a",
        "bar": 1
    }, {
        "foo": "b",
        "bar": 2
    }, {
        "foo": "c",
        "bar": 2
    }]
    actual = fromdicts(data)
    expect = (('foo', 'bar'), ('a', 1), ('b', 2), ('c', 2))
    ieq(expect, actual)
    ieq(expect, actual)  # verify can iterate twice
Example #9
0
def test_fromdicts_2():

    data = [{
        "foo": "a",
        "bar": 1
    }, {
        "foo": "b"
    }, {
        "foo": "c",
        "bar": 2,
        "baz": True
    }]
    actual = fromdicts(data)
    expect = (('foo', 'bar', 'baz'), ('a', 1, None), ('b', None, None),
              ('c', 2, True))
    ieq(expect, actual)
    ieq(expect, actual)  # verify can iterate twice