def test_truncate_not_too_big(self):
df = pd.DataFrame({"foo": ["foo", "bar", "baz"]})
expected = ProcessResult(df.copy())
result = ProcessResult(df)
result.truncate_in_place_if_too_big()
self.assertEqual(result, expected)
def test_truncate_too_big_remove_unused_categories(self):
result_df = pd.DataFrame({"A": ["x", "y", "z", "z"]}, dtype="category")
result = ProcessResult(result_df)
result.truncate_in_place_if_too_big()
assert_frame_equal(
result.dataframe, pd.DataFrame({"A": ["x", "y"]}, dtype="category")
)
def test_truncate_too_big_and_error(self):
expected_df = pd.DataFrame({"foo": ["bar", "baz"]})
expected = ProcessResult(
dataframe=expected_df, error="Some error\nTruncated output from 3 rows to 2"
)
result_df = pd.DataFrame({"foo": ["bar", "baz", "moo"]})
result = ProcessResult(result_df, error="Some error")
result.truncate_in_place_if_too_big()
self.assertEqual(result, expected)
def test_truncate_too_big_no_error(self):
expected_df = pd.DataFrame({"foo": ["bar", "baz"]})
expected = ProcessResult(
dataframe=expected_df,
errors=[
RenderError(
I18nMessage(
"py.cjwkernel.pandas.types.ProcessResult.truncate_in_place_if_too_big.warning",
{"old_number": 3, "new_number": 2},
None,
)
)
],
)
result_df = pd.DataFrame({"foo": ["bar", "baz", "moo"]})
result = ProcessResult(result_df, errors=[])
result.truncate_in_place_if_too_big()
self.assertEqual(result, expected)
async def fetch(params):
# We delve into pd.read_html()'s innards, below. Part of that means some
# first-use initialization.
pd.io.html._importers()
table = None
url: str = params["url"].strip()
tablenum: int = params["tablenum"] - 1 # 1-based for user
if tablenum < 0:
return ProcessResult(error="Table number must be at least 1")
result = None
try:
async with moduleutils.spooled_data_from_url(url) as (spool, headers,
charset):
# pandas.read_html() does automatic type conversion, but we prefer
# our own. Delve into its innards so we can pass all the conversion
# kwargs we want.
with moduleutils.wrap_text(spool, charset) as textio:
tables = pd.io.html._parse(
# Positional arguments:
flavor="html5lib", # force algorithm, for reproducibility
io=textio,
match=".+",
attrs=None,
encoding=None, # textio is already decoded
displayed_only=False, # avoid dud feature: it ignores CSS
# Required kwargs that pd.read_html() would set by default:
header=None,
skiprows=None,
# Now the reason we used pd.io.html._parse() instead of
# pd.read_html(): we get to pass whatever kwargs we want to
# TextParser.
#
# kwargs we get to add as a result of this hack:
na_filter=False, # do not autoconvert
dtype=str, # do not autoconvert
)
except asyncio.TimeoutError:
return ProcessResult(error=f"Timeout fetching {url}")
except aiohttp.InvalidURL:
return ProcessResult(error=f"Invalid URL")
except aiohttp.ClientResponseError as err:
return ProcessResult(error=("Error from server: %d %s" %
(err.status, err.message)))
except aiohttp.ClientError as err:
return ProcessResult(error=str(err))
except ValueError:
return ProcessResult(
error="Did not find any <table> tags on that page")
except IndexError:
# pandas.read_html() gives this unhelpful error message....
return ProcessResult(error="Table has no columns")
if not tables:
return ProcessResult(
error="Did not find any <table> tags on that page")
if tablenum >= len(tables):
return ProcessResult(
error=(f"The maximum table number on this page is {len(tables)}"))
# pd.read_html() guarantees unique colnames
table = tables[tablenum]
merge_colspan_headers_in_place(table)
moduleutils.autocast_dtypes_in_place(table)
if len(table) == 0:
# read_html() produces an empty Index. We want a RangeIndex.
table.reset_index(drop=True, inplace=True)
result = ProcessResult(dataframe=table)
result.truncate_in_place_if_too_big()
return result