def test_to_arrow_empty_dataframe(self):
fd, filename = tempfile.mkstemp()
os.close(fd)
# Remove the file. Then we'll test that ProcessResult.to_arrow() does
# not write it (because the result is an error)
os.unlink(filename)
try:
result = ProcessResult.coerce("bad, bad error").to_arrow(Path(filename))
self.assertEqual(
result,
atypes.RenderResult(
atypes.ArrowTable(None, atypes.TableMetadata(0, [])),
[
atypes.RenderError(
atypes.I18nMessage.TODO_i18n("bad, bad error"), []
)
],
{},
),
)
with self.assertRaises(FileNotFoundError):
open(filename)
finally:
try:
os.unlink(filename)
except FileNotFoundError:
pass
def test_to_arrow_normal_dataframe(self):
fd, filename = tempfile.mkstemp()
try:
process_result = ProcessResult.coerce(pd.DataFrame({"A": [1, 2]}))
result = process_result.to_arrow(Path(filename))
self.assertEqual(
result,
atypes.RenderResult(
[],
{},
),
)
with pa.ipc.open_file(filename) as reader:
table = reader.read_all()
assert_arrow_table_equals(
table,
make_table(
make_column(
# Whatever .format ProcessResult.coerce() gave
"A",
[1, 2],
format=process_result.columns[0].type.format,
)
),
)
finally:
os.unlink(filename)
def test_to_arrow_normal_dataframe(self):
fd, filename = tempfile.mkstemp()
os.close(fd)
# Remove the file. Then we'll test that ProcessResult.to_arrow() does
# not write it (because the result is an error)
os.unlink(filename)
try:
process_result = ProcessResult.coerce(pd.DataFrame({"A": [1, 2]}))
result = process_result.to_arrow(Path(filename))
self.assertEqual(
result,
atypes.RenderResult(
atypes.ArrowTable(
Path(filename),
pyarrow.table({"A": [1, 2]}),
atypes.TableMetadata(
2,
[
atypes.Column(
"A",
ColumnType.Number(
# Whatever .format
# ProcessResult.coerce() gave
process_result.columns[0].type.format),
)
],
),
),
[],
{},
),
)
finally:
os.unlink(filename)
def __render_arrow(
*,
table: types.ArrowTable,
params: Dict[str, Any],
tab_name: str,
fetch_result: Optional[types.FetchResult],
output_path: Path,
) -> types.RenderResult:
"""Render using `cjwkernel.types` data types.
Write to `output_path`.
This will typically call `render()`.
"""
# call render()
raw_result = render(
table.table,
params,
output_path,
columns=table.metadata.columns,
settings=settings,
tab_name=tab_name,
fetch_result=fetch_result,
)
# coerce result
# TODO let module output column types. (Currently, the lack of column types
# means this is only useful for fetch modules that don't output number
# formats.)
table = types.ArrowTable.from_arrow_file_with_inferred_metadata(
output_path,
fallback_column_types={c.name: c.type
for c in table.metadata.columns},
)
errors = []
# TODO support more output types? Or develop the One True Types (maybe
# types.RenderResult) and force modules to output it.
if isinstance(raw_result, list):
# List of I18nMessage errors
errors = [
# TODO don't use coerce_I18nMessage? At least, don't use ptypes.
# Do any modules even require coerce? Or do they all correctly
# output tuples-or-text? Is it only unit tests that output
# non-I18nMessage tuples?
types.RenderError(ptypes.coerce_I18nMessage(message))
for message in raw_result
]
elif raw_result is None:
errors = []
return types.RenderResult(table, errors)
def test_to_arrow_empty_dataframe(self):
fd, filename = tempfile.mkstemp()
# We'll test that ProcessResult.to_arrow() writes empty bytes on error
os.write(fd, b"to-remove")
os.close(fd)
try:
result = ProcessResult.coerce("bad, bad error").to_arrow(Path(filename))
self.assertEqual(
result,
atypes.RenderResult(
[RenderError(TODO_i18n("bad, bad error"), [])],
{},
),
)
assert_arrow_table_equals(
load_untrusted_arrow_file_with_columns(Path(filename))[0], make_table()
)
finally:
os.unlink(filename)