def test_fetch_result_from_thrift_happy_path(self):
with tempfile.NamedTemporaryFile(dir=str(self.basedir)) as tf:
self.assertEqual(
types.FetchResult.from_thrift(
ttypes.FetchResult(
Path(tf.name).name,
[types.RenderError(types.I18nMessage("hi")).to_thrift()],
),
self.basedir,
),
types.FetchResult(
Path(tf.name), [types.RenderError(types.I18nMessage("hi"))]
),
)
def test_render_error_to_dict(self):
self.assertEqual(
fields._render_error_to_dict(
types.RenderError(
types.I18nMessage("err", {}),
[
types.QuickFix(
types.I18nMessage("click", {}, "cjwmodule"),
types.QuickFixAction.PrependStep("filter", {"x": "y"}),
)
],
)
),
{
"message": {"id": "err", "arguments": {}},
"quickFixes": [
{
"buttonText": {
"id": "click",
"arguments": {},
"source": "cjwmodule",
},
"action": {
"type": "prependStep",
"moduleSlug": "filter",
"partialParams": {"x": "y"},
},
}
],
},
)
def test_render_error_to_thrift(self):
self.assertEqual(
types.RenderError(
types.I18nMessage("foo", {}),
[
types.QuickFix(
types.I18nMessage("click"),
types.QuickFixAction.PrependStep("filter", {"x": "y"}),
)
],
).to_thrift(),
ttypes.RenderError(
ttypes.I18nMessage("foo", {}, ttypes.I18nMessageSource()),
[
ttypes.QuickFix(
ttypes.I18nMessage("click", {}, ttypes.I18nMessageSource()),
ttypes.QuickFixAction(
prepend_step=ttypes.PrependStepQuickFixAction(
"filter", ttypes.RawParams('{"x":"y"}')
)
),
)
],
),
)
def test_to_arrow_empty_dataframe(self):
fd, filename = tempfile.mkstemp()
os.close(fd)
# Remove the file. Then we'll test that ProcessResult.to_arrow() does
# not write it (because the result is an error)
os.unlink(filename)
try:
result = ProcessResult.coerce("bad, bad error").to_arrow(Path(filename))
self.assertEqual(
result,
atypes.RenderResult(
atypes.ArrowTable(None, atypes.TableMetadata(0, [])),
[
atypes.RenderError(
atypes.I18nMessage.TODO_i18n("bad, bad error"), []
)
],
{},
),
)
with self.assertRaises(FileNotFoundError):
open(filename)
finally:
try:
os.unlink(filename)
except FileNotFoundError:
pass
def test_render_error_to_thrift(self):
self.assertEqual(
types.arrow_render_error_to_thrift(
types.RenderError(
I18nMessage("foo", {}, None),
[
types.QuickFix(
I18nMessage("click", {}, None),
types.QuickFixAction.PrependStep(
"filter", {"x": "y"}),
)
],
)),
ttypes.RenderError(
ttypes.I18nMessage("foo", {}, None),
[
ttypes.QuickFix(
ttypes.I18nMessage("click", {}, None),
ttypes.QuickFixAction(
prepend_step=ttypes.PrependStepQuickFixAction(
"filter", {"x": ttypes.Json(
string_value="y")})),
)
],
),
)
def fetch_pandas(
params: Dict[str, Any],
secrets: Dict[str, Any],
last_fetch_result: Optional[types.FetchResult],
input_table_parquet_path: Optional[Path],
output_path: Path,
) -> Union[ptypes.ProcessResult, types.FetchResult]:
"""
Call `fetch()` and validate the result.
Module authors should not replace this function: they should replace
`fetch()` instead.
This function validates the `fetch()` return value, to raise a helpful
`ValueError` if the module code is buggy.
"""
spec = inspect.getfullargspec(fetch)
kwargs = {}
varkw = bool(spec.varkw) # if True, function accepts **kwargs
kwonlyargs = spec.kwonlyargs
if varkw or "secrets" in kwonlyargs:
kwargs["secrets"] = secrets
if varkw or "get_input_dataframe" in kwonlyargs:
async def get_input_dataframe():
if input_table_parquet_path is None:
return None
else:
return __parquet_to_pandas(input_table_parquet_path)
kwargs["get_input_dataframe"] = get_input_dataframe
if varkw or "get_stored_dataframe" in kwonlyargs:
async def get_stored_dataframe():
if last_fetch_result is None:
return None
else:
return __parquet_to_pandas(last_fetch_result.path)
kwargs["get_stored_dataframe"] = get_stored_dataframe
if varkw or "output_path" in kwonlyargs:
kwargs["output_path"] = output_path
result = fetch(params, **kwargs)
if asyncio.iscoroutine(result):
result = asyncio.run(result)
if (isinstance(result, tuple) and len(result) == 2
and isinstance(result[0], Path) and isinstance(result[1], str)):
return types.FetchResult(
result[0],
[types.RenderError(types.I18nMessage.TODO_i18n(result[1]))])
elif isinstance(result, Path):
return types.FetchResult(result)
else:
return ptypes.ProcessResult.coerce(result)
def __render_arrow(
*,
table: types.ArrowTable,
params: Dict[str, Any],
tab_name: str,
fetch_result: Optional[types.FetchResult],
output_path: Path,
) -> types.RenderResult:
"""Render using `cjwkernel.types` data types.
Write to `output_path`.
This will typically call `render()`.
"""
# call render()
raw_result = render(
table.table,
params,
output_path,
columns=table.metadata.columns,
settings=settings,
tab_name=tab_name,
fetch_result=fetch_result,
)
# coerce result
# TODO let module output column types. (Currently, the lack of column types
# means this is only useful for fetch modules that don't output number
# formats.)
table = types.ArrowTable.from_arrow_file_with_inferred_metadata(
output_path,
fallback_column_types={c.name: c.type
for c in table.metadata.columns},
)
errors = []
# TODO support more output types? Or develop the One True Types (maybe
# types.RenderResult) and force modules to output it.
if isinstance(raw_result, list):
# List of I18nMessage errors
errors = [
# TODO don't use coerce_I18nMessage? At least, don't use ptypes.
# Do any modules even require coerce? Or do they all correctly
# output tuples-or-text? Is it only unit tests that output
# non-I18nMessage tuples?
types.RenderError(ptypes.coerce_I18nMessage(message))
for message in raw_result
]
elif raw_result is None:
errors = []
return types.RenderResult(table, errors)
def test_to_arrow_quick_fixes(self):
fd, filename = tempfile.mkstemp()
os.close(fd)
# Remove the file. Then we'll test that ProcessResult.to_arrow() does
# not write it (because the result is an error)
os.unlink(filename)
try:
result = ProcessResult(
error="bad, bad error",
quick_fixes=[
QuickFix(
"button foo",
"prependModule",
["converttotext", {"colnames": ["A", "B"]}],
),
QuickFix(
"button bar",
"prependModule",
["converttonumber", {"colnames": ["A", "B"]}],
),
],
).to_arrow(Path(filename))
self.assertEqual(
result.errors,
[
atypes.RenderError(
atypes.I18nMessage.TODO_i18n("bad, bad error"),
[
atypes.QuickFix(
atypes.I18nMessage.TODO_i18n("button foo"),
atypes.QuickFixAction.PrependStep(
"converttotext", {"colnames": ["A", "B"]}
),
),
atypes.QuickFix(
atypes.I18nMessage.TODO_i18n("button bar"),
atypes.QuickFixAction.PrependStep(
"converttonumber", {"colnames": ["A", "B"]}
),
),
],
)
],
)
with self.assertRaises(FileNotFoundError):
open(filename)
finally:
try:
os.unlink(filename)
except FileNotFoundError:
pass
