def test_coerce_dict_legacy_with_quickfix_dict(self):
dataframe = pd.DataFrame({"A": [1, 2]})
result = ProcessResult.coerce(
{
"dataframe": dataframe,
"error": "an error",
"json": {"foo": "bar"},
"quick_fixes": [
{
"text": "Hi",
"action": "prependModule",
"args": ["texttodate", {"column": "created_at"}],
}
],
}
)
expected = ProcessResult(
dataframe,
errors=[
RenderError(
TODO_i18n("an error"),
[
QuickFix(
TODO_i18n("Hi"),
QuickFixAction.PrependStep(
"texttodate", {"column": "created_at"}
),
)
],
)
],
json={"foo": "bar"},
)
self.assertEqual(result, expected)
def test_from_string_with_quick_fix(self):
self.assertEqual(
coerce_RenderError(
{
"message": "error",
"quickFixes": [
dict(
text="button text",
action="prependModule",
args=["converttotext", {"colnames": ["A", "B"]}],
)
],
}
),
RenderError(
TODO_i18n("error"),
[
QuickFix(
TODO_i18n("button text"),
QuickFixAction.PrependStep(
"converttotext", {"colnames": ["A", "B"]}
),
)
],
),
)
def test_coerce_dict_quickfix_multiple(self):
dataframe = pd.DataFrame({"A": [1, 2]})
result = ProcessResult.coerce({
"dataframe":
dataframe,
"errors": [
{
"message":
"an error",
"quickFixes": [
dict(
text="Hi",
action="prependModule",
args=["texttodate", {
"column": "created_at"
}],
),
dict(
text=("message.id", {}),
action="prependModule",
args=["texttodate", {
"column": "created_at"
}],
),
],
},
"other error",
],
"json": {
"foo": "bar"
},
})
expected = ProcessResult(
dataframe,
errors=[
RenderError(
TODO_i18n("an error"),
[
QuickFix(
TODO_i18n("Hi"),
QuickFixAction.PrependStep(
"texttodate", {"column": "created_at"}),
),
QuickFix(
I18nMessage("message.id", {}, None),
QuickFixAction.PrependStep(
"texttodate", {"column": "created_at"}),
),
],
),
RenderError(TODO_i18n("other error")),
],
json={"foo": "bar"},
)
self.assertEqual(result, expected)
def test_coerce_dict_i18n(self):
expected = ProcessResult(
errors=[
RenderError(
TODO_i18n("an error"),
[
QuickFix(
I18nMessage("message.id", {}, None),
QuickFixAction.PrependStep(
"texttodate", {"column": "created_at"}
),
)
],
)
]
)
result = ProcessResult.coerce(
{
"message": "an error",
"quickFixes": [
dict(
text=("message.id", {}),
action="prependModule",
args=["texttodate", {"column": "created_at"}],
)
],
}
)
self.assertEqual(result, expected)
def test_list_from_dict(self):
result = coerce_RenderError_list({
"message": "error",
"quickFixes": []
})
expected = [RenderError(TODO_i18n("error"))]
self.assertEqual(result, expected)
def test_execute_migrate_params_module_error_gives_default_params(self):
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
create_module_zipfile(
"mod",
spec_kwargs={
"loads_data":
True,
"parameters": [{
"id_name": "x",
"type": "string",
"default": "def"
}],
},
python_code=textwrap.dedent("""
import json
def render(table, params): return "params: " + json.dumps(params)
def migrate_params(params): cause_module_error() # NameError
"""),
)
step = tab.steps.create(order=0,
slug="step-1",
module_id_name="mod",
params={"x": "good"})
self._execute(workflow)
step.refresh_from_db()
self.assertEqual(
step.cached_render_result_errors,
[RenderError(TODO_i18n('params: {"x": "def"}'))],
)
def test_fetch_return_tuple_path_and_error(self):
with tempfile_context(dir=self.basedir) as outfile:
async def fetch(params):
outfile.write_text("xyz")
return outfile, "foo"
result = self._test_fetch(fetch, output_filename=outfile.name)
self.assertEqual(result.errors, [FetchError(TODO_i18n("foo"))])
def test_default_render_returns_fetch_result(self):
# Functionality used by libraryofcongress
#
# TODO nix this functionality.
with ModuleTestEnv() as env:
with parquet_file({"A": [2]}, dir=env.basedir) as parquet_path:
outcome = env.call_render(
make_table(),
{},
fetch_result=FetchResult(
path=parquet_path,
errors=[FetchError(TODO_i18n("A warning"))]),
)
self.assertEqual(
outcome.result,
RenderResult([RenderError(TODO_i18n("A warning"))]))
assert_arrow_table_equals(outcome.read_table(),
make_table(make_column("A", [2])))
def test_fetch_return_error(self):
async def fetch(params):
return "bad things"
with tempfile_context(dir=self.basedir) as outfile:
result = self._test_fetch(fetch, output_filename=outfile.name)
self.assertEqual(result.errors,
[FetchError(TODO_i18n("bad things"))])
self.assertEqual(outfile.read_bytes(), b"")
def test_list_from_list_of_string_and_tuples(self):
self.assertEqual(
coerce_RenderError_list(
["error", ("my_id", {}), ("my_other_id", {"this": "one"})]
),
[
RenderError(TODO_i18n("error")),
RenderError(I18nMessage("my_id", {}, None)),
RenderError(I18nMessage("my_other_id", {"this": "one"}, None)),
],
)
def test_truncate_too_big_and_error(self):
expected_df = pd.DataFrame({"foo": ["bar", "baz"]})
expected = ProcessResult(
dataframe=expected_df,
errors=[
RenderError(TODO_i18n("Some error")),
RenderError(
I18nMessage(
"py.cjwkernel.pandas.types.ProcessResult.truncate_in_place_if_too_big.warning",
{"old_number": 3, "new_number": 2},
None,
)
),
],
)
result_df = pd.DataFrame({"foo": ["bar", "baz", "moo"]})
result = ProcessResult(result_df, errors=[RenderError(TODO_i18n("Some error"))])
result.truncate_in_place_if_too_big()
self.assertEqual(result, expected)
def test_execute_mark_unreachable(self, send_update):
future_none = asyncio.Future()
future_none.set_result(None)
send_update.return_value = future_none
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
create_module_zipfile(
"mod",
spec_kwargs={"loads_data": True},
python_code=
'def render(table, params): return "error, not warning"',
)
step1 = tab.steps.create(order=0, slug="step-1", module_id_name="mod")
step2 = tab.steps.create(order=1, slug="step-2", module_id_name="mod")
step3 = tab.steps.create(order=2, slug="step-3", module_id_name="mod")
self._execute(workflow)
# step1: error
step1.refresh_from_db()
with open_cached_render_result(step1.cached_render_result) as result:
self.assertEqual(result.path.read_bytes(), b"")
self.assertEqual(
step1.cached_render_result.errors,
[RenderError(TODO_i18n("error, not warning"))],
)
# step2, step3: unreachable (no errors, no table data)
step2.refresh_from_db()
self.assertEqual(step2.cached_render_result.status, "unreachable")
with open_cached_render_result(step2.cached_render_result) as result:
self.assertEqual(result.path.read_bytes(), b"")
self.assertEqual(step2.cached_render_result.errors, [])
step3.refresh_from_db()
with open_cached_render_result(step3.cached_render_result) as result:
self.assertEqual(result.path.read_bytes(), b"")
self.assertEqual(step3.cached_render_result.errors, [])
send_update.assert_called_with(
workflow.id,
clientside.Update(
steps={
step3.id:
clientside.StepUpdate(
render_result=step3.cached_render_result,
module_slug="mod")
}),
)
def test_coerce_dict_legacy(self):
dataframe = pd.DataFrame({"A": [1, 2]})
result = ProcessResult.coerce(
{
"dataframe": dataframe,
"error": "an error",
"json": {"foo": "bar"},
"quick_fixes": [],
}
)
expected = ProcessResult(
dataframe,
[RenderError(TODO_i18n("an error"), [])],
json={"foo": "bar"},
)
self.assertEqual(result, expected)
def test_to_arrow_empty_dataframe(self):
fd, filename = tempfile.mkstemp()
# We'll test that ProcessResult.to_arrow() writes empty bytes on error
os.write(fd, b"to-remove")
os.close(fd)
try:
result = ProcessResult.coerce("bad, bad error").to_arrow(Path(filename))
self.assertEqual(
result,
atypes.RenderResult(
[RenderError(TODO_i18n("bad, bad error"), [])],
{},
),
)
assert_arrow_table_equals(
load_untrusted_arrow_file_with_columns(Path(filename))[0], make_table()
)
finally:
os.unlink(filename)
def test_default_render_returns_fetch_result(self):
# Functionality used by libraryofcongress
with ExitStack() as ctx:
input_arrow_table = ctx.enter_context(
arrow_table_context({"A": [1]}, dir=self.basedir))
parquet_filename = Path(
ctx.enter_context(parquet_file({"A": [2]},
dir=self.basedir)).name).name
out_filename = ctx.enter_context(
tempfile_context(dir=self.basedir)).name
thrift_result = module.render_thrift(
ttypes.RenderRequest(
str(self.basedir),
arrow_arrow_table_to_thrift(input_arrow_table),
{}, # params
ttypes.Tab("tab-1", "Tab 1"),
ttypes.FetchResult(
parquet_filename,
[
ttypes.RenderError(
ttypes.I18nMessage(
"TODO_i18n",
{
"text":
ttypes.I18nArgument(
string_value="A warning")
},
None,
),
[],
)
],
),
out_filename,
))
result = thrift_render_result_to_arrow(thrift_result, self.basedir)
assert_render_result_equals(
result,
RenderResult(
arrow_table({"A": [2]}),
[RenderError(TODO_i18n("A warning"))],
),
)
def test_to_arrow_empty_dataframe(self):
fd, filename = tempfile.mkstemp()
os.close(fd)
# Remove the file. Then we'll test that ProcessResult.to_arrow() does
# not write it (because the result is an error)
os.unlink(filename)
try:
result = ProcessResult.coerce("bad, bad error").to_arrow(
Path(filename))
self.assertEqual(
result,
atypes.RenderResult(
atypes.ArrowTable(None, None, TableMetadata(0, [])),
[RenderError(TODO_i18n("bad, bad error"), [])],
{},
),
)
with self.assertRaises(FileNotFoundError):
open(filename)
finally:
try:
os.unlink(filename)
except FileNotFoundError:
pass
def test_list_from_list_of_string(self):
self.assertEqual(
coerce_RenderError_list(["error"]),
[RenderError(TODO_i18n("error"))],
)
def test_coerce_tuple_dataframe_str_none(self):
df = pd.DataFrame({"foo": ["bar"]})
expected = ProcessResult(df, [RenderError(TODO_i18n("hi"))])
result = ProcessResult.coerce((df, "hi", None))
self.assertEqual(result, expected)
def test_list_from_list_with_quick_fixes(self):
self.assertEqual(
coerce_RenderError_list(
[
{
"message": ("my id", {}),
"quickFixes": [
dict(
text="button text",
action="prependModule",
args=["converttotext", {"colnames": ["A", "B"]}],
)
],
},
{
"message": ("my other id", {"other": "this"}),
"quickFixes": [
dict(
text=("quick fix id", {"fix": "that"}),
action="prependModule",
args=["convert-date", {"colnames": ["C", "D"]}],
),
dict(
text=("another quick fix id", {"fix": "that"}),
action="prependModule",
args=["converttonumber", {"colnames": ["E", "F"]}],
),
],
},
]
),
[
RenderError(
I18nMessage("my id", {}, None),
[
QuickFix(
TODO_i18n("button text"),
QuickFixAction.PrependStep(
"converttotext", {"colnames": ["A", "B"]}
),
)
],
),
RenderError(
I18nMessage("my other id", {"other": "this"}, None),
[
QuickFix(
I18nMessage("quick fix id", {"fix": "that"}, None),
QuickFixAction.PrependStep(
"convert-date", {"colnames": ["C", "D"]}
),
),
QuickFix(
I18nMessage("another quick fix id", {"fix": "that"}, None),
QuickFixAction.PrependStep(
"converttonumber", {"colnames": ["E", "F"]}
),
),
],
),
],
)
def test_list_from_nonempty_string(self):
result = coerce_RenderError_list("hello")
expected = [RenderError(TODO_i18n("hello"))]
self.assertEqual(result, expected)
def test_coerce_str(self):
expected = ProcessResult(errors=[RenderError(TODO_i18n("yay"))])
result = ProcessResult.coerce("yay")
self.assertEqual(result, expected)
def test_coerce_tuple_none_str_dict(self):
expected = ProcessResult(errors=[RenderError(TODO_i18n("hi"))], json={"a": "b"})
result = ProcessResult.coerce((None, "hi", {"a": "b"}))
self.assertEqual(result, expected)
def test_coerce_tuple_none_str_none(self):
expected = ProcessResult(errors=[RenderError(TODO_i18n("hi"))])
result = ProcessResult.coerce((None, "hi", None))
self.assertEqual(result, expected)
