def test_coerce_dict_legacy_with_quickfix_dict(self): dataframe = pd.DataFrame({"A": [1, 2]}) result = ProcessResult.coerce( { "dataframe": dataframe, "error": "an error", "json": {"foo": "bar"}, "quick_fixes": [ { "text": "Hi", "action": "prependModule", "args": ["texttodate", {"column": "created_at"}], } ], } ) expected = ProcessResult( dataframe, errors=[ RenderError( TODO_i18n("an error"), [ QuickFix( TODO_i18n("Hi"), QuickFixAction.PrependStep( "texttodate", {"column": "created_at"} ), ) ], ) ], json={"foo": "bar"}, ) self.assertEqual(result, expected)
def test_from_string_with_quick_fix(self): self.assertEqual( coerce_RenderError( { "message": "error", "quickFixes": [ dict( text="button text", action="prependModule", args=["converttotext", {"colnames": ["A", "B"]}], ) ], } ), RenderError( TODO_i18n("error"), [ QuickFix( TODO_i18n("button text"), QuickFixAction.PrependStep( "converttotext", {"colnames": ["A", "B"]} ), ) ], ), )
def test_coerce_dict_quickfix_multiple(self): dataframe = pd.DataFrame({"A": [1, 2]}) result = ProcessResult.coerce({ "dataframe": dataframe, "errors": [ { "message": "an error", "quickFixes": [ dict( text="Hi", action="prependModule", args=["texttodate", { "column": "created_at" }], ), dict( text=("message.id", {}), action="prependModule", args=["texttodate", { "column": "created_at" }], ), ], }, "other error", ], "json": { "foo": "bar" }, }) expected = ProcessResult( dataframe, errors=[ RenderError( TODO_i18n("an error"), [ QuickFix( TODO_i18n("Hi"), QuickFixAction.PrependStep( "texttodate", {"column": "created_at"}), ), QuickFix( I18nMessage("message.id", {}, None), QuickFixAction.PrependStep( "texttodate", {"column": "created_at"}), ), ], ), RenderError(TODO_i18n("other error")), ], json={"foo": "bar"}, ) self.assertEqual(result, expected)
def test_coerce_dict_i18n(self): expected = ProcessResult( errors=[ RenderError( TODO_i18n("an error"), [ QuickFix( I18nMessage("message.id", {}, None), QuickFixAction.PrependStep( "texttodate", {"column": "created_at"} ), ) ], ) ] ) result = ProcessResult.coerce( { "message": "an error", "quickFixes": [ dict( text=("message.id", {}), action="prependModule", args=["texttodate", {"column": "created_at"}], ) ], } ) self.assertEqual(result, expected)
def test_list_from_dict(self): result = coerce_RenderError_list({ "message": "error", "quickFixes": [] }) expected = [RenderError(TODO_i18n("error"))] self.assertEqual(result, expected)
def test_execute_migrate_params_module_error_gives_default_params(self): workflow = Workflow.create_and_init() tab = workflow.tabs.first() create_module_zipfile( "mod", spec_kwargs={ "loads_data": True, "parameters": [{ "id_name": "x", "type": "string", "default": "def" }], }, python_code=textwrap.dedent(""" import json def render(table, params): return "params: " + json.dumps(params) def migrate_params(params): cause_module_error() # NameError """), ) step = tab.steps.create(order=0, slug="step-1", module_id_name="mod", params={"x": "good"}) self._execute(workflow) step.refresh_from_db() self.assertEqual( step.cached_render_result_errors, [RenderError(TODO_i18n('params: {"x": "def"}'))], )
def test_fetch_return_tuple_path_and_error(self): with tempfile_context(dir=self.basedir) as outfile: async def fetch(params): outfile.write_text("xyz") return outfile, "foo" result = self._test_fetch(fetch, output_filename=outfile.name) self.assertEqual(result.errors, [FetchError(TODO_i18n("foo"))])
def test_default_render_returns_fetch_result(self): # Functionality used by libraryofcongress # # TODO nix this functionality. with ModuleTestEnv() as env: with parquet_file({"A": [2]}, dir=env.basedir) as parquet_path: outcome = env.call_render( make_table(), {}, fetch_result=FetchResult( path=parquet_path, errors=[FetchError(TODO_i18n("A warning"))]), ) self.assertEqual( outcome.result, RenderResult([RenderError(TODO_i18n("A warning"))])) assert_arrow_table_equals(outcome.read_table(), make_table(make_column("A", [2])))
def test_fetch_return_error(self): async def fetch(params): return "bad things" with tempfile_context(dir=self.basedir) as outfile: result = self._test_fetch(fetch, output_filename=outfile.name) self.assertEqual(result.errors, [FetchError(TODO_i18n("bad things"))]) self.assertEqual(outfile.read_bytes(), b"")
def test_list_from_list_of_string_and_tuples(self): self.assertEqual( coerce_RenderError_list( ["error", ("my_id", {}), ("my_other_id", {"this": "one"})] ), [ RenderError(TODO_i18n("error")), RenderError(I18nMessage("my_id", {}, None)), RenderError(I18nMessage("my_other_id", {"this": "one"}, None)), ], )
def test_truncate_too_big_and_error(self): expected_df = pd.DataFrame({"foo": ["bar", "baz"]}) expected = ProcessResult( dataframe=expected_df, errors=[ RenderError(TODO_i18n("Some error")), RenderError( I18nMessage( "py.cjwkernel.pandas.types.ProcessResult.truncate_in_place_if_too_big.warning", {"old_number": 3, "new_number": 2}, None, ) ), ], ) result_df = pd.DataFrame({"foo": ["bar", "baz", "moo"]}) result = ProcessResult(result_df, errors=[RenderError(TODO_i18n("Some error"))]) result.truncate_in_place_if_too_big() self.assertEqual(result, expected)
def test_execute_mark_unreachable(self, send_update): future_none = asyncio.Future() future_none.set_result(None) send_update.return_value = future_none workflow = Workflow.create_and_init() tab = workflow.tabs.first() create_module_zipfile( "mod", spec_kwargs={"loads_data": True}, python_code= 'def render(table, params): return "error, not warning"', ) step1 = tab.steps.create(order=0, slug="step-1", module_id_name="mod") step2 = tab.steps.create(order=1, slug="step-2", module_id_name="mod") step3 = tab.steps.create(order=2, slug="step-3", module_id_name="mod") self._execute(workflow) # step1: error step1.refresh_from_db() with open_cached_render_result(step1.cached_render_result) as result: self.assertEqual(result.path.read_bytes(), b"") self.assertEqual( step1.cached_render_result.errors, [RenderError(TODO_i18n("error, not warning"))], ) # step2, step3: unreachable (no errors, no table data) step2.refresh_from_db() self.assertEqual(step2.cached_render_result.status, "unreachable") with open_cached_render_result(step2.cached_render_result) as result: self.assertEqual(result.path.read_bytes(), b"") self.assertEqual(step2.cached_render_result.errors, []) step3.refresh_from_db() with open_cached_render_result(step3.cached_render_result) as result: self.assertEqual(result.path.read_bytes(), b"") self.assertEqual(step3.cached_render_result.errors, []) send_update.assert_called_with( workflow.id, clientside.Update( steps={ step3.id: clientside.StepUpdate( render_result=step3.cached_render_result, module_slug="mod") }), )
def test_coerce_dict_legacy(self): dataframe = pd.DataFrame({"A": [1, 2]}) result = ProcessResult.coerce( { "dataframe": dataframe, "error": "an error", "json": {"foo": "bar"}, "quick_fixes": [], } ) expected = ProcessResult( dataframe, [RenderError(TODO_i18n("an error"), [])], json={"foo": "bar"}, ) self.assertEqual(result, expected)
def test_to_arrow_empty_dataframe(self): fd, filename = tempfile.mkstemp() # We'll test that ProcessResult.to_arrow() writes empty bytes on error os.write(fd, b"to-remove") os.close(fd) try: result = ProcessResult.coerce("bad, bad error").to_arrow(Path(filename)) self.assertEqual( result, atypes.RenderResult( [RenderError(TODO_i18n("bad, bad error"), [])], {}, ), ) assert_arrow_table_equals( load_untrusted_arrow_file_with_columns(Path(filename))[0], make_table() ) finally: os.unlink(filename)
def test_default_render_returns_fetch_result(self): # Functionality used by libraryofcongress with ExitStack() as ctx: input_arrow_table = ctx.enter_context( arrow_table_context({"A": [1]}, dir=self.basedir)) parquet_filename = Path( ctx.enter_context(parquet_file({"A": [2]}, dir=self.basedir)).name).name out_filename = ctx.enter_context( tempfile_context(dir=self.basedir)).name thrift_result = module.render_thrift( ttypes.RenderRequest( str(self.basedir), arrow_arrow_table_to_thrift(input_arrow_table), {}, # params ttypes.Tab("tab-1", "Tab 1"), ttypes.FetchResult( parquet_filename, [ ttypes.RenderError( ttypes.I18nMessage( "TODO_i18n", { "text": ttypes.I18nArgument( string_value="A warning") }, None, ), [], ) ], ), out_filename, )) result = thrift_render_result_to_arrow(thrift_result, self.basedir) assert_render_result_equals( result, RenderResult( arrow_table({"A": [2]}), [RenderError(TODO_i18n("A warning"))], ), )
def test_to_arrow_empty_dataframe(self): fd, filename = tempfile.mkstemp() os.close(fd) # Remove the file. Then we'll test that ProcessResult.to_arrow() does # not write it (because the result is an error) os.unlink(filename) try: result = ProcessResult.coerce("bad, bad error").to_arrow( Path(filename)) self.assertEqual( result, atypes.RenderResult( atypes.ArrowTable(None, None, TableMetadata(0, [])), [RenderError(TODO_i18n("bad, bad error"), [])], {}, ), ) with self.assertRaises(FileNotFoundError): open(filename) finally: try: os.unlink(filename) except FileNotFoundError: pass
def test_list_from_list_of_string(self): self.assertEqual( coerce_RenderError_list(["error"]), [RenderError(TODO_i18n("error"))], )
def test_coerce_tuple_dataframe_str_none(self): df = pd.DataFrame({"foo": ["bar"]}) expected = ProcessResult(df, [RenderError(TODO_i18n("hi"))]) result = ProcessResult.coerce((df, "hi", None)) self.assertEqual(result, expected)
def test_list_from_list_with_quick_fixes(self): self.assertEqual( coerce_RenderError_list( [ { "message": ("my id", {}), "quickFixes": [ dict( text="button text", action="prependModule", args=["converttotext", {"colnames": ["A", "B"]}], ) ], }, { "message": ("my other id", {"other": "this"}), "quickFixes": [ dict( text=("quick fix id", {"fix": "that"}), action="prependModule", args=["convert-date", {"colnames": ["C", "D"]}], ), dict( text=("another quick fix id", {"fix": "that"}), action="prependModule", args=["converttonumber", {"colnames": ["E", "F"]}], ), ], }, ] ), [ RenderError( I18nMessage("my id", {}, None), [ QuickFix( TODO_i18n("button text"), QuickFixAction.PrependStep( "converttotext", {"colnames": ["A", "B"]} ), ) ], ), RenderError( I18nMessage("my other id", {"other": "this"}, None), [ QuickFix( I18nMessage("quick fix id", {"fix": "that"}, None), QuickFixAction.PrependStep( "convert-date", {"colnames": ["C", "D"]} ), ), QuickFix( I18nMessage("another quick fix id", {"fix": "that"}, None), QuickFixAction.PrependStep( "converttonumber", {"colnames": ["E", "F"]} ), ), ], ), ], )
def test_list_from_nonempty_string(self): result = coerce_RenderError_list("hello") expected = [RenderError(TODO_i18n("hello"))] self.assertEqual(result, expected)
def test_coerce_str(self): expected = ProcessResult(errors=[RenderError(TODO_i18n("yay"))]) result = ProcessResult.coerce("yay") self.assertEqual(result, expected)
def test_coerce_tuple_none_str_dict(self): expected = ProcessResult(errors=[RenderError(TODO_i18n("hi"))], json={"a": "b"}) result = ProcessResult.coerce((None, "hi", {"a": "b"})) self.assertEqual(result, expected)
def test_coerce_tuple_none_str_none(self): expected = ProcessResult(errors=[RenderError(TODO_i18n("hi"))]) result = ProcessResult.coerce((None, "hi", None)) self.assertEqual(result, expected)