def test_fetch_result_from_thrift_happy_path(self): with tempfile.NamedTemporaryFile(dir=str(self.basedir)) as tf: self.assertEqual( types.FetchResult.from_thrift( ttypes.FetchResult( Path(tf.name).name, [types.RenderError(types.I18nMessage("hi")).to_thrift()], ), self.basedir, ), types.FetchResult( Path(tf.name), [types.RenderError(types.I18nMessage("hi"))] ), )
def test_render_error_to_dict(self): self.assertEqual( fields._render_error_to_dict( types.RenderError( types.I18nMessage("err", {}), [ types.QuickFix( types.I18nMessage("click", {}, "cjwmodule"), types.QuickFixAction.PrependStep("filter", {"x": "y"}), ) ], ) ), { "message": {"id": "err", "arguments": {}}, "quickFixes": [ { "buttonText": { "id": "click", "arguments": {}, "source": "cjwmodule", }, "action": { "type": "prependStep", "moduleSlug": "filter", "partialParams": {"x": "y"}, }, } ], }, )
def test_render_error_to_thrift(self): self.assertEqual( types.RenderError( types.I18nMessage("foo", {}), [ types.QuickFix( types.I18nMessage("click"), types.QuickFixAction.PrependStep("filter", {"x": "y"}), ) ], ).to_thrift(), ttypes.RenderError( ttypes.I18nMessage("foo", {}, ttypes.I18nMessageSource()), [ ttypes.QuickFix( ttypes.I18nMessage("click", {}, ttypes.I18nMessageSource()), ttypes.QuickFixAction( prepend_step=ttypes.PrependStepQuickFixAction( "filter", ttypes.RawParams('{"x":"y"}') ) ), ) ], ), )
def test_to_arrow_empty_dataframe(self): fd, filename = tempfile.mkstemp() os.close(fd) # Remove the file. Then we'll test that ProcessResult.to_arrow() does # not write it (because the result is an error) os.unlink(filename) try: result = ProcessResult.coerce("bad, bad error").to_arrow(Path(filename)) self.assertEqual( result, atypes.RenderResult( atypes.ArrowTable(None, atypes.TableMetadata(0, [])), [ atypes.RenderError( atypes.I18nMessage.TODO_i18n("bad, bad error"), [] ) ], {}, ), ) with self.assertRaises(FileNotFoundError): open(filename) finally: try: os.unlink(filename) except FileNotFoundError: pass
def test_render_error_to_thrift(self): self.assertEqual( types.arrow_render_error_to_thrift( types.RenderError( I18nMessage("foo", {}, None), [ types.QuickFix( I18nMessage("click", {}, None), types.QuickFixAction.PrependStep( "filter", {"x": "y"}), ) ], )), ttypes.RenderError( ttypes.I18nMessage("foo", {}, None), [ ttypes.QuickFix( ttypes.I18nMessage("click", {}, None), ttypes.QuickFixAction( prepend_step=ttypes.PrependStepQuickFixAction( "filter", {"x": ttypes.Json( string_value="y")})), ) ], ), )
def fetch_pandas( params: Dict[str, Any], secrets: Dict[str, Any], last_fetch_result: Optional[types.FetchResult], input_table_parquet_path: Optional[Path], output_path: Path, ) -> Union[ptypes.ProcessResult, types.FetchResult]: """ Call `fetch()` and validate the result. Module authors should not replace this function: they should replace `fetch()` instead. This function validates the `fetch()` return value, to raise a helpful `ValueError` if the module code is buggy. """ spec = inspect.getfullargspec(fetch) kwargs = {} varkw = bool(spec.varkw) # if True, function accepts **kwargs kwonlyargs = spec.kwonlyargs if varkw or "secrets" in kwonlyargs: kwargs["secrets"] = secrets if varkw or "get_input_dataframe" in kwonlyargs: async def get_input_dataframe(): if input_table_parquet_path is None: return None else: return __parquet_to_pandas(input_table_parquet_path) kwargs["get_input_dataframe"] = get_input_dataframe if varkw or "get_stored_dataframe" in kwonlyargs: async def get_stored_dataframe(): if last_fetch_result is None: return None else: return __parquet_to_pandas(last_fetch_result.path) kwargs["get_stored_dataframe"] = get_stored_dataframe if varkw or "output_path" in kwonlyargs: kwargs["output_path"] = output_path result = fetch(params, **kwargs) if asyncio.iscoroutine(result): result = asyncio.run(result) if (isinstance(result, tuple) and len(result) == 2 and isinstance(result[0], Path) and isinstance(result[1], str)): return types.FetchResult( result[0], [types.RenderError(types.I18nMessage.TODO_i18n(result[1]))]) elif isinstance(result, Path): return types.FetchResult(result) else: return ptypes.ProcessResult.coerce(result)
def __render_arrow( *, table: types.ArrowTable, params: Dict[str, Any], tab_name: str, fetch_result: Optional[types.FetchResult], output_path: Path, ) -> types.RenderResult: """Render using `cjwkernel.types` data types. Write to `output_path`. This will typically call `render()`. """ # call render() raw_result = render( table.table, params, output_path, columns=table.metadata.columns, settings=settings, tab_name=tab_name, fetch_result=fetch_result, ) # coerce result # TODO let module output column types. (Currently, the lack of column types # means this is only useful for fetch modules that don't output number # formats.) table = types.ArrowTable.from_arrow_file_with_inferred_metadata( output_path, fallback_column_types={c.name: c.type for c in table.metadata.columns}, ) errors = [] # TODO support more output types? Or develop the One True Types (maybe # types.RenderResult) and force modules to output it. if isinstance(raw_result, list): # List of I18nMessage errors errors = [ # TODO don't use coerce_I18nMessage? At least, don't use ptypes. # Do any modules even require coerce? Or do they all correctly # output tuples-or-text? Is it only unit tests that output # non-I18nMessage tuples? types.RenderError(ptypes.coerce_I18nMessage(message)) for message in raw_result ] elif raw_result is None: errors = [] return types.RenderResult(table, errors)
def test_to_arrow_quick_fixes(self): fd, filename = tempfile.mkstemp() os.close(fd) # Remove the file. Then we'll test that ProcessResult.to_arrow() does # not write it (because the result is an error) os.unlink(filename) try: result = ProcessResult( error="bad, bad error", quick_fixes=[ QuickFix( "button foo", "prependModule", ["converttotext", {"colnames": ["A", "B"]}], ), QuickFix( "button bar", "prependModule", ["converttonumber", {"colnames": ["A", "B"]}], ), ], ).to_arrow(Path(filename)) self.assertEqual( result.errors, [ atypes.RenderError( atypes.I18nMessage.TODO_i18n("bad, bad error"), [ atypes.QuickFix( atypes.I18nMessage.TODO_i18n("button foo"), atypes.QuickFixAction.PrependStep( "converttotext", {"colnames": ["A", "B"]} ), ), atypes.QuickFix( atypes.I18nMessage.TODO_i18n("button bar"), atypes.QuickFixAction.PrependStep( "converttonumber", {"colnames": ["A", "B"]} ), ), ], ) ], ) with self.assertRaises(FileNotFoundError): open(filename) finally: try: os.unlink(filename) except FileNotFoundError: pass