def test_render_error_from_thrift(self): self.assertEqual( types.thrift_render_error_to_arrow( ttypes.RenderError( ttypes.I18nMessage("foo", {}, None), [ ttypes.QuickFix( ttypes.I18nMessage("click", {}, None), ttypes.QuickFixAction( prepend_step=ttypes.PrependStepQuickFixAction( "filter", {"x": ttypes.Json(string_value="y")})), ) ], )), types.RenderError( I18nMessage("foo", {}, None), [ types.QuickFix( I18nMessage("click", {}, None), types.QuickFixAction.PrependStep("filter", {"x": "y"}), ) ], ), )
def test_render_error_to_thrift(self): self.assertEqual( types.arrow_render_error_to_thrift( types.RenderError( I18nMessage("foo", {}, None), [ types.QuickFix( I18nMessage("click", {}, None), types.QuickFixAction.PrependStep("filter", {"x": "y"}), ) ], ) ), ttypes.RenderError( ttypes.I18nMessage("foo", {}, None), [ ttypes.QuickFix( ttypes.I18nMessage("click", {}, None), ttypes.QuickFixAction( prepend_step=ttypes.PrependStepQuickFixAction( "filter", ttypes.RawParams('{"x":"y"}') ) ), ) ], ), )
def test_list_from_list_of_string_and_tuples(self): self.assertEqual( coerce_RenderError_list( ["error", ("my_id", {}), ("my_other_id", {"this": "one"})] ), [ RenderError(TODO_i18n("error")), RenderError(I18nMessage("my_id", {}, None)), RenderError(I18nMessage("my_other_id", {"this": "one"}, None)), ], )
def test_coerce_with_source_library(self): self.assertEqual( coerce_I18nMessage(("my_id", { "hello": "there" }, "cjwmodule")), I18nMessage("my_id", {"hello": "there"}, "cjwmodule"), )
def test_coerce_from_tuple(self): self.assertEqual( coerce_I18nMessage(("my_id", { "hello": "there" })), I18nMessage("my_id", {"hello": "there"}, None), )
def test_coerce_with_source_library_none(self): self.assertEqual( coerce_I18nMessage(("my_id", { "hello": "there" }, None)), I18nMessage("my_id", {"hello": "there"}, None), )
def test_from_message_2tuple(self): self.assertEqual( coerce_RenderError(("my_id", { "hello": "there" })), RenderError(I18nMessage("my_id", {"hello": "there"}, None)), )
def coerce_I18nMessage(value: mtypes.Message) -> I18nMessage: if isinstance(value, str): return TODO_i18n(value) elif isinstance(value, tuple): if len(value) < 2 or len(value) > 3: raise ValueError("This tuple cannot be coerced to I18nMessage: %s" % value) if not isinstance(value[0], str): raise ValueError( "Message ID must be string, got %s" % type(value[0]).__name__ ) if not isinstance(value[1], dict): raise ValueError( "Message arguments must be a dict, got %s" % type(value[1]).__name__ ) if len(value) == 3: source = value[2] if source not in ["module", "cjwmodule", "cjwparse", None]: raise ValueError("Invalid i18n message source %r" % source) else: source = None return I18nMessage(value[0], value[1], source) else: raise ValueError( "%s is of type %s, which cannot be coerced to I18nMessage" % (value, type(value).__name__) )
def test_coerce_tuple_dataframe_i18n_none(self): df = pd.DataFrame({"foo": ["bar"]}) expected = ProcessResult( df, [RenderError(I18nMessage("message.id", {"param1": "a"}, None))] ) result = ProcessResult.coerce((df, ("message.id", {"param1": "a"}), None)) self.assertEqual(result, expected)
def test_render_deprecated_parquet_warning(self): message = I18nMessage("TODO_i18n", {"text": "truncated table"}, None) fetch_errors = [RenderError(message)] with parquet_file({"A": [1, 2], "B": [3, 4]}) as fetched_path: table, errors = call_render(P(), FetchResult(fetched_path, fetch_errors)) assert_arrow_table_equals(table, {"A": [1, 2], "B": [3, 4]}) self.assertEqual(errors, [message])
def test_from_message_3tuple(self): self.assertEqual( coerce_RenderError(("my_id", { "hello": "there" }, "cjwmodule")), RenderError(I18nMessage("my_id", {"hello": "there"}, "cjwmodule")), )
def test_render_fetch_error(self): message = I18nMessage("x", {"y": "z"}, None) fetch_errors = [RenderError(message)] with tempfile_context() as empty_path: table, errors = call_render(P(), FetchResult(empty_path, fetch_errors)) assert_arrow_table_equals(table, None) self.assertEqual(errors, [message])
def test_list_index_out_of_range(self): # Pandas' read_csv() freaks out on even the simplest examples.... # # Today's exhibit: # pd.read_csv(io.StringIO('A\n,,'), index_col=False) # raises IndexError: list index out of range result = render_arrow(csv="A\n,,", has_header_row=True) assert_arrow_table_equals(result.table, { "A": [""], "Column 2": [""], "Column 3": [""] }) self.assertEqual( result.errors, [ I18nMessage( "util.colnames.warnings.default", { "n_columns": 2, "first_colname": "Column 2" }, "cjwmodule", ) ], )
def test_render_deprecated_parquet_has_header_false(self): # This behavior is totally awful, but we support it for backwards # compatibility. # # Back in the day, we parsed during fetch. But has_header can change # between fetch and render. We were lazy, so we made fetch() follow the # most-common path: has_header=True. Then, in render(), we would "undo" # the change if has_header=False. This was lossy. It took a lot of time # to figure it out. It was _never_ wise to code this. Now we need to # support these lossy, mangled files. with parquet_file({"A": [1, 2], "B": [3, 4]}) as fetched_path: table, errors = call_render(P(has_header=False), FetchResult(fetched_path)) assert_arrow_table_equals(table, {"A": [1, 2], "B": [3, 4]}) self.assertEqual( errors, [ I18nMessage( "TODO_i18n", { "text": "Please re-download this file to disable header-row handling" }, None, ) ], )
def test_coerce_dict_i18n(self): expected = ProcessResult( errors=[ RenderError( TODO_i18n("an error"), [ QuickFix( I18nMessage("message.id", {}, None), QuickFixAction.PrependStep( "texttodate", {"column": "created_at"} ), ) ], ) ] ) result = ProcessResult.coerce( { "message": "an error", "quickFixes": [ dict( text=("message.id", {}), action="prependModule", args=["texttodate", {"column": "created_at"}], ) ], } ) self.assertEqual(result, expected)
def TODO_i18n(text: str) -> I18nMessage: """Build an I18nMessage that "translates" into English only. The message has id "TODO_i18n" and one argument, "text", in English. Long-term, all these messages should disappear; but this helps us migrate by letting us code without worrying about translation. """ return I18nMessage("TODO_i18n", {"text": text}, None)
def test_coerce_tuple_none_i18n_none(self): expected = ProcessResult(errors=[ RenderError(I18nMessage("message.id", {"param1": "a"}, None)) ]) result = ProcessResult.coerce((None, ("message.id", { "param1": "a" }), None)) self.assertEqual(result, expected)
def test_coerce_3tuple_i18n(self): self.assertEqual( ProcessResult.coerce(("my_id", {"hello": "there"}, "cjwmodule")), ProcessResult( errors=[ RenderError(I18nMessage("my_id", {"hello": "there"}, "cjwmodule")) ] ), )
def test_coerce_dict_quickfix_multiple(self): dataframe = pd.DataFrame({"A": [1, 2]}) result = ProcessResult.coerce({ "dataframe": dataframe, "errors": [ { "message": "an error", "quickFixes": [ dict( text="Hi", action="prependModule", args=["texttodate", { "column": "created_at" }], ), dict( text=("message.id", {}), action="prependModule", args=["texttodate", { "column": "created_at" }], ), ], }, "other error", ], "json": { "foo": "bar" }, }) expected = ProcessResult( dataframe, errors=[ RenderError( TODO_i18n("an error"), [ QuickFix( TODO_i18n("Hi"), QuickFixAction.PrependStep( "texttodate", {"column": "created_at"}), ), QuickFix( I18nMessage("message.id", {}, None), QuickFixAction.PrependStep( "texttodate", {"column": "created_at"}), ), ], ), RenderError(TODO_i18n("other error")), ], json={"foo": "bar"}, ) self.assertEqual(result, expected)
def thrift_i18n_message_to_arrow(value: ttypes.I18nMessage) -> I18nMessage: if value.source not in [None, "module", "cjwmodule", "cjwparse"]: raise ValueError("Invalid message source %r" % value.source) return I18nMessage( value.id, { k: _thrift_i18n_argument_to_arrow(v) for k, v in value.arguments.items() }, value.source, )
def test_fetch_invalid_url(self): with call_fetch("htt://blah") as result: self.assertEqual(result.path.read_bytes(), b"") self.assertEqual( result.errors, [ RenderError( I18nMessage("http.errors.HttpErrorInvalidUrl", {}, "cjwmodule")) ], )
def _dict_to_i18n_message(value: Dict[str, Any]) -> I18nMessage: arguments = value["arguments"] # Compatibility for https://www.pivotaltracker.com/story/show/174865394 # DELETEME when there are no CachedRenderResults from before 2020-10-01 if not value.get("source") and value["id"] in ( "py.renderer.execute.types.PromptingError.WrongColumnType.as_quick_fixes.general", "py.renderer.execute.types.PromptingError.WrongColumnType.as_error_message.general", ): for key in ("found_type", "best_wanted_type"): if arguments.get(key) == "datetime": arguments = {**arguments, key: "timestamp"} return I18nMessage(value["id"], arguments, value.get("source"))
def test_quick_fix_to_thrift(self): self.assertEqual( types.arrow_quick_fix_to_thrift( types.QuickFix( I18nMessage("click", {}, None), types.QuickFixAction.PrependStep("filter", {"x": "y"}), )), ttypes.QuickFix( ttypes.I18nMessage("click", {}, None), ttypes.QuickFixAction( prepend_step=ttypes.PrependStepQuickFixAction( "filter", {"x": ttypes.Json(string_value="y")})), ), )
def test_render_error(self): with self._file(b"A,B\nx,y", suffix=".json") as path: table, errors = do_render({"file": path, "has_header": True}) assert_arrow_table_equals(table, {}) self.assertEqual( errors, [ I18nMessage( "TODO_i18n", {"text": "JSON parse error at byte 0: Invalid value."}, None, ) ], )
def test_i18n_message_from_thrift_source_none(self): self.assertEqual( types.thrift_i18n_message_to_arrow( ttypes.I18nMessage( "modules.x.y", { "a": ttypes.I18nArgument(string_value="s"), "b": ttypes.I18nArgument(i32_value=12345678), "c": ttypes.I18nArgument(double_value=0.123), }, None, ) ), I18nMessage("modules.x.y", {"a": "s", "b": 12345678, "c": 0.123}, None), )
def trans( message_id: str, *, default: str, arguments: Dict[str, Union[int, float, str]] = {}, ) -> I18nMessage: """Build an I18nMessage, marking it for translation. Use this function (instead of constructing `I18nMessage` directly) and the string will be marked for translation. Workbench's tooling will extract messages from all `trans()` calls and send them to translators. The `default` argument informs the translation pipeline; it is not sent directly to users on production. """ return I18nMessage(message_id, arguments, None)
def test_duplicate_column_names_renamed(self): result = render_arrow(csv="A,A\na,b", has_header_row=True) assert_arrow_table_equals(result.table, {"A": ["a"], "A 2": ["b"]}) self.assertEqual( result.errors, [ I18nMessage( "util.colnames.warnings.numbered", { "n_columns": 1, "first_colname": "A 2" }, "cjwmodule", ) ], )
def test_i18n_message_to_thrift_source_library(self): self.assertEqual( types.arrow_i18n_message_to_thrift( I18nMessage( "modules.x.y", {"a": "s", "b": 12345678, "c": 0.123}, "cjwmodule" ) ), ttypes.I18nMessage( "modules.x.y", { "a": ttypes.I18nArgument(string_value="s"), "b": ttypes.I18nArgument(i32_value=12345678), "c": ttypes.I18nArgument(double_value=0.123), }, "cjwmodule", ), )
def test_quick_fix_from_thrift(self): self.assertEqual( types.thrift_quick_fix_to_arrow( ttypes.QuickFix( ttypes.I18nMessage("click", {}, None), ttypes.QuickFixAction( prepend_step=ttypes.PrependStepQuickFixAction( "filter", ttypes.RawParams('{"x":"y"}') ) ), ) ), types.QuickFix( I18nMessage("click", {}, None), types.QuickFixAction.PrependStep("filter", {"x": "y"}), ), )
def test_redirect_loop(self): url1 = self.build_url("/url1.csv") url2 = self.build_url("/url2.csv") self.mock_http_response = itertools.cycle([ MockHttpResponse(302, [("Location", url2)]), MockHttpResponse(302, [("Location", url1)]), ]) with call_fetch(url1) as result: self.assertEqual(result.path.read_bytes(), b"") self.assertEqual( result.errors, [ RenderError( I18nMessage("http.errors.HttpErrorTooManyRedirects", {}, "cjwmodule")) ], )