def test_render_error_from_thrift(self):
self.assertEqual(
types.thrift_render_error_to_arrow(
ttypes.RenderError(
ttypes.I18nMessage("foo", {}, None),
[
ttypes.QuickFix(
ttypes.I18nMessage("click", {}, None),
ttypes.QuickFixAction(
prepend_step=ttypes.PrependStepQuickFixAction(
"filter",
{"x": ttypes.Json(string_value="y")})),
)
],
)),
types.RenderError(
I18nMessage("foo", {}, None),
[
types.QuickFix(
I18nMessage("click", {}, None),
types.QuickFixAction.PrependStep("filter", {"x": "y"}),
)
],
),
)
def test_render_error_to_thrift(self):
self.assertEqual(
types.arrow_render_error_to_thrift(
types.RenderError(
I18nMessage("foo", {}, None),
[
types.QuickFix(
I18nMessage("click", {}, None),
types.QuickFixAction.PrependStep("filter", {"x": "y"}),
)
],
)
),
ttypes.RenderError(
ttypes.I18nMessage("foo", {}, None),
[
ttypes.QuickFix(
ttypes.I18nMessage("click", {}, None),
ttypes.QuickFixAction(
prepend_step=ttypes.PrependStepQuickFixAction(
"filter", ttypes.RawParams('{"x":"y"}')
)
),
)
],
),
)
def test_list_from_list_of_string_and_tuples(self):
self.assertEqual(
coerce_RenderError_list(
["error", ("my_id", {}), ("my_other_id", {"this": "one"})]
),
[
RenderError(TODO_i18n("error")),
RenderError(I18nMessage("my_id", {}, None)),
RenderError(I18nMessage("my_other_id", {"this": "one"}, None)),
],
)
def test_coerce_with_source_library(self):
self.assertEqual(
coerce_I18nMessage(("my_id", {
"hello": "there"
}, "cjwmodule")),
I18nMessage("my_id", {"hello": "there"}, "cjwmodule"),
)
def test_coerce_from_tuple(self):
self.assertEqual(
coerce_I18nMessage(("my_id", {
"hello": "there"
})),
I18nMessage("my_id", {"hello": "there"}, None),
)
def test_coerce_with_source_library_none(self):
self.assertEqual(
coerce_I18nMessage(("my_id", {
"hello": "there"
}, None)),
I18nMessage("my_id", {"hello": "there"}, None),
)
def test_from_message_2tuple(self):
self.assertEqual(
coerce_RenderError(("my_id", {
"hello": "there"
})),
RenderError(I18nMessage("my_id", {"hello": "there"}, None)),
)
def coerce_I18nMessage(value: mtypes.Message) -> I18nMessage:
if isinstance(value, str):
return TODO_i18n(value)
elif isinstance(value, tuple):
if len(value) < 2 or len(value) > 3:
raise ValueError("This tuple cannot be coerced to I18nMessage: %s" % value)
if not isinstance(value[0], str):
raise ValueError(
"Message ID must be string, got %s" % type(value[0]).__name__
)
if not isinstance(value[1], dict):
raise ValueError(
"Message arguments must be a dict, got %s" % type(value[1]).__name__
)
if len(value) == 3:
source = value[2]
if source not in ["module", "cjwmodule", "cjwparse", None]:
raise ValueError("Invalid i18n message source %r" % source)
else:
source = None
return I18nMessage(value[0], value[1], source)
else:
raise ValueError(
"%s is of type %s, which cannot be coerced to I18nMessage"
% (value, type(value).__name__)
)
def test_coerce_tuple_dataframe_i18n_none(self):
df = pd.DataFrame({"foo": ["bar"]})
expected = ProcessResult(
df, [RenderError(I18nMessage("message.id", {"param1": "a"}, None))]
)
result = ProcessResult.coerce((df, ("message.id", {"param1": "a"}), None))
self.assertEqual(result, expected)
def test_render_deprecated_parquet_warning(self):
message = I18nMessage("TODO_i18n", {"text": "truncated table"}, None)
fetch_errors = [RenderError(message)]
with parquet_file({"A": [1, 2], "B": [3, 4]}) as fetched_path:
table, errors = call_render(P(), FetchResult(fetched_path, fetch_errors))
assert_arrow_table_equals(table, {"A": [1, 2], "B": [3, 4]})
self.assertEqual(errors, [message])
def test_from_message_3tuple(self):
self.assertEqual(
coerce_RenderError(("my_id", {
"hello": "there"
}, "cjwmodule")),
RenderError(I18nMessage("my_id", {"hello": "there"}, "cjwmodule")),
)
def test_render_fetch_error(self):
message = I18nMessage("x", {"y": "z"}, None)
fetch_errors = [RenderError(message)]
with tempfile_context() as empty_path:
table, errors = call_render(P(), FetchResult(empty_path, fetch_errors))
assert_arrow_table_equals(table, None)
self.assertEqual(errors, [message])
def test_list_index_out_of_range(self):
# Pandas' read_csv() freaks out on even the simplest examples....
#
# Today's exhibit:
# pd.read_csv(io.StringIO('A\n,,'), index_col=False)
# raises IndexError: list index out of range
result = render_arrow(csv="A\n,,", has_header_row=True)
assert_arrow_table_equals(result.table, {
"A": [""],
"Column 2": [""],
"Column 3": [""]
})
self.assertEqual(
result.errors,
[
I18nMessage(
"util.colnames.warnings.default",
{
"n_columns": 2,
"first_colname": "Column 2"
},
"cjwmodule",
)
],
)
def test_render_deprecated_parquet_has_header_false(self):
# This behavior is totally awful, but we support it for backwards
# compatibility.
#
# Back in the day, we parsed during fetch. But has_header can change
# between fetch and render. We were lazy, so we made fetch() follow the
# most-common path: has_header=True. Then, in render(), we would "undo"
# the change if has_header=False. This was lossy. It took a lot of time
# to figure it out. It was _never_ wise to code this. Now we need to
# support these lossy, mangled files.
with parquet_file({"A": [1, 2], "B": [3, 4]}) as fetched_path:
table, errors = call_render(P(has_header=False),
FetchResult(fetched_path))
assert_arrow_table_equals(table, {"A": [1, 2], "B": [3, 4]})
self.assertEqual(
errors,
[
I18nMessage(
"TODO_i18n",
{
"text":
"Please re-download this file to disable header-row handling"
},
None,
)
],
)
def test_coerce_dict_i18n(self):
expected = ProcessResult(
errors=[
RenderError(
TODO_i18n("an error"),
[
QuickFix(
I18nMessage("message.id", {}, None),
QuickFixAction.PrependStep(
"texttodate", {"column": "created_at"}
),
)
],
)
]
)
result = ProcessResult.coerce(
{
"message": "an error",
"quickFixes": [
dict(
text=("message.id", {}),
action="prependModule",
args=["texttodate", {"column": "created_at"}],
)
],
}
)
self.assertEqual(result, expected)
def TODO_i18n(text: str) -> I18nMessage:
"""Build an I18nMessage that "translates" into English only.
The message has id "TODO_i18n" and one argument, "text", in English.
Long-term, all these messages should disappear; but this helps us
migrate by letting us code without worrying about translation.
"""
return I18nMessage("TODO_i18n", {"text": text}, None)
def test_coerce_tuple_none_i18n_none(self):
expected = ProcessResult(errors=[
RenderError(I18nMessage("message.id", {"param1": "a"}, None))
])
result = ProcessResult.coerce((None, ("message.id", {
"param1": "a"
}), None))
self.assertEqual(result, expected)
def test_coerce_3tuple_i18n(self):
self.assertEqual(
ProcessResult.coerce(("my_id", {"hello": "there"}, "cjwmodule")),
ProcessResult(
errors=[
RenderError(I18nMessage("my_id", {"hello": "there"}, "cjwmodule"))
]
),
)
def test_coerce_dict_quickfix_multiple(self):
dataframe = pd.DataFrame({"A": [1, 2]})
result = ProcessResult.coerce({
"dataframe":
dataframe,
"errors": [
{
"message":
"an error",
"quickFixes": [
dict(
text="Hi",
action="prependModule",
args=["texttodate", {
"column": "created_at"
}],
),
dict(
text=("message.id", {}),
action="prependModule",
args=["texttodate", {
"column": "created_at"
}],
),
],
},
"other error",
],
"json": {
"foo": "bar"
},
})
expected = ProcessResult(
dataframe,
errors=[
RenderError(
TODO_i18n("an error"),
[
QuickFix(
TODO_i18n("Hi"),
QuickFixAction.PrependStep(
"texttodate", {"column": "created_at"}),
),
QuickFix(
I18nMessage("message.id", {}, None),
QuickFixAction.PrependStep(
"texttodate", {"column": "created_at"}),
),
],
),
RenderError(TODO_i18n("other error")),
],
json={"foo": "bar"},
)
self.assertEqual(result, expected)
def thrift_i18n_message_to_arrow(value: ttypes.I18nMessage) -> I18nMessage:
if value.source not in [None, "module", "cjwmodule", "cjwparse"]:
raise ValueError("Invalid message source %r" % value.source)
return I18nMessage(
value.id,
{
k: _thrift_i18n_argument_to_arrow(v)
for k, v in value.arguments.items()
},
value.source,
)
def test_fetch_invalid_url(self):
with call_fetch("htt://blah") as result:
self.assertEqual(result.path.read_bytes(), b"")
self.assertEqual(
result.errors,
[
RenderError(
I18nMessage("http.errors.HttpErrorInvalidUrl", {},
"cjwmodule"))
],
)
def _dict_to_i18n_message(value: Dict[str, Any]) -> I18nMessage:
arguments = value["arguments"]
# Compatibility for https://www.pivotaltracker.com/story/show/174865394
# DELETEME when there are no CachedRenderResults from before 2020-10-01
if not value.get("source") and value["id"] in (
"py.renderer.execute.types.PromptingError.WrongColumnType.as_quick_fixes.general",
"py.renderer.execute.types.PromptingError.WrongColumnType.as_error_message.general",
):
for key in ("found_type", "best_wanted_type"):
if arguments.get(key) == "datetime":
arguments = {**arguments, key: "timestamp"}
return I18nMessage(value["id"], arguments, value.get("source"))
def test_quick_fix_to_thrift(self):
self.assertEqual(
types.arrow_quick_fix_to_thrift(
types.QuickFix(
I18nMessage("click", {}, None),
types.QuickFixAction.PrependStep("filter", {"x": "y"}),
)),
ttypes.QuickFix(
ttypes.I18nMessage("click", {}, None),
ttypes.QuickFixAction(
prepend_step=ttypes.PrependStepQuickFixAction(
"filter", {"x": ttypes.Json(string_value="y")})),
),
)
def test_render_error(self):
with self._file(b"A,B\nx,y", suffix=".json") as path:
table, errors = do_render({"file": path, "has_header": True})
assert_arrow_table_equals(table, {})
self.assertEqual(
errors,
[
I18nMessage(
"TODO_i18n",
{"text": "JSON parse error at byte 0: Invalid value."},
None,
)
],
)
def test_i18n_message_from_thrift_source_none(self):
self.assertEqual(
types.thrift_i18n_message_to_arrow(
ttypes.I18nMessage(
"modules.x.y",
{
"a": ttypes.I18nArgument(string_value="s"),
"b": ttypes.I18nArgument(i32_value=12345678),
"c": ttypes.I18nArgument(double_value=0.123),
},
None,
)
),
I18nMessage("modules.x.y", {"a": "s", "b": 12345678, "c": 0.123}, None),
)
def trans(
message_id: str,
*,
default: str,
arguments: Dict[str, Union[int, float, str]] = {},
) -> I18nMessage:
"""Build an I18nMessage, marking it for translation.
Use this function (instead of constructing `I18nMessage` directly) and the
string will be marked for translation. Workbench's tooling will extract
messages from all `trans()` calls and send them to translators.
The `default` argument informs the translation pipeline; it is not sent
directly to users on production.
"""
return I18nMessage(message_id, arguments, None)
def test_duplicate_column_names_renamed(self):
result = render_arrow(csv="A,A\na,b", has_header_row=True)
assert_arrow_table_equals(result.table, {"A": ["a"], "A 2": ["b"]})
self.assertEqual(
result.errors,
[
I18nMessage(
"util.colnames.warnings.numbered",
{
"n_columns": 1,
"first_colname": "A 2"
},
"cjwmodule",
)
],
)
def test_i18n_message_to_thrift_source_library(self):
self.assertEqual(
types.arrow_i18n_message_to_thrift(
I18nMessage(
"modules.x.y", {"a": "s", "b": 12345678, "c": 0.123}, "cjwmodule"
)
),
ttypes.I18nMessage(
"modules.x.y",
{
"a": ttypes.I18nArgument(string_value="s"),
"b": ttypes.I18nArgument(i32_value=12345678),
"c": ttypes.I18nArgument(double_value=0.123),
},
"cjwmodule",
),
)
def test_quick_fix_from_thrift(self):
self.assertEqual(
types.thrift_quick_fix_to_arrow(
ttypes.QuickFix(
ttypes.I18nMessage("click", {}, None),
ttypes.QuickFixAction(
prepend_step=ttypes.PrependStepQuickFixAction(
"filter", ttypes.RawParams('{"x":"y"}')
)
),
)
),
types.QuickFix(
I18nMessage("click", {}, None),
types.QuickFixAction.PrependStep("filter", {"x": "y"}),
),
)
def test_redirect_loop(self):
url1 = self.build_url("/url1.csv")
url2 = self.build_url("/url2.csv")
self.mock_http_response = itertools.cycle([
MockHttpResponse(302, [("Location", url2)]),
MockHttpResponse(302, [("Location", url1)]),
])
with call_fetch(url1) as result:
self.assertEqual(result.path.read_bytes(), b"")
self.assertEqual(
result.errors,
[
RenderError(
I18nMessage("http.errors.HttpErrorTooManyRedirects",
{}, "cjwmodule"))
],
)