def render_thrift(request: ttypes.RenderRequest) -> ttypes.RenderResult:
"""
Render using Thrift data types.
This function will convert to `cjwkernel.types` (opening Arrow tables in
the process), call `render_arrow()`, and then convert the result back to
Thrift. This uses very little RAM.
Module authors may overwrite this function to avoid reading or writing the
data table entirely -- for instance, a "change number format" module may
not need to read any data, so it could operate on the Thrift layer. Most
modules _do_ look at table data, so they should not overwrite this
function.
"""
basedir = Path(request.basedir)
arrow_table = thrift_arrow_table_to_arrow(request.input_table,
basedir,
trusted=True)
params = thrift_params_to_arrow(request.params, basedir)
params_dict = params.params
if request.fetch_result is None:
fetch_result = None
else:
fetch_result = thrift_fetch_result_to_arrow(request.fetch_result,
basedir)
arrow_result: types.RenderResult = __render_by_signature(
table=arrow_table,
params=params_dict,
tab_name=request.tab.name,
fetch_result=fetch_result,
output_path=basedir / request.output_filename,
)
return arrow_render_result_to_thrift(arrow_result)
def test_params_filename_from_thrift_happy_path(self):
with tempfile.NamedTemporaryFile(dir=self.basedir) as tf:
path = Path(tf.name)
path.write_bytes(b"")
self.assertEqual(
types.thrift_params_to_arrow(
{"A": ttypes.ParamValue(filename_value=path.name)},
self.basedir),
types.Params({"A": path}),
)
def fetch_thrift(request: ttypes.FetchRequest) -> ttypes.FetchResult:
basedir = Path(request.basedir)
arrow_result = fetch_arrow(
thrift_params_to_arrow(request.params, basedir).params,
thrift_raw_params_to_arrow(request.secrets).params,
(None if request.last_fetch_result is None else
thrift_fetch_result_to_arrow(request.last_fetch_result, basedir)),
(None if request.input_table_parquet_filename is None else basedir /
request.input_table_parquet_filename),
basedir / request.output_filename,
)
return arrow_fetch_result_to_thrift(arrow_result)
def test_params_from_thrift(self):
self.assertEqual(
types.thrift_params_to_arrow(
{
"str": ttypes.ParamValue(string_value="s"),
"int": ttypes.ParamValue(integer_value=2),
"float": ttypes.ParamValue(float_value=1.2),
"null": ttypes.ParamValue(),
"bool": ttypes.ParamValue(boolean_value=False),
"column": ttypes.ParamValue(
column_value=ttypes.Column(
"A",
ttypes.ColumnType(
number_type=ttypes.ColumnTypeNumber(format="{:,.2f}")
),
)
),
"listofmaps": ttypes.ParamValue(
list_value=[
ttypes.ParamValue(
map_value={
"A": ttypes.ParamValue(string_value="a"),
"B": ttypes.ParamValue(string_value="b"),
}
),
ttypes.ParamValue(
map_value={
"C": ttypes.ParamValue(string_value="c"),
"D": ttypes.ParamValue(string_value="d"),
}
),
]
),
"tab": ttypes.ParamValue(string_value="TODO tabs"),
},
self.basedir,
),
types.Params(
{
"str": "s",
"int": 2,
"float": 1.2,
"null": None,
"bool": False,
"column": types.Column(
"A", types.ColumnType.Number(format="{:,.2f}")
),
"listofmaps": [{"A": "a", "B": "b"}, {"C": "c", "D": "d"}],
"tab": "TODO tabs",
}
),
)
def test_params_filename_from_thrift_file_not_found_is_error(self):
with self.assertRaisesRegexp(ValueError, "file must exist"):
types.thrift_params_to_arrow(
{"A": ttypes.ParamValue(filename_value="does_not_exist")},
self.basedir)
