def test_render_exception(self):
module = self.kernel.compile(
MockPath(
["foo.py"],
b"import os\ndef render(table, params): raise RuntimeError('fail')",
),
"foo",
)
with self.assertRaises(ModuleExitedError) as cm:
with arrow_table_context({"A": [1]},
dir=self.basedir) as input_table:
input_table.path.chmod(0o644)
with self.chroot_context.tempfile_context(
prefix="output-", dir=self.basedir) as output_path:
self.kernel.render(
module,
self.chroot_context,
self.basedir,
input_table,
types.Params({
"m": 2.5,
"s": "XX"
}),
types.Tab("tab-1", "Tab 1"),
None,
output_filename=output_path.name,
)
self.assertEquals(cm.exception.exit_code, 1) # Python exit code
self.assertRegex(cm.exception.log, r"\bRuntimeError\b")
self.assertRegex(cm.exception.log, r"\bfail\b")
# Regression test: [2019-10-02], the "pyspawner_main()->spawn_child()"
# process would raise _another_ exception while exiting. It would try to
# close an already-closed socket.
self.assertNotRegex(cm.exception.log, r"Bad file descriptor")
def test_fetch_happy_path(self):
module = self.kernel.compile(
MockPath(
["foo.py"],
textwrap.dedent("""
import pandas as pd
def fetch(params):
return pd.DataFrame({"A": [params["a"]]})
""").encode("utf-8"),
),
"foo",
)
with self.chroot_context.tempfile_context(
prefix="output-", dir=self.basedir) as output_path:
result = self.kernel.fetch(
module,
self.chroot_context,
self.basedir,
types.Params({"a": 1}),
{},
None,
None,
output_filename=output_path.name,
)
self.assertEquals(result.errors, [])
table = pyarrow.parquet.read_pandas(str(result.path))
self.assertEquals(table.to_pydict(), {"A": [1]})
def test_params_to_thrift(self):
self.assertEqual(
types.Params({
"str":
"s",
"int":
2,
"float":
1.2,
"null":
None,
"bool":
False,
"column":
types.Column("A", types.ColumnType.Number(format="{:,.2f}")),
"listofmaps": [{
"A": "a",
"B": "b"
}, {
"C": "c",
"D": "d"
}],
"tab":
"TODO tabs",
}).to_thrift(),
{
"str":
ttypes.ParamValue(string_value="s"),
"int":
ttypes.ParamValue(integer_value=2),
"float":
ttypes.ParamValue(float_value=1.2),
"null":
ttypes.ParamValue(),
"bool":
ttypes.ParamValue(boolean_value=False),
"column":
ttypes.ParamValue(column_value=ttypes.Column(
"A",
ttypes.ColumnType(number_type=ttypes.ColumnTypeNumber(
format="{:,.2f}")),
)),
"listofmaps":
ttypes.ParamValue(list_value=[
ttypes.ParamValue(
map_value={
"A": ttypes.ParamValue(string_value="a"),
"B": ttypes.ParamValue(string_value="b"),
}),
ttypes.ParamValue(
map_value={
"C": ttypes.ParamValue(string_value="c"),
"D": ttypes.ParamValue(string_value="d"),
}),
]),
"tab":
ttypes.ParamValue(string_value="TODO tabs"),
},
)
def test_params_filename_to_thrift(self):
path = self.basedir / "x.bin"
self.assertEqual(
types.Params({
"A": path
}).to_thrift(),
{"A": ttypes.ParamValue(filename_value="x.bin")},
)
def test_params_filename_from_thrift_happy_path(self):
with tempfile.NamedTemporaryFile(dir=self.basedir) as tf:
path = Path(tf.name)
path.write_bytes(b"")
self.assertEqual(
types.Params.from_thrift(
{"A": ttypes.ParamValue(filename_value=path.name)}, self.basedir
),
types.Params({"A": path}),
)
def test_render_happy_path(self):
module = self.kernel.compile(
MockPath(
["foo.py"],
b"import pandas as pd\ndef render(table, params): return pd.DataFrame({'A': table['A'] * params['m'], 'B': table['B'] + params['s']})",
),
"foo",
)
with arrow_table_context(
{
"A": [1, 2, 3],
"B": ["a", "b", "c"]
},
columns=[
types.Column("A", types.ColumnType.Number("{:,d}")),
types.Column("B", types.ColumnType.Text()),
],
dir=self.basedir,
) as input_table:
input_table.path.chmod(0o644)
with self.chroot_context.tempfile_context(
prefix="output-", dir=self.basedir) as output_path:
result = self.kernel.render(
module,
self.chroot_context,
self.basedir,
input_table,
types.Params({
"m": 2.5,
"s": "XX"
}),
types.Tab("tab-1", "Tab 1"),
None,
output_filename=output_path.name,
)
self.assertEquals(
result.table.table.to_pydict(),
{
"A": [2.5, 5.0, 7.5],
"B": ["aXX", "bXX", "cXX"]
},
)
def test_render_kill_timeout(self):
mod = _compile(
"foo", "import time\ndef render(table, params):\n time.sleep(2)")
with patch.object(self.kernel, "render_timeout", 0.001):
with self.assertRaises(ModuleTimeoutError):
with arrow_table_context({"A": [1]},
dir=self.basedir) as input_table:
input_table.path.chmod(0o644)
with self.chroot_context.tempfile_context(
prefix="output-", dir=self.basedir) as output_path:
self.kernel.render(
mod,
self.chroot_context,
self.basedir,
input_table,
types.Params({}),
types.Tab("tab-1", "Tab 1"),
None,
output_filename=output_path.name,
)
def test_render_killed_hard_out_of_memory(self):
# This is similar to out-of-memory kill (but with different exit_code).
# Testing out-of-memory is slow because we have to force the kernel to,
# er, run out of memory. On a typical dev machine, that means filling
# swap space -- gumming up the whole system. Not practical.
#
# In case of out-of-memory, the Linux out-of-memory killer will find
# and kill a process using SIGKILL.
#
# So let's simulate that SIGKILL.
module = self.kernel.compile(
MockPath(
["foo.py"],
b"import os\nimport time\ndef render(table, params): os.kill(os.getpid(), 9); time.sleep(1)",
),
"foo",
)
with self.assertRaises(ModuleExitedError) as cm:
with arrow_table_context({"A": [1]},
dir=self.basedir) as input_table:
input_table.path.chmod(0o644)
with tempfile_context(prefix="output-",
dir=self.basedir) as output_path:
result = self.kernel.render(
module,
self.basedir,
input_table,
types.Params({
"m": 2.5,
"s": "XX"
}),
types.Tab("tab-1", "Tab 1"),
None,
output_filename=output_path.name,
)
print(repr(result))
self.assertEquals(cm.exception.exit_code, -9) # SIGKILL
self.assertEquals(cm.exception.log, "")
