def test_render_exception(self): module = self.kernel.compile( MockPath( ["foo.py"], b"import os\ndef render(table, params): raise RuntimeError('fail')", ), "foo", ) with self.assertRaises(ModuleExitedError) as cm: with arrow_table_context({"A": [1]}, dir=self.basedir) as input_table: input_table.path.chmod(0o644) with self.chroot_context.tempfile_context( prefix="output-", dir=self.basedir) as output_path: self.kernel.render( module, self.chroot_context, self.basedir, input_table, types.Params({ "m": 2.5, "s": "XX" }), types.Tab("tab-1", "Tab 1"), None, output_filename=output_path.name, ) self.assertEquals(cm.exception.exit_code, 1) # Python exit code self.assertRegex(cm.exception.log, r"\bRuntimeError\b") self.assertRegex(cm.exception.log, r"\bfail\b") # Regression test: [2019-10-02], the "pyspawner_main()->spawn_child()" # process would raise _another_ exception while exiting. It would try to # close an already-closed socket. self.assertNotRegex(cm.exception.log, r"Bad file descriptor")
def test_fetch_happy_path(self): module = self.kernel.compile( MockPath( ["foo.py"], textwrap.dedent(""" import pandas as pd def fetch(params): return pd.DataFrame({"A": [params["a"]]}) """).encode("utf-8"), ), "foo", ) with self.chroot_context.tempfile_context( prefix="output-", dir=self.basedir) as output_path: result = self.kernel.fetch( module, self.chroot_context, self.basedir, types.Params({"a": 1}), {}, None, None, output_filename=output_path.name, ) self.assertEquals(result.errors, []) table = pyarrow.parquet.read_pandas(str(result.path)) self.assertEquals(table.to_pydict(), {"A": [1]})
def test_params_to_thrift(self): self.assertEqual( types.Params({ "str": "s", "int": 2, "float": 1.2, "null": None, "bool": False, "column": types.Column("A", types.ColumnType.Number(format="{:,.2f}")), "listofmaps": [{ "A": "a", "B": "b" }, { "C": "c", "D": "d" }], "tab": "TODO tabs", }).to_thrift(), { "str": ttypes.ParamValue(string_value="s"), "int": ttypes.ParamValue(integer_value=2), "float": ttypes.ParamValue(float_value=1.2), "null": ttypes.ParamValue(), "bool": ttypes.ParamValue(boolean_value=False), "column": ttypes.ParamValue(column_value=ttypes.Column( "A", ttypes.ColumnType(number_type=ttypes.ColumnTypeNumber( format="{:,.2f}")), )), "listofmaps": ttypes.ParamValue(list_value=[ ttypes.ParamValue( map_value={ "A": ttypes.ParamValue(string_value="a"), "B": ttypes.ParamValue(string_value="b"), }), ttypes.ParamValue( map_value={ "C": ttypes.ParamValue(string_value="c"), "D": ttypes.ParamValue(string_value="d"), }), ]), "tab": ttypes.ParamValue(string_value="TODO tabs"), }, )
def test_params_filename_to_thrift(self): path = self.basedir / "x.bin" self.assertEqual( types.Params({ "A": path }).to_thrift(), {"A": ttypes.ParamValue(filename_value="x.bin")}, )
def test_params_filename_from_thrift_happy_path(self): with tempfile.NamedTemporaryFile(dir=self.basedir) as tf: path = Path(tf.name) path.write_bytes(b"") self.assertEqual( types.Params.from_thrift( {"A": ttypes.ParamValue(filename_value=path.name)}, self.basedir ), types.Params({"A": path}), )
def test_render_happy_path(self): module = self.kernel.compile( MockPath( ["foo.py"], b"import pandas as pd\ndef render(table, params): return pd.DataFrame({'A': table['A'] * params['m'], 'B': table['B'] + params['s']})", ), "foo", ) with arrow_table_context( { "A": [1, 2, 3], "B": ["a", "b", "c"] }, columns=[ types.Column("A", types.ColumnType.Number("{:,d}")), types.Column("B", types.ColumnType.Text()), ], dir=self.basedir, ) as input_table: input_table.path.chmod(0o644) with self.chroot_context.tempfile_context( prefix="output-", dir=self.basedir) as output_path: result = self.kernel.render( module, self.chroot_context, self.basedir, input_table, types.Params({ "m": 2.5, "s": "XX" }), types.Tab("tab-1", "Tab 1"), None, output_filename=output_path.name, ) self.assertEquals( result.table.table.to_pydict(), { "A": [2.5, 5.0, 7.5], "B": ["aXX", "bXX", "cXX"] }, )
def test_render_kill_timeout(self): mod = _compile( "foo", "import time\ndef render(table, params):\n time.sleep(2)") with patch.object(self.kernel, "render_timeout", 0.001): with self.assertRaises(ModuleTimeoutError): with arrow_table_context({"A": [1]}, dir=self.basedir) as input_table: input_table.path.chmod(0o644) with self.chroot_context.tempfile_context( prefix="output-", dir=self.basedir) as output_path: self.kernel.render( mod, self.chroot_context, self.basedir, input_table, types.Params({}), types.Tab("tab-1", "Tab 1"), None, output_filename=output_path.name, )
def test_render_killed_hard_out_of_memory(self): # This is similar to out-of-memory kill (but with different exit_code). # Testing out-of-memory is slow because we have to force the kernel to, # er, run out of memory. On a typical dev machine, that means filling # swap space -- gumming up the whole system. Not practical. # # In case of out-of-memory, the Linux out-of-memory killer will find # and kill a process using SIGKILL. # # So let's simulate that SIGKILL. module = self.kernel.compile( MockPath( ["foo.py"], b"import os\nimport time\ndef render(table, params): os.kill(os.getpid(), 9); time.sleep(1)", ), "foo", ) with self.assertRaises(ModuleExitedError) as cm: with arrow_table_context({"A": [1]}, dir=self.basedir) as input_table: input_table.path.chmod(0o644) with tempfile_context(prefix="output-", dir=self.basedir) as output_path: result = self.kernel.render( module, self.basedir, input_table, types.Params({ "m": 2.5, "s": "XX" }), types.Tab("tab-1", "Tab 1"), None, output_filename=output_path.name, ) print(repr(result)) self.assertEquals(cm.exception.exit_code, -9) # SIGKILL self.assertEquals(cm.exception.log, "")