def execute_spark(self, cell, output_var, samplemethod, maxrows, samplefraction, session_name, coerce): (success, out, mimetype) = self.spark_controller.run_command(Command(cell), session_name) if not success: if conf.spark_statement_errors_are_fatal(): if conf.shutdown_session_on_spark_statement_errors(): self.spark_controller.cleanup() raise SparkStatementException(out) self.ipython_display.send_error(out) else: if isinstance(out, string_types): if mimetype == MIMETYPE_TEXT_HTML: self.ipython_display.html(out) else: self.ipython_display.write(out) else: self.ipython_display.display(out) if output_var is not None: spark_store_command = self._spark_store_command( output_var, samplemethod, maxrows, samplefraction, coerce) df = self.spark_controller.run_command(spark_store_command, session_name) self.shell.user_ns[output_var] = df
def test_spark_fatal_spark_statement_exception(): conf.override_all({ "all_errors_are_fatal": True, }) line = "" cell = "some spark code" spark_controller.run_command = MagicMock(side_effect=SparkStatementException('Oh no!')) magic.spark(line, cell)