def execute_spark(self, cell, output_var, samplemethod, maxrows,
samplefraction, session_name, coerce):
(success, out,
mimetype) = self.spark_controller.run_command(Command(cell),
session_name)
if not success:
if conf.spark_statement_errors_are_fatal():
if conf.shutdown_session_on_spark_statement_errors():
self.spark_controller.cleanup()
raise SparkStatementException(out)
self.ipython_display.send_error(out)
else:
if isinstance(out, string_types):
if mimetype == MIMETYPE_TEXT_HTML:
self.ipython_display.html(out)
else:
self.ipython_display.write(out)
else:
self.ipython_display.display(out)
if output_var is not None:
spark_store_command = self._spark_store_command(
output_var, samplemethod, maxrows, samplefraction, coerce)
df = self.spark_controller.run_command(spark_store_command,
session_name)
self.shell.user_ns[output_var] = df
def test_spark_fatal_spark_statement_exception():
conf.override_all({
"all_errors_are_fatal": True,
})
line = ""
cell = "some spark code"
spark_controller.run_command = MagicMock(side_effect=SparkStatementException('Oh no!'))
magic.spark(line, cell)