def __init__(self, shell, data=None): # You must call the parent constructor super(SparkMagicBase, self).__init__(shell) self.logger = Log("SparkMagics") self.ipython_display = IpythonDisplay() self.spark_controller = SparkController(self.ipython_display) try: should_serialize = conf.serialize() if should_serialize: self.logger.debug("Serialization enabled.") self.magics_home_path = get_magics_home_path() path_to_serialize = join_paths(self.magics_home_path, "state.json") self.logger.debug( "Will serialize to {}.".format(path_to_serialize)) self.spark_controller = SparkController( self.ipython_display, serialize_path=path_to_serialize) else: self.logger.debug("Serialization NOT enabled.") except KeyError: self.logger.error("Could not read env vars for serialization.") self.logger.debug("Initialized spark magics.")
def _setup(): global client_manager, controller, ipython_display client_manager = MagicMock() ipython_display = MagicMock() controller = SparkController(ipython_display) controller.client_manager = client_manager
def _setup(): global client_manager, controller, ipython_display client_manager = MagicMock() ipython_display = MagicMock() spark_events = MagicMock() controller = SparkController(ipython_display) controller.session_manager = client_manager controller.spark_events = spark_events
def __init__(self, shell, data=None, spark_events=None): # You must call the parent constructor super(SparkMagicBase, self).__init__(shell) self.logger = Log("SparkMagics") self.ipython_display = IpythonDisplay() self.spark_controller = SparkController(self.ipython_display) self.logger.debug("Initialized spark magics.") if spark_events is None: spark_events = SparkEvents() spark_events.emit_library_loaded_event()