def _setup(): global client_manager, controller, ipython_display client_manager = MagicMock() ipython_display = MagicMock() controller = SparkController(ipython_display) controller.client_manager = client_manager
def __init__(self, shell, data=None): # You must call the parent constructor super(SparkMagicBase, self).__init__(shell) self.logger = Log("SparkMagics") self.ipython_display = IpythonDisplay() self.spark_controller = SparkController(self.ipython_display) try: should_serialize = conf.serialize() if should_serialize: self.logger.debug("Serialization enabled.") self.magics_home_path = get_magics_home_path() path_to_serialize = join_paths(self.magics_home_path, "state.json") self.logger.debug("Will serialize to {}.".format(path_to_serialize)) self.spark_controller = SparkController(self.ipython_display, serialize_path=path_to_serialize) else: self.logger.debug("Serialization NOT enabled.") except KeyError: self.logger.error("Could not read env vars for serialization.") self.logger.debug("Initialized spark magics.")
def __init__(self, shell, data=None): # You must call the parent constructor super(SparkMagicBase, self).__init__(shell) self.logger = Log("SparkMagics") self.ipython_display = IpythonDisplay() self.spark_controller = SparkController(self.ipython_display) try: should_serialize = conf.serialize() if should_serialize: self.logger.debug("Serialization enabled.") self.magics_home_path = get_magics_home_path() path_to_serialize = join_paths(self.magics_home_path, "state.json") self.logger.debug( "Will serialize to {}.".format(path_to_serialize)) self.spark_controller = SparkController( self.ipython_display, serialize_path=path_to_serialize) else: self.logger.debug("Serialization NOT enabled.") except KeyError: self.logger.error("Could not read env vars for serialization.") self.logger.debug("Initialized spark magics.")
def _setup(): global client_manager, client_factory, controller client_manager = MagicMock() client_factory = MagicMock() controller = SparkController() controller.client_manager = client_manager controller.client_factory = client_factory
def _setup(): global client_manager, controller, ipython_display client_manager = MagicMock() ipython_display = MagicMock() spark_events = MagicMock() controller = SparkController(ipython_display) controller.session_manager = client_manager controller.spark_events = spark_events
def __init__(self, shell, data=None, spark_events=None): # You must call the parent constructor super(SparkMagicBase, self).__init__(shell) self.logger = Log("SparkMagics") self.ipython_display = IpythonDisplay() self.spark_controller = SparkController(self.ipython_display) self.logger.debug("Initialized spark magics.") if spark_events is None: spark_events = SparkEvents() spark_events.emit_library_loaded_event()
class SparkMagicBase(Magics): def __init__(self, shell, data=None): # You must call the parent constructor super(SparkMagicBase, self).__init__(shell) self.logger = Log("SparkMagics") self.ipython_display = IpythonDisplay() self.spark_controller = SparkController(self.ipython_display) try: should_serialize = conf.serialize() if should_serialize: self.logger.debug("Serialization enabled.") self.magics_home_path = get_magics_home_path() path_to_serialize = join_paths(self.magics_home_path, "state.json") self.logger.debug( "Will serialize to {}.".format(path_to_serialize)) self.spark_controller = SparkController( self.ipython_display, serialize_path=path_to_serialize) else: self.logger.debug("Serialization NOT enabled.") except KeyError: self.logger.error("Could not read env vars for serialization.") self.logger.debug("Initialized spark magics.") def execute_sqlquery(self, sqlquery, session, output_var, quiet): try: df = self.spark_controller.run_cell_sql(sqlquery, session) if output_var is not None: self.shell.user_ns[output_var] = df if quiet: return None else: return df except DataFrameParseException as e: self.ipython_display.send_error(e.out) return None @staticmethod def print_endpoint_info(info_sessions): sessions_info = [" {}".format(i) for i in info_sessions] print("""Info for endpoint: Sessions: {} """.format("\n".join(sessions_info)))
class SparkMagicBase(Magics): def __init__(self, shell, data=None): # You must call the parent constructor super(SparkMagicBase, self).__init__(shell) self.logger = Log("SparkMagics") self.ipython_display = IpythonDisplay() self.spark_controller = SparkController(self.ipython_display) try: should_serialize = conf.serialize() if should_serialize: self.logger.debug("Serialization enabled.") self.magics_home_path = get_magics_home_path() path_to_serialize = join_paths(self.magics_home_path, "state.json") self.logger.debug("Will serialize to {}.".format(path_to_serialize)) self.spark_controller = SparkController(self.ipython_display, serialize_path=path_to_serialize) else: self.logger.debug("Serialization NOT enabled.") except KeyError: self.logger.error("Could not read env vars for serialization.") self.logger.debug("Initialized spark magics.") def execute_sqlquery(self, sqlquery, session, output_var, quiet): try: df = self.spark_controller.run_cell_sql(sqlquery, session) if output_var is not None: self.shell.user_ns[output_var] = df if quiet: return None else: return df except DataFrameParseException as e: self.ipython_display.send_error(e.out) return None @staticmethod def print_endpoint_info(info_sessions): sessions_info = [" {}".format(i) for i in info_sessions] print("""Info for endpoint: Sessions: {} """.format("\n".join(sessions_info)))
class SparkMagicBase(Magics): def __init__(self, shell, data=None, spark_events=None): # You must call the parent constructor super(SparkMagicBase, self).__init__(shell) self.logger = Log("SparkMagics") self.ipython_display = IpythonDisplay() self.spark_controller = SparkController(self.ipython_display) self.logger.debug("Initialized spark magics.") if spark_events is None: spark_events = SparkEvents() spark_events.emit_library_loaded_event() def execute_sqlquery(self, cell, samplemethod, maxrows, samplefraction, session, output_var, quiet): sqlquery = self._sqlquery(cell, samplemethod, maxrows, samplefraction) df = self.spark_controller.run_sqlquery(sqlquery, session) if output_var is not None: self.shell.user_ns[output_var] = df if quiet: return None else: return df @staticmethod def _sqlquery(cell, samplemethod, maxrows, samplefraction): return SQLQuery(cell, samplemethod, maxrows, samplefraction) @staticmethod def print_endpoint_info(info_sessions): sessions_info = [" {}".format(i) for i in info_sessions] print("""Info for endpoint: Sessions: {} """.format("\n".join(sessions_info)))