def __init__(self, **kwargs): super(SparkKernelBase, self).__init__(**kwargs) self.logger = Log(self.client_name) self.already_ran_once = False self._fatal_error = None # Disable warnings for test env in HDI requests.packages.urllib3.disable_warnings()
class SparkKernelBase(IPythonKernel): fatal_error_suggestion = "The code failed because of a fatal error:\n\t{}.\n\nSome things to try:\n" \ "a) Make sure Spark has enough available resources for Jupyter to create a Spark context."\ "\nb) Contact your Jupyter administrator to make sure the Spark magics library is " \ "configured correctly." \ "\nc) Restart the kernel." # Required by Jupyter - Override implementation = None implementation_version = None language = None language_version = None language_info = None banner = None # Override username_conf_name = None password_conf_name = None url_conf_name = None session_language = None client_name = None def __init__(self, **kwargs): super(SparkKernelBase, self).__init__(**kwargs) self.logger = Log(self.client_name) self.already_ran_once = False self._fatal_error = None # Disable warnings for test env in HDI requests.packages.urllib3.disable_warnings() def do_execute(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False): if self._fatal_error is not None: self._abort_with_fatal_error(self._fatal_error) if not self.already_ran_once: (username, password, url) = self._get_configuration() self._initialize_magics(username, password, url) # Modify code by prepending spark magic text if code.lower().startswith("%sql\n") or code.lower().startswith("%sql "): code = "%%spark -c sql\n{}".format(code[5:]) elif code.lower().startswith("%%sql\n") or code.lower().startswith("%%sql "): code = "%%spark -c sql\n{}".format(code[6:]) elif code.lower().startswith("%hive\n") or code.lower().startswith("%hive "): code = "%%spark -c hive\n{}".format(code[6:]) elif code.lower().startswith("%%hive\n") or code.lower().startswith("%%hive "): code = "%%spark -c hive\n{}".format(code[7:]) else: code = "%%spark\n{}".format(code) return self._execute_cell(code, silent, store_history, user_expressions, allow_stdin) def do_shutdown(self, restart): # Cleanup if self.already_ran_once: code = "%spark cleanup" self._execute_cell_for_user(code, True, False) self.already_ran_once = False return self._do_shutdown_ipykernel(restart) def _initialize_magics(self, username, password, url): connection_string = get_connection_string(url, username, password) register_magics_code = "%load_ext remotespark" self._execute_cell(register_magics_code, True, False, shutdown_if_error=True, log_if_error="Failed to load the Spark magics library.") self.logger.debug("Loaded magics.") add_endpoint_code = "%spark add {} {} {} skip".format( self.client_name, self.session_language, connection_string) self._execute_cell(add_endpoint_code, True, False, shutdown_if_error=True, log_if_error="Failed to create a Livy session.") self.logger.debug("Added endpoint.") self.already_ran_once = True def _get_configuration(self): try: username = get_configuration(self.username_conf_name) password = get_configuration(self.password_conf_name) url = get_configuration(self.url_conf_name) return username, password, url except KeyError: message = "Please set configuration for '{}', '{}', '{} to initialize Kernel.".format( self.username_conf_name, self.password_conf_name, self.url_conf_name) self._abort_with_fatal_error(message) def _execute_cell(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False, shutdown_if_error=False, log_if_error=None): reply_content = self._execute_cell_for_user(code, silent, store_history, user_expressions, allow_stdin) if shutdown_if_error and reply_content[u"status"] == u"error": error_from_reply = reply_content[u"evalue"] if log_if_error is not None: message = "{}\nException details:\n\t\"{}\"".format(log_if_error, error_from_reply) self._abort_with_fatal_error(message) return reply_content def _execute_cell_for_user(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False): return super(SparkKernelBase, self).do_execute(code, silent, store_history, user_expressions, allow_stdin) def _do_shutdown_ipykernel(self, restart): return super(SparkKernelBase, self).do_shutdown(restart) def _abort_with_fatal_error(self, message): self._fatal_error = message error = get_configuration(Constants.fatal_error_suggestion, self.fatal_error_suggestion).format(message) self.logger.error(error) self._send_error(error) raise ValueError(message) def _send_error(self, error): stream_content = {"name": "stderr", "text": error} self._ipython_send_error(stream_content) def _ipython_send_error(self, stream_content): self.send_response(self.iopub_socket, "stream", stream_content)