def _print_endpoint_info(self, info_sessions, current_session_id): if info_sessions: info_sessions = sorted(info_sessions, key=lambda s: s.id) html = get_sessions_info_html(info_sessions, current_session_id) self.ipython_display.html(html) else: self.ipython_display.html(u'No active sessions.')
def start(self, create_sql_context=True): """Start the session against actual livy server.""" self._spark_events.emit_session_creation_start_event(self.guid, self.kind) try: r = self._http_client.post_session(self.properties) self.id = r[u"id"] self.status = str(r[u"state"]) self.ipython_display.writeln(u"Creating SparkContext as 'sc'") # Start heartbeat thread to keep Livy interactive session alive. self._start_heartbeat_thread() # We wait for livy_session_startup_timeout_seconds() for the session to start up. try: self.wait_for_idle(conf.livy_session_startup_timeout_seconds()) except LivyClientTimeoutException: raise LivyClientTimeoutException(u"Session {} did not start up in {} seconds." .format(self.id, conf.livy_session_startup_timeout_seconds())) html = get_sessions_info_html([self], self.id) self.ipython_display.html(html) if create_sql_context: self.create_sql_context() except Exception as e: self._spark_events.emit_session_creation_end_event(self.guid, self.kind, self.id, self.status, False, e.__class__.__name__, str(e)) raise else: self._spark_events.emit_session_creation_end_event(self.guid, self.kind, self.id, self.status, True, "", "")
def start(self): """Start the session against actual livy server.""" self._spark_events.emit_session_creation_start_event( self.guid, self.kind) self._printed_resource_warning = False try: r = self._http_client.post_session(self.properties) self.id = r[u"id"] self.status = str(r[u"state"]) self.ipython_display.writeln(u"Starting Spark application") # Start heartbeat thread to keep Livy interactive session alive. self._start_heartbeat_thread() # We wait for livy_session_startup_timeout_seconds() for the session to start up. try: self.wait_for_idle(conf.livy_session_startup_timeout_seconds()) except LivyClientTimeoutException: raise LivyClientTimeoutException( u"Session {} did not start up in {} seconds.".format( self.id, conf.livy_session_startup_timeout_seconds())) html = get_sessions_info_html([self], self.id) self.ipython_display.html(html) command = Command("spark") (success, out, mimetype) = command.execute(self) if success: self.ipython_display.writeln( u"SparkSession available as 'spark'.") self.sql_context_variable_name = "spark" else: command = Command("sqlContext") (success, out, mimetype) = command.execute(self) if success: self.ipython_display.writeln( u"SparkContext available as 'sc'.") if ("hive" in out.lower()): self.ipython_display.writeln( u"HiveContext available as 'sqlContext'.") else: self.ipython_display.writeln( u"SqlContext available as 'sqlContext'.") self.sql_context_variable_name = "sqlContext" else: raise SqlContextNotFoundException( u"Neither SparkSession nor HiveContext/SqlContext is available." ) except Exception as e: self._spark_events.emit_session_creation_end_event( self.guid, self.kind, self.id, self.status, False, e.__class__.__name__, str(e)) raise else: self._spark_events.emit_session_creation_end_event( self.guid, self.kind, self.id, self.status, True, "", "")
def start(self): """Start the session against actual livy server.""" self._spark_events.emit_session_creation_start_event(self.guid, self.kind) self._printed_resource_warning = False try: r = self._http_client.post_session(self.properties) self.id = r[u"id"] self.status = str(r[u"state"]) self.ipython_display.writeln(u"Starting Spark application") # Start heartbeat thread to keep Livy interactive session alive. self._start_heartbeat_thread() # We wait for livy_session_startup_timeout_seconds() for the session to start up. try: self.wait_for_idle(conf.livy_session_startup_timeout_seconds()) except LivyClientTimeoutException: raise LivyClientTimeoutException(u"Session {} did not start up in {} seconds." .format(self.id, conf.livy_session_startup_timeout_seconds())) html = get_sessions_info_html([self], self.id) self.ipython_display.html(html) command = Command("spark") (success, out) = command.execute(self) if success: self.ipython_display.writeln(u"SparkSession available as 'spark'.") self.sql_context_variable_name = "spark" else: command = Command("sqlContext") (success, out) = command.execute(self) if success: self.ipython_display.writeln(u"SparkContext available as 'sc'.") if ("hive" in out.lower()): self.ipython_display.writeln(u"HiveContext available as 'sqlContext'.") else: self.ipython_display.writeln(u"SqlContext available as 'sqlContext'.") self.sql_context_variable_name = "sqlContext" else: raise SqlContextNotFoundException(u"Neither SparkSession nor HiveContext/SqlContext is available.") except Exception as e: self._spark_events.emit_session_creation_end_event(self.guid, self.kind, self.id, self.status, False, e.__class__.__name__, str(e)) raise else: self._spark_events.emit_session_creation_end_event(self.guid, self.kind, self.id, self.status, True, "", "")
def start(self, create_sql_context=True): """Start the session against actual livy server.""" self._spark_events.emit_session_creation_start_event( self.guid, self.kind) try: r = self._http_client.post_session(self.properties) self.id = r[u"id"] self.status = str(r[u"state"]) self.ipython_display.writeln(u"Creating SparkContext as 'sc'") # Start heartbeat thread to keep Livy interactive session alive. self._start_heartbeat_thread() # We wait for livy_session_startup_timeout_seconds() for the session to start up. try: self.wait_for_idle(conf.livy_session_startup_timeout_seconds()) except LivyClientTimeoutException: raise LivyClientTimeoutException( u"Session {} did not start up in {} seconds.".format( self.id, conf.livy_session_startup_timeout_seconds())) html = get_sessions_info_html([self], self.id) self.ipython_display.html(html) if create_sql_context: self.create_sql_context() except Exception as e: self._spark_events.emit_session_creation_end_event( self.guid, self.kind, self.id, self.status, False, e.__class__.__name__, str(e)) raise else: self._spark_events.emit_session_creation_end_event( self.guid, self.kind, self.id, self.status, True, "", "")
def start(self): """Start the session against actual livy server.""" self._spark_events.emit_session_creation_start_event( self.guid, self.kind) self._printed_resource_warning = False try: connection_file = os.path.basename(ipykernel.get_connection_file()) if 'kernel' in connection_file: kernel_id = connection_file.split('-', 1)[1].split('.')[0] self.properties['conf'][ 'spark.yarn.appMasterEnv.HOPSWORKS_KERNEL_ID'] = kernel_id self.properties['conf'][ 'spark.executorEnv.HOPSWORKS_KERNEL_ID'] = kernel_id if 'hops.util' in sys.modules: util.attach_jupyter_configuration_to_notebook(kernel_id) r = self._http_client.post_session(self.properties) self.id = r[u"id"] self.status = str(r[u"state"]) self.ipython_display.writeln(u"Starting Spark application") # Start heartbeat thread to keep Livy interactive session alive. self._start_heartbeat_thread() # We wait for livy_session_startup_timeout_seconds() for the session to start up. try: self.wait_for_idle(conf.livy_session_startup_timeout_seconds()) except LivyClientTimeoutException: raise LivyClientTimeoutException( u"Session {} did not start up in {} seconds.".format( self.id, conf.livy_session_startup_timeout_seconds())) html = get_sessions_info_html([self], self.id) self.ipython_display.html(html) command = Command("spark") (success, out, mimetype) = command.execute(self) if success: self.ipython_display.writeln( u"SparkSession available as 'spark'.") self.sql_context_variable_name = "spark" else: command = Command("sqlContext") (success, out, mimetype) = command.execute(self) if success: self.ipython_display.writeln( u"SparkContext available as 'sc'.") if ("hive" in out.lower()): self.ipython_display.writeln( u"HiveContext available as 'sqlContext'.") else: self.ipython_display.writeln( u"SqlContext available as 'sqlContext'.") self.sql_context_variable_name = "sqlContext" else: raise SqlContextNotFoundException( u"Neither SparkSession nor HiveContext/SqlContext is available." ) except Exception as e: self._spark_events.emit_session_creation_end_event( self.guid, self.kind, self.id, self.status, False, e.__class__.__name__, str(e)) raise else: self._spark_events.emit_session_creation_end_event( self.guid, self.kind, self.id, self.status, True, "", "")