def __init__(self, http_client, properties, ipython_display, session_id=-1, spark_events=None, heartbeat_timeout=0, heartbeat_thread=None): super(LivySession, self).__init__() assert constants.LIVY_KIND_PARAM in list(properties.keys()) kind = properties[constants.LIVY_KIND_PARAM] should_heartbeat = False if heartbeat_timeout > 0: should_heartbeat = True properties[ constants.LIVY_HEARTBEAT_TIMEOUT_PARAM] = heartbeat_timeout elif constants.LIVY_HEARTBEAT_TIMEOUT_PARAM in list(properties.keys()): properties.pop(constants.LIVY_HEARTBEAT_TIMEOUT_PARAM) self.properties = properties self.ipython_display = ipython_display self._should_heartbeat = should_heartbeat self._user_passed_heartbeat_thread = heartbeat_thread if spark_events is None: spark_events = SparkEvents() self._spark_events = spark_events self._policy = ConfigurableRetryPolicy( retry_seconds_to_sleep_list=[0.2, 0.5, 0.5, 1, 1, 2], max_retries=5000) wait_for_idle_timeout_seconds = conf.wait_for_idle_timeout_seconds() assert wait_for_idle_timeout_seconds > 0 self.logger = SparkLog(u"LivySession") kind = kind.lower() if kind not in constants.SESSION_KINDS_SUPPORTED: raise BadUserDataException( u"Session of kind '{}' not supported. Session must be of kinds {}." .format(kind, ", ".join(constants.SESSION_KINDS_SUPPORTED))) self._app_id = None self._logs = u"" self._http_client = http_client self._wait_for_idle_timeout_seconds = wait_for_idle_timeout_seconds self._printed_resource_warning = False self.kind = kind self.id = session_id self.session_info = u"" self._heartbeat_thread = None if session_id == -1: self.status = constants.NOT_STARTED_SESSION_STATUS else: self.status = constants.BUSY_SESSION_STATUS self._start_heartbeat_thread()
def __init__(self, http_client, properties, ipython_display, session_id=-1, sql_created=None, spark_events=None, should_heartbeat=False, heartbeat_thread=None): super(LivySession, self).__init__() assert u"kind" in list(properties.keys()) kind = properties[u"kind"] self.properties = properties self.ipython_display = ipython_display self._should_heartbeat = should_heartbeat self._user_passed_heartbeat_thread = heartbeat_thread if spark_events is None: spark_events = SparkEvents() self._spark_events = spark_events status_sleep_seconds = conf.status_sleep_seconds() statement_sleep_seconds = conf.statement_sleep_seconds() wait_for_idle_timeout_seconds = conf.wait_for_idle_timeout_seconds() assert status_sleep_seconds > 0 assert statement_sleep_seconds > 0 assert wait_for_idle_timeout_seconds > 0 if session_id == -1 and sql_created is True: raise BadUserDataException( u"Cannot indicate sql state without session id.") self.logger = SparkLog(u"LivySession") kind = kind.lower() if kind not in constants.SESSION_KINDS_SUPPORTED: raise BadUserDataException( u"Session of kind '{}' not supported. Session must be of kinds {}." .format(kind, ", ".join(constants.SESSION_KINDS_SUPPORTED))) self._app_id = None self._logs = u"" self._http_client = http_client self._status_sleep_seconds = status_sleep_seconds self._statement_sleep_seconds = statement_sleep_seconds self._wait_for_idle_timeout_seconds = wait_for_idle_timeout_seconds self.kind = kind self.id = session_id self.created_sql_context = sql_created self._heartbeat_thread = None if session_id == -1: self.status = constants.NOT_STARTED_SESSION_STATUS sql_created = False else: self.status = constants.BUSY_SESSION_STATUS self._start_heartbeat_thread()
def __init__(self, http_client, properties, ipython_display, session_id=-1, spark_events=None, heartbeat_timeout=0, heartbeat_thread=None): super(LivySession, self).__init__() assert constants.LIVY_KIND_PARAM in list(properties.keys()) kind = properties[constants.LIVY_KIND_PARAM] should_heartbeat = False if heartbeat_timeout > 0: should_heartbeat = True properties[constants.LIVY_HEARTBEAT_TIMEOUT_PARAM] = heartbeat_timeout elif constants.LIVY_HEARTBEAT_TIMEOUT_PARAM in list(properties.keys()): properties.pop(constants.LIVY_HEARTBEAT_TIMEOUT_PARAM) self.properties = properties self.ipython_display = ipython_display self._should_heartbeat = should_heartbeat self._user_passed_heartbeat_thread = heartbeat_thread if spark_events is None: spark_events = SparkEvents() self._spark_events = spark_events status_sleep_seconds = conf.status_sleep_seconds() statement_sleep_seconds = conf.statement_sleep_seconds() wait_for_idle_timeout_seconds = conf.wait_for_idle_timeout_seconds() assert status_sleep_seconds > 0 assert statement_sleep_seconds > 0 assert wait_for_idle_timeout_seconds > 0 self.logger = SparkLog(u"LivySession") kind = kind.lower() if kind not in constants.SESSION_KINDS_SUPPORTED: raise BadUserDataException(u"Session of kind '{}' not supported. Session must be of kinds {}." .format(kind, ", ".join(constants.SESSION_KINDS_SUPPORTED))) self._app_id = None self._logs = u"" self._http_client = http_client self._status_sleep_seconds = status_sleep_seconds self._statement_sleep_seconds = statement_sleep_seconds self._wait_for_idle_timeout_seconds = wait_for_idle_timeout_seconds self._printed_resource_warning = False self.kind = kind self.id = session_id self.session_info = u"" self._heartbeat_thread = None if session_id == -1: self.status = constants.NOT_STARTED_SESSION_STATUS else: self.status = constants.BUSY_SESSION_STATUS self._start_heartbeat_thread()
def __init__(self, http_client, properties, ipython_display, session_id=-1, sql_created=None, spark_events=None, should_heartbeat=False, heartbeat_thread=None): super(LivySession, self).__init__() assert u"kind" in list(properties.keys()) kind = properties[u"kind"] self.properties = properties self.ipython_display = ipython_display self._should_heartbeat = should_heartbeat self._user_passed_heartbeat_thread = heartbeat_thread if spark_events is None: spark_events = SparkEvents() self._spark_events = spark_events status_sleep_seconds = conf.status_sleep_seconds() statement_sleep_seconds = conf.statement_sleep_seconds() wait_for_idle_timeout_seconds = conf.wait_for_idle_timeout_seconds() assert status_sleep_seconds > 0 assert statement_sleep_seconds > 0 assert wait_for_idle_timeout_seconds > 0 if session_id == -1 and sql_created is True: raise BadUserDataException(u"Cannot indicate sql state without session id.") self.logger = SparkLog(u"LivySession") kind = kind.lower() if kind not in constants.SESSION_KINDS_SUPPORTED: raise BadUserDataException(u"Session of kind '{}' not supported. Session must be of kinds {}." .format(kind, ", ".join(constants.SESSION_KINDS_SUPPORTED))) self._app_id = None self._logs = u"" self._http_client = http_client self._status_sleep_seconds = status_sleep_seconds self._statement_sleep_seconds = statement_sleep_seconds self._wait_for_idle_timeout_seconds = wait_for_idle_timeout_seconds self.kind = kind self.id = session_id self.created_sql_context = sql_created self._heartbeat_thread = None if session_id == -1: self.status = constants.NOT_STARTED_SESSION_STATUS sql_created = False else: self.status = constants.BUSY_SESSION_STATUS self._start_heartbeat_thread()