def _setup(): global spark_events, guid1, guid2, guid3, time_stamp spark_events = SparkEvents() spark_events.handler = MagicMock() spark_events.get_utc_date_time = MagicMock() spark_events._verify_language_ok = MagicMock() time_stamp = spark_events.get_utc_date_time() guid1 = generate_uuid() guid2 = generate_uuid() guid3 = generate_uuid()
def __init__(self, shell, data=None, spark_events=None): # You must call the parent constructor super(SparkMagicBase, self).__init__(shell) self.logger = SparkLog(u"SparkMagics") self.ipython_display = IpythonDisplay() self.spark_controller = SparkController(self.ipython_display) self.logger.debug("Initialized spark magics.") if spark_events is None: spark_events = SparkEvents() spark_events.emit_library_loaded_event()
def __init__(self, shell, data=None, spark_events=None): # You must call the parent constructor super(KernelMagics, self).__init__(shell, data) self.session_name = u"session_name" self.session_started = False # In order to set these following 3 properties, call %%_do_not_call_change_language -l language self.language = u"" self.endpoint = None self.fatal_error = False self.fatal_error_message = u"" if spark_events is None: spark_events = SparkEvents() self._spark_events = spark_events
def __init__(self, query, samplemethod=None, maxrows=None, samplefraction=None, spark_events=None): super(SQLQuery, self).__init__() if samplemethod is None: samplemethod = conf.default_samplemethod() if maxrows is None: maxrows = conf.default_maxrows() if samplefraction is None: samplefraction = conf.default_samplefraction() if samplemethod not in {u'take', u'sample'}: raise BadUserDataException(u'samplemethod (-m) must be one of (take, sample)') if not isinstance(maxrows, int): raise BadUserDataException(u'maxrows (-n) must be an integer') if not 0.0 <= samplefraction <= 1.0: raise BadUserDataException(u'samplefraction (-r) must be a float between 0.0 and 1.0') self.query = query self.samplemethod = samplemethod self.maxrows = maxrows self.samplefraction = samplefraction if spark_events is None: spark_events = SparkEvents() self._spark_events = spark_events
def __init__(self, http_client, properties, ipython_display, session_id=-1, spark_events=None, heartbeat_timeout=0, heartbeat_thread=None): super(LivySession, self).__init__() assert constants.LIVY_KIND_PARAM in list(properties.keys()) kind = properties[constants.LIVY_KIND_PARAM] should_heartbeat = False if heartbeat_timeout > 0: should_heartbeat = True properties[ constants.LIVY_HEARTBEAT_TIMEOUT_PARAM] = heartbeat_timeout elif constants.LIVY_HEARTBEAT_TIMEOUT_PARAM in list(properties.keys()): properties.pop(constants.LIVY_HEARTBEAT_TIMEOUT_PARAM) self.properties = properties self.ipython_display = ipython_display self._should_heartbeat = should_heartbeat self._user_passed_heartbeat_thread = heartbeat_thread if spark_events is None: spark_events = SparkEvents() self._spark_events = spark_events self._policy = ConfigurableRetryPolicy( retry_seconds_to_sleep_list=[0.2, 0.5, 0.5, 1, 1, 2], max_retries=5000) wait_for_idle_timeout_seconds = conf.wait_for_idle_timeout_seconds() assert wait_for_idle_timeout_seconds > 0 self.logger = SparkLog(u"LivySession") kind = kind.lower() if kind not in constants.SESSION_KINDS_SUPPORTED: raise BadUserDataException( u"Session of kind '{}' not supported. Session must be of kinds {}." .format(kind, ", ".join(constants.SESSION_KINDS_SUPPORTED))) self._app_id = None self._user = None self._logs = u"" self._http_client = http_client self._wait_for_idle_timeout_seconds = wait_for_idle_timeout_seconds self._printed_resource_warning = False self.kind = kind self.id = session_id self.session_info = u"" self._heartbeat_thread = None if session_id == -1: self.status = constants.NOT_STARTED_SESSION_STATUS else: self.status = constants.BUSY_SESSION_STATUS self._start_heartbeat_thread()
def _get_spark_events(self): spark_events = getattr(self, 'spark_events', None) if spark_events is None: return SparkEvents() return spark_events
def test_magic_verify_language_ok_error(): SparkEvents()._verify_language_ok('NYARGLEBARGLE')
def test_magic_verify_language_ok(): for language in constants.SESSION_KINDS_SUPPORTED: SparkEvents()._verify_language_ok(language)