def _setup():
global spark_events, guid1, guid2, guid3, time_stamp
spark_events = SparkEvents()
spark_events.handler = MagicMock()
spark_events.get_utc_date_time = MagicMock()
spark_events._verify_language_ok = MagicMock()
time_stamp = spark_events.get_utc_date_time()
guid1 = generate_uuid()
guid2 = generate_uuid()
guid3 = generate_uuid()
def __init__(self, shell, data=None, spark_events=None):
# You must call the parent constructor
super(SparkMagicBase, self).__init__(shell)
self.logger = SparkLog(u"SparkMagics")
self.ipython_display = IpythonDisplay()
self.spark_controller = SparkController(self.ipython_display)
self.logger.debug("Initialized spark magics.")
if spark_events is None:
spark_events = SparkEvents()
spark_events.emit_library_loaded_event()
def __init__(self, shell, data=None, spark_events=None):
# You must call the parent constructor
super(KernelMagics, self).__init__(shell, data)
self.session_name = u"session_name"
self.session_started = False
# In order to set these following 3 properties, call %%_do_not_call_change_language -l language
self.language = u""
self.endpoint = None
self.fatal_error = False
self.fatal_error_message = u""
if spark_events is None:
spark_events = SparkEvents()
self._spark_events = spark_events
def _setup():
global spark_events, guid1, guid2, guid3, time_stamp
spark_events = SparkEvents()
spark_events.handler = MagicMock()
spark_events.get_utc_date_time = MagicMock()
spark_events._verify_language_ok = MagicMock()
time_stamp = spark_events.get_utc_date_time()
guid1 = generate_uuid()
guid2 = generate_uuid()
guid3 = generate_uuid()
def __init__(self, query, samplemethod=None, maxrows=None, samplefraction=None, spark_events=None):
super(SQLQuery, self).__init__()
if samplemethod is None:
samplemethod = conf.default_samplemethod()
if maxrows is None:
maxrows = conf.default_maxrows()
if samplefraction is None:
samplefraction = conf.default_samplefraction()
if samplemethod not in {u'take', u'sample'}:
raise BadUserDataException(u'samplemethod (-m) must be one of (take, sample)')
if not isinstance(maxrows, int):
raise BadUserDataException(u'maxrows (-n) must be an integer')
if not 0.0 <= samplefraction <= 1.0:
raise BadUserDataException(u'samplefraction (-r) must be a float between 0.0 and 1.0')
self.query = query
self.samplemethod = samplemethod
self.maxrows = maxrows
self.samplefraction = samplefraction
if spark_events is None:
spark_events = SparkEvents()
self._spark_events = spark_events
def __init__(self,
http_client,
properties,
ipython_display,
session_id=-1,
spark_events=None,
heartbeat_timeout=0,
heartbeat_thread=None):
super(LivySession, self).__init__()
assert constants.LIVY_KIND_PARAM in list(properties.keys())
kind = properties[constants.LIVY_KIND_PARAM]
should_heartbeat = False
if heartbeat_timeout > 0:
should_heartbeat = True
properties[
constants.LIVY_HEARTBEAT_TIMEOUT_PARAM] = heartbeat_timeout
elif constants.LIVY_HEARTBEAT_TIMEOUT_PARAM in list(properties.keys()):
properties.pop(constants.LIVY_HEARTBEAT_TIMEOUT_PARAM)
self.properties = properties
self.ipython_display = ipython_display
self._should_heartbeat = should_heartbeat
self._user_passed_heartbeat_thread = heartbeat_thread
if spark_events is None:
spark_events = SparkEvents()
self._spark_events = spark_events
self._policy = ConfigurableRetryPolicy(
retry_seconds_to_sleep_list=[0.2, 0.5, 0.5, 1, 1, 2],
max_retries=5000)
wait_for_idle_timeout_seconds = conf.wait_for_idle_timeout_seconds()
assert wait_for_idle_timeout_seconds > 0
self.logger = SparkLog(u"LivySession")
kind = kind.lower()
if kind not in constants.SESSION_KINDS_SUPPORTED:
raise BadUserDataException(
u"Session of kind '{}' not supported. Session must be of kinds {}."
.format(kind, ", ".join(constants.SESSION_KINDS_SUPPORTED)))
self._app_id = None
self._user = None
self._logs = u""
self._http_client = http_client
self._wait_for_idle_timeout_seconds = wait_for_idle_timeout_seconds
self._printed_resource_warning = False
self.kind = kind
self.id = session_id
self.session_info = u""
self._heartbeat_thread = None
if session_id == -1:
self.status = constants.NOT_STARTED_SESSION_STATUS
else:
self.status = constants.BUSY_SESSION_STATUS
self._start_heartbeat_thread()
def _get_spark_events(self):
spark_events = getattr(self, 'spark_events', None)
if spark_events is None:
return SparkEvents()
return spark_events
def test_magic_verify_language_ok_error():
SparkEvents()._verify_language_ok('NYARGLEBARGLE')
def test_magic_verify_language_ok():
for language in constants.SESSION_KINDS_SUPPORTED:
SparkEvents()._verify_language_ok(language)
