class SparkMagicBase(Magics):
def __init__(self, shell, data=None, spark_events=None):
# You must call the parent constructor
super(SparkMagicBase, self).__init__(shell)
self.logger = SparkLog(u"SparkMagics")
self.ipython_display = IpythonDisplay()
self.spark_controller = SparkController(self.ipython_display)
self.logger.debug("Initialized spark magics.")
if spark_events is None:
spark_events = SparkEvents()
spark_events.emit_library_loaded_event()
def execute_spark(self, cell, output_var, samplemethod, maxrows,
samplefraction, session_name):
(success,
out) = self.spark_controller.run_command(Command(cell), session_name)
if not success:
self.ipython_display.send_error(out)
else:
self.ipython_display.write(out)
if output_var is not None:
spark_store_command = self._spark_store_command(
output_var, samplemethod, maxrows, samplefraction)
df = self.spark_controller.run_command(spark_store_command,
session_name)
self.shell.user_ns[output_var] = df
@staticmethod
def _spark_store_command(output_var, samplemethod, maxrows,
samplefraction):
return SparkStoreCommand(output_var, samplemethod, maxrows,
samplefraction)
def execute_sqlquery(self, cell, samplemethod, maxrows, samplefraction,
session, output_var, quiet):
sqlquery = self._sqlquery(cell, samplemethod, maxrows, samplefraction)
df = self.spark_controller.run_sqlquery(sqlquery, session)
if output_var is not None:
self.shell.user_ns[output_var] = df
if quiet:
return None
else:
return df
@staticmethod
def _sqlquery(cell, samplemethod, maxrows, samplefraction):
return SQLQuery(cell, samplemethod, maxrows, samplefraction)
def _print_endpoint_info(self, info_sessions, current_session_id):
if info_sessions:
info_sessions = sorted(info_sessions, key=lambda s: s.id)
html = get_sessions_info_html(info_sessions, current_session_id)
self.ipython_display.html(html)
else:
self.ipython_display.html(u'No active sessions.')
class SparkMagicBase(Magics):
def __init__(self, shell, data=None, spark_events=None):
# You must call the parent constructor
super(SparkMagicBase, self).__init__(shell)
self.logger = SparkLog(u"SparkMagics")
self.ipython_display = IpythonDisplay()
self.spark_controller = SparkController(self.ipython_display)
self.logger.debug("Initialized spark magics.")
if spark_events is None:
spark_events = SparkEvents()
spark_events.emit_library_loaded_event()
def execute_sqlquery(self, cell, samplemethod, maxrows, samplefraction,
session, output_var, quiet):
sqlquery = self._sqlquery(cell, samplemethod, maxrows, samplefraction)
df = self.spark_controller.run_sqlquery(sqlquery, session)
if output_var is not None:
self.shell.user_ns[output_var] = df
if quiet:
return None
else:
return df
@staticmethod
def _sqlquery(cell, samplemethod, maxrows, samplefraction):
return SQLQuery(cell, samplemethod, maxrows, samplefraction)
def _print_endpoint_info(self, info_sessions, current_session_id):
if info_sessions:
info_sessions = sorted(info_sessions, key=lambda s: s.id)
html = u"""<table>
<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr>""" + \
u"".join([SparkMagicBase._session_row_html(session, current_session_id) for session in info_sessions]) + \
u"</table>"
self.ipython_display.html(html)
else:
self.ipython_display.html(u'No active sessions.')
@staticmethod
def _session_row_html(session, current_session_id):
return u"""<tr><td>{0}</td><td>{1}</td><td>{2}</td><td>{3}</td><td>{4}</td><td>{5}</td><td>{6}</td></tr>""".format(
session.id, session.get_app_id(), session.kind, session.status,
SparkMagicBase._link(u'Link', session.get_spark_ui_url()),
SparkMagicBase._link(u'Link', session.get_driver_log_url()),
u"" if current_session_id is None
or current_session_id != session.id else u"✔")
@staticmethod
def _link(text, url):
if url is not None:
return u"""<a target="_blank" href="{1}">{0}</a>""".format(
text, url)
else:
return u""
class SparkMagicBase(Magics):
def __init__(self, shell, data=None, spark_events=None):
# You must call the parent constructor
super(SparkMagicBase, self).__init__(shell)
self.logger = SparkLog(u"SparkMagics")
self.ipython_display = IpythonDisplay()
self.spark_controller = SparkController(self.ipython_display)
self.logger.debug("Initialized spark magics.")
if spark_events is None:
spark_events = SparkEvents()
spark_events.emit_library_loaded_event()
def execute_sqlquery(self, cell, samplemethod, maxrows, samplefraction,
session, output_var, quiet):
sqlquery = self._sqlquery(cell, samplemethod, maxrows, samplefraction)
df = self.spark_controller.run_sqlquery(sqlquery, session)
if output_var is not None:
self.shell.user_ns[output_var] = df
if quiet:
return None
else:
return df
@staticmethod
def _sqlquery(cell, samplemethod, maxrows, samplefraction):
return SQLQuery(cell, samplemethod, maxrows, samplefraction)
def _print_endpoint_info(self, info_sessions, current_session_id):
if info_sessions:
info_sessions = sorted(info_sessions, key=lambda s: s.id)
html = u"""<table>
<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr>""" + \
u"".join([SparkMagicBase._session_row_html(session, current_session_id) for session in info_sessions]) + \
u"</table>"
self.ipython_display.html(html)
else:
self.ipython_display.html(u'No active sessions.')
@staticmethod
def _session_row_html(session, current_session_id):
return u"""<tr><td>{0}</td><td>{1}</td><td>{2}</td><td>{3}</td><td>{4}</td><td>{5}</td><td>{6}</td></tr>""".format(
session.id, session.get_app_id(), session.kind, session.status,
SparkMagicBase._link(u'Link', session.get_spark_ui_url()), SparkMagicBase._link(u'Link', session.get_driver_log_url()),
u"" if current_session_id is None or current_session_id != session.id else u"✔"
)
@staticmethod
def _link(text, url):
if url is not None:
return u"""<a target="_blank" href="{1}">{0}</a>""".format(text, url)
else:
return u""
class DataGraph(object):
"""This does not use the table version of plotly because it freezes up the browser for >60 rows. Instead, we use
pandas df HTML representation."""
def __init__(self, display=None):
if display is None:
self.display = IpythonDisplay()
else:
self.display = display
def render(self, df, encoding, output):
with output:
max_rows = pd.get_option("display.max_rows")
max_cols = pd.get_option("display.max_columns")
show_dimensions = pd.get_option("display.show_dimensions")
# This will hide the index column for pandas df.
self.display.html("""
<style>
table.dataframe.hideme thead th:first-child {
display: none;
}
table.dataframe.hideme tbody th {
display: none;
}
</style>
""")
self.display.html(df.to_html(max_rows=max_rows, max_cols=max_cols,
show_dimensions=show_dimensions, notebook=True, classes="hideme"))
@staticmethod
def display_logarithmic_x_axis():
return False
@staticmethod
def display_logarithmic_y_axis():
return False
@staticmethod
def display_x():
return False
@staticmethod
def display_y():
return False
class SparkMagicBase(Magics):
def __init__(self, shell, data=None, spark_events=None):
# You must call the parent constructor
super(SparkMagicBase, self).__init__(shell)
self.logger = SparkLog(u"SparkMagics")
self.ipython_display = IpythonDisplay()
self.spark_controller = SparkController(self.ipython_display)
self.logger.debug("Initialized spark magics.")
if spark_events is None:
spark_events = SparkEvents()
spark_events.emit_library_loaded_event()
def execute_sqlquery(self, cell, samplemethod, maxrows, samplefraction,
session, output_var, quiet):
sqlquery = self._sqlquery(cell, samplemethod, maxrows, samplefraction)
df = self.spark_controller.run_sqlquery(sqlquery, session)
if output_var is not None:
self.shell.user_ns[output_var] = df
if quiet:
return None
else:
return df
@staticmethod
def _sqlquery(cell, samplemethod, maxrows, samplefraction):
return SQLQuery(cell, samplemethod, maxrows, samplefraction)
def _print_endpoint_info(self, info_sessions, current_session_id):
if info_sessions:
info_sessions = sorted(info_sessions, key=lambda s: s.id)
html = get_sessions_info_html(info_sessions, current_session_id)
self.ipython_display.html(html)
else:
self.ipython_display.html(u'No active sessions.')
class SparkMagicBase(Magics):
_STRING_VAR_TYPE = 'str'
_PANDAS_DATAFRAME_VAR_TYPE = 'df'
_ALLOWED_LOCAL_TO_SPARK_TYPES = [
_STRING_VAR_TYPE, _PANDAS_DATAFRAME_VAR_TYPE
]
def __init__(self, shell, data=None, spark_events=None):
# You must call the parent constructor
super(SparkMagicBase, self).__init__(shell)
self.logger = SparkLog(u"SparkMagics")
self.ipython_display = IpythonDisplay()
self.spark_controller = SparkController(self.ipython_display)
self.logger.debug(u'Initialized spark magics.')
if spark_events is None:
spark_events = SparkEvents()
spark_events.emit_library_loaded_event()
def do_send_to_spark(self, cell, input_variable_name, var_type,
output_variable_name, max_rows, session_name):
try:
input_variable_value = self.shell.user_ns[input_variable_name]
except KeyError:
raise BadUserDataException(
u'Variable named {} not found.'.format(input_variable_name))
if input_variable_value is None:
raise BadUserDataException(
u'Value of {} is None!'.format(input_variable_name))
if not output_variable_name:
output_variable_name = input_variable_name
if not max_rows:
max_rows = conf.default_maxrows()
input_variable_type = var_type.lower()
if input_variable_type == self._STRING_VAR_TYPE:
command = SendStringToSparkCommand(input_variable_name,
input_variable_value,
output_variable_name)
elif input_variable_type == self._PANDAS_DATAFRAME_VAR_TYPE:
command = SendPandasDfToSparkCommand(input_variable_name,
input_variable_value,
output_variable_name,
max_rows)
else:
raise BadUserDataException(
u'Invalid or incorrect -t type. Available are: [{}]'.format(
u','.join(self._ALLOWED_LOCAL_TO_SPARK_TYPES)))
(success, result,
mime_type) = self.spark_controller.run_command(command, None)
if not success:
self.ipython_display.send_error(result)
else:
self.ipython_display.write(
u'Successfully passed \'{}\' as \'{}\' to Spark'
u' kernel'.format(input_variable_name, output_variable_name))
def execute_spark(self,
cell,
output_var,
samplemethod,
maxrows,
samplefraction,
session_name,
coerce,
output_handler=None):
output_handler = output_handler or SparkOutputHandler(
html=self.ipython_display.html,
text=self.ipython_display.write,
default=self.ipython_display.display)
(success, out,
mimetype) = self.spark_controller.run_command(Command(cell),
session_name)
if not success:
if conf.shutdown_session_on_spark_statement_errors():
self.spark_controller.cleanup()
raise SparkStatementException(out)
else:
if isinstance(out, string_types):
if mimetype == MIMETYPE_TEXT_HTML:
output_handler.html(out)
else:
output_handler.text(out)
else:
output_handler.default(out)
if output_var is not None:
spark_store_command = self._spark_store_command(
output_var, samplemethod, maxrows, samplefraction, coerce)
df = self.spark_controller.run_command(spark_store_command,
session_name)
self.shell.user_ns[output_var] = df
@staticmethod
def _spark_store_command(output_var, samplemethod, maxrows, samplefraction,
coerce):
return SparkStoreCommand(output_var,
samplemethod,
maxrows,
samplefraction,
coerce=coerce)
def execute_sqlquery(self, cell, samplemethod, maxrows, samplefraction,
session, output_var, quiet, coerce):
sqlquery = self._sqlquery(cell, samplemethod, maxrows, samplefraction,
coerce)
df = self.spark_controller.run_sqlquery(sqlquery, session)
if output_var is not None:
self.shell.user_ns[output_var] = df
if quiet:
return None
else:
return df
@staticmethod
def _sqlquery(cell, samplemethod, maxrows, samplefraction, coerce):
return SQLQuery(cell,
samplemethod,
maxrows,
samplefraction,
coerce=coerce)
def _print_endpoint_info(self, info_sessions, current_session_id):
if info_sessions:
info_sessions = sorted(info_sessions, key=lambda s: s.id)
html = get_sessions_info_html(info_sessions, current_session_id)
self.ipython_display.html(html)
else:
self.ipython_display.html(u'No active sessions.')
class SparkMagicBase(Magics):
def __init__(self, shell, data=None, spark_events=None):
# You must call the parent constructor
super(SparkMagicBase, self).__init__(shell)
self.logger = SparkLog(u"SparkMagics")
self.ipython_display = IpythonDisplay()
self.spark_controller = SparkController(self.ipython_display)
self.logger.debug("Initialized spark magics.")
if spark_events is None:
spark_events = SparkEvents()
spark_events.emit_library_loaded_event()
def execute_final(self, cell, output_var, samplemethod, maxrows,
samplefraction, session_name, coerce):
(success,
out) = self.spark_controller.run_command(Command(cell), session_name)
if not success:
self.ipython_display.send_error(out)
else:
self.ipython_display.write(out)
if output_var is not None:
spark_store_command = self._spark_store_command(
output_var, samplemethod, maxrows, samplefraction, coerce)
df = self.spark_controller.run_command(spark_store_command,
session_name)
self.shell.user_ns[output_var] = df
def execute_spark(self, cell, output_var, samplemethod, maxrows,
samplefraction, session_name, coerce):
if "lagom as" in cell:
self.ipython_display.send_error(
"You are not allowed to do the following: 'import maggy.experiment.lagom as ...'. Please, just use 'import maggy.experiment as experiment' (or something else)"
)
raise
elif ".lagom" in cell:
client = Client(self.spark_controller, self.session_name, 5,
self.ipython_display)
try:
client.start_heartbeat()
if DEBUG:
self.ipython_display.writeln("Started heartbeating...")
self.execute_final(cell, output_var, samplemethod, maxrows,
samplefraction, session_name, coerce)
except:
raise
finally:
# 4. Kill thread before leaving current scope
client.stop()
try:
client.close()
except:
if DEBUG:
print("Socket already closed by maggy server.")
pass
else:
self.execute_final(cell, output_var, samplemethod, maxrows,
samplefraction, session_name, coerce)
@staticmethod
def _spark_store_command(output_var, samplemethod, maxrows, samplefraction,
coerce):
return SparkStoreCommand(output_var,
samplemethod,
maxrows,
samplefraction,
coerce=coerce)
def execute_sqlquery(self, cell, samplemethod, maxrows, samplefraction,
session, output_var, quiet, coerce):
sqlquery = self._sqlquery(cell, samplemethod, maxrows, samplefraction,
coerce)
df = self.spark_controller.run_sqlquery(sqlquery, session)
if output_var is not None:
self.shell.user_ns[output_var] = df
if quiet:
return None
else:
return df
@staticmethod
def _sqlquery(cell, samplemethod, maxrows, samplefraction, coerce):
return SQLQuery(cell,
samplemethod,
maxrows,
samplefraction,
coerce=coerce)
def _print_endpoint_info(self, info_sessions, current_session_id):
if info_sessions:
info_sessions = sorted(info_sessions, key=lambda s: s.id)
html = get_sessions_info_html(info_sessions, current_session_id)
self.ipython_display.html(html)
else:
self.ipython_display.html(u'No active sessions.')
class SparkMagicBase(Magics):
def __init__(self, shell, data=None, spark_events=None):
# You must call the parent constructor
super(SparkMagicBase, self).__init__(shell)
self.logger = SparkLog(u"SparkMagics")
self.ipython_display = IpythonDisplay()
self.spark_controller = SparkController(self.ipython_display)
self.logger.debug("Initialized spark magics.")
if spark_events is None:
spark_events = SparkEvents()
spark_events.emit_library_loaded_event()
def _get_session_name_by_session(self, session):
session_name = self.spark_controller.session_manager.get_session_name_by_id(
session.id)
# 如果session不存在,则将session激活并加入session_list
if not session_name:
session_name = session.session_name
if session_name:
self.spark_controller.session_manager.add_session(
session_name, session)
session.already_start()
return session_name
else:
return session_name
return None
def init_livy_session(self, language="python"):
'''
执行sql时自动初始化sql
:return:
'''
return self.__get_or_create_session(language)
def __get_or_create_session(self, language):
proxy_user = getpass.getuser()
self.session_language = language
endpoint = build_endpoint(self.session_language)
kernel_instance_id = id(self.shell.kernel)
session_name_seleted = self.spark_controller.generate_livy_session_name(
kernel_instance_id)
properties = conf.get_session_properties(self.session_language)
properties["proxyUser"] = proxy_user
properties["session_language"] = self.session_language
properties["session_name"] = session_name_seleted
session_info_list = self.spark_controller.get_all_sessions_endpoint(
endpoint)
for session in session_info_list:
# session kind 必须一致
if session.kind != properties['kind']:
continue
# 区分pyspark 及 pyspark3
if session.session_language != properties['session_language']:
continue
session_name = self._get_session_name_by_session(session)
if session_name == session_name_seleted:
if session.status in constants.HEALTHY_SESSION_STATUS:
return session_name_seleted
elif session.status in constants.FINAL_STATEMENT_STATUS:
# FINAL, recreate new session
self.spark_controller.add_session(session_name_seleted,
endpoint, False,
properties)
return session_name_seleted
else:
# 如果livy中没有session，则创建session
self.spark_controller.add_session(session_name_seleted, endpoint,
False, properties)
return session_name_seleted
def execute_spark(self, cell, output_var, samplemethod, maxrows,
samplefraction, session_name, coerce):
(success,
out) = self.spark_controller.run_command(Command(cell), session_name)
if not success:
self.ipython_display.send_error(out)
else:
if isinstance(out, string_types):
self.ipython_display.write(out)
elif isinstance(out, dict):
df = convert_data_struct_to_dataframe(out)
html = df.fillna('NULL').astype(str).to_html(notebook=True)
self.ipython_display.html(html)
else:
self.ipython_display.write(out)
if output_var is not None:
spark_store_command = self._spark_store_command(
output_var, samplemethod, maxrows, samplefraction, coerce)
df = self.spark_controller.run_command(spark_store_command,
session_name)
self.shell.user_ns[output_var] = df
@staticmethod
def _spark_store_command(output_var, samplemethod, maxrows, samplefraction,
coerce):
return SparkStoreCommand(output_var,
samplemethod,
maxrows,
samplefraction,
coerce=coerce)
def execute_sqlquery(self, cell, samplemethod, maxrows, samplefraction,
session, output_var, quiet, coerce):
sqlquery = self._sqlquery(cell, samplemethod, maxrows, samplefraction,
coerce)
df = self.spark_controller.run_sqlquery(sqlquery, session)
if output_var is not None:
self.shell.user_ns[output_var] = df
if quiet:
return None
else:
return df
@staticmethod
def _sqlquery(cell, samplemethod, maxrows, samplefraction, coerce):
return SQLQuery(cell,
samplemethod,
maxrows,
samplefraction,
coerce=coerce)
def _print_endpoint_info(self, info_sessions, current_session_id):
if info_sessions:
info_sessions = sorted(info_sessions, key=lambda s: s.id)
html = get_sessions_info_html(info_sessions, current_session_id)
self.ipython_display.html(html)
else:
self.ipython_display.html(u'No active sessions.')
class SparkMagicBase(Magics):
_STRING_VAR_TYPE = 'str'
_PANDAS_DATAFRAME_VAR_TYPE = 'df'
_ALLOWED_LOCAL_TO_SPARK_TYPES = [
_STRING_VAR_TYPE, _PANDAS_DATAFRAME_VAR_TYPE
]
def __init__(self, shell, data=None, spark_events=None):
# You must call the parent constructor
super(SparkMagicBase, self).__init__(shell)
self.logger = SparkLog(u"SparkMagics")
self.ipython_display = IpythonDisplay()
self.spark_controller = SparkController(self.ipython_display)
self.logger.debug(u'Initialized spark magics.')
if spark_events is None:
spark_events = SparkEvents()
spark_events.emit_library_loaded_event()
def do_send_to_spark(self, cell, input_variable_name, var_type,
output_variable_name, max_rows, session_name):
try:
input_variable_value = self.shell.user_ns[input_variable_name]
except KeyError:
raise BadUserDataException(
u'Variable named {} not found.'.format(input_variable_name))
if input_variable_value is None:
raise BadUserDataException(
u'Value of {} is None!'.format(input_variable_name))
if not output_variable_name:
output_variable_name = input_variable_name
if not max_rows:
max_rows = conf.default_maxrows()
input_variable_type = var_type.lower()
if input_variable_type == self._STRING_VAR_TYPE:
command = SendStringToSparkCommand(input_variable_name,
input_variable_value,
output_variable_name)
elif input_variable_type == self._PANDAS_DATAFRAME_VAR_TYPE:
command = SendPandasDfToSparkCommand(input_variable_name,
input_variable_value,
output_variable_name,
max_rows)
else:
raise BadUserDataException(
u'Invalid or incorrect -t type. Available are: [{}]'.format(
u','.join(self._ALLOWED_LOCAL_TO_SPARK_TYPES)))
(success, result,
mime_type) = self.spark_controller.run_command(command, None)
if not success:
self.ipython_display.send_error(result)
else:
self.ipython_display.write(
u'Successfully passed \'{}\' as \'{}\' to Spark'
u' kernel'.format(input_variable_name, output_variable_name))
def execute_final(self, cell, output_var, samplemethod, maxrows,
samplefraction, session_name, coerce):
(success, out,
mimetype) = self.spark_controller.run_command(Command(cell),
session_name)
if not success:
if conf.shutdown_session_on_spark_statement_errors():
self.spark_controller.cleanup()
raise SparkStatementException(out)
else:
if isinstance(out, string_types):
if mimetype == MIMETYPE_TEXT_HTML:
self.ipython_display.html(out)
else:
self.ipython_display.write(out)
else:
self.ipython_display.display(out)
if output_var is not None:
spark_store_command = self._spark_store_command(
output_var, samplemethod, maxrows, samplefraction, coerce)
df = self.spark_controller.run_command(spark_store_command,
session_name)
self.shell.user_ns[output_var] = df
def execute_spark(self, cell, output_var, samplemethod, maxrows,
samplefraction, session_name, coerce):
if "lagom as" in cell:
self.ipython_display.send_error(
"You are not allowed to do the following: 'import maggy.experiment.lagom as ...'. Please, just use 'import maggy.experiment as experiment' (or something else)"
)
raise
elif ".lagom" in cell:
client = Client(self.spark_controller, self.session_name, 5,
self.ipython_display)
try:
client.start_heartbeat()
if DEBUG:
self.ipython_display.writeln("Started heartbeating...")
self.execute_final(cell, output_var, samplemethod, maxrows,
samplefraction, session_name, coerce)
except:
raise
finally:
# 4. Kill thread before leaving current scope
client.stop()
try:
client.close()
except:
if DEBUG:
print("Socket already closed by maggy server.")
pass
else:
self.execute_final(cell, output_var, samplemethod, maxrows,
samplefraction, session_name, coerce)
@staticmethod
def _spark_store_command(output_var, samplemethod, maxrows, samplefraction,
coerce):
return SparkStoreCommand(output_var,
samplemethod,
maxrows,
samplefraction,
coerce=coerce)
def execute_sqlquery(self, cell, samplemethod, maxrows, samplefraction,
session, output_var, quiet, coerce):
sqlquery = self._sqlquery(cell, samplemethod, maxrows, samplefraction,
coerce)
df = self.spark_controller.run_sqlquery(sqlquery, session)
if output_var is not None:
self.shell.user_ns[output_var] = df
if quiet:
return None
else:
return df
@staticmethod
def _sqlquery(cell, samplemethod, maxrows, samplefraction, coerce):
return SQLQuery(cell,
samplemethod,
maxrows,
samplefraction,
coerce=coerce)
def _print_endpoint_info(self, info_sessions, current_session_id):
if info_sessions:
info_sessions = sorted(info_sessions, key=lambda s: s.id)
html = get_sessions_info_html(info_sessions, current_session_id)
self.ipython_display.html(html)
else:
self.ipython_display.html(u'No active sessions.')
