class ScriptRunner(object):
def __init__(self, report, main_dg, sidebar_dg, widget_states,
request_queue):
"""Initialize the ScriptRunner.
(The ScriptRunner won't start executing until start() is called.)
Parameters
----------
report : Report
The ReportSession's report.
main_dg : DeltaGenerator
The ReportSession's main DeltaGenerator.
sidebar_dg : DeltaGenerator
The ReportSession's sidebar DeltaGenerator.
widget_states : streamlit.proto.Widget_pb2.WidgetStates
The ReportSession's current widget states
request_queue : ScriptRequestQueue
The queue that the ReportSession is publishing ScriptRequests to.
ScriptRunner will continue running until the queue is empty,
and then shut down.
"""
self._report = report
self._main_dg = main_dg
self._sidebar_dg = sidebar_dg
self._request_queue = request_queue
self._widgets = Widgets()
self._widgets.set_state(widget_states)
self.on_event = Signal(doc="""Emitted when a ScriptRunnerEvent occurs.
This signal is *not* emitted on the same thread that the
ScriptRunner was created on.
Parameters
----------
event : ScriptRunnerEvent
exception : BaseException | None
Our compile error. Set only for the
SCRIPT_STOPPED_WITH_COMPILE_ERROR event.
widget_states : streamlit.proto.Widget_pb2.WidgetStates | None
The ScriptRunner's final WidgetStates. Set only for the
SHUTDOWN event.
""")
# Set to true when we process a SHUTDOWN request
self._shutdown_requested = False
# Set to true while we're executing. Used by
# maybe_handle_execution_control_request.
self._execing = False
# This is initialized in start()
self._script_thread = None
def start(self):
"""Start a new thread to process the ScriptEventQueue.
This must be called only once.
"""
if self._script_thread is not None:
raise Exception("ScriptRunner was already started")
self._script_thread = ReportThread(
main_dg=self._main_dg,
sidebar_dg=self._sidebar_dg,
widgets=self._widgets,
target=self._process_request_queue,
name="ScriptRunner.scriptThread",
)
self._script_thread.start()
def _process_request_queue(self):
"""Process the ScriptRequestQueue and then exits.
This is run in a separate thread.
"""
LOGGER.debug("Beginning script thread")
while not self._shutdown_requested and self._request_queue.has_request:
request, data = self._request_queue.dequeue()
if request == ScriptRequest.STOP:
LOGGER.debug("Ignoring STOP request while not running")
elif request == ScriptRequest.SHUTDOWN:
LOGGER.debug("Shutting down")
self._shutdown_requested = True
elif request == ScriptRequest.RERUN:
self._run_script(data)
else:
raise RuntimeError("Unrecognized ScriptRequest: %s" % request)
# Send a SHUTDOWN event before exiting. This includes the widget values
# as they existed after our last successful script run, which the
# ReportSession will pass on to the next ScriptRunner that gets
# created.
self.on_event.send(ScriptRunnerEvent.SHUTDOWN,
widget_states=self._widgets.get_state())
def _is_in_script_thread(self):
"""True if the calling function is running in the script thread"""
return self._script_thread == threading.current_thread()
def maybe_handle_execution_control_request(self):
if not self._is_in_script_thread():
# We can only handle execution_control_request if we're on the
# script execution thread. However, it's possible for deltas to
# be enqueued (and, therefore, for this function to be called)
# in separate threads, so we check for that here.
return
if not self._execing:
# If the _execing flag is not set, we're not actually inside
# an exec() call. This happens when our script exec() completes,
# we change our state to STOPPED, and a statechange-listener
# enqueues a new ForwardEvent
return
# Pop the next request from our queue.
request, data = self._request_queue.dequeue()
if request is None:
return
LOGGER.debug("Received ScriptRequest: %s", request)
if request == ScriptRequest.STOP:
raise StopException()
elif request == ScriptRequest.SHUTDOWN:
self._shutdown_requested = True
raise StopException()
elif request == ScriptRequest.RERUN:
raise RerunException(data)
else:
raise RuntimeError("Unrecognized ScriptRequest: %s" % request)
def _install_tracer(self):
"""Install function that runs before each line of the script."""
def trace_calls(frame, event, arg):
self.maybe_handle_execution_control_request()
return trace_calls
# Python interpreters are not required to implement sys.settrace.
if hasattr(sys, "settrace"):
sys.settrace(trace_calls)
@contextmanager
def _set_execing_flag(self):
"""A context for setting the ScriptRunner._execing flag.
Used by maybe_handle_execution_control_request to ensure that
we only handle requests while we're inside an exec() call
"""
if self._execing:
raise RuntimeError("Nested set_execing_flag call")
self._execing = True
try:
yield
finally:
self._execing = False
def _run_script(self, rerun_data):
"""Run our script.
Parameters
----------
rerun_data: RerunData
The RerunData to use.
"""
assert self._is_in_script_thread()
LOGGER.debug("Running script %s", rerun_data)
# Reset delta generator so it starts from index 0.
import streamlit as st
st._reset(self._main_dg, self._sidebar_dg)
self.on_event.send(ScriptRunnerEvent.SCRIPT_STARTED)
# Compile the script. Any errors thrown here will be surfaced
# to the user via a modal dialog in the frontend, and won't result
# in their previous report disappearing.
try:
# Python 3 got rid of the native execfile() command, so we read
# the file, compile it, and exec() it. This implementation is
# compatible with both 2 and 3.
with open(self._report.script_path) as f:
filebody = f.read()
if config.get_option("runner.magicEnabled"):
filebody = magic.add_magic(filebody, self._report.script_path)
code = compile(
filebody,
# Pass in the file path so it can show up in exceptions.
self._report.script_path,
# We're compiling entire blocks of Python, so we need "exec"
# mode (as opposed to "eval" or "single").
mode="exec",
# Don't inherit any flags or "future" statements.
flags=0,
dont_inherit=1,
# Parameter not supported in Python2:
# optimize=-1,
)
except BaseException as e:
# We got a compile error. Send an error event and bail immediately.
LOGGER.debug("Fatal script error: %s" % e)
self.on_event.send(
ScriptRunnerEvent.SCRIPT_STOPPED_WITH_COMPILE_ERROR,
exception=e)
return
# If we get here, we've successfully compiled our script. The next step
# is to run it. Errors thrown during execution will be shown to the
# user as ExceptionElements.
# Update Report.argv
if rerun_data.argv is not None:
argv = rerun_data.argv
self._report.argv = rerun_data.argv
else:
argv = self._report.argv
# Update the Widget singleton with the new widget_state
if rerun_data.widget_state is not None:
self._widgets.set_state(rerun_data.widget_state)
if config.get_option("runner.installTracer"):
self._install_tracer()
# This will be set to a RerunData instance if our execution
# is interrupted by a RerunException.
rerun_with_data = None
try:
# Create fake module. This gives us a name global namespace to
# execute the code in.
module = _new_module("__main__")
# Install the fake module as the __main__ module. This allows
# the pickle module to work inside the user's code, since it now
# can know the module where the pickled objects stem from.
# IMPORTANT: This means we can't use "if __name__ == '__main__'" in
# our code, as it will point to the wrong module!!!
sys.modules["__main__"] = module
# Make it look like command-line args were set to whatever the user
# asked them to be via the GUI.
# IMPORTANT: This means we can't count on sys.argv in our code ---
# but we already knew it from the argv surgery in cli.py.
# TODO: Remove this feature when we implement interactivity!
# This is not robust in a multi-user environment.
sys.argv = argv
# Add special variables to the module's globals dict.
module.__dict__["__file__"] = self._report.script_path
with modified_sys_path(self._report), self._set_execing_flag():
exec(code, module.__dict__)
except RerunException as e:
rerun_with_data = e.rerun_data
except StopException:
pass
except BaseException as e:
# Show exceptions in the Streamlit report.
LOGGER.debug(e)
import streamlit as st
st.exception(e) # This is OK because we're in the script thread.
# TODO: Clean up the stack trace, so it doesn't include
# ScriptRunner.
finally:
self._widgets.reset_triggers()
self.on_event.send(ScriptRunnerEvent.SCRIPT_STOPPED_WITH_SUCCESS)
# Use _log_if_error() to make sure we never ever ever stop running the
# script without meaning to.
_log_if_error(_clean_problem_modules)
if rerun_with_data is not None:
self._run_script(rerun_with_data)
class ScriptRunner(object):
def __init__(self, report):
"""Initialize.
Parameters
----------
report : Report
The report with the script to run.
"""
self._report = report
self._event_queue = ScriptEventQueue()
self._state = ScriptState.STOPPED
self._last_run_data = RerunData(argv=report.argv,
widget_state=WidgetStates())
self._widgets = Widgets()
self.run_on_save = config.get_option('server.runOnSave')
self.on_state_changed = Signal(
doc="""Emitted when the script's execution state state changes.
Parameters
----------
state : ScriptState
""")
self.on_file_change_not_handled = Signal(
doc="Emitted when the file is modified and we haven't handled it.")
self.on_script_compile_error = Signal(
doc="""Emitted if our script fails to compile. (*Not* emitted
for normal exceptions thrown while a script is running.)
Parameters
----------
exception : Exception
The exception that was thrown
""")
self._local_sources_watcher = LocalSourcesWatcher(
self._report, self._on_source_file_changed)
# Will be set to true when we process a SHUTDOWN event
self._shutdown_requested = False
self._script_thread = None
self._ctx = None
# Set to true while we're executing. Used by
# maybe_handle_execution_control_request.
self._execing = False
@property
def widgets(self):
"""
Returns
-------
Widgets
Our widget state dictionary
"""
return self._widgets
def start_run_loop(self, ctx):
"""Starts the ScriptRunner's thread.
This must be called only once.
Parameters
----------
ctx : ReportContext
The ReportContext that owns this ScriptRunner. This will be
attached to the ScriptRunner's runloop thread.
"""
assert self._script_thread is None, 'Already started!'
self._ctx = ctx
# start our thread
self._script_thread = ReportThread(ctx,
target=self._loop,
name='ScriptRunner.loop')
self._script_thread.start()
def _set_state(self, new_state):
if self._state == new_state:
return
LOGGER.debug('Scriptrunner state: %s -> %s' % (self._state, new_state))
self._state = new_state
self.on_state_changed.send(self._state)
def is_running(self):
return self._state == ScriptState.RUNNING
def is_shutdown(self):
return self._state == ScriptState.IS_SHUTDOWN
def request_rerun(self, argv=None, widget_state=None):
"""Signal that we're interested in running the script.
If the script is not already running, it will be started immediately.
Otherwise, a rerun will be requested.
Parameters
----------
argv : dict | None
The new command line arguments to run the script with, or None
to use the argv from the previous run of the script.
widget_state : dict | None
The widget state dictionary to run the script with, or None
to use the widget state from the previous run of the script.
"""
if self.is_shutdown():
LOGGER.warning('Discarding RERUN event after shutdown')
return
self._event_queue.enqueue(ScriptEvent.RERUN,
RerunData(argv, widget_state))
def request_stop(self):
if self.is_shutdown():
LOGGER.warning('Discarding STOP event after shutdown')
return
self._event_queue.enqueue(ScriptEvent.STOP)
def request_shutdown(self):
if self.is_shutdown():
LOGGER.warning('Discarding SHUTDOWN event after shutdown')
return
self._event_queue.enqueue(ScriptEvent.SHUTDOWN)
def maybe_handle_execution_control_request(self):
if self._script_thread != threading.current_thread():
# We can only handle execution_control_request if we're on the
# script execution thread. However, it's possible for deltas to
# be enqueued (and, therefore, for this function to be called)
# in separate threads, so we check for that here.
return
if not self._execing:
# If the _execing flag is not set, we're not actually inside
# an exec() call. This happens when our script exec() completes,
# we change our state to STOPPED, and a statechange-listener
# enqueues a new ForwardEvent
return
# Pop the next event from our queue. Don't block if there's no event
event, event_data = self._event_queue.dequeue_nowait()
if event is None:
return
LOGGER.debug('Received ScriptEvent: %s', event)
if event == ScriptEvent.STOP:
raise StopException()
elif event == ScriptEvent.SHUTDOWN:
self._shutdown_requested = True
raise StopException()
elif event == ScriptEvent.RERUN:
raise RerunException(event_data)
else:
raise RuntimeError('Unrecognized ScriptEvent: %s' % event)
def _on_source_file_changed(self):
"""One of our source files changed. Schedule a rerun if appropriate."""
if self.run_on_save:
self._event_queue.enqueue(ScriptEvent.RERUN,
RerunData(argv=None, widget_state=None))
else:
self.on_file_change_not_handled.send()
def _loop(self):
"""Our run loop.
Continually pops events from the event_queue. Ends when we receive
a SHUTDOWN event.
"""
while not self._shutdown_requested:
assert self._state == ScriptState.STOPPED
# Dequeue our next event. If the event queue is empty, the thread
# will go to sleep, and awake when there's a new event.
event, event_data = self._event_queue.dequeue()
if event == ScriptEvent.STOP:
LOGGER.debug('Ignoring STOP event while not running')
elif event == ScriptEvent.SHUTDOWN:
LOGGER.debug('Shutting down')
self._shutdown_requested = True
elif event == ScriptEvent.RERUN:
self._run_script(event_data)
else:
raise RuntimeError('Unrecognized ScriptEvent: %s' % event)
# Release resources
self._local_sources_watcher.close()
self._set_state(ScriptState.IS_SHUTDOWN)
def _install_tracer(self):
"""Install function that runs before each line of the script."""
def trace_calls(frame, event, arg):
self.maybe_handle_execution_control_request()
return trace_calls
# Python interpreters are not required to implement sys.settrace.
if hasattr(sys, 'settrace'):
sys.settrace(trace_calls)
@contextmanager
def _set_execing_flag(self):
"""A context for setting the ScriptRunner._execing flag.
Used by maybe_handle_execution_control_request to ensure that
we only handle requests while we're inside an exec() call
"""
if self._execing:
raise RuntimeError('Nested set_execing_flag call')
self._execing = True
try:
yield
finally:
self._execing = False
def _run_script(self, rerun_data):
"""Run our script.
Parameters
----------
rerun_data: RerunData
The RerunData to use.
"""
assert self._state == ScriptState.STOPPED
LOGGER.debug('Running script %s', rerun_data)
# Reset delta generator so it starts from index 0.
import streamlit as st
st._reset()
self._set_state(ScriptState.RUNNING)
# Compile the script. Any errors thrown here will be surfaced
# to the user via a modal dialog, and won't result in their
# previous report disappearing.
try:
# Python 3 got rid of the native execfile() command, so we now read
# the file, compile it, and exec() it. This implementation is
# compatible with both 2 and 3.
with open(self._report.script_path) as f:
filebody = f.read()
if config.get_option('runner.magicEnabled'):
filebody = magic.add_magic(filebody, self._report.script_path)
code = compile(
filebody,
# Pass in the file path so it can show up in exceptions.
self._report.script_path,
# We're compiling entire blocks of Python, so we need "exec"
# mode (as opposed to "eval" or "single").
'exec',
# Don't inherit any flags or "future" statements.
flags=0,
dont_inherit=1,
# Parameter not supported in Python2:
# optimize=-1,
)
except BaseException as e:
# We got a compile error. Send the exception onto the client
# as a SessionEvent and bail immediately.
LOGGER.debug('Fatal script error: %s' % e)
self.on_script_compile_error.send(e)
self._set_state(ScriptState.STOPPED)
return
# If we get here, we've successfully compiled our script. The next step
# is to run it. Errors thrown during execution will be shown to the
# user as ExceptionElements.
# Get our argv and widget_state for this run, defaulting to
# self._last_run_data for missing values.
# Also update self._last_run_data for the next run.
argv = rerun_data.argv or self._last_run_data.argv
widget_state = rerun_data.widget_state or \
self._last_run_data.widget_state
self._last_run_data = RerunData(argv,
reset_widget_triggers(widget_state))
# Update the Widget singleton with the new widget_state
self._ctx.widgets.set_state(widget_state)
if config.get_option('runner.installTracer'):
self._install_tracer()
# This will be set to a RerunData instance if our execution
# is interrupted by a RerunException.
rerun_with_data = None
try:
# Create fake module. This gives us a name global namespace to
# execute the code in.
module = _new_module('__main__')
# Install the fake module as the __main__ module. This allows
# the pickle module to work inside the user's code, since it now
# can know the module where the pickled objects stem from.
# IMPORTANT: This means we can't use "if __name__ == '__main__'" in
# our code, as it will point to the wrong module!!!
sys.modules['__main__'] = module
# Make it look like command-line args were set to whatever the user
# asked them to be via the GUI.
# IMPORTANT: This means we can't count on sys.argv in our code ---
# but we already knew it from the argv surgery in cli.py.
# TODO: Remove this feature when we implement interactivity!
# This is not robust in a multi-user environment.
sys.argv = argv
# Add special variables to the module's globals dict.
module.__dict__['__file__'] = self._report.script_path
with modified_sys_path(self._report), self._set_execing_flag():
exec(code, module.__dict__)
except RerunException as e:
rerun_with_data = e.rerun_data
except StopException:
pass
except BaseException as e:
# Show exceptions in the Streamlit report.
LOGGER.debug(e)
st.exception(e) # This is OK because we're in the script thread.
# TODO: Clean up the stack trace, so it doesn't include
# ScriptRunner.
finally:
self._set_state(ScriptState.STOPPED)
# Use _log_if_error() to make sure we never ever ever stop running the
# script without meaning to.
_log_if_error(self._local_sources_watcher.update_watched_modules)
_log_if_error(_clean_problem_modules)
if rerun_with_data is not None:
self._run_script(rerun_with_data)