def get_file_watcher_class():
watcher_type = config.get_option("server.fileWatcherType")
if watcher_type == "auto":
if watchdog_available:
return EventBasedFileWatcher
else:
from streamlit.watcher.PollingFileWatcher import PollingFileWatcher
return PollingFileWatcher
elif watcher_type == "watchdog" and watchdog_available:
return EventBasedFileWatcher
elif watcher_type == "poll":
from streamlit.watcher.PollingFileWatcher import PollingFileWatcher
return PollingFileWatcher
else:
return None
def _enqueue(self, msg: ForwardMsg) -> None:
"""Enqueue a ForwardMsg to our browser queue.
This private function is called by ScriptRunContext only.
It may be called from the script thread OR the main thread.
"""
# Whenever we enqueue a ForwardMsg, we also handle any pending
# execution control request. This means that a script can be
# cleanly interrupted and stopped inside most `st.foo` calls.
#
# (If "runner.installTracer" is true, then we'll actually be
# handling these requests in a callback called after every Python
# instruction instead.)
if not config.get_option("runner.installTracer"):
self._maybe_handle_execution_control_request()
# Pass the message up to our associated AppSession.
self._enqueue_forward_msg(msg)
def __init__(self, name="md5", hasher=None):
self.hashes = dict()
self.name = name
# The number of the bytes in the hash.
self.size = 0
# An ever increasing counter.
self._counter = 0
if hasher:
self.hasher = hasher
else:
self.hasher = hashlib.new(name)
self._folder_black_list = FolderBlackList(
config.get_option("server.folderWatchBlacklist"))
def __init__(self, report, on_file_changed):
self._report = report
self._on_file_changed = on_file_changed
self._is_closed = False
self._folder_blacklist = config.get_option(
"server.folderWatchBlacklist")
# Blacklist some additional folders, using glob syntax.
self._folder_blacklist.extend(DEFAULT_FOLDER_BLACKLIST)
# A dict of filepath -> WatchedModule.
self._watched_modules = {}
self._register_watcher(
self._report.script_path,
module_name=None, # Only the root script has None here.
)
def _enqueue_forward_msg(self, msg: ForwardMsg) -> None:
"""Enqueue a new ForwardMsg to our browser queue.
This can be called on both the main thread and a ScriptRunner
run thread.
Parameters
----------
msg : ForwardMsg
The message to enqueue
"""
if not config.get_option("client.displayEnabled"):
return
self._session_data.enqueue(msg)
if self._message_enqueued_callback:
self._message_enqueued_callback()
async def get(self):
ok, msg = await self._callback()
if ok:
self.write(msg)
self.set_status(200)
# Tornado will set the _xsrf cookie automatically for the page on
# request for the document. However, if the server is reset and
# server.enableXsrfProtection is updated, the browser does not reload the document.
# Manually setting the cookie on /healthz since it is pinged when the
# browser is disconnected from the server.
if config.get_option("server.enableXsrfProtection"):
self.set_cookie("_xsrf", self.xsrf_token)
else:
# 503 = SERVICE_UNAVAILABLE
self.set_status(503)
self.write(msg)
def __init__(self, ioloop, script_path, command_line):
"""Initialize the ReportSession.
Parameters
----------
ioloop : tornado.ioloop.IOLoop
The Tornado IOLoop that we're running within.
script_path : str
Path of the Python file from which this report is generated.
command_line : str
Command line as input by the user.
"""
# Each ReportSession gets a unique ID
self.id = ReportSession._next_id
ReportSession._next_id += 1
self._ioloop = ioloop
self._report = Report(script_path, command_line)
self._state = ReportSessionState.REPORT_NOT_RUNNING
self._main_dg = DeltaGenerator(enqueue=self.enqueue,
container=BlockPath.MAIN)
self._sidebar_dg = DeltaGenerator(enqueue=self.enqueue,
container=BlockPath.SIDEBAR)
self._widget_states = WidgetStates()
self._local_sources_watcher = LocalSourcesWatcher(
self._report, self._on_source_file_changed)
self._sent_initialize_message = False
self._storage = None
self._maybe_reuse_previous_run = False
self._run_on_save = config.get_option("server.runOnSave")
# The ScriptRequestQueue is the means by which we communicate
# with the active ScriptRunner.
self._script_request_queue = ScriptRequestQueue()
self._scriptrunner = None
LOGGER.debug("ReportSession initialized (id=%s)", self.id)
def start_listening(app):
"""Takes the server start listening at the configured port.
In case the port is already taken it tries listening to the next available
port. It will error after MAX_PORT_SEARCH_RETRIES attempts.
"""
call_count = 0
while call_count < MAX_PORT_SEARCH_RETRIES:
port = config.get_option("server.port")
try:
app.listen(port)
break # It worked! So let's break out of the loop.
except (OSError, socket.error) as e:
if e.errno == errno.EADDRINUSE:
if server_port_is_manually_set():
LOGGER.error("Port %s is already in use", port)
sys.exit(1)
else:
LOGGER.debug(
"Port %s already in use, trying to use the next one.",
port)
port += 1
# Save port 3000 because it is used for the development
# server in the front end.
if port == 3000:
port += 1
config._set_option(
"server.port", port,
config.ConfigOption.STREAMLIT_DEFINITION)
call_count += 1
else:
raise
if call_count >= MAX_PORT_SEARCH_RETRIES:
raise RetriesExceeded(
"Cannot start Streamlit server. Port %s is already in use, and "
"Streamlit was unable to find a free port after %s attempts.",
port, MAX_PORT_SEARCH_RETRIES)
def test_blacklist(self, fob):
prev_blacklist = config.get_option('server.folderWatchBlacklist')
config.set_option('server.folderWatchBlacklist',
[os.path.dirname(DUMMY_MODULE_1.__file__)])
lso = LocalSourcesWatcher.LocalSourcesWatcher(REPORT, CALLBACK)
fob.assert_called_once()
sys.modules['DUMMY_MODULE_1'] = DUMMY_MODULE_1
fob.reset_mock()
lso.update_watched_modules()
fob.assert_not_called()
# Reset the config object.
config.set_option('server.folderWatchBlacklist', prev_blacklist)
def _on_script_finished(self, ctx: ScriptRunContext) -> None:
"""Called when our script finishes executing, even if it finished
early with an exception. We perform post-run cleanup here.
"""
self._session_state.reset_triggers()
self._session_state.cull_nonexistent(ctx.widget_ids_this_run)
# Signal that the script has finished. (We use SCRIPT_STOPPED_WITH_SUCCESS
# even if we were stopped with an exception.)
self.on_event.send(ScriptRunnerEvent.SCRIPT_STOPPED_WITH_SUCCESS)
# Delete expired files now that the script has run and files in use
# are marked as active.
in_memory_file_manager.del_expired_files()
# Force garbage collection to run, to help avoid memory use building up
# This is usually not an issue, but sometimes GC takes time to kick in and
# causes apps to go over resource limits, and forcing it to run between
# script runs is low cost, since we aren't doing much work anyway.
if config.get_option("runner.postScriptGC"):
gc.collect(2)
def activate(self, show_instructions=True):
"""Activate Streamlit.
Used by `streamlit activate`.
"""
try:
self.load()
except RuntimeError:
pass
if self.activation:
if self.activation.is_valid:
_exit("Already activated")
else:
_exit("Activation not valid. Please run "
"`streamlit activate reset` then `streamlit activate`")
else:
activated = False
while not activated:
# Don't stop execution to show an interactive prompt
# when in headless mode, as this makes it harder for
# people to deploy Streamlit apps.
if config.get_option("server.headless"):
email = ""
else:
email = click.prompt(
text=EMAIL_PROMPT,
prompt_suffix="",
default="",
show_default=False,
)
self.activation = _verify_email(email)
if self.activation.is_valid:
self.save()
click.secho(TELEMETRY_TEXT)
if show_instructions:
click.secho(INSTRUCTIONS_TEXT)
activated = True
else: # pragma: nocover
LOGGER.error("Please try again.")
def __init__(self, ioloop, script_path, command_line,
uploaded_file_manager):
"""Initialize the ReportSession.
Parameters
----------
ioloop : tornado.ioloop.IOLoop
The Tornado IOLoop that we're running within.
script_path : str
Path of the Python file from which this report is generated.
command_line : str
Command line as input by the user.
uploaded_file_manager : UploadedFileManager
The server's UploadedFileManager.
"""
# Each ReportSession has a unique string ID.
self.id = str(uuid.uuid4())
self._ioloop = ioloop
self._report = Report(script_path, command_line)
self._uploaded_file_mgr = uploaded_file_manager
self._state = ReportSessionState.REPORT_NOT_RUNNING
self._widget_states = WidgetStates()
self._local_sources_watcher = LocalSourcesWatcher(
self._report, self._on_source_file_changed)
self._sent_initialize_message = False
self._storage = None
self._maybe_reuse_previous_run = False
self._run_on_save = config.get_option("server.runOnSave")
# The ScriptRequestQueue is the means by which we communicate
# with the active ScriptRunner.
self._script_request_queue = ScriptRequestQueue()
self._scriptrunner = None
LOGGER.debug("ReportSession initialized (id=%s)", self.id)
def test_get_set_and_complex_config_options(self):
"""Verify that changing one option changes another, dependent one.
This also implicitly tests simple and complex ConfigOptions as well as
get_option() and set_option().
"""
# Some useful variables.
DUMMY_VAL_1, DUMMY_VAL_2, DUMMY_VAL_3 = 'Steven', 'Vincent', 'Buscemi'
# Set up both options.
config._create_option('_test.independentOption',
description='This option can change at will',
default_val=DUMMY_VAL_1)
@config._create_option('_test.dependentOption')
def _test_dependent_option():
"""Depend on the value of _test.independentOption."""
return config.get_option('_test.independentOption')
# Check that the default values are good.
self.assertEqual(config.get_option('_test.independentOption'),
DUMMY_VAL_1)
self.assertEqual(config.get_option('_test.dependentOption'),
DUMMY_VAL_1)
self.assertEqual(config.get_where_defined('_test.independentOption'),
ConfigOption.DEFAULT_DEFINITION)
self.assertEqual(config.get_where_defined('_test.dependentOption'),
ConfigOption.DEFAULT_DEFINITION)
# Override the independent option. Both update!
config.set_option('_test.independentOption', DUMMY_VAL_2)
self.assertEqual(config.get_option('_test.independentOption'),
DUMMY_VAL_2)
self.assertEqual(config.get_option('_test.dependentOption'),
DUMMY_VAL_2)
self.assertEqual(config.get_where_defined('_test.independentOption'),
config._USER_DEFINED)
self.assertEqual(config.get_where_defined('_test.dependentOption'),
ConfigOption.DEFAULT_DEFINITION)
# Override the dependent option. Only that updates!
config.set_option('_test.dependentOption', DUMMY_VAL_3)
self.assertEqual(config.get_option('_test.independentOption'),
DUMMY_VAL_2)
self.assertEqual(config.get_option('_test.dependentOption'),
DUMMY_VAL_3)
self.assertEqual(config.get_where_defined('_test.independentOption'),
config._USER_DEFINED)
self.assertEqual(config.get_where_defined('_test.dependentOption'),
config._USER_DEFINED)
def request_rerun(self, client_state: Optional[ClientState]) -> None:
"""Signal that we're interested in running the script.
If the script is not already running, it will be started immediately.
Otherwise, a rerun will be requested.
Parameters
----------
client_state : streamlit.proto.ClientState_pb2.ClientState | None
The ClientState protobuf to run the script with, or None
to use previous client state.
"""
if self._state == AppSessionState.SHUTDOWN_REQUESTED:
LOGGER.warning("Discarding rerun request after shutdown")
return
if client_state:
rerun_data = RerunData(client_state.query_string,
client_state.widget_states)
else:
rerun_data = RerunData()
if self._scriptrunner is not None:
if bool(config.get_option("runner.fastReruns")):
# If fastReruns is enabled, we don't send rerun requests to our
# existing ScriptRunner. Instead, we tell it to shut down. We'll
# then spin up a new ScriptRunner, below, to handle the rerun
# immediately.
self._scriptrunner.request_stop()
self._scriptrunner = None
else:
# fastReruns is not enabled. Send our ScriptRunner a rerun
# request. If the request is accepted, we're done.
success = self._scriptrunner.request_rerun(rerun_data)
if success:
return
# If we are here, then either we have no ScriptRunner, or our
# current ScriptRunner is shutting down and cannot handle a rerun
# request - so we'll create and start a new ScriptRunner.
self._create_scriptrunner(rerun_data)
def start(self, on_started):
"""Start the server.
Parameters
----------
on_started : callable
A callback that will be called when the server's run-loop
has started, and the server is ready to begin receiving clients.
"""
if self._state != State.INITIAL:
raise RuntimeError("Server has already been started")
LOGGER.debug("Starting server...")
app = self._create_app()
port = config.get_option("server.port")
app.listen(port)
LOGGER.debug("Server started on port %s", port)
self._ioloop.spawn_callback(self._loop_coroutine, on_started)
def test_config_blacklist(self, fob, _):
"""Test server.folderWatchBlacklist"""
prev_blacklist = config.get_option("server.folderWatchBlacklist")
config.set_option("server.folderWatchBlacklist",
[os.path.dirname(DUMMY_MODULE_1.__file__)])
lso = local_sources_watcher.LocalSourcesWatcher(REPORT, NOOP_CALLBACK)
fob.assert_called_once()
sys.modules["DUMMY_MODULE_1"] = DUMMY_MODULE_1
fob.reset_mock()
lso.update_watched_modules()
fob.assert_not_called()
# Reset the config object.
config.set_option("server.folderWatchBlacklist", prev_blacklist)
def enqueue(self, msg):
"""Enqueues a new ForwardMsg to our browser queue.
Parameters
----------
msg : ForwardMsg
The message to enqueue
Returns
-------
bool
True if the message was enqueued, or False if
client.displayEnabled is not set
"""
if not config.get_option('client.displayEnabled'):
return False
self._scriptrunner.maybe_handle_execution_control_request()
self._report.enqueue(msg)
return True
def _s3_upload_files(self, files, progress_coroutine):
set_private_acl = config.get_option('s3.requireLoginToView')
for i, (path, data) in enumerate(files):
mime_type = mimetypes.guess_type(path)[0]
if not mime_type:
mime_type = 'application/octet-stream'
if set_private_acl and path.startswith('report'):
acl = 'private'
else:
acl = 'public-read'
self._s3_client.put_object(Bucket=self._bucketname,
Body=data,
Key=self._s3_key(path),
ContentType=mime_type,
ACL=acl)
LOGGER.debug('Uploaded: "%s"', path)
if progress_coroutine:
yield progress_coroutine(math.ceil(100 * (i + 1) / len(files)))
else:
yield
def on_message(self, payload):
if not self._session:
return
msg = BackMsg()
try:
msg.ParseFromString(payload)
msg_type = msg.WhichOneof("type")
LOGGER.debug("Received the following back message:\n%s", msg)
if msg_type == "cloud_upload":
yield self._session.handle_save_request(self)
elif msg_type == "rerun_script":
self._session.handle_rerun_script_request()
elif msg_type == "clear_cache":
self._session.handle_clear_cache_request()
elif msg_type == "set_run_on_save":
self._session.handle_set_run_on_save_request(msg.set_run_on_save)
elif msg_type == "stop_report":
self._session.handle_stop_script_request()
elif msg_type == "update_widgets":
self._session.handle_rerun_script_request(
widget_state=msg.update_widgets
)
elif msg_type == "close_connection":
if config.get_option("global.developmentMode"):
Server.get_current().stop()
else:
LOGGER.warning(
"Client tried to close connection when "
"not in development mode"
)
else:
LOGGER.warning('No handler for "%s"', msg_type)
except BaseException as e:
LOGGER.error(e)
self._session.enqueue_exception(e)
def is_url_from_allowed_origins(url: str) -> bool:
"""Return True if URL is from allowed origins (for CORS purpose).
Allowed origins:
1. localhost
2. The internal and external IP addresses of the machine where this
function was called from.
If `server.enableCORS` is False, this allows all origins.
"""
if not config.get_option("server.enableCORS"):
# Allow everything when CORS is disabled.
return True
hostname = url_util.get_hostname(url)
allowed_domains = [ # List[Union[str, Callable[[], Optional[str]]]]
# Check localhost first.
"localhost",
"0.0.0.0",
"127.0.0.1",
# Try to avoid making unecessary HTTP requests by checking if the user
# manually specified a server address.
_get_server_address_if_manually_set,
# Then try the options that depend on HTTP requests or opening sockets.
net_util.get_internal_ip,
net_util.get_external_ip,
]
for allowed_domain in allowed_domains:
if callable(allowed_domain):
allowed_domain = allowed_domain()
if allowed_domain is None:
continue
if hostname == allowed_domain:
return True
return False
def _populate_theme_msg(msg: CustomThemeConfig) -> None:
theme_opts = config.get_options_for_section("theme")
if (theme.check_theme_completeness(theme_opts) !=
theme.ThemeCompleteness.FULLY_DEFINED):
return
for option_name, option_val in theme_opts.items():
# We don't set the "font" option here as it needs to be converted
# from string -> enum.
if option_name != "font" and option_val is not None:
setattr(msg, to_snake_case(option_name), option_val)
font_map = {
"sans serif": msg.FontFamily.SANS_SERIF,
"serif": msg.FontFamily.SERIF,
"monospace": msg.FontFamily.MONOSPACE,
}
msg.font = font_map.get(
config.get_option("theme.font"),
msg.FontFamily.SANS_SERIF,
)
def setup_formatter(logger):
"""Set up the console formatter for a given logger."""
# Deregister any previous console loggers.
if hasattr(logger, "streamlit_console_handler"):
logger.removeHandler(logger.streamlit_console_handler)
logger.streamlit_console_handler = logging.StreamHandler()
if config._config_file_has_been_parsed:
# logger is required in ConfigOption.set_value
# Getting the config option before the config file has been parsed
# can create an infinite loop
message_format = config.get_option("logger.messageFormat")
else:
message_format = DEFAULT_LOG_MESSAGE
formatter = logging.Formatter(fmt=message_format)
formatter.default_msec_format = "%s.%03d"
logger.streamlit_console_handler.setFormatter(formatter)
# Register the new console logger.
logger.addHandler(logger.streamlit_console_handler)
def get_url(host_ip: str) -> str:
"""Get the URL for any app served at the given host_ip.
Parameters
----------
host_ip : str
The IP address of the machine that is running the Streamlit Server.
Returns
-------
str
The URL.
"""
port = _get_browser_address_bar_port()
base_path = config.get_option("server.baseUrlPath").strip("/")
if base_path:
base_path = "/" + base_path
host_ip = host_ip.strip("/")
return f"http://{host_ip}:{port}{base_path}"
def _maybe_print_use_warning():
"""Print a warning if Streamlit is imported but not being run with `streamlit run`.
The warning is printed only once.
"""
global _use_warning_has_been_displayed
if not _use_warning_has_been_displayed:
_use_warning_has_been_displayed = True
warning = _click.style("Warning:", bold=True, fg="yellow")
if _env_util.is_repl():
_LOGGER.warning(
f"\n {warning} to view a Streamlit app on a browser, use Streamlit in a file and\n run it with the following command:\n\n streamlit run [FILE_NAME] [ARGUMENTS]"
)
elif not _is_running_with_streamlit and _config.get_option(
"global.showWarningOnDirectExecution"):
script_name = _sys.argv[0]
_LOGGER.warning(
f"\n {warning} to view this Streamlit app on a browser, run it with the following\n command:\n\n streamlit run {script_name} [ARGUMENTS]"
)
def _maybe_print_repl_warning():
global _repl_warning_has_been_displayed
if not _repl_warning_has_been_displayed:
_repl_warning_has_been_displayed = True
if _env_util.is_repl():
_LOGGER.warning(
_textwrap.dedent(
"""
Will not generate Streamlit app
To generate an app, use Streamlit in a file and run it with:
$ streamlit run [FILE_NAME] [ARGUMENTS]
"""
)
)
elif _config.get_option("global.showWarningOnDirectExecution"):
script_name = _sys.argv[0]
_LOGGER.warning(
_textwrap.dedent(
"""
Will not generate Streamlit App
To generate an App, run this file with:
$ streamlit run %s [ARGUMENTS]
"""
),
script_name,
)
def get_url(cls, host_ip):
"""Get the URL for any app served at the given host_ip.
Parameters
----------
host_ip : str
The IP address of the machine that is running the Streamlit Server.
Returns
-------
str
The URL.
"""
port = _get_browser_address_bar_port()
base_path = config.get_option("server.baseUrlPath").strip("/")
if base_path:
base_path = "/" + base_path
return "http://%(host_ip)s:%(port)s%(base_path)s" % {
"host_ip": host_ip.strip("/"),
"port": port,
"base_path": base_path,
}
def toggle_metrics(self):
self._metrics = {}
if config.get_option("global.metrics"):
try:
import prometheus_client
except ImportError as e:
raise ImportError(
"prometheus-client is not installed. pip install prometheus-client"
)
self.generate_latest = prometheus_client.generate_latest
existing_metrics = (prometheus_client.registry.REGISTRY.
_names_to_collectors.keys())
for kind, metric, doc, labels in self._raw_metrics:
if metric in existing_metrics:
continue
p = getattr(prometheus_client, kind)
self._metrics[metric] = p(metric, doc, labels)
else:
self.generate_latest = lambda: ""
for _, metric, _, _ in self._raw_metrics:
self._metrics[metric] = MockMetric()
def on_message(self, payload):
msg = BackMsg()
try:
msg.ParseFromString(payload)
LOGGER.debug('Received the following back message:\n%s', msg)
msg_type = msg.WhichOneof('type')
if msg_type == 'cloud_upload':
yield self._session.handle_save_request(self)
elif msg_type == 'rerun_script':
self._session.handle_rerun_script_request(
command_line=msg.rerun_script)
elif msg_type == 'clear_cache':
self._session.handle_clear_cache_request()
elif msg_type == 'set_run_on_save':
self._session.handle_set_run_on_save_request(msg.set_run_on_save)
elif msg_type == 'stop_report':
self._session.handle_stop_script_request()
elif msg_type == 'update_widgets':
self._session.handle_rerun_script_request(
widget_state=msg.update_widgets)
elif msg_type == 'close_connection':
if config.get_option('global.developmentMode'):
Server.get_current().stop()
else:
LOGGER.warning(
'Client tried to close connection when '
'not in development mode')
else:
LOGGER.warning('No handler for "%s"', msg_type)
except BaseException as e:
LOGGER.error(e)
self._session.enqueue_exception(e)
def has_message_reference(self, msg, session, report_run_count):
"""Return True if a session has a reference to a message.
Parameters
----------
msg : ForwardMsg
session : ReportSession
report_run_count : int
The number of times the session's report has run
Returns
-------
bool
"""
populate_hash_if_needed(msg)
entry = self._entries.get(msg.hash, None)
if entry is None or not entry.has_session_ref(session):
return False
# Ensure we're not expired
age = entry.get_session_ref_age(session, report_run_count)
return age <= config.get_option("global.maxCachedMessageAge")
def test_upload_file_default_values(self):
self.assertEqual(200, config.get_option("server.maxUploadSize"))
