def test_add_rows_rerun(self):
rq = ReportQueue()
self.assertTrue(rq.is_empty())
rq.enqueue(INIT_MSG)
# Simulate rerun
for i in range(2):
TEXT_DELTA_MSG1.delta.id = 0
rq.enqueue(TEXT_DELTA_MSG1)
DF_DELTA_MSG.delta.id = 1
rq.enqueue(DF_DELTA_MSG)
ADD_ROWS_MSG.delta.id = 1
rq.enqueue(ADD_ROWS_MSG)
queue = rq.flush()
self.assertEqual(len(queue), 3)
self.assertTrue(queue[0].initialize.config.sharing_enabled)
self.assertEqual(queue[1].delta.id, 0)
self.assertEqual(queue[1].delta.new_element.text.body, 'text1')
self.assertEqual(queue[2].delta.id, 1)
col0 = queue[2].delta.new_element.data_frame.data.cols[0].int64s.data
col1 = queue[2].delta.new_element.data_frame.data.cols[1].int64s.data
self.assertEqual(col0, [0, 1, 2, 3, 4, 5])
self.assertEqual(col1, [10, 11, 12, 13, 14, 15])
def test_simple_add_rows(self):
rq = ReportQueue()
self.assertTrue(rq.is_empty())
rq.enqueue(INIT_MSG)
TEXT_DELTA_MSG1.metadata.delta_id = 0
rq.enqueue(TEXT_DELTA_MSG1)
DF_DELTA_MSG.metadata.delta_id = 1
rq.enqueue(DF_DELTA_MSG)
ADD_ROWS_MSG.metadata.delta_id = 1
rq.enqueue(ADD_ROWS_MSG)
queue = rq.flush()
self.assertEqual(len(queue), 3)
self.assertTrue(queue[0].initialize.config.sharing_enabled)
self.assertEqual(queue[1].metadata.delta_id, 0)
self.assertEqual(queue[1].delta.new_element.text.body, "text1")
self.assertEqual(queue[2].metadata.delta_id, 1)
col0 = queue[2].delta.new_element.data_frame.data.cols[0].int64s.data
col1 = queue[2].delta.new_element.data_frame.data.cols[1].int64s.data
self.assertEqual(col0, [0, 1, 2, 3, 4, 5])
self.assertEqual(col1, [10, 11, 12, 13, 14, 15])
def test_multiple_containers(self):
"""Deltas should only be coalesced if they're in the same container"""
rq = ReportQueue()
self.assertTrue(rq.is_empty())
rq.enqueue(INIT_MSG)
def enqueue_deltas(container, path):
# We deep-copy the protos because we mutate each one
# multiple times.
msg = copy.deepcopy(TEXT_DELTA_MSG1)
msg.delta.id = 0
msg.delta.parent_block.container = container
msg.delta.parent_block.path[:] = path
rq.enqueue(msg)
msg = copy.deepcopy(DF_DELTA_MSG)
msg.delta.id = 1
msg.delta.parent_block.container = container
msg.delta.parent_block.path[:] = path
rq.enqueue(msg)
msg = copy.deepcopy(ADD_ROWS_MSG)
msg.delta.id = 1
msg.delta.parent_block.container = container
msg.delta.parent_block.path[:] = path
rq.enqueue(msg)
enqueue_deltas(BlockPath.MAIN, [])
enqueue_deltas(BlockPath.SIDEBAR, [0, 0, 1])
def assert_deltas(container, path, idx):
self.assertEqual(0, queue[idx].delta.id)
self.assertEqual(container,
queue[idx].delta.parent_block.container)
self.assertEqual(path, queue[idx].delta.parent_block.path)
self.assertEqual('text1', queue[idx].delta.new_element.text.body)
self.assertEqual(1, queue[idx + 1].delta.id)
self.assertEqual(container,
queue[idx + 1].delta.parent_block.container)
self.assertEqual(path, queue[idx + 1].delta.parent_block.path)
col0 = queue[idx + 1] \
.delta.new_element.data_frame.data.cols[0].int64s.data
col1 = queue[idx + 1] \
.delta.new_element.data_frame.data.cols[1].int64s.data
self.assertEqual([0, 1, 2, 3, 4, 5], col0)
self.assertEqual([10, 11, 12, 13, 14, 15], col1)
queue = rq.flush()
self.assertEqual(5, len(queue))
self.assertTrue(queue[0].initialize.config.sharing_enabled)
assert_deltas(BlockPath.MAIN, [], 1)
assert_deltas(BlockPath.SIDEBAR, [0, 0, 1], 3)
def test_simple_enqueue(self):
rq = ReportQueue()
self.assertTrue(rq.is_empty())
rq.enqueue(INIT_MSG)
self.assertFalse(rq.is_empty())
queue = rq.flush()
self.assertTrue(rq.is_empty())
self.assertEqual(len(queue), 1)
self.assertTrue(queue[0].initialize.config.sharing_enabled)
def test_enqueue_two(self):
rq = ReportQueue()
self.assertTrue(rq.is_empty())
rq.enqueue(INIT_MSG)
TEXT_DELTA_MSG1.delta.id = 0
rq.enqueue(TEXT_DELTA_MSG1)
queue = rq.flush()
self.assertEqual(len(queue), 2)
self.assertTrue(queue[0].initialize.config.sharing_enabled)
self.assertEqual(queue[1].delta.id, 0)
self.assertEqual(queue[1].delta.new_element.text.body, 'text1')
def test_replace_element(self):
rq = ReportQueue()
self.assertTrue(rq.is_empty())
rq.enqueue(INIT_MSG)
TEXT_DELTA_MSG1.metadata.delta_id = 0
rq.enqueue(TEXT_DELTA_MSG1)
TEXT_DELTA_MSG2.metadata.delta_id = 0
rq.enqueue(TEXT_DELTA_MSG2)
queue = rq.flush()
self.assertEqual(len(queue), 2)
self.assertTrue(queue[0].initialize.config.sharing_enabled)
self.assertEqual(queue[1].metadata.delta_id, 0)
self.assertEqual(queue[1].delta.new_element.text.body, "text2")
class Report(object):
"""
Contains parameters related to running a report, and also houses
the two ReportQueues (master_queue and browser_queue) that are used
to deliver messages to a connected browser, and to serialize the
running report.
"""
@classmethod
def get_url(cls, host_ip):
"""Get the URL for any app served at the given host_ip.
Parameters
----------
host_ip : str
The IP address of the machine that is running the Streamlit Server.
Returns
-------
str
The URL.
"""
port = _get_browser_address_bar_port()
base_path = config.get_option("server.baseUrlPath").strip("/")
if base_path:
base_path = "/" + base_path
return "http://%(host_ip)s:%(port)s%(base_path)s" % {
"host_ip": host_ip.strip("/"),
"port": port,
"base_path": base_path,
}
def __init__(self, script_path, command_line):
"""Constructor.
Parameters
----------
script_path : str
Path of the Python file from which this app is generated.
command_line : string
Command line as input by the user
"""
basename = os.path.basename(script_path)
self.script_path = os.path.abspath(script_path)
self.script_folder = os.path.dirname(self.script_path)
self.name = os.path.splitext(basename)[0]
# The master queue contains all messages that comprise the report.
# If the user chooses to share a saved version of the report,
# we serialize the contents of the master queue.
self._master_queue = ReportQueue()
# The browser queue contains messages that haven't yet been
# delivered to the browser. Periodically, the server flushes
# this queue and delivers its contents to the browser.
self._browser_queue = ReportQueue()
self.report_id = None
self.generate_new_id()
self.command_line = command_line
def get_debug(self):
return {"master queue": self._master_queue.get_debug()}
def enqueue(self, msg):
self._master_queue.enqueue(msg)
self._browser_queue.enqueue(msg)
def clear(self):
# Master_queue retains its initial message; browser_queue is
# completely cleared.
initial_msg = self._master_queue.get_initial_msg()
self._master_queue.clear()
if initial_msg:
self._master_queue.enqueue(initial_msg)
self._browser_queue.clear()
def flush_browser_queue(self):
"""Clears our browser queue and returns the messages it contained.
The Server calls this periodically to deliver new messages
to the browser connected to this report.
This doesn't affect the master_queue.
Returns
-------
list[ForwardMsg]
The messages that were removed from the queue and should
be delivered to the browser.
"""
return self._browser_queue.flush()
def generate_new_id(self):
"""Randomly generate an ID representing this report's execution."""
# Convert to str for Python2
self.report_id = str(
base58.b58encode(uuid.uuid4().bytes).decode("utf-8"))
def serialize_running_report_to_files(self):
"""Return a running report as an easily-serializable list of tuples.
Returns
-------
list of tuples
See `CloudStorage.save_report_files()` for schema. But as to the
output of this method, it's just a manifest pointing to the Server
so browsers who go to the shareable report URL can connect to it
live.
"""
LOGGER.debug("Serializing running report")
manifest = self._build_manifest(
status=StaticManifest.RUNNING,
external_server_ip=net_util.get_external_ip(),
internal_server_ip=net_util.get_internal_ip(),
)
return [("reports/%s/manifest.pb" % self.report_id,
manifest.SerializeToString())]
def serialize_final_report_to_files(self):
"""Return the report as an easily-serializable list of tuples.
Returns
-------
list of tuples
See `CloudStorage.save_report_files()` for schema. But as to the
output of this method, it's (1) a simple manifest and (2) a bunch
of serialized ForwardMsgs.
"""
LOGGER.debug("Serializing final report")
messages = [
copy.deepcopy(msg) for msg in self._master_queue
if _should_save_report_msg(msg)
]
first_delta_index = 0
num_deltas = 0
for idx in range(len(messages)):
if messages[idx].HasField("delta"):
if num_deltas == 0:
first_delta_index = idx
num_deltas += 1
manifest = self._build_manifest(status=StaticManifest.DONE,
num_messages=len(messages))
# Build a list of message tuples: (message_location, serialized_message)
message_tuples = [(
"reports/%(id)s/%(idx)s.pb" % {
"id": self.report_id,
"idx": msg_idx
},
msg.SerializeToString(),
) for msg_idx, msg in enumerate(messages)]
manifest_tuples = [(
"reports/%(id)s/manifest.pb" % {
"id": self.report_id
},
manifest.SerializeToString(),
)]
# Manifest must be at the end, so clients don't connect and read the
# manifest while the deltas haven't been saved yet.
return message_tuples + manifest_tuples
def _build_manifest(
self,
status,
num_messages=None,
external_server_ip=None,
internal_server_ip=None,
):
"""Build a manifest dict for this report.
Parameters
----------
status : StaticManifest.ServerStatus
The report status. If the script is still executing, then the
status should be RUNNING. Otherwise, DONE.
num_messages : int or None
Set only when status is DONE. The number of ForwardMsgs that this report
is made of.
external_server_ip : str or None
Only when status is RUNNING. The IP of the Server's websocket.
internal_server_ip : str or None
Only when status is RUNNING. The IP of the Server's websocket.
Returns
-------
StaticManifest
A StaticManifest protobuf message
"""
manifest = StaticManifest()
manifest.name = self.name
manifest.server_status = status
if status == StaticManifest.RUNNING:
manifest.external_server_ip = external_server_ip
manifest.internal_server_ip = internal_server_ip
manifest.configured_server_address = config.get_option(
"browser.serverAddress")
# Don't use _get_browser_address_bar_port() here, since we want the
# websocket port, not the web server port. (These are the same in
# prod, but different in dev)
manifest.server_port = config.get_option("browser.serverPort")
manifest.server_base_path = config.get_option("server.baseUrlPath")
else:
manifest.num_messages = num_messages
return manifest
class Report(object):
"""
Contains parameters related to running a report, and also houses
the two ReportQueues (master_queue and browser_queue) that are used
to deliver messages to a connected browser, and to serialize the
running report.
"""
@classmethod
def get_url(cls, host_ip):
"""Get the URL for any app served at the given host_ip.
Parameters
----------
host_ip : str
The IP address of the machine that is running the Streamlit Server.
Returns
-------
str
The URL.
"""
port = _get_browser_address_bar_port()
return "http://%(host_ip)s:%(port)s" % {
"host_ip": host_ip,
"port": port
}
def __init__(self, script_path, argv):
"""Constructor.
Parameters
----------
script_path : str
Path of the Python file from which this app is generated.
argv : list of str
Command-line arguments to run the script with.
"""
basename = os.path.basename(script_path)
self.script_path = os.path.abspath(script_path)
self.script_folder = os.path.dirname(self.script_path)
self.argv = argv
self.name = os.path.splitext(basename)[0]
# The master queue contains all messages that comprise the report.
# If the user chooses to share a saved version of the report,
# we serialize the contents of the master queue.
self._master_queue = ReportQueue()
# The browser queue contains messages that haven't yet been
# delivered to the browser. Periodically, the server flushes
# this queue and delivers its contents to the browser.
self._browser_queue = ReportQueue()
self.report_id = None
self.generate_new_id()
def get_debug(self):
return {"master queue": self._master_queue.get_debug()}
def parse_argv_from_command_line(self, cmd_line_str):
"""Parses an argv dict for this script from a command line string.
Parameters
----------
cmd_line_str : str
The string to parse.
Returns
-------
dict
An argv dict, suitable for executing this Report with.
"""
import shlex
cmd_line_list = shlex.split(cmd_line_str)
new_script_path = os.path.abspath(cmd_line_list[0])
if new_script_path != self.script_path:
raise ValueError("Cannot change script from %s to %s" %
(self.script_path, cmd_line_list[0]))
self.argv = cmd_line_list
def enqueue(self, msg):
self._master_queue.enqueue(msg)
self._browser_queue.enqueue(msg)
def clear(self):
# Master_queue retains its initial message; browser_queue is
# completely cleared.
initial_msg = self._master_queue.get_initial_msg()
self._master_queue.clear()
if initial_msg:
self._master_queue.enqueue(initial_msg)
self._browser_queue.clear()
def flush_browser_queue(self):
"""Clears our browser queue and returns the messages it contained.
The Server calls this periodically to deliver new messages
to the browser connected to this report.
This doesn't affect the master_queue.
Returns
-------
list[ForwardMsg]
The messages that were removed from the queue and should
be delivered to the browser.
"""
return self._browser_queue.flush()
def generate_new_id(self):
"""Randomly generate an ID representing this report's execution."""
# Convert to str for Python2
self.report_id = str(
base58.b58encode(uuid.uuid4().bytes).decode("utf-8"))
def serialize_running_report_to_files(self):
"""Return a running report as an easily-serializable list of tuples.
Returns
-------
list of tuples
See `CloudStorage.save_report_files()` for schema. But as to the
output of this method, it's just a manifest pointing to the Server
so browsers who go to the shareable report URL can connect to it
live.
"""
LOGGER.debug("Serializing running report")
manifest = self._build_manifest(
status="running",
external_server_ip=util.get_external_ip(),
internal_server_ip=util.get_internal_ip(),
)
manifest_json = json.dumps(manifest).encode("utf-8")
return [("reports/%s/manifest.json" % self.report_id, manifest_json)]
def serialize_final_report_to_files(self):
"""Return the report as an easily-serializable list of tuples.
Returns
-------
list of tuples
See `CloudStorage.save_report_files()` for schema. But as to the
output of this method, it's (1) a simple manifest and (2) a bunch
of serialized ForwardMsgs.
"""
LOGGER.debug("Serializing final report")
messages = [
copy.deepcopy(msg) for msg in self._master_queue
if _should_save_report_msg(msg)
]
first_delta_index = 0
num_deltas = 0
for idx in range(len(messages)):
if messages[idx].HasField("delta"):
messages[idx].metadata.delta_id = num_deltas
if num_deltas == 0:
first_delta_index = idx
num_deltas += 1
manifest = self._build_manifest(
status="done",
num_messages=len(messages),
first_delta_index=first_delta_index,
num_deltas=num_deltas,
)
manifest_json = json.dumps(manifest).encode("utf-8")
# Build a list of message tuples: (message_location, serialized_message)
message_tuples = [(
"reports/%(id)s/%(idx)s.pb" % {
"id": self.report_id,
"idx": msg_idx
},
msg.SerializeToString(),
) for msg_idx, msg in enumerate(messages)]
manifest_tuples = [("reports/%(id)s/manifest.json" % {
"id": self.report_id
}, manifest_json)]
# Manifest must be at the end, so clients don't connect and read the
# manifest while the deltas haven't been saved yet.
return message_tuples + manifest_tuples
def _build_manifest(
self,
status,
num_messages=None,
first_delta_index=None,
num_deltas=None,
external_server_ip=None,
internal_server_ip=None,
):
"""Build a manifest dict for this report.
Parameters
----------
status : 'done' or 'running'
The report status. If the script is still executing, then the
status should be RUNNING. Otherwise, DONE.
num_messages : int or None
Set only when status is DONE. The number of ForwardMsgs that this report
is made of.
first_delta_index : int or None
Set only when status is DONE. The index of our first Delta message
num_deltas : int or None
Set only when status is DONE. The number of Delta messages in the report
external_server_ip : str or None
Only when status is RUNNING. The IP of the Server's websocket.
internal_server_ip : str or None
Only when status is RUNNING. The IP of the Server's websocket.
Returns
-------
dict
The actual manifest. Schema:
- localId: str,
- numMessages: int or None,
- firstDeltaIndex: int or None,
- numDeltas: int or None,
- serverStatus: 'running' or 'done',
- externalServerIP: str or None,
- internalServerIP: str or None,
- serverPort: int
"""
if status == "running":
configured_server_address = config.get_option(
"browser.serverAddress")
else:
configured_server_address = None
return dict(
name=self.name,
numMessages=num_messages,
firstDeltaIndex=first_delta_index,
numDeltas=num_deltas,
serverStatus=status,
configuredServerAddress=configured_server_address,
externalServerIP=external_server_ip,
internalServerIP=internal_server_ip,
# Don't use _get_browser_address_bar_port() here, since we want the
# websocket port, not the web server port. (These are the same in
# prod, but different in dev)
serverPort=config.get_option("browser.serverPort"),
)
