def setUp(self, override_root=True):
self.report_queue = ReportQueue()
if override_root:
dg = self.new_delta_generator()
setattr(threading.current_thread(), REPORT_CONTEXT_ATTR_NAME,
MockReportContext(dg))
def __init__(self, script_name):
"""Initializes the ScriptRunner for the given script_name"""
# DeltaGenerator deltas will be enqueued into self.report_queue.
self.report_queue = ReportQueue()
def enqueue_fn(msg):
self.report_queue.enqueue(msg)
self.maybe_handle_execution_control_request()
self.script_request_queue = ScriptRequestQueue()
script_path = os.path.join(os.path.dirname(__file__), "test_data",
script_name)
super(TestScriptRunner, self).__init__(
session_id="test session id",
report=Report(script_path, "test command line"),
enqueue_forward_msg=enqueue_fn,
widget_states=WidgetStates(),
request_queue=self.script_request_queue,
)
# Accumulates uncaught exceptions thrown by our run thread.
self.script_thread_exceptions = []
# Accumulates all ScriptRunnerEvents emitted by us.
self.events = []
def record_event(event, **kwargs):
self.events.append(event)
self.on_event.connect(record_event, weak=False)
def __init__(self, script_name):
"""Initializes the ScriptRunner for the given script_name"""
# DeltaGenerator deltas will be enqueued into self.report_queue.
self.report_queue = ReportQueue()
def enqueue_fn(msg):
self.report_queue.enqueue(msg)
self.maybe_handle_execution_control_request()
return True
self.main_dg = DeltaGenerator(enqueue_fn, container=BlockPath.MAIN)
self.sidebar_dg = DeltaGenerator(enqueue_fn,
container=BlockPath.SIDEBAR)
self.script_request_queue = ScriptRequestQueue()
script_path = os.path.join(os.path.dirname(__file__), 'test_data',
script_name)
super(TestScriptRunner,
self).__init__(report=Report(script_path, []),
main_dg=self.main_dg,
sidebar_dg=self.sidebar_dg,
widget_states=WidgetStates(),
request_queue=self.script_request_queue)
# Accumulates uncaught exceptions thrown by our run thread.
self.script_thread_exceptions = []
# Accumulates all ScriptRunnerEvents emitted by us.
self.events = []
def record_event(event, **kwargs):
self.events.append(event)
self.on_event.connect(record_event, weak=False)
class DeltaGeneratorTestCase(unittest.TestCase):
def setUp(self, override_root=True):
self.report_queue = ReportQueue()
if override_root:
dg = self.new_delta_generator()
setattr(threading.current_thread(), REPORT_CONTEXT_ATTR_NAME,
MockReportContext(dg))
def tearDown(self):
self.report_queue._clear()
if hasattr(threading.current_thread(), REPORT_CONTEXT_ATTR_NAME):
delattr(threading.current_thread(), REPORT_CONTEXT_ATTR_NAME)
def new_delta_generator(self, *args, **kwargs):
def enqueue_fn(msg):
self.report_queue.enqueue(msg)
return True
if len(args) > 0:
enqueue = args[0]
args = args[1:]
elif 'enqueue' in kwargs:
enqueue = kwargs.pop('enqueue')
else:
enqueue = enqueue_fn
return DeltaGenerator(enqueue, *args, **kwargs)
def get_message_from_queue(self, index=-1):
return self.report_queue._queue[index]
def get_delta_from_queue(self, index=-1):
return self.report_queue._queue[index].delta
def __init__(self, script_path, command_line):
"""Constructor.
Parameters
----------
script_path : str
Path of the Python file from which this app is generated.
command_line : string
Command line as input by the user
"""
basename = os.path.basename(script_path)
self.script_path = os.path.abspath(script_path)
self.script_folder = os.path.dirname(self.script_path)
self.name = os.path.splitext(basename)[0]
# The master queue contains all messages that comprise the report.
# If the user chooses to share a saved version of the report,
# we serialize the contents of the master queue.
self._master_queue = ReportQueue()
# The browser queue contains messages that haven't yet been
# delivered to the browser. Periodically, the server flushes
# this queue and delivers its contents to the browser.
self._browser_queue = ReportQueue()
self.report_id = None
self.generate_new_id()
self.command_line = command_line
def setUp(self):
self._report_queue = ReportQueue()
def enqueue(msg):
self._report_queue.enqueue(msg)
return True
self._dg = DeltaGenerator(enqueue)
class DeltaGeneratorTestCase(unittest.TestCase):
def setUp(self, override_root=True):
self.report_queue = ReportQueue()
if override_root:
main_dg = self.new_delta_generator()
sidebar_dg = self.new_delta_generator(container=BlockPath.SIDEBAR)
setattr(
threading.current_thread(),
REPORT_CONTEXT_ATTR_NAME,
ReportContext(
main_dg=main_dg,
sidebar_dg=sidebar_dg,
widgets=Widgets(),
widget_ids_this_run=_WidgetIDSet(),
uploaded_file_mgr=None,
),
)
def tearDown(self):
self.report_queue._clear()
if hasattr(threading.current_thread(), REPORT_CONTEXT_ATTR_NAME):
delattr(threading.current_thread(), REPORT_CONTEXT_ATTR_NAME)
def new_delta_generator(self, *args, **kwargs):
def enqueue_fn(msg):
self.report_queue.enqueue(msg)
return True
if len(args) > 0:
enqueue = args[0]
args = args[1:]
elif "enqueue" in kwargs:
enqueue = kwargs.pop("enqueue")
else:
enqueue = enqueue_fn
return DeltaGenerator(enqueue, *args, **kwargs)
def get_message_from_queue(self, index=-1):
"""Get a ForwardMsg proto from the queue, by index.
Returns
-------
ForwardMsg
"""
return self.report_queue._queue[index]
def get_delta_from_queue(self, index=-1):
"""Get a Delta proto from the queue, by index.
Returns
-------
Delta
"""
return self.report_queue._queue[index].delta
def setUp(self, override_root=True):
self.report_queue = ReportQueue()
if override_root:
main_dg = self.new_delta_generator()
sidebar_dg = self.new_delta_generator()
setattr(threading.current_thread(),
REPORT_CONTEXT_ATTR_NAME,
ReportContext(main_dg=main_dg, sidebar_dg=sidebar_dg,
widgets=Widgets()))
class DeltaGeneratorTestCase(unittest.TestCase):
def setUp(self, override_root=True):
self.report_queue = ReportQueue()
self.override_root = override_root
self.orig_report_ctx = None
if self.override_root:
self.orig_report_ctx = get_report_ctx()
add_report_ctx(
threading.current_thread(),
ReportContext(
session_id="test session id",
enqueue=self.report_queue.enqueue,
widgets=Widgets(),
widget_ids_this_run=_WidgetIDSet(),
uploaded_file_mgr=UploadedFileManager(),
),
)
def tearDown(self):
self.clear_queue()
if self.override_root:
add_report_ctx(threading.current_thread(), self.orig_report_ctx)
def get_message_from_queue(self, index=-1):
"""Get a ForwardMsg proto from the queue, by index.
Returns
-------
ForwardMsg
"""
return self.report_queue._queue[index]
def get_delta_from_queue(self, index=-1):
"""Get a Delta proto from the queue, by index.
Returns
-------
Delta
"""
deltas = self.get_all_deltas_from_queue()
return deltas[index]
def get_all_deltas_from_queue(self):
"""Return all the delta messages in our ReportQueue"""
return [
msg.delta for msg in self.report_queue._queue
if msg.HasField("delta")
]
def clear_queue(self):
self.report_queue._clear()
def setUp(self, override_root=True):
self.report_queue = ReportQueue()
self.override_root = override_root
self.orig_report_ctx = None
if self.override_root:
self.orig_report_ctx = get_report_ctx()
add_report_ctx(
threading.current_thread(),
ReportContext(
enqueue=self.report_queue.enqueue,
widgets=Widgets(),
widget_ids_this_run=_WidgetIDSet(),
uploaded_file_mgr=UploadedFileManager(),
),
)
def setUp(self, override_root=True):
self.report_queue = ReportQueue()
if override_root:
main_dg = self.new_delta_generator()
sidebar_dg = self.new_delta_generator(container=BlockPath.SIDEBAR)
setattr(
threading.current_thread(),
REPORT_CONTEXT_ATTR_NAME,
ReportContext(
main_dg=main_dg,
sidebar_dg=sidebar_dg,
widgets=Widgets(),
widget_ids_this_run=_WidgetIDSet(),
),
)
def test_replace_element(self):
rq = ReportQueue()
self.assertTrue(rq.is_empty())
rq.enqueue(INIT_MSG)
TEXT_DELTA_MSG1.delta.id = 0
rq.enqueue(TEXT_DELTA_MSG1)
TEXT_DELTA_MSG2.delta.id = 0
rq.enqueue(TEXT_DELTA_MSG2)
queue = rq.flush()
self.assertEqual(len(queue), 2)
self.assertTrue(queue[0].initialize.config.sharing_enabled)
self.assertEqual(queue[1].delta.id, 0)
self.assertEqual(queue[1].delta.new_element.text.body, 'text2')
def test_simple_enqueue(self):
rq = ReportQueue()
self.assertTrue(rq.is_empty())
rq.enqueue(INIT_MSG)
self.assertFalse(rq.is_empty())
queue = rq.flush()
self.assertTrue(rq.is_empty())
self.assertEqual(len(queue), 1)
self.assertTrue(queue[0].initialize.config.sharing_enabled)
def test_enqueue_three(self):
rq = ReportQueue()
self.assertTrue(rq.is_empty())
rq.enqueue(INIT_MSG)
TEXT_DELTA_MSG1.metadata.delta_id = 0
rq.enqueue(TEXT_DELTA_MSG1)
TEXT_DELTA_MSG2.metadata.delta_id = 1
rq.enqueue(TEXT_DELTA_MSG2)
queue = rq.flush()
self.assertEqual(len(queue), 3)
self.assertTrue(queue[0].initialize.config.sharing_enabled)
self.assertEqual(queue[1].metadata.delta_id, 0)
self.assertEqual(queue[1].delta.new_element.text.body, "text1")
self.assertEqual(queue[2].metadata.delta_id, 1)
self.assertEqual(queue[2].delta.new_element.text.body, "text2")
def test_multiple_containers(self):
"""Deltas should only be coalesced if they're in the same container"""
rq = ReportQueue()
self.assertTrue(rq.is_empty())
rq.enqueue(INIT_MSG)
def enqueue_deltas(container, path):
# We deep-copy the protos because we mutate each one
# multiple times.
msg = copy.deepcopy(TEXT_DELTA_MSG1)
msg.delta.id = 0
msg.delta.parent_block.container = container
msg.delta.parent_block.path[:] = path
rq.enqueue(msg)
msg = copy.deepcopy(DF_DELTA_MSG)
msg.delta.id = 1
msg.delta.parent_block.container = container
msg.delta.parent_block.path[:] = path
rq.enqueue(msg)
msg = copy.deepcopy(ADD_ROWS_MSG)
msg.delta.id = 1
msg.delta.parent_block.container = container
msg.delta.parent_block.path[:] = path
rq.enqueue(msg)
enqueue_deltas(BlockPath.MAIN, [])
enqueue_deltas(BlockPath.SIDEBAR, [0, 0, 1])
def assert_deltas(container, path, idx):
self.assertEqual(0, queue[idx].delta.id)
self.assertEqual(container,
queue[idx].delta.parent_block.container)
self.assertEqual(path, queue[idx].delta.parent_block.path)
self.assertEqual('text1', queue[idx].delta.new_element.text.body)
self.assertEqual(1, queue[idx + 1].delta.id)
self.assertEqual(container,
queue[idx + 1].delta.parent_block.container)
self.assertEqual(path, queue[idx + 1].delta.parent_block.path)
col0 = queue[idx + 1] \
.delta.new_element.data_frame.data.cols[0].int64s.data
col1 = queue[idx + 1] \
.delta.new_element.data_frame.data.cols[1].int64s.data
self.assertEqual([0, 1, 2, 3, 4, 5], col0)
self.assertEqual([10, 11, 12, 13, 14, 15], col1)
queue = rq.flush()
self.assertEqual(5, len(queue))
self.assertTrue(queue[0].initialize.config.sharing_enabled)
assert_deltas(BlockPath.MAIN, [], 1)
assert_deltas(BlockPath.SIDEBAR, [0, 0, 1], 3)
class TestScriptRunner(ScriptRunner):
"""Subclasses ScriptRunner to provide some testing features."""
def __init__(self, script_name):
"""Initializes the ScriptRunner for the given script_name"""
# DeltaGenerator deltas will be enqueued into self.report_queue.
self.report_queue = ReportQueue()
def enqueue_fn(msg):
self.report_queue.enqueue(msg)
self.maybe_handle_execution_control_request()
return True
self.main_dg = DeltaGenerator(enqueue=enqueue_fn, container=BlockPath.MAIN)
self.sidebar_dg = DeltaGenerator(
enqueue=enqueue_fn, container=BlockPath.SIDEBAR
)
self.script_request_queue = ScriptRequestQueue()
script_path = os.path.join(os.path.dirname(__file__), "test_data", script_name)
super(TestScriptRunner, self).__init__(
report=Report(script_path, "test command line"),
main_dg=self.main_dg,
sidebar_dg=self.sidebar_dg,
widget_states=WidgetStates(),
request_queue=self.script_request_queue,
)
# Accumulates uncaught exceptions thrown by our run thread.
self.script_thread_exceptions = []
# Accumulates all ScriptRunnerEvents emitted by us.
self.events = []
def record_event(event, **kwargs):
self.events.append(event)
self.on_event.connect(record_event, weak=False)
@property
def widget_states(self):
"""
Returns
-------
WidgetStates
A WidgetStates protobuf object
"""
return self._widgets.get_state()
def enqueue_rerun(self, argv=None, widget_state=None):
self.script_request_queue.enqueue(
ScriptRequest.RERUN, RerunData(widget_state=widget_state)
)
def enqueue_stop(self):
self.script_request_queue.enqueue(ScriptRequest.STOP)
def enqueue_shutdown(self):
self.script_request_queue.enqueue(ScriptRequest.SHUTDOWN)
def _process_request_queue(self):
try:
super(TestScriptRunner, self)._process_request_queue()
except BaseException as e:
self.script_thread_exceptions.append(e)
def join(self):
"""Joins the run thread, if it was started"""
if self._script_thread is not None:
self._script_thread.join()
def deltas(self):
"""Returns the delta messages in our ReportQueue"""
return [msg.delta for msg in self.report_queue._queue if msg.HasField("delta")]
def get_widget_id(self, widget_type, label):
"""Returns the id of the widget with the specified type and label"""
for delta in self.deltas():
new_element = getattr(delta, "new_element", None)
widget = getattr(new_element, widget_type, None)
widget_label = getattr(widget, "label", None)
if widget_label == label:
return widget.id
return None
def test_add_rows_rerun(self):
rq = ReportQueue()
self.assertTrue(rq.is_empty())
rq.enqueue(INIT_MSG)
# Simulate rerun
for i in range(2):
TEXT_DELTA_MSG1.delta.id = 0
rq.enqueue(TEXT_DELTA_MSG1)
DF_DELTA_MSG.delta.id = 1
rq.enqueue(DF_DELTA_MSG)
ADD_ROWS_MSG.delta.id = 1
rq.enqueue(ADD_ROWS_MSG)
queue = rq.flush()
self.assertEqual(len(queue), 3)
self.assertTrue(queue[0].initialize.config.sharing_enabled)
self.assertEqual(queue[1].delta.id, 0)
self.assertEqual(queue[1].delta.new_element.text.body, 'text1')
self.assertEqual(queue[2].delta.id, 1)
col0 = queue[2].delta.new_element.data_frame.data.cols[0].int64s.data
col1 = queue[2].delta.new_element.data_frame.data.cols[1].int64s.data
self.assertEqual(col0, [0, 1, 2, 3, 4, 5])
self.assertEqual(col1, [10, 11, 12, 13, 14, 15])
class Report(object):
"""
Contains parameters related to running a report, and also houses
the two ReportQueues (master_queue and browser_queue) that are used
to deliver messages to a connected browser, and to serialize the
running report.
"""
@classmethod
def get_url(cls, host_ip):
"""Get the URL for any app served at the given host_ip.
Parameters
----------
host_ip : str
The IP address of the machine that is running the Streamlit Server.
Returns
-------
str
The URL.
"""
port = _get_browser_address_bar_port()
return "http://%(host_ip)s:%(port)s" % {
"host_ip": host_ip,
"port": port
}
def __init__(self, script_path, argv):
"""Constructor.
Parameters
----------
script_path : str
Path of the Python file from which this app is generated.
argv : list of str
Command-line arguments to run the script with.
"""
basename = os.path.basename(script_path)
self.script_path = os.path.abspath(script_path)
self.script_folder = os.path.dirname(self.script_path)
self.argv = argv
self.name = os.path.splitext(basename)[0]
# The master queue contains all messages that comprise the report.
# If the user chooses to share a saved version of the report,
# we serialize the contents of the master queue.
self._master_queue = ReportQueue()
# The browser queue contains messages that haven't yet been
# delivered to the browser. Periodically, the server flushes
# this queue and delivers its contents to the browser.
self._browser_queue = ReportQueue()
self.report_id = None
self.generate_new_id()
def get_debug(self):
return {"master queue": self._master_queue.get_debug()}
def parse_argv_from_command_line(self, cmd_line_str):
"""Parses an argv dict for this script from a command line string.
Parameters
----------
cmd_line_str : str
The string to parse.
Returns
-------
dict
An argv dict, suitable for executing this Report with.
"""
import shlex
cmd_line_list = shlex.split(cmd_line_str)
new_script_path = os.path.abspath(cmd_line_list[0])
if new_script_path != self.script_path:
raise ValueError("Cannot change script from %s to %s" %
(self.script_path, cmd_line_list[0]))
self.argv = cmd_line_list
def enqueue(self, msg):
self._master_queue.enqueue(msg)
self._browser_queue.enqueue(msg)
def clear(self):
# Master_queue retains its initial message; browser_queue is
# completely cleared.
initial_msg = self._master_queue.get_initial_msg()
self._master_queue.clear()
if initial_msg:
self._master_queue.enqueue(initial_msg)
self._browser_queue.clear()
def flush_browser_queue(self):
"""Clears our browser queue and returns the messages it contained.
The Server calls this periodically to deliver new messages
to the browser connected to this report.
This doesn't affect the master_queue.
Returns
-------
list[ForwardMsg]
The messages that were removed from the queue and should
be delivered to the browser.
"""
return self._browser_queue.flush()
def generate_new_id(self):
"""Randomly generate an ID representing this report's execution."""
# Convert to str for Python2
self.report_id = str(
base58.b58encode(uuid.uuid4().bytes).decode("utf-8"))
def serialize_running_report_to_files(self):
"""Return a running report as an easily-serializable list of tuples.
Returns
-------
list of tuples
See `CloudStorage.save_report_files()` for schema. But as to the
output of this method, it's just a manifest pointing to the Server
so browsers who go to the shareable report URL can connect to it
live.
"""
LOGGER.debug("Serializing running report")
manifest = self._build_manifest(
status="running",
external_server_ip=util.get_external_ip(),
internal_server_ip=util.get_internal_ip(),
)
manifest_json = json.dumps(manifest).encode("utf-8")
return [("reports/%s/manifest.json" % self.report_id, manifest_json)]
def serialize_final_report_to_files(self):
"""Return the report as an easily-serializable list of tuples.
Returns
-------
list of tuples
See `CloudStorage.save_report_files()` for schema. But as to the
output of this method, it's (1) a simple manifest and (2) a bunch
of serialized ForwardMsgs.
"""
LOGGER.debug("Serializing final report")
messages = [
copy.deepcopy(msg) for msg in self._master_queue
if _should_save_report_msg(msg)
]
first_delta_index = 0
num_deltas = 0
for idx in range(len(messages)):
if messages[idx].HasField("delta"):
messages[idx].metadata.delta_id = num_deltas
if num_deltas == 0:
first_delta_index = idx
num_deltas += 1
manifest = self._build_manifest(
status="done",
num_messages=len(messages),
first_delta_index=first_delta_index,
num_deltas=num_deltas,
)
manifest_json = json.dumps(manifest).encode("utf-8")
# Build a list of message tuples: (message_location, serialized_message)
message_tuples = [(
"reports/%(id)s/%(idx)s.pb" % {
"id": self.report_id,
"idx": msg_idx
},
msg.SerializeToString(),
) for msg_idx, msg in enumerate(messages)]
manifest_tuples = [("reports/%(id)s/manifest.json" % {
"id": self.report_id
}, manifest_json)]
# Manifest must be at the end, so clients don't connect and read the
# manifest while the deltas haven't been saved yet.
return message_tuples + manifest_tuples
def _build_manifest(
self,
status,
num_messages=None,
first_delta_index=None,
num_deltas=None,
external_server_ip=None,
internal_server_ip=None,
):
"""Build a manifest dict for this report.
Parameters
----------
status : 'done' or 'running'
The report status. If the script is still executing, then the
status should be RUNNING. Otherwise, DONE.
num_messages : int or None
Set only when status is DONE. The number of ForwardMsgs that this report
is made of.
first_delta_index : int or None
Set only when status is DONE. The index of our first Delta message
num_deltas : int or None
Set only when status is DONE. The number of Delta messages in the report
external_server_ip : str or None
Only when status is RUNNING. The IP of the Server's websocket.
internal_server_ip : str or None
Only when status is RUNNING. The IP of the Server's websocket.
Returns
-------
dict
The actual manifest. Schema:
- localId: str,
- numMessages: int or None,
- firstDeltaIndex: int or None,
- numDeltas: int or None,
- serverStatus: 'running' or 'done',
- externalServerIP: str or None,
- internalServerIP: str or None,
- serverPort: int
"""
if status == "running":
configured_server_address = config.get_option(
"browser.serverAddress")
else:
configured_server_address = None
return dict(
name=self.name,
numMessages=num_messages,
firstDeltaIndex=first_delta_index,
numDeltas=num_deltas,
serverStatus=status,
configuredServerAddress=configured_server_address,
externalServerIP=external_server_ip,
internalServerIP=internal_server_ip,
# Don't use _get_browser_address_bar_port() here, since we want the
# websocket port, not the web server port. (These are the same in
# prod, but different in dev)
serverPort=config.get_option("browser.serverPort"),
)
class Report(object):
"""
Contains parameters related to running a report, and also houses
the two ReportQueues (master_queue and browser_queue) that are used
to deliver messages to a connected browser, and to serialize the
running report.
"""
@classmethod
def get_url(cls, host_ip):
"""Get the URL for any app served at the given host_ip.
Parameters
----------
host_ip : str
The IP address of the machine that is running the Streamlit Server.
Returns
-------
str
The URL.
"""
port = _get_browser_address_bar_port()
base_path = config.get_option("server.baseUrlPath").strip("/")
if base_path:
base_path = "/" + base_path
return "http://%(host_ip)s:%(port)s%(base_path)s" % {
"host_ip": host_ip.strip("/"),
"port": port,
"base_path": base_path,
}
def __init__(self, script_path, command_line):
"""Constructor.
Parameters
----------
script_path : str
Path of the Python file from which this app is generated.
command_line : string
Command line as input by the user
"""
basename = os.path.basename(script_path)
self.script_path = os.path.abspath(script_path)
self.script_folder = os.path.dirname(self.script_path)
self.name = os.path.splitext(basename)[0]
# The master queue contains all messages that comprise the report.
# If the user chooses to share a saved version of the report,
# we serialize the contents of the master queue.
self._master_queue = ReportQueue()
# The browser queue contains messages that haven't yet been
# delivered to the browser. Periodically, the server flushes
# this queue and delivers its contents to the browser.
self._browser_queue = ReportQueue()
self.report_id = None
self.generate_new_id()
self.command_line = command_line
def get_debug(self):
return {"master queue": self._master_queue.get_debug()}
def enqueue(self, msg):
self._master_queue.enqueue(msg)
self._browser_queue.enqueue(msg)
def clear(self):
# Master_queue retains its initial message; browser_queue is
# completely cleared.
initial_msg = self._master_queue.get_initial_msg()
self._master_queue.clear()
if initial_msg:
self._master_queue.enqueue(initial_msg)
self._browser_queue.clear()
def flush_browser_queue(self):
"""Clears our browser queue and returns the messages it contained.
The Server calls this periodically to deliver new messages
to the browser connected to this report.
This doesn't affect the master_queue.
Returns
-------
list[ForwardMsg]
The messages that were removed from the queue and should
be delivered to the browser.
"""
return self._browser_queue.flush()
def generate_new_id(self):
"""Randomly generate an ID representing this report's execution."""
# Convert to str for Python2
self.report_id = str(
base58.b58encode(uuid.uuid4().bytes).decode("utf-8"))
def serialize_running_report_to_files(self):
"""Return a running report as an easily-serializable list of tuples.
Returns
-------
list of tuples
See `CloudStorage.save_report_files()` for schema. But as to the
output of this method, it's just a manifest pointing to the Server
so browsers who go to the shareable report URL can connect to it
live.
"""
LOGGER.debug("Serializing running report")
manifest = self._build_manifest(
status=StaticManifest.RUNNING,
external_server_ip=net_util.get_external_ip(),
internal_server_ip=net_util.get_internal_ip(),
)
return [("reports/%s/manifest.pb" % self.report_id,
manifest.SerializeToString())]
def serialize_final_report_to_files(self):
"""Return the report as an easily-serializable list of tuples.
Returns
-------
list of tuples
See `CloudStorage.save_report_files()` for schema. But as to the
output of this method, it's (1) a simple manifest and (2) a bunch
of serialized ForwardMsgs.
"""
LOGGER.debug("Serializing final report")
messages = [
copy.deepcopy(msg) for msg in self._master_queue
if _should_save_report_msg(msg)
]
first_delta_index = 0
num_deltas = 0
for idx in range(len(messages)):
if messages[idx].HasField("delta"):
if num_deltas == 0:
first_delta_index = idx
num_deltas += 1
manifest = self._build_manifest(status=StaticManifest.DONE,
num_messages=len(messages))
# Build a list of message tuples: (message_location, serialized_message)
message_tuples = [(
"reports/%(id)s/%(idx)s.pb" % {
"id": self.report_id,
"idx": msg_idx
},
msg.SerializeToString(),
) for msg_idx, msg in enumerate(messages)]
manifest_tuples = [(
"reports/%(id)s/manifest.pb" % {
"id": self.report_id
},
manifest.SerializeToString(),
)]
# Manifest must be at the end, so clients don't connect and read the
# manifest while the deltas haven't been saved yet.
return message_tuples + manifest_tuples
def _build_manifest(
self,
status,
num_messages=None,
external_server_ip=None,
internal_server_ip=None,
):
"""Build a manifest dict for this report.
Parameters
----------
status : StaticManifest.ServerStatus
The report status. If the script is still executing, then the
status should be RUNNING. Otherwise, DONE.
num_messages : int or None
Set only when status is DONE. The number of ForwardMsgs that this report
is made of.
external_server_ip : str or None
Only when status is RUNNING. The IP of the Server's websocket.
internal_server_ip : str or None
Only when status is RUNNING. The IP of the Server's websocket.
Returns
-------
StaticManifest
A StaticManifest protobuf message
"""
manifest = StaticManifest()
manifest.name = self.name
manifest.server_status = status
if status == StaticManifest.RUNNING:
manifest.external_server_ip = external_server_ip
manifest.internal_server_ip = internal_server_ip
manifest.configured_server_address = config.get_option(
"browser.serverAddress")
# Don't use _get_browser_address_bar_port() here, since we want the
# websocket port, not the web server port. (These are the same in
# prod, but different in dev)
manifest.server_port = config.get_option("browser.serverPort")
manifest.server_base_path = config.get_option("server.baseUrlPath")
else:
manifest.num_messages = num_messages
return manifest
def test_simple_add_rows(self):
rq = ReportQueue()
self.assertTrue(rq.is_empty())
rq.enqueue(INIT_MSG)
TEXT_DELTA_MSG1.metadata.delta_id = 0
rq.enqueue(TEXT_DELTA_MSG1)
DF_DELTA_MSG.metadata.delta_id = 1
rq.enqueue(DF_DELTA_MSG)
ADD_ROWS_MSG.metadata.delta_id = 1
rq.enqueue(ADD_ROWS_MSG)
queue = rq.flush()
self.assertEqual(len(queue), 3)
self.assertTrue(queue[0].initialize.config.sharing_enabled)
self.assertEqual(queue[1].metadata.delta_id, 0)
self.assertEqual(queue[1].delta.new_element.text.body, "text1")
self.assertEqual(queue[2].metadata.delta_id, 1)
col0 = queue[2].delta.new_element.data_frame.data.cols[0].int64s.data
col1 = queue[2].delta.new_element.data_frame.data.cols[1].int64s.data
self.assertEqual(col0, [0, 1, 2, 3, 4, 5])
self.assertEqual(col1, [10, 11, 12, 13, 14, 15])
class DataFrameStylingTest(unittest.TestCase):
"""Tests marshalling of pandas.Styler dataframe styling data
with both st.dataframe and st.table.
"""
def setUp(self):
self._report_queue = ReportQueue()
def enqueue(msg):
self._report_queue.enqueue(msg)
return True
self._dg = DeltaGenerator(enqueue)
@parameterized.expand([('dataframe', 'data_frame'), ('table', 'table')])
def test_unstyled_has_no_style(self, element, proto):
"""A DataFrame with an unmodified Styler should result in a protobuf
with no styling data
"""
df = pd.DataFrame({'A': [1, 2, 3, 4, 5]})
getattr(self._dg, element)(df.style)
proto_df = getattr(self._get_element(), proto)
rows, cols = df.shape
for row in range(rows):
for col in range(cols):
style = get_cell_style(proto_df, col, row)
self.assertEqual(style.display_value, '')
self.assertEqual(style.has_display_value, False)
self.assertEqual(len(style.css), 0)
@parameterized.expand([('dataframe', 'data_frame'), ('table', 'table')])
def test_format(self, element, proto):
"""Tests DataFrame.style.format()"""
values = [0.1, 0.2, 0.3352, np.nan]
display_values = ['10.00%', '20.00%', '33.52%', 'nan%']
df = pd.DataFrame({'A': values})
get_delta = getattr(self._dg, element)
get_delta(df.style.format('{:.2%}'))
proto_df = getattr(self._get_element(), proto)
self._assert_column_display_values(proto_df, 0, display_values)
@parameterized.expand([('dataframe', 'data_frame'), ('table', 'table')])
def test_css_styling(self, element, proto):
"""Tests DataFrame.style css styling"""
values = [-1, 1]
css_values = [{css_s('color', 'red')},
{
css_s('color', 'black'),
css_s('background-color', 'yellow')
}]
df = pd.DataFrame({'A': values})
get_delta = getattr(self._dg, element)
get_delta(
df.style.highlight_max(color='yellow').applymap(
lambda val: 'color: red' if val < 0 else 'color: black'))
proto_df = getattr(self._get_element(), proto)
self._assert_column_css_styles(proto_df, 0, css_values)
@parameterized.expand([('dataframe', 'data_frame'), ('table', 'table')])
def test_add_styled_rows(self, element, proto):
"""Add rows should preserve existing styles and append new styles"""
df1 = pd.DataFrame([5, 6])
df2 = pd.DataFrame([7, 8])
css_values = [
{css_s('color', 'red')},
{css_s('color', 'red')},
{css_s('color', 'black')},
{css_s('color', 'black')},
]
get_delta = getattr(self._dg, element)
x = get_delta(df1.style.applymap(lambda val: 'color: red'))
x.add_rows(df2.style.applymap(lambda val: 'color: black'))
proto_df = getattr(self._get_element(), proto)
self._assert_column_css_styles(proto_df, 0, css_values)
@parameterized.expand([('dataframe', 'data_frame'), ('table', 'table')])
def test_add_styled_rows_to_unstyled_rows(self, element, proto):
"""Adding styled rows to unstyled rows should work"""
df1 = pd.DataFrame([5, 6])
df2 = pd.DataFrame([7, 8])
css_values = [
set(),
set(),
{css_s('color', 'black')},
{css_s('color', 'black')},
]
x = getattr(self._dg, element)(df1)
x.add_rows(df2.style.applymap(lambda val: 'color: black'))
proto_df = getattr(self._get_element(), proto)
self._assert_column_css_styles(proto_df, 0, css_values)
@parameterized.expand([('dataframe', 'data_frame'), ('table', 'table')])
def test_add_unstyled_rows_to_styled_rows(self, element, proto):
"""Adding unstyled rows to styled rows should work"""
df1 = pd.DataFrame([5, 6])
df2 = pd.DataFrame([7, 8])
css_values = [
{css_s('color', 'black')},
{css_s('color', 'black')},
set(),
set(),
]
get_delta = getattr(self._dg, element)
x = get_delta(df1.style.applymap(lambda val: 'color: black'))
x.add_rows(df2)
proto_df = getattr(self._get_element(), proto)
self._assert_column_css_styles(proto_df, 0, css_values)
def _get_element(self):
"""Returns the most recent element in the DeltaGenerator queue"""
return self._report_queue._queue[-1].delta.new_element
def _assert_column_display_values(self, proto_df, col, display_values):
"""Asserts that cells in a column have the given display_values"""
for row in range(len(display_values)):
style = get_cell_style(proto_df, col, row)
self.assertEqual(style.has_display_value, display_values[row]
is not None)
self.assertEqual(style.display_value, display_values[row])
def _assert_column_css_styles(self, proto_df, col, expected_styles):
"""Asserts that cells in a column have the given expected_styles
expected_styles : List[Set[serialized_proto_str]]
"""
for row in range(len(expected_styles)):
proto_cell_style = get_cell_style(proto_df, col, row)
# throw the `repeated CSSStyle styles` into a set of serialized strings
cell_styles = set(
(proto_to_str(css) for css in proto_cell_style.css))
self.assertEqual(expected_styles[row], cell_styles)
