def test_rank_symbol_theme_layer(self, mock_send):
"""
:type mock_send:Mock
:type layer:RankSymbolThemeLayer
:param mock_send:
:return:
"""
layer = RankSymbolThemeLayer(name='test1',
data=[{
'name': '北京市',
'value': 23014.59
}, {
'name': '天津市',
'value': 16538.189999999999
}, {
'name': '河北省',
'value': 29806.110000000001
}],
address_key='name',
value_key='value')
layer._map = MapView()
comm = Comm()
comm.kernel = Kernel()
layer.comm = comm
layer.name = "test"
expected = {
'method': 'update',
'state': {
'name': 'test'
},
'buffer_paths': []
}
mock_send.assert_called_with(data=expected, buffers=[])
def test_heat_layer(self, mock_send):
"""
:type mock_send:Mock
:param mock_send:
:return:
"""
layer = HeatLayer(heat_points=[[
39.86369678193944, 116.25519414479928, 33.46073333534289
], [39.85730998494624, 116.21477193093531, 20.440150687928877]])
layer._map = MapView()
comm = Comm()
comm.kernel = Kernel()
layer.comm = comm
layer.radius = 30
expected = {
'method': 'update',
'state': {
'radius': 30
},
'buffer_paths': []
}
mock_send.assert_called_with(data=expected, buffers=[])
layer.blur = 50
expected_blur = {
'method': 'update',
'state': {
'blur': 50
},
'buffer_paths': []
}
mock_send.assert_called_with(data=expected_blur, buffers=[])
self.assertEqual(mock_send.call_count, 2)
def send(self, code, data): #{5}
comm = Comm(target_name=str(self.uid),
data={
"code": code,
"data": json_encoder.serialize_json(data)
})
comm.close()
def test_cloud_tile_layer(self, mock_send):
"""
:type mock_send:Mock
:param mock_send:
:return:
"""
layer = CloudTileLayer()
layer._map = MapView()
comm = Comm()
comm.kernel = Kernel()
layer.comm = comm
layer.map_name = 'quanguo'
expected = {
'method': 'update',
'state': {
'map_name': 'quanguo'
},
'buffer_paths': []
}
mock_send.assert_called_with(data=expected, buffers=[])
layer.type = 'web'
expectedVisibility = {
'method': 'update',
'state': {
'type': 'web'
},
'buffer_paths': []
}
mock_send.assert_called_with(data=expectedVisibility, buffers=[])
self.assertEqual(mock_send.call_count, 2)
def open(self):
"""Open a comm to the frontend if one isn't already open."""
if self.comm is None:
args = dict(target_name='ipython.widget', data=self.get_state())
if self._model_id is not None:
args['comm_id'] = self._model_id
self.comm = Comm(**args)
def __init__(self):
self._comm = Comm(target_name='@wwtelescope/jupyterlab:research', data={})
self._comm.on_msg(self._on_message_received)
self._comm.open()
self._send_msg(event='trigger') # get bidirectional updates flowing
BaseWWTWidget.__init__(self)
def locate_jpd_comm(da_id, app, app_path):
global local_comms
comm = local_comms.get(da_id, None )
if comm is None:
comm = Comm(target_name='jpd_%s' %da_id,
data={'jpd_type':'inform',
'da_id':da_id})
@comm.on_msg
def callback(msg, app=app, da_id=da_id, app_path=app_path):
# TODO see if app and da_id args needed for this closure
content = msg['content']
data = content['data']
stem = data['stem']
args = data['args']
#app_path = "app/endpoints/%s" % da_id
response, mimetype = app.process_view(stem, args, app_path)
comm.send({'jpd_type':'response',
'da_id':da_id,
'app':str(app),
'response':response,
'mimetype':mimetype})
local_comms[da_id] = comm
return comm
def make_comm() -> None:
global _JupyterComm
J_LOGGER.info("IPYTHON: Registering Comms")
comm_target_name = COMM_NAME
jupyter_comm = Comm(target_name=comm_target_name)
def _get_command(msg) -> Optional[str]:
return msg["content"]["data"].get("command", None)
@jupyter_comm.on_msg
def _recv(msg):
if _get_command(msg) == "merge_notebooks":
J_LOGGER.info("GOT UPDATE STATUS")
merge_notebooks(jupyter_comm, msg["content"]["data"])
return
J_LOGGER.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
J_LOGGER.info(msg)
J_LOGGER.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
# store comm for access in this thread later
_JupyterComm = jupyter_comm
J_LOGGER.info("==> Success")
return _JupyterComm
def buttonCallback(self, *args):
if len(args) > 0:
args[0].actionPerformed()
arguments = dict(target_name='beaker.tag.run')
comm = Comm(**arguments)
msg = {'runByTag': args[0].tag}
state = {'state': msg}
comm.send(data=state, buffers=[])
def __init__(self):
_maybe_perpetrate_mega_kernel_hack()
self._comm = Comm(target_name='@wwtelescope/jupyterlab:research',
data={})
self._comm.on_msg(self._on_comm_message_received)
self._comm.open()
super(WWTLabApplication, self).__init__()
def initialize_comms():
if in_jupyter():
for target in pyflyby_comm_targets:
_register_target(target)
from ipykernel.comm import Comm
comm = Comm(target_name=INIT_COMMS)
msg = {"type": INIT_COMMS}
logger.debug("Requesting frontend to (re-)initialize comms")
comm.send(msg)
def update_graph(self):
data = {
"updateGraph": True,
"lossData": self.loss_data,
"accuracyData": self.accuracy_data,
"epochNumber": self.epoch_number,
}
my_comm = Comm(target_name="epochData", data=data)
my_comm.send(data=data)
def display_statistics(self):
data = {
"runTime": self.total_runtime,
"totalLoss": (sum(self.total_losses) / float(len(self.total_losses))),
"totalAccuracy": (
sum(self.total_accuracy) / float(len(self.total_accuracy))
),
}
my_comm = Comm(target_name="totalData", data=data)
my_comm.send(data=data)
def _send_comm_message(self, msg_type, content):
"""
Sends a ipykernel.Comm message to the KBaseJobs channel with the given msg_type
and content. These just get encoded into the message itself.
"""
msg = {'msg_type': msg_type, 'content': content}
if self._comm is None:
self._comm = Comm(target_name='KBaseJobs', data={})
self._comm.on_msg(self._handle_comm_message)
self._comm.send(msg)
def urlArg(self, argName):
arguments = dict(target_name='beakerx.geturlarg')
comm = Comm(**arguments)
state = {'name': 'URL_ARG', 'arg_name': argName}
data = {'state': state, 'url': self._url, 'type': 'python'}
comm.send(data=data, buffers=[])
data = self._queue.get()
params = json.loads(data)
return params['argValue']
def create_comm(target: str,
data: dict = None,
callback: callable = None,
**kwargs):
"""Create ipykernel message comm."""
# create comm on python site
comm = Comm(target_name=target, data=data, **kwargs)
comm.on_msg(callback)
return comm
class AppViewer(object):
_dash_comm = Comm(target_name='dash_viewer')
def __init__(self, port=8050):
self.server_process = None
self.uid = str(uuid.uuid4())
self.port = port
self.stderr_queue = StdErrorQueue()
def show(self, app):
def run():
# Serve App
sys.stdout = self.stderr_queue
sys.stderr = self.stderr_queue
app.run_server(debug=False, port=self.port)
# Terminate any existing server process
self.terminate()
# Start new server process in separate process
self.server_process = multiprocessing.Process(target=run)
self.server_process.daemon = True
self.server_process.start()
# Wait for server to start
started = False
retries = 0
while not started and retries < 100:
try:
out = self.stderr_queue.queue.get(timeout=.1)
try:
out = out.decode()
except AttributeError:
pass
if 'Running on' in out:
started = True
except Empty:
retries += 1
pass
if started:
# Update front-end extension
self._dash_comm.send({
'type': 'show',
'uid': self.uid,
'url': 'http://localhost:{}'.format(self.port)
})
else:
# Failed to start development server
raise ConnectionError('Unable to start Dash server')
def terminate(self):
if self.server_process:
self.server_process.terminate()
def __init__(self, session=None, json=None, auth=None):
self.session = session
self.id = json.get('id')
self.auth = auth
if self.session.lgn.ipython_enabled:
from ipykernel.comm import Comm
self.comm = Comm('lightning', {'id': self.id})
self.comm_handlers = {}
self.comm.on_msg(self._handle_comm_message)
def set_comm(self, midas_instance_name: str, logger_id: str):
if self.is_in_ipynb:
self.comm = Comm(target_name=MIDAS_CELL_COMM_NAME)
self.comm.send({
"type": "initialize",
"name": midas_instance_name,
"loggerId": logger_id
})
self.comm.on_msg(self.handle_msg)
else:
self.comm = MockComm()
def open(self):
"""Open a comm to the frontend if one isn't already open."""
if self.comm is None:
state, buffer_paths, buffers = _remove_buffers(self.get_state())
args = dict(target_name='jupyter.widget',
data={'state': state, 'buffer_paths': buffer_paths},
buffers=buffers)
if self._model_id is not None:
args['comm_id'] = self._model_id
self.comm = Comm(**args)
def display_progress(self):
data = {
"totalProgress": (self.total_progress / (self.sample_amount * self.epochs))
* 100,
"currentProgress": (self.current_progress / self.sample_amount) * 100,
"runTime": self.total_runtime,
"loss": self.loss,
"accuracy": self.accuracy,
"epochs": self.epochs,
}
my_comm = Comm(target_name="batchData", data=data)
my_comm.send(data=data)
def stream_df_as_arrow(df, spark, limit=5000):
# df = df.repartition(2000)#.cache()
# cos = pa.output_stream(sink,compression='gzip')
# get the aliases and change the names of the columns to show the aliases
# this avoids duplicate columns
aliased_schema = get_schema_with_aliases(df)
renamed_schema_fields = []
for i in range(len(aliased_schema)):
aliased_field = aliased_schema[i]
fieldname: str = ""
if aliased_field["tablealias"] != "":
fieldname = aliased_field["tablealias"] + "." + aliased_field[
"name"]
else:
fieldname = aliased_field["name"]
renamed_schema_fields.append(fieldname)
comm = Comm(target_name="inspect_df")
# see if we have any results
renamed_df = df.toDF(*renamed_schema_fields)
row_iterator = renamed_df.toLocalIterator()
row_num = 0
row_buff = []
chunk_size = 500
for row in row_iterator:
if (row_num > 2000):
break
row_num += 1
logging.debug(row_num)
row_buff.append(row)
if row_num % chunk_size == 0:
batches = spark.createDataFrame(
row_buff, renamed_df.schema)._collectAsArrow()
if len(batches) > 0:
sink = pa.BufferOutputStream()
writer = pa.RecordBatchStreamWriter(sink, batches[0].schema)
for batch in batches:
writer.write_batch(batch)
comm.send(data="test", buffers=[sink.getvalue()])
row_buff = []
# send the last batch
batches = spark.createDataFrame(row_buff,
renamed_df.schema)._collectAsArrow()
if len(batches) > 0:
sink = pa.BufferOutputStream()
writer = pa.RecordBatchStreamWriter(sink, batches[0].schema)
for batch in batches:
writer.write_batch(batch)
comm.send(data="test", buffers=[sink.getvalue()])
comm.close(data="closing comm")
def get_comms(target_name):
''' Create a Jupyter comms object for a specific target, that can
be used to update Bokeh documents in the notebook.
Args:
target_name (str) : the target name the Comms object should connect to
Returns
Jupyter Comms
'''
from ipykernel.comm import Comm
return Comm(target_name=target_name, data={})
def test_map_view(self, mock_send):
"""
:type mock_send:Mock
:tpye map:SuperMapMap
:param mock_send:
:return:
"""
map = MapView()
comm = Comm()
comm.kernel = Kernel()
map.comm = comm
map.mapName = 'quanguo'
self.assertIsInstance(map.default_tiles, CloudTileLayer)
def setup(self, conn):
"""Set up the plugin connection."""
self.comm = self.comm or Comm(target_name="imjoy_comm_" + self.id,
data={"id": self.id})
def on_disconnect():
if not conn.opt.daemon:
conn.exit(1)
def emit(msg):
"""Emit a message to the socketio server."""
self.comm.send(msg)
def comm_plugin_message(msg):
"""Handle plugin message."""
data = msg["content"]["data"]
# emit({'type': 'logging', 'details': data})
# if not self.conn.executed:
# self.emit({'type': 'message', 'data': {"type": "interfaceSetAsRemote"}})
if data["type"] == "import":
emit({"type": "importSuccess", "url": data["url"]})
elif data["type"] == "disconnect":
conn.abort.set()
try:
if "exit" in conn.interface and callable(
conn.interface["exit"]):
conn.interface["exit"]()
except Exception as exc: # pylint: disable=broad-except
logger.error("Error when exiting: %s", exc)
elif data["type"] == "execute":
if not conn.executed:
self.queue.put(data)
else:
logger.debug("Skip execution")
emit({"type": "executeSuccess"})
elif data["type"] == "message":
_data = data["data"]
self.queue.put(_data)
logger.debug("Added task to the queue")
self.comm.on_msg(comm_plugin_message)
self.comm.on_close(on_disconnect)
conn.default_exit = lambda: None
conn.emit = emit
emit({"type": "initialized", "dedicatedThread": True})
logger.info("Plugin %s initialized", conn.opt.id)
def open(self):
"""Open a comm to the frontend if one isn't already open."""
if self.comm is None:
state, buffer_keys, buffers = self._split_state_buffers(self.get_state())
args = dict(target_name='jupyter.widget', data=state)
if self._model_id is not None:
args['comm_id'] = self._model_id
self.comm = Comm(**args)
if buffers:
# FIXME: workaround ipykernel missing binary message support in open-on-init
# send state with binary elements as second message
self.send_state()
def _register_client(self, data):
r"""
Called when the frontend sends a 'register' message.
Open a comm in the opposite direction to this specific widget and wrap
it in a client object.
Note that client are currently never deregistered. This is usually not
a big issue since all connections are essentially blocking and so
inactive clients do not consume bandwidth to the frontend.
"""
from ipykernel.comm import Comm
comm = Comm(data["target"], {})
self._clients.add(self._create_client(comm))
def _on_action_details(self, msg):
params = msg['content']['data']['content']
graphics_object = None
for item in self.chart.graphics_list:
if item.uid == params['itemId']:
graphics_object = item
action_type = params['params']['actionType']
if action_type == 'onclick' or action_type == 'onkey':
self.details = GraphicsActionObject(graphics_object, params['params'])
arguments = dict(target_name='beakerx.tag.run')
comm = Comm(**arguments)
msg = {'runByTag': params['params']['tag']}
state = {'state': msg}
comm.send(data=state, buffers=[])
def get_comms(target_name: str) -> Comm:
''' Create a Jupyter comms object for a specific target, that can
be used to update Bokeh documents in the Jupyter notebook.
Args:
target_name (str) : the target name the Comms object should connect to
Returns
Jupyter Comms
'''
# NOTE: must defer all IPython imports inside functions
from ipykernel.comm import Comm
return Comm(target_name=target_name, data={})
def __init__(self):
"""Constructor"""
self._calls = 0
self._callbacks = {}
# Push the Javascript to the front-end.
with open(
os.path.join(os.path.split(__file__)[0], 'backend_context.js'),
'r') as f:
display(Javascript(data=f.read()))
# Open communication with the front-end.
self._comm = Comm(target_name='BrowserContext')
self._comm.on_msg(self._on_msg)
