def postprocess(cell_outputs, saved_cell_outputs, verbose=True, cell_n=None):
"""Unify outputs and saved_outputs to make them comparable.
Side effects:
1) Print outputs
2) Reinitiate logging object
"""
if saved_cell_outputs is None:
saved_cell_outputs = defaultdict(lambda: None)
cell_outputs["stderr"], cell_outputs["stdout"] = (
cell_outputs["stderr"].getvalue(),
cell_outputs["stdout"].getvalue(),
)
if verbose:
print(
"-----> stderr of Cell %d:" % cell_n,
cell_outputs["stderr"],
"-----> stdout of Cell %d:" % cell_n,
cell_outputs["stdout"],
sep="\n",
)
root_logger = logging.getLogger()
# Reinitiate the logging object
for handl in root_logger.handlers:
root_logger.removeHandler(handl)
logging.basicConfig()
if saved_cell_outputs["stdout"] is None:
saved_cell_outputs["stdout"] = ""
if saved_cell_outputs["stderr"] is None:
saved_cell_outputs["stderr"] = ""
cell_outputs["stderr"] = escape_ansi(cell_outputs["stderr"])
if cell_outputs["display_data"] is None:
cell_outputs["display_data"] = ""
if saved_cell_outputs["display_data"] is None:
saved_cell_outputs["display_data"] = "''"
# elif isinstance(cell_outputs, str):
# cell_outputs = "'" + cell_outputs + "'"
celltest_disp = DisplayFormatter()
cell_outputs["display_data"] = celltest_disp.format(
cell_outputs["display_data"], include="text/plain")[0]["text/plain"]
return cell_outputs, saved_cell_outputs
class ReactivePythonKernel(Kernel):
implementation = 'reactivepy'
implementation_version = __version__
language_info = {
'name': 'python',
'version': sys.version.split()[0],
'nbconvert_exporter': 'python',
'mimetype': 'text/x-python',
'file_extension': '.py'
}
banner = ''
# measured in seconds. This value is intended to delay the same amount as
# 60fps does between frames
REGULAR_GENERATOR_DELAY = 0.016666
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._eventloop = IOLoop.current()
self._key = generate_id(size=32).encode('utf-8')
self._execution_lock = Lock(loop=self._eventloop.asyncio_loop)
self._inner_state_lock = Lock(loop=self._eventloop.asyncio_loop)
self._dep_tracker = DependencyTracker()
self._exec_unit_container = ExecUnitContainer()
self.formatter = DisplayFormatter()
self.ns_manager = BuiltInManager()
self.initialize_builtins()
self._execution_ctx = Executor(self._exec_unit_container,
ns_manager=self.ns_manager)
self.KernelTB = ultratb.AutoFormattedTB(mode='Plain',
color_scheme='LightBG',
tb_offset=1,
debugger_cls=None)
self._execution_queue = Queue(loop=self._eventloop.asyncio_loop)
self._registered_generators = dict()
self._eventloop.spawn_callback(self._execution_loop)
def initialize_builtins(self):
self.ns_manager.add_builtin('var_dependency_graph',
self._var_dependency_graph)
self.ns_manager.add_builtin('cell_dependency_graph',
self._cell_dependency_graph)
def _var_dependency_graph(self):
h = Digraph()
for start_node in self._dep_tracker.get_nodes():
if "-" in start_node:
for dest_node in self._dep_tracker.get_neighbors(start_node):
if "-" in dest_node:
first = start_node[:start_node.find('-')]
second = dest_node[:dest_node.find('-')]
for char in "[]":
first = first.replace(char, "")
second = second.replace(char, "")
h.edge(first, second)
return h
def _cell_dependency_graph(self):
h = Digraph()
for start_node in self._dep_tracker.get_nodes():
if "-" in start_node:
for dest_node in self._dep_tracker.get_neighbors(start_node):
if "-" in dest_node:
first = self._exec_unit_container.get_by_display_id(
start_node).pinning_cell
second = self._exec_unit_container.get_by_display_id(
dest_node).pinning_cell
h.edge(first, second)
return h
async def _run_single_async_iter_step(self, item, exec_unit):
# Make sure only single variable is important for execution
# unit
assert len(exec_unit.code_obj.output_vars) == 1
target_id = list(exec_unit.code_obj.output_vars)[0].get_name()
request_id, async_gen_obj, insert_id = self._registered_generators[
target_id]
async with self._inner_state_lock:
if request_id != item.request.msg_id and insert_id < item.id:
# self._log(item.request,
# f"Cancelling ({request_id}, {insert_id}) for ({item.request.msg_id}, {item.id}) 1")
await async_gen_obj.aclose()
self._registered_generators[
target_id] = item.request.msg_id, item.async_gen_obj, item.id
elif insert_id > item.id:
return None
try:
exec_result = await self._execution_ctx.run_coroutine(
anext(item.async_gen_obj),
target_id,
nohandle_exceptions=(StopAsyncIteration, ))
except StopAsyncIteration as e:
# iteration is complete
raise e
# Schedule next instance of iterator
# Compute execution units which depend on this unit,
# and schedule a new block to run which contains them
descendants = self._dep_tracker.get_descendants(exec_unit.display_id)
descendant_exec_units = list(
map(
lambda display_id: self._exec_unit_container.get_by_display_id(
display_id), descendants))
next_iter_round = ExecBlock(item.request,
exec_unit,
descendant_exec_units,
async_gen_obj=item.async_gen_obj)
await self._execution_queue.put(next_iter_round)
return exec_result
async def _first_async_iter_step(self, item: ExecBlock,
exec_unit: ExecutionUnitInfo,
exec_result: ExecutionResult):
# self._log(item.request, "Before inner state lock")
async with self._inner_state_lock:
# self._log(item.request, "After inner state lock")
if exec_result.target_id in self._registered_generators:
request_id, async_gen_obj, insert_id = self._registered_generators[
exec_result.target_id]
if insert_id < item.id:
# self._log(
# item.request, f"Cancelling ({request_id}, {insert_id}) for ({item.request.msg_id}, {item.id}) 2")
await async_gen_obj.aclose()
else:
raise ValueError(f"Very confuse. {insert_id} vs {item.id}")
# self._log(
# item.request,
# f"Storing {(item.request.msg_id, exec_result.output, item.id)}")
self._registered_generators[
exec_result.
target_id] = item.request.msg_id, exec_result.output, item.id
# Get first value and replace async gen with first
# value or None
async_gen_obj = exec_result.output
try:
exec_result.output = await anext(exec_result.output)
except Exception:
# DO not schedule further updates on failure
return False
# Schedule next instance of iterator
# Compute execution units which depend on this unit,
# and schedule a new block to run which contains
# them
descendants = self._dep_tracker.get_descendants(exec_unit.display_id)
descendant_exec_units = list(
map(
lambda display_id: self._exec_unit_container.get_by_display_id(
display_id), descendants))
next_iter_round = ExecBlock(item.request,
exec_unit,
descendant_exec_units,
async_gen_obj=async_gen_obj)
await self._execution_queue.put(next_iter_round)
return True
async def _execution_loop(self):
while True:
item: ExecBlock = await self._execution_queue.get()
# self._log(item.request, f"Handling {item.request.msg_id}")
if item is None:
# Use None to signal end
# self._log(item.request, 'Breaking execution loop!')
break
for exec_unit in ([item.current] + item.descendants):
# async with self._execution_lock:
# self._log(item.request, exec_unit.display_id)
is_current = exec_unit == item.current
is_iter = item.is_iter_exec and is_current
if is_iter:
# self._log(item.request, "Stepping iter")
try:
exec_result = await self._run_single_async_iter_step(
item, exec_unit)
except StopAsyncIteration:
# self._log(item.request,
# 'Step failed, breaking exec unit loop!')
break
else:
# self._log(item.request, "Executing normally")
# Execute code normally
exec_result = self._execution_ctx.run_cell(
exec_unit.code_obj.code, exec_unit.display_id)
if exec_result.output is not None and not is_iter:
is_awaitable, is_gen, is_async_gen = inspect_output_attrs(
exec_result.output)
# self._log(item.request,
# f"{is_awaitable} {is_gen} {is_async_gen}")
# These flags should be disjoint (never multiple True at same time)
# but this layout allows for transforming a regular generator into an async generator
# with a small timeout, and then passing it to the
# async_gen branch
# If the output is awaitable, wait for it and then replace
# the old output
if is_awaitable:
exec_result.output = await exec_result.output
# If the output is a regular generator, wrap it in an async
# generator that will add a very small delay
if is_gen:
exec_result.output = convert_gen_to_async(
exec_result.output,
ReactivePythonKernel.REGULAR_GENERATOR_DELAY)()
is_async_gen = True
is_gen = False
# If it is an async generator, either replace the old
# generator that was active for this target name or
# register this completely new generator. Remove the old
# generator by canceling it
if is_async_gen:
# self._log(item.request, "Is now async gen!")
setup_succeeded = await self._first_async_iter_step(
item, exec_unit, exec_result)
if not setup_succeeded:
break
if is_awaitable or is_gen or is_async_gen:
self._execution_ctx.update_ns(
{exec_result.target_id: exec_result.output})
if not item.request.silent:
self._output_exec_results(exec_unit, item.request,
is_current and not is_iter,
exec_result)
self._execution_queue.task_done()
def _update_existing_exec_unit(self, code_obj, cell_id):
# 4a. If it is a redefinition, get the old execution unit
current_exec_unit = self._exec_unit_container.get_by_display_id(
code_obj.display_id)
if current_exec_unit.is_pinned and current_exec_unit.pinning_cell != cell_id:
raise RedefiningOwnedCellException
old_code_obj = current_exec_unit.code_obj
new_input_vars = set(code_obj.input_vars)
old_input_vars = set(old_code_obj.input_vars)
# 4b. Compute the sets of edges to be added and deleted if the
# variable dependencies of the cell have changed
to_delete = old_input_vars - new_input_vars
to_add = new_input_vars - old_input_vars
# 4c. Actually replace old code object, delete old edges,
# add new edges
exec_unit = self._exec_unit_container.get_by_display_id(
code_obj.display_id)
exec_unit.code_obj = code_obj
for sym in to_delete:
code_object_id = self._exec_unit_container.get_by_symbol(
sym).code_obj.display_id
self._dep_tracker.delete_edge(code_object_id, code_obj.display_id)
for sym in to_add:
code_object_id = self._exec_unit_container.get_by_symbol(
sym).code_obj.display_id
self._dep_tracker.add_edge(code_object_id, code_obj.display_id)
# 4d. Get the updated execution unit and return it
return self._exec_unit_container.get_by_display_id(code_obj.display_id)
def _create_new_exec_unit(self, code_obj, cell_id):
# 4a. Create new execution unit to hold code object + display
# id + cell id data
current_exec_unit = self._exec_unit_container.register(
ExecutionUnitInfo(code_obj, pinning_cell=cell_id))
# 4b. Add new node
self._dep_tracker.add_node(code_obj.display_id)
# 4c. For each variable it depends on, find the complete set of
# defining variables (all the variables that were defined in
# the same code block), and create a dependency to them
for sym in code_obj.input_vars:
dep = self._exec_unit_container.get_by_symbol(sym)
if dep is not None:
code_object_id = dep.code_obj.display_id
self._dep_tracker.add_edge(code_object_id, code_obj.display_id)
else:
raise DefinitionNotFoundException(str(sym))
return current_exec_unit
async def _inner_execute_request_callback(self, stream, ident, parent):
# COPIED FROM IPYKERNEL/KERNELBASE.PY
# async with self._execution_lock:
try:
request = RequestInfo(parent, ident)
except BaseException:
self.log.error(f"Got bad msg: {parent}")
return
# Re-broadcast our input for the benefit of listening clients, and
# start computing output
if not request.silent:
request.execution_count = self.execution_count = 1 + self.execution_count
self._publish_execute_input(request.code, parent,
request.execution_count)
stop_on_error = request.content.get('stop_on_error', True)
response_meta = self.init_metadata(parent)
reply_content = await self.do_execute(request)
# Flush output before sending the reply.
sys.stdout.flush()
sys.stderr.flush()
# FIXME: on rare occasions, the flush doesn't seem to make it to the
# clients... This seems to mitigate the problem, but we definitely need
# to better understand what's going on.
if self._execute_sleep:
time.sleep(self._execute_sleep)
# Send the reply.
reply_content = json_clean(reply_content)
response_meta = self.finish_metadata(parent, response_meta,
reply_content)
reply_msg = self.session.send(stream,
u'execute_reply',
reply_content,
parent,
metadata=response_meta,
ident=ident)
self.log.debug(f"{reply_msg}")
if not request.silent and reply_msg['content'][
'status'] == u'error' and stop_on_error:
self._abort_queues()
def execute_request(self, stream, ident, parent):
"""handle an execute_request"""
self._eventloop.spawn_callback(self._inner_execute_request_callback,
stream, ident, parent)
def _update_kernel_state(self, code_obj, cell_id, deleted_cell_ids):
# 3. If deletedCells was passed, then unpin all execution units
# that were previously attached to a cell
if deleted_cell_ids is not None:
for to_delete_cell_id in deleted_cell_ids:
self._exec_unit_container.unpin_exec_unit(to_delete_cell_id)
self._dep_tracker.start_transaction()
self._exec_unit_container.start_transaction()
try:
# 4. Test if the code is new or a redefinition
if self._exec_unit_container.contains_display_id(
code_obj.display_id):
current_exec_unit = self._update_existing_exec_unit(
code_obj, cell_id)
else:
current_exec_unit = self._create_new_exec_unit(
code_obj, cell_id)
except Exception as e:
# rollback changes made to dep graph
self._dep_tracker.rollback()
self._exec_unit_container.rollback()
raise e
else:
self._dep_tracker.commit()
self._exec_unit_container.commit()
return current_exec_unit
def _output_exec_results(self, exec_unit: ExecutionUnitInfo,
request: RequestInfo, is_current: bool,
exec_result: ExecutionResult):
# 6b. Determine whether the current execution unit will be
# directly display or update
message_mode = 'update_display_data' if not is_current else 'display_data'
# 6c. Create rich outputs for captured output
if exec_result.output is not None:
data, md = self.formatter.format(exec_result.output)
else:
data, md = {}, {}
stdout, stderr = exec_result.stdout.getvalue(
), exec_result.stderr.getvalue()
# 6d. For the captured output value, stdout, and stderr
# send appropriate responses back to the front-end
if len(stdout) > 0:
self.session.send(self.iopub_socket,
message_mode,
content={
'data': {
'text/plain': stdout
},
'metadata': {},
'transient': {
'display_id': exec_unit.stdout_display_id
}
},
parent=request.parent)
if exec_result.output is not None:
self.session.send(self.iopub_socket,
message_mode,
content={
'data': data,
'metadata': md,
'transient': {
'display_id': exec_unit.display_id
}
},
parent=request.parent)
if len(stderr) > 0:
self.session.send(self.iopub_socket,
'stream',
content={
'name': 'stderr',
'text': stderr
},
parent=request.parent)
def _log(self, request, text):
self.session.send(self.iopub_socket,
'stream',
content={
'name': 'stdout',
'text': text + '\n'
},
parent=request.parent)
async def do_execute(self, request: RequestInfo):
try:
# 1. Create code object
code_obj = CodeObject(request.code, self._key,
self._execution_ctx.ns_manager)
# 2. Extract metadata (both are optional)
cell_id = request.metadata[
'cellId'] if 'cellId' in request.metadata else None
deleted_cell_ids = request.metadata[
'deletedCells'] if 'deletedCells' in request.metadata else None
# Doesn't need lock because the code inside _update_kernel_state
# will never await. If that ever changes, uncomment lock below
async with self._inner_state_lock:
current_exec_unit = self._update_kernel_state(
code_obj, cell_id, deleted_cell_ids)
# 5. Compute the dependant execution units which must be rerun, and
# add the current execution unit to the front of the list
descendants = self._dep_tracker.get_descendants(
code_obj.display_id)
descendant_exec_units = list(
map(
lambda display_id: self._exec_unit_container.
get_by_display_id(display_id), descendants))
block = ExecBlock(request, current_exec_unit,
descendant_exec_units)
await self._execution_queue.put(block)
except Exception:
etype, value, tb = sys.exc_info()
stb = self.KernelTB.structured_traceback(etype, value, tb)
formatted_lines = self.KernelTB.stb2text(stb).splitlines()
error_content = {
'ename': str(etype),
'evalue': str(value),
'traceback': formatted_lines
}
if not request.silent:
self.session.send(self.iopub_socket,
'error',
content=error_content,
parent=request.parent)
error_content['status'] = 'error'
return error_content
return {
'status': 'ok',
'execution_count': request.execution_count,
}
class ReactivePythonKernel(Kernel):
implementation = 'reactivepy'
implementation_version = __version__
language_info = {
'name': 'python',
'version': sys.version.split()[0],
'nbconvert_exporter': 'python',
'mimetype': 'text/x-python',
'file_extension': '.py'
}
banner = ''
# measured in seconds. This value is intended to delay the same amount as
# 60fps does between frames
REGULAR_GENERATOR_DELAY = 0.016666
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._key = generate_id(size=32).encode('utf-8')
self._eventloop = IOLoop.current().asyncio_loop
self._inner_state_lock = Lock(loop=self._eventloop)
self._dep_tracker = DependencyTracker()
self._exec_unit_container = ExecUnitContainer()
self.formatter = DisplayFormatter()
self.ns_manager = BuiltInManager()
self.initialize_builtins()
self._execution_ctx = Executor(self._exec_unit_container,
ns_manager=self.ns_manager)
self.KernelTB = ultratb.AutoFormattedTB(mode='Plain',
color_scheme='LightBG',
tb_offset=1,
debugger_cls=None)
# mapping from variable name (target id) to (request id, generator
# object)
self._registered_generators = dict()
def initialize_builtins(self):
self.ns_manager.add_builtin('var_dependency_graph',
self._var_dependency_graph)
self.ns_manager.add_builtin('cell_dependency_graph',
self._cell_dependency_graph)
def _var_dependency_graph(self):
h = Digraph()
for start_node in self._dep_tracker.get_nodes():
if "-" in start_node:
for dest_node in self._dep_tracker.get_neighbors(start_node):
if "-" in dest_node:
first = start_node[:start_node.find('-')]
second = dest_node[:dest_node.find('-')]
for char in "[]":
first = first.replace(char, "")
second = second.replace(char, "")
h.edge(first, second)
return h
def _cell_dependency_graph(self):
h = Digraph()
for start_node in self._dep_tracker.get_nodes():
if "-" in start_node:
for dest_node in self._dep_tracker.get_neighbors(start_node):
if "-" in dest_node:
first = self._exec_unit_container.get_by_display_id(
start_node).pinning_cell
second = self._exec_unit_container.get_by_display_id(
dest_node).pinning_cell
h.edge(first, second)
return h
def _update_existing_exec_unit(self, code_obj, cell_id):
# 1. If it is a redefinition, get the old execution unit
current_exec_unit = self._exec_unit_container.get_by_display_id(
code_obj.display_id)
# 2. If the cell is owned and the owning cell is not the cell that sent the execute request, raise an exception
if current_exec_unit.is_pinned and current_exec_unit.pinning_cell != cell_id:
raise RedefiningOwnedCellException
old_code_obj = current_exec_unit.code_obj
new_input_vars = set(code_obj.input_vars)
old_input_vars = set(old_code_obj.input_vars)
# 3. Compute the sets of edges to be added and deleted if the
# variable dependencies of the cell have changed
to_delete = old_input_vars - new_input_vars
to_add = new_input_vars - old_input_vars
# 4. Actually replace old code object, delete old edges,
# add new edges
exec_unit = self._exec_unit_container.get_by_display_id(
code_obj.display_id)
exec_unit.code_obj = code_obj
for sym in to_delete:
code_object_id = self._exec_unit_container.get_by_symbol(
sym).code_obj.display_id
self._dep_tracker.delete_edge(code_object_id, code_obj.display_id)
for sym in to_add:
code_object_id = self._exec_unit_container.get_by_symbol(
sym).code_obj.display_id
self._dep_tracker.add_edge(code_object_id, code_obj.display_id)
# 5. Get the updated execution unit and return it
return self._exec_unit_container.get_by_display_id(code_obj.display_id)
def _create_new_exec_unit(self, code_obj, cell_id):
# 1. Create new execution unit to hold code object + display
# id + cell id data
current_exec_unit = self._exec_unit_container.register(
ExecutionUnitInfo(code_obj, pinning_cell=cell_id))
# 2. Add new node
self._dep_tracker.add_node(code_obj.display_id)
# 3. For each variable it depends on, find the complete set of
# defining variables (all the variables that were defined in
# the same code block), and create a dependency to them
for sym in code_obj.input_vars:
dep = self._exec_unit_container.get_by_symbol(sym)
if dep is not None:
code_object_id = dep.code_obj.display_id
self._dep_tracker.add_edge(code_object_id, code_obj.display_id)
else:
raise DefinitionNotFoundException(str(sym))
return current_exec_unit
async def _inner_execute_request_callback(self, request: RequestInfo):
# Re-broadcast our input for the benefit of listening clients, and
# start computing output
if not request.silent:
request.execution_count = self.execution_count = 1 + self.execution_count
self._publish_execute_input(request.code, request.parent,
request.execution_count)
stop_on_error = request.content.get('stop_on_error', True)
response_meta = self.init_metadata(request.parent)
request.response_meta = response_meta
request.stop_on_error = stop_on_error
await self.do_execute(request)
def _complete_execute_request(self, request: RequestInfo, reply_content):
# Flush output before sending the reply.
sys.stdout.flush()
sys.stderr.flush()
# FIXME: on rare occasions, the flush doesn't seem to make it to the
# clients... This seems to mitigate the problem, but we definitely need
# to better understand what's going on.
if self._execute_sleep:
time.sleep(self._execute_sleep)
# Send the reply.
reply_content = json_clean(reply_content)
request.response_meta = self.finish_metadata(request.parent,
request.response_meta,
reply_content)
reply_msg = self.session.send(request.stream,
u'execute_reply',
reply_content,
request.parent,
metadata=request.response_meta,
ident=request.ident)
if not request.silent and reply_msg['content'][
'status'] == u'error' and request.stop_on_error:
self._abort_queues()
def execute_request(self, stream, ident, parent):
"""handle an execute_request"""
try:
request = RequestInfo(stream, parent, ident)
except BaseException:
self.log.error("Got bad msg: ")
self.log.error("%s", parent)
return
self._eventloop.create_task(
self._inner_execute_request_callback(request))
def _update_kernel_state(self, code_obj, cell_id, deleted_cell_ids):
# 1. If deletedCells was passed, then unpin all execution units
# that were previously attached to a cell
if deleted_cell_ids is not None:
for to_delete_cell_id in deleted_cell_ids:
self._exec_unit_container.unpin_exec_unit(to_delete_cell_id)
self._dep_tracker.start_transaction()
self._exec_unit_container.start_transaction()
try:
# 2. Test if the code is new or a redefinition
if self._exec_unit_container.contains_display_id(
code_obj.display_id):
current_exec_unit = self._update_existing_exec_unit(
code_obj, cell_id)
else:
current_exec_unit = self._create_new_exec_unit(
code_obj, cell_id)
except Exception as e:
# 3a. rollback changes made to dep graph
self._dep_tracker.rollback()
self._exec_unit_container.rollback()
raise e
else:
# 3b. commit changes made to dep graph
self._dep_tracker.commit()
self._exec_unit_container.commit()
return current_exec_unit
def _output_exec_results(self, exec_unit: ExecutionUnitInfo,
request: RequestInfo, is_current: bool,
exec_result: ExecutionResult):
# 6b. Determine whether the current execution unit will be
# directly display or update
message_mode = 'update_display_data' if not is_current else 'display_data'
# 6c. Create rich outputs for captured output
if exec_result.output is not None:
data, md = self.formatter.format(exec_result.output)
else:
data, md = {}, {}
stdout, stderr = exec_result.stdout.getvalue(
), exec_result.stderr.getvalue()
# 6d. For the captured output value, stdout, and stderr
# send appropriate responses back to the front-end
if len(stdout) > 0:
self.session.send(self.iopub_socket,
message_mode,
content={
'data': {
'text/plain': stdout
},
'metadata': {},
'transient': {
'display_id': exec_unit.stdout_display_id
}
},
parent=request.parent)
if exec_result.output is not None:
self.session.send(self.iopub_socket,
message_mode,
content={
'data': data,
'metadata': md,
'transient': {
'display_id': exec_unit.display_id
}
},
parent=request.parent)
if len(stderr) > 0:
self.session.send(self.iopub_socket,
'stream',
content={
'name': 'stderr',
'text': stderr
},
parent=request.parent)
async def do_execute(self, request: RequestInfo):
try:
# 1. Create code object
code_obj = CodeObject(request.code, self._key,
self._execution_ctx.ns_manager)
# 2. Extract metadata (both are optional)
cell_id = request.metadata[
'cellId'] if 'cellId' in request.metadata else None
deleted_cell_ids = request.metadata[
'deletedCells'] if 'deletedCells' in request.metadata else None
# 3. Update internal graph
current_exec_unit = self._update_kernel_state(
code_obj, cell_id, deleted_cell_ids)
# 4. Compute the dependant execution units which must be rerun, and
# add the current execution unit to the front of the list
descendants = list(
map(self._exec_unit_container.get_by_display_id,
self._dep_tracker.get_descendants(code_obj.display_id)))
is_awaitable, is_gen, is_async_gen = False, False, False
current_result = self._execution_ctx.run_cell(
current_exec_unit.code_obj.code, current_exec_unit.display_id)
if current_result.output is not None:
is_awaitable, is_gen, is_async_gen = inspect_output_attrs(
current_result.output)
# If the output is awaitable, wait for it and then replace
# the old output
if is_awaitable:
current_result.output = await current_result.output
# If the output is a regular generator, wrap it in an async
# generator that will add a very small delay
if is_gen:
current_result.output = convert_gen_to_async(
current_result.output,
ReactivePythonKernel.REGULAR_GENERATOR_DELAY)()
is_async_gen = True
is_gen = False
if is_async_gen:
async_gen_obj = current_result.output
current_result.output = await anext(
current_result.output, None)
if is_awaitable or is_gen or is_async_gen:
self._execution_ctx.update_ns(
{current_result.target_id: current_result.output})
if not request.silent:
self._output_exec_results(current_exec_unit, request, True,
current_result)
self._complete_execute_request(
request, {
'status': 'ok',
'execution_count': request.execution_count
})
# If the result output was not an async generator,
# then execute descendants in this "thread" of control.
# The `_start_new_async_iter` will handle running descendants and output their stuff.
if not is_async_gen:
# Return control to event loop before executing descendants
await asyncio.sleep(0)
for exec_unit in descendants:
await self._run_descendant(exec_unit, request)
# If it is an async generator, either replace the old
# generator that was active for this target name or
# register this completely new generator. Remove the old
# generator by canceling it
if is_async_gen:
await self._start_new_async_iter(current_exec_unit,
async_gen_obj,
current_result.target_id,
request)
except Exception:
etype, value, tb = sys.exc_info()
stb = self.KernelTB.structured_traceback(etype, value, tb)
formatted_lines = self.KernelTB.stb2text(stb).splitlines()
error_content = {
'ename': str(etype),
'evalue': str(value),
'traceback': formatted_lines
}
if not request.silent:
self.session.send(self.iopub_socket,
'error',
content=error_content,
parent=request.parent)
error_content['status'] = 'error'
self._complete_execute_request(request, error_content)
return
async def _run_descendant(self, exec_unit: ExecutionUnitInfo,
request: RequestInfo):
""" Run descendant of an iterator or execute request.
Also output new values to front end.
"""
result = self._execution_ctx.run_cell(exec_unit.code_obj.code,
exec_unit.display_id)
# Set vars ahead to provide defaults
is_async_gen = False
async_gen_obj = None
if result.output is not None:
is_awaitable, is_gen, is_async_gen = inspect_output_attrs(
result.output)
# If the output is awaitable, wait for it and then replace
# the old output
if is_awaitable:
result.output = await result.output
# If the output is a regular generator, wrap it in an async
# generator that will add a very small delay
if is_gen:
result.output = convert_gen_to_async(
result.output,
ReactivePythonKernel.REGULAR_GENERATOR_DELAY)()
is_async_gen = True
is_gen = False
# If it is an async generator, either replace the old
# generator that was active for this target name or
# register this completely new generator. Remove the old
# generator by canceling it
if is_async_gen:
async_gen_obj = result.output
result.output = await anext(result.output, None)
if is_awaitable or is_gen or is_async_gen:
self._execution_ctx.update_ns(
{result.target_id: result.output})
if not request.silent:
self._output_exec_results(exec_unit, request, False, result)
if is_async_gen:
await self._start_new_async_iter(exec_unit, async_gen_obj,
result.target_id, request)
async def _start_new_async_iter(self, exec_unit: ExecutionUnitInfo,
async_gen_obj, target_id: str,
request: RequestInfo):
"""Attempt to start a new async generator running.
This will check for previous generators for this same variable, and cancel them.
"""
# Check if a previous iterator exists for this variable.
if exec_unit.display_id in self._registered_generators:
old_start, old_task = self._registered_generators[
exec_unit.display_id]
new_start = now()
if new_start > old_start:
old_task.cancel()
new_task = self._eventloop.create_task(
self._run_async_iter(new_start, exec_unit, async_gen_obj,
target_id, request))
self._registered_generators[
exec_unit.display_id] = new_start, new_task
else:
new_start = now()
new_task = self._eventloop.create_task(
self._run_async_iter(new_start, exec_unit, async_gen_obj,
target_id, request))
self._registered_generators[
exec_unit.display_id] = new_start, new_task
async def _run_async_iter(self, started, exec_unit: ExecutionUnitInfo,
async_gen_obj, target_id: str,
request: RequestInfo):
try:
while True:
# Get next value and update namespace
try:
exec_result = await self._execution_ctx.run_coroutine(
anext(async_gen_obj),
target_id,
nohandle_exceptions=(StopAsyncIteration,
GeneratorExit))
except StopAsyncIteration as e:
# Generator is complete
break
except GeneratorExit as e:
# Generator cancelled
break
# Output values to the front end
self._output_exec_results(exec_unit, request, False,
exec_result)
descendant_exec_units = list(
map(
self._exec_unit_container.get_by_display_id,
self._dep_tracker.get_descendants(
exec_unit.display_id)))
# For each descendant, run them
for descendant in descendant_exec_units:
await self._run_descendant(descendant, request)
# Return control to eventloop in between steps of generator
await asyncio.sleep(0)
except Exception:
etype, value, tb = sys.exc_info()
stb = self.KernelTB.structured_traceback(etype, value, tb)
formatted_lines = self.KernelTB.stb2text(stb).splitlines()
error_content = {
'ename': str(etype),
'evalue': str(value),
'traceback': formatted_lines
}
if not request.silent:
self.session.send(self.iopub_socket,
'error',
content=error_content,
parent=request.parent)
class PickyKernel(Kernel):
"""A kernel that accepts kernel magics which configure the environment"""
implementation = "picky"
implementation_version = __version__
# This banner only shows on `jupyter console` (at the command line).
banner = """Pick, the kernel for choosy users! ⛏
Read more about it at https://github.com/nteract/pick
"""
# NOTE: This may not match the underlying kernel we launch. However, we need a default
# for the initial launch.
# TODO: Dynamically send over the underlying kernel's kernel_info_reply later
language_info = {
"name": "python",
"version": sys.version.split()[0],
"mimetype": "text/x-python",
"codemirror_mode": {
"name": "ipython",
"version": sys.version_info[0]
},
"pygments_lexer": "ipython3",
"nbconvert_exporter": "python",
"file_extension": ".py",
}
default_kernel = None
default_config = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Ensure the kernel we work with uses Futures on recv, so we can await them
self.context = ctx = Context.instance()
# Our subscription to messages from the kernel we launch
self.iosub = ctx.socket(zmq.SUB)
self.iosub.subscribe = b""
# From kernelapp.py, shell_streams are typically shell_stream, control_stream
self.shell_stream = self.shell_streams[0]
# Start with no child kernel
self.child_kernel = None
self.kernel_config = None
self.acquiring_kernel = asyncio.Lock()
self.kernel_launched = asyncio.Event()
self.display_formatter = DisplayFormatter()
def start(self):
"""Start the PickyKernel and its event loop"""
super().start()
loop = IOLoop.current()
# Collect and send all IOPub messages, for all time
# TODO: Check errors from this loop and restart as needed (or shut down the kernel)
loop.add_callback(self.relay_iopub_messages)
async def relay_iopub_messages(self):
"""Relay messages received by the Picky Kernel
to the consumer client (e.g. notebook)
"""
while True:
msg = await self.iosub.recv_multipart()
# Send the message up to the consumer (for example, the notebook)
self.iopub_socket.send_multipart(msg)
async def start_kernel(self, name=None, config=None):
# Create a connection file that is named as a child of this kernel
base, ext = os.path.splitext(self.parent.connection_file)
connection_file = "{base}-child{ext}".format(
base=base,
ext=ext,
)
subkernel = subkernels.get_subkernel(name)
try:
km = await subkernel.launch(
config=config,
session=self.session,
context=self.context,
connection_file=connection_file,
)
except Exception as err:
self.log.error(err)
kernel = KernelProxy(manager=km, shell_upstream=self.shell_stream)
self.iosub.connect(kernel.iopub_url)
# Make sure the kernel is really started. We do that by using
# kernel_info_requests here.
#
# In the future we should incorporate kernel logs (output of the kernel process), then
# and send back all the information back to the user as display output.
# Create a temporary KernelClient for waiting for the kernel to start
kc = km.client()
kc.start_channels()
# Wait for kernel info reply on shell channel
while True:
self.log.debug("querying kernel info")
kc.kernel_info(reply=False)
try:
msg = await kc.shell_channel.get_msg(timeout=1)
except Empty:
pass
else:
if msg["msg_type"] == "kernel_info_reply":
# Now we know the kernel is (mostly) ready.
# However, most kernels are not quite ready at this point to send
# on more execution.
#
# Do we wait for a further status: idle on iopub?
# Wait for idle?
# Wait for particular logs from the stdout of the kernel process?
break
if not await kc.is_alive():
# TODO: Emit child kernel death message into the notebook output
self.log.error("Kernel died while launching")
raise RuntimeError(
"Kernel died before replying to kernel_info")
# Wait before sending another kernel info request
await asyncio.sleep(0.1)
# Flush IOPub channel on our (temporary) kernel client
while True:
try:
msg = await kc.iopub_channel.get_msg(timeout=0.2)
except Empty:
break
# Clean up our temporary kernel client
kc.stop_channels()
# Inform all waiters for the kernel that it is ready
self.kernel_launched.set()
return kernel
async def get_kernel(self):
"""Get a launched child kernel"""
# Ensure that the kernel is launched
await self.kernel_launched.wait()
if self.child_kernel is None:
self.log.error("the child kernel was not available")
return self.child_kernel
async def queue_before_relay(self, stream, ident, parent):
"""Queue messages before sending between the child and parent kernels."""
if not self.kernel_launched.is_set():
self._publish_status("busy")
kernel = await self.get_kernel()
self._publish_status("idle")
self.session.send(kernel.shell, parent, ident=ident)
def display(self, obj, parent, display_id=False, update=False):
"""Publish a rich format of an object from our picky kernel, associated with the parent message"""
data, metadata = self.display_formatter.format(obj)
if metadata is None:
metadata = {}
transient = {}
if display_id:
transient = {"display_id": display_id}
content = {"data": data, "metadata": metadata, "transient": transient}
self.session.send(
self.iopub_socket,
"update_display_data" if update else "display_data",
content,
parent=parent,
ident=self._topic("display_data"),
)
def _publish_error(self, exc_info, parent=None):
"""send error on IOPub"""
exc_type, exception, traceback = exc_info
content = {
"ename": exc_type.__name__,
"evalue": str(exception),
"traceback": format_tb(traceback),
}
self.session.send(
self.iopub_socket,
"error",
content,
parent=parent,
ident=self._topic("error"),
)
def _publish_execute_reply_error(self, exc_info, ident, parent):
exc_type, exception, traceback = exc_info
reply_content = {
"status": "error",
"ename": exc_type.__name__,
"evalue": str(exception),
"traceback": format_tb(traceback),
# Since this isn't the underlying kernel
"execution_count": None,
}
self.session.send(
self.shell_stream,
"execute_reply",
reply_content,
parent=parent,
metadata={"status": "error"},
ident=ident,
)
def _publish_status(self, status, parent=None):
"""send status (busy/idle) on IOPub"""
self.session.send(
self.iopub_socket,
"status",
{"execution_state": status},
parent=parent or self._parent_header,
ident=self._topic("status"),
metadata={"picky": True},
)
async def async_execute_request(self, stream, ident, parent):
"""process an execution request, sending it on to the child kernel or launching one
if it has not been started.
"""
# Announce that we are busy handling this request
self._publish_status("busy")
# Only one kernel can be acquired at the same time
async with self.acquiring_kernel:
# Check for configuration code first
content = parent["content"]
code = content["code"]
config, kernel_name = self.parse_cell(content["code"])
has_config = bool(config)
if has_config:
# Report back that we'll be running the config code
# NOTE: We are, for the time being, ignoring the silent flag, store_history, etc.
self._publish_execute_input(code, parent, self.execution_count)
# User is attempting to run a cell with config after the kernel is started,
# so we inform them and bail
if has_config and self.child_kernel is not None:
self.display(
Markdown(f"""## Kernel already configured and launched.
You can only run the `%%kernel.{kernel_name}` cell at the top of your notebook and the
start of your session. Please **restart your kernel** and run the cell again if
you want to change configuration.
"""),
parent=parent,
)
# Complete the "execution request" so the jupyter client (e.g. the notebook) thinks
# execution is finished
reply_content = {
"status": "error",
# Since our result is not part of `In` or `Out`, ensure
# that the execution count is unset
"execution_count": None,
"user_expressions": {},
"payload": {},
}
metadata = {
"parametrized-kernel": True,
"status": reply_content["status"],
}
self.session.send(
stream,
"execute_reply",
reply_content,
parent,
metadata=metadata,
ident=ident,
)
self._publish_status("idle")
return
kernel_display_id = hexlify(os.urandom(8)).decode("ascii")
# Launching a custom kernel
if self.child_kernel is None and has_config:
# Start a kernel now
# If there's config set, we launch with that config
# If the user is requesting the kernel with config, launch it!
self.display(
Markdown("Launching customized runtime..."),
display_id=kernel_display_id,
parent=parent,
)
try:
self.child_kernel = await self.start_kernel(
kernel_name, config)
except PickRegistrationException as err:
# Get access to the exception info prior to doing any potential awaiting
exc_info = sys.exc_info()
self.log.info(exc_info)
separator = "\n"
self.display(
Markdown(
f"""## There is no kernel magic named `{kernel_name}`
These are the available kernels:
{separator.join([f"* `{kernel}`" for kernel in subkernels.list_subkernels()])}
"""),
display_id=kernel_display_id,
update=True,
parent=parent,
)
self.log.error(err)
self._publish_execute_reply_error(exc_info,
ident=ident,
parent=parent)
self._publish_status("idle", parent=parent)
return
except Exception as err:
self.log.error(err)
exc_info = sys.exc_info()
self._publish_error(exc_info, parent=parent)
self._publish_execute_reply_error(exc_info,
ident=ident,
parent=parent)
self._publish_status("idle", parent=parent)
return
self.display(
Markdown("Runtime ready!"),
display_id=kernel_display_id,
parent=parent,
update=True,
)
# Complete the "execution request" so the jupyter client (e.g. the notebook) thinks
# execution is finished
reply_content = {
"status": "ok",
# Our kernel setup is always the zero-th execution (In[] starts at 1)
"execution_count": 0,
# Note: user_expressions are not supported on our kernel creation magic
"user_expressions": {},
"payload": {},
}
metadata = {
"parametrized-kernel": True,
"status": reply_content["status"],
}
self.session.send(
stream,
"execute_reply",
reply_content,
parent,
metadata=metadata,
ident=ident,
)
# With that, we're all done launching the customized kernel and
# pushing updates on the kernel to the user.
return
# Start the default kernel to run code
if self.child_kernel is None:
self.display(
Markdown("Preparing default kernel..."),
parent=parent,
display_id=kernel_display_id,
)
self.child_kernel = await self.start_kernel(
self.default_kernel, self.default_config)
self.display(
# Wipe out the previous message.
# NOTE: The Jupyter notebook frontend ignores the case of an empty output for
# an update_display_data so we have to publish some empty content instead.
Markdown(""),
parent=parent,
display_id=kernel_display_id,
update=True,
)
self.session.send(self.child_kernel.shell, parent, ident=ident)
def parse_cell(self, cell):
if not cell.startswith("%%kernel."):
return None, None
try:
# Split off our config from the kernel magic name
magic_name, raw_config = cell.split("\n", 1)
kernel_name = magic_name.split("%%kernel.")[1]
return raw_config, kernel_name
except Exception as err:
self.log.error(err)
return None, None
def _log_task_exceptions(self, task, *args, **kwargs):
try:
task_exception = task.exception()
if task_exception:
self.log.error(task_exception)
self.log.error(task.get_stack(limit=5))
except asyncio.CancelledError as err:
self.log.error(err)
except Exception as err:
self.log.error(err)
def relay_execute_to_kernel(self, stream, ident, parent):
# Shove this execution onto the task queue
task = asyncio.create_task(
self.async_execute_request(stream, ident, parent))
task.add_done_callback(partial(self._log_task_exceptions, task))
def relay_to_kernel(self, stream, ident, parent):
# relay_to_kernel is synchronous, and we rely on an asynchronous start
# so we create each kernel message as a task...
task = asyncio.create_task(
self.queue_before_relay(stream, ident, parent))
task.add_done_callback(partial(self._log_task_exceptions, task))
execute_request = relay_execute_to_kernel
inspect_request = relay_to_kernel
complete_request = relay_to_kernel
class EchoKernel(Kernel):
implementation = 'Echo'
implementation_version = '1.0'
language = 'no-op'
language_version = '0.1'
language_info = {
'name': 'Any text',
'mimetype': 'text/plain',
'file_extension': '.txt',
}
banner = "Echo kernel - as useful as a parrot"
def __init__(self, **k):
# connect with kdb
super().__init__(**k)
self._formatter = DisplayFormatter()
def do_execute(self,
code,
silent,
store_history=True,
user_expressions=None,
allow_stdin=False):
if not silent:
magic_code, code = strip_magic(code)
stream_content = {
'name': 'stdout',
'text': str(user_expressions) + '\n'
}
self.send_response(self.iopub_socket, 'stream', stream_content)
stream_content = {'name': 'stdout', 'text': '===\n'}
self.send_response(self.iopub_socket, 'stream', stream_content)
stream_content = {'name': 'stdout', 'text': code}
self.send_response(self.iopub_socket, 'stream', stream_content)
o = pd.DataFrame(dict(
key=list('abc'),
val=[code] * 3,
))
# stream_content = {'data': {'text/html': '<b>hihihi</b>'},
# 'metadata': {}}
# self.send_response(self.iopub_socket, 'display_data', stream_content)
data, metadata = self._formatter.format(o)
stream_content = {
'data': data,
'metadata': metadata,
# 'data': {
# 'text/html': '<code>bibidi</code>',
# },
# 'metadata': {
# },
'execution_count': self.execution_count,
}
self.send_response(
self.iopub_socket,
'execute_result',
stream_content,
)
return {
'status': 'ok',
# The base class increments the execution count
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {},
}