def make_comm() -> None:
global _JupyterComm
J_LOGGER.info("IPYTHON: Registering Comms")
comm_target_name = COMM_NAME
jupyter_comm = Comm(target_name=comm_target_name)
def _get_command(msg) -> Optional[str]:
return msg["content"]["data"].get("command", None)
@jupyter_comm.on_msg
def _recv(msg):
if _get_command(msg) == "merge_notebooks":
J_LOGGER.info("GOT UPDATE STATUS")
merge_notebooks(jupyter_comm, msg["content"]["data"])
return
J_LOGGER.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
J_LOGGER.info(msg)
J_LOGGER.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
# store comm for access in this thread later
_JupyterComm = jupyter_comm
J_LOGGER.info("==> Success")
return _JupyterComm
def send(file_name: str):
J_LOGGER.info(f"Executing all cells in file: {file_name}...")
file_name = str(Path(file_name).absolute())
request_obj = ExecuteAllRequest(file_name=file_name)
jupyter_server.request_notebook_command(request_obj)
J_LOGGER.info("... Complete")
def register_server(notebook_path: str, port_number: int) -> None:
J_LOGGER.info("Registering notebook {notebook} on port {port}",
notebook=notebook_path,
port=port_number)
_REGISTERED_SERVERS[notebook_path] = port_number
J_LOGGER.debug("Updated notebook mappings: {}", _REGISTERED_SERVERS)
def start_notebook_server_in_thread(notebook_name: str,
server,
file_watcher_enabled: bool = False,
status_widget=None):
"""
Args:
notebook_name: The name of the notebook you want to be syncing in this process.
file_watcher_enabled: If you're going to fire off events from a file watcher in your editor (like in PyCharm),
then you don't need to enable this. It will just use the same HTTP requests as normal
"""
notebook_path = Path(notebook_name).absolute()
if not status_widget:
status_widget = widgets.Text()
status_widget.style.description_width = "300px"
display(status_widget)
if file_watcher_enabled:
assert False, "Currently unsupported."
from watchdog.observers import Observer
from jupyter_ascending.watchers.file_watcher import NotebookEventHandler
event_handler = NotebookEventHandler(str(notebook_path.absolute()),
file_watcher_enabled)
file_observer = Observer()
abs_path = str(notebook_path.parent.absolute())
file_observer.schedule(event_handler, abs_path, recursive=False)
file_watcher_thread = threading.Thread(target=file_observer.start,
args=tuple())
file_watcher_thread.start()
# TODO: This might be a race condition if a bunch of these started at once...
notebook_server_port = find_free_port()
notebook_executor = HTTPServer(
("localhost", notebook_server_port),
NotebookKernelRequestHandler,
)
notebook_executor_thread = threading.Thread(
target=notebook_executor.serve_forever, args=tuple())
notebook_executor_thread.start()
J_LOGGER.info("IPYTHON: Registering notebook {}", notebook_path)
request(
EXECUTE_HOST_URL,
server.register_notebook_server.__name__,
# Params
notebook_path=str(notebook_path),
port_number=notebook_server_port,
)
J_LOGGER.info("==> Success")
make_comm()
return status_widget
def handle_get_status_request(data: dict) -> str:
J_LOGGER.info("Attempting get_status")
comm = get_comm()
comm.send({"command": "get_status"})
J_LOGGER.info("Sent get_status")
return f"Updating status"
def send(file_name: str):
if f".{SYNC_EXTENSION}.py" not in file_name:
return
J_LOGGER.info(f"Syncing File: {file_name}...")
file_name = str(Path(file_name).absolute())
with open(file_name, "r") as reader:
raw_result = reader.read()
request_obj = SyncRequest(file_name=file_name, contents=raw_result)
jupyter_server.request_notebook_command(request_obj)
J_LOGGER.info("... Complete")
def do_POST(self):
# Process request
request = self.rfile.read(int(self.headers["Content-Length"])).decode()
J_LOGGER.info("{} processing request:\n\t\t{}", name, request)
response = dispatch(request, methods=methods)
J_LOGGER.info("Got Response:\n\t\t{}", response)
# Return response
self.send_response(response.http_status)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(str(response).encode())
def start_server_in_thread():
try:
server_executor = HTTPServer(EXECUTE_HOST_LOCATION, JupyterServerRequestHandler)
except OSError:
print(f"It appears you already are using {EXECUTE_HOST_LOCATION}")
print("Use the environment variable: 'JUPYTER_ASCENDING_EXECUTE_PORT' to set a new port")
return
server_executor_thread = threading.Thread(target=server_executor.serve_forever, args=tuple())
server_executor_thread.start()
J_LOGGER.info("Successfully started multiplexer server")
return server_executor
def _recv(msg):
if _get_command(msg) == "merge_notebooks":
J_LOGGER.info("GOT UPDATE STATUS")
merge_notebooks(jupyter_comm, msg["content"]["data"])
return
J_LOGGER.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
J_LOGGER.info(msg)
J_LOGGER.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
def send(file_name: str, line_number: int, *args, **kwargs):
J_LOGGER.debug("Starting execute request")
# Always pass absolute path
file_name = str(Path(file_name).absolute())
request_obj = partial(ExecuteRequest, file_name=file_name, contents="")
with open(file_name, "r") as reader:
lines = reader.readlines()
cell_index = _find_cell_number(lines, line_number)
final_request = request_obj(cell_index=cell_index)
J_LOGGER.info(f"Sending request with {final_request}")
jupyter_server.request_notebook_command(final_request)
J_LOGGER.info("... Complete")
def start_notebook_server_in_thread(notebook_name: str,
server,
status_widget=None):
"""
Args:
notebook_name: The name of the notebook you want to be syncing in this process.
"""
notebook_path = Path(notebook_name).absolute()
if not status_widget:
status_widget = widgets.Text()
status_widget.style.description_width = "300px"
display(status_widget)
# TODO: This might be a race condition if a bunch of these started at once...
notebook_server_port = find_free_port()
notebook_executor = HTTPServer(
("localhost", notebook_server_port),
NotebookKernelRequestHandler,
)
notebook_executor_thread = threading.Thread(
target=notebook_executor.serve_forever, args=tuple())
notebook_executor_thread.start()
J_LOGGER.info("IPYTHON: Registering notebook {}", notebook_path)
request(
EXECUTE_HOST_URL,
server.register_notebook_server.__name__,
# Params
notebook_path=str(notebook_path),
port_number=notebook_server_port,
)
J_LOGGER.info("==> Success")
make_comm()
return status_widget
def set_everything_up():
# Note that this is also called from javascript after a kernel restart
J_LOGGER.info("Loading Ipython...")
# Start the server if it's the right name.
notebook_name = get_name_from_python()
J_LOGGER.info("IPYTHON: Loading {notebook}", notebook=notebook_name)
if f".{SYNC_EXTENSION}.ipynb" not in notebook_name:
J_LOGGER.info("IPYTHON: Note loading {notebook} because name does not match", notebook=notebook_name)
return
J_LOGGER.info("IPYTHON LOAD: " + time.ctime() + ": " + notebook_name)
jupyter_notebook.start_notebook_server_in_thread(notebook_name, jupyter_server)
def merge_notebooks(comm: Comm, result: Dict[str, Any]) -> None:
javascript_cells = result["javascript_cells"]
current_notebook = NotebookContents(cells=[
JupyterCell(
index=i,
cell_type=x["cell_type"],
source=x["source"],
output=get_output_text(x),
# metadata=x["metadata"],
) for i, x in enumerate(javascript_cells)
])
new_notebook = NotebookContents(
cells=[JupyterCell(**x) for x in result["new_notebook"]])
opcodes = opcode_merge_cell_contents(current_notebook, new_notebook)
J_LOGGER.info("Performing Opcodes...")
J_LOGGER.info(opcodes)
net_shift = 0
for op_action in opcodes:
net_shift = perform_op_code(comm, op_action, current_notebook,
new_notebook, net_shift)
def load_jupyter_server_extension(ipython):
ipython.log.info("LOADING SERVER")
J_LOGGER.info("SERVER LOAD: " + time.ctime())
server = jupyter_server.start_server_in_thread()
if not server:
return
# HACK:
# A bit of a hack to make sure the server gets shutdown when we're done here.
# Had some problems with hanging servers
#
# I think this doesn't quite work if we don't confirm that we want the server shutdown.
# Oh well for now...
ORIGINAL = None
def shutdown_from_signal(*args, **kwargs):
if ORIGINAL:
ORIGINAL(*args, **kwargs)
J_LOGGER.info("SERVER: Shutting down server")
server.shutdown()
ORIGINAL = signal.signal(signal.SIGINT, shutdown_from_signal)
def load_ipython_extension(ipython):
J_LOGGER.info("Loading Ipython...")
# Add %start_notebook_syncing
ipython.register_magics(SyncMagic)
# Start the server if it's the right name.
notebook_name = get_name_from_python()
J_LOGGER.info("IPYTHON: Loading {notebook}", notebook=notebook_name)
if ".synced.ipynb" not in notebook_name:
J_LOGGER.info(
"IPYTHON: Note loading {notebook} because name does not match",
notebook=notebook_name)
return
J_LOGGER.info("IPYTHON LOAD: " + time.ctime() + ": " + notebook_name)
jupyter_notebook.start_notebook_server_in_thread(notebook_name,
jupyter_server)
def perform_op_code(
comm: Comm,
op_action: OpCodeAction,
current_notebook: NotebookContents,
updated_notebook: NotebookContents,
net_shift: int,
) -> int:
"""
net_shift (int): Tracks the net shift of previous op codes since we can't apply all the operations at the same time to jupyter,
since it does not have that kind of editting model.
So what we do is make sure that as we delete and insert, we keep track of the shifts that have happened thus far.
Given this shift, we will shift the actions that we tell Jupyter notebook to do.
"""
if op_action.op_code == OpCodes.EQUAL:
pass
elif op_action.op_code == OpCodes.DELETE:
J_LOGGER.info(f"Performing Delete: {op_action}")
# Since deletion is a bit goofy for jupyter, so it has to be adjusted by net shift thus far.
cells_to_delete = [x + net_shift for x in range(*op_action.current)]
comm.send({
"command": "op_code__delete_cells",
"cell_indices": cells_to_delete
})
net_shift = net_shift - len(cells_to_delete)
elif op_action.op_code == OpCodes.INSERT:
J_LOGGER.info(f"Performing Insert: {op_action}")
cells_to_insert = list(range(*op_action.updated))
for cell_number in cells_to_insert:
comm.send({
"command":
"op_code__insert_cell",
"cell_number":
cell_number,
"cell_type":
updated_notebook.cells[cell_number].cell_type,
"cell_contents":
updated_notebook.cells[cell_number].joined_source,
})
net_shift = net_shift + len(cells_to_insert)
elif op_action.op_code == OpCodes.REPLACE:
# Keep track of what the current cells looked like before.
current_cells = list(range(*op_action.current))
updated_cells = list(range(*op_action.updated))
for cell_number in updated_cells:
# If we have current cells we're replacing, do that.
if current_cells:
current_cells.pop(0)
comm.send({
"command":
"op_code__replace_cell",
"cell_number":
cell_number,
"cell_type":
updated_notebook.cells[cell_number].cell_type,
"cell_contents":
updated_notebook.cells[cell_number].joined_source,
})
# Otherwise, we have new cells to insert so we don't overwrite existing cells
else:
net_shift = perform_op_code(
comm,
OpCodeAction(
op_code=OpCodes.INSERT,
# NOTE: This is intentionally the last index for both of these
current_start_idx=op_action.current_final_idx,
current_final_idx=op_action.current_final_idx,
updated_start_idx=cell_number,
updated_final_idx=cell_number + 1,
),
current_notebook,
updated_notebook,
net_shift,
)
# If we have cells left over from the replace (i.e. 1-4 replaced with 1-2),
# then we need to delete the rest of them.
if current_cells:
net_shift = perform_op_code(
comm,
OpCodeAction(
op_code=OpCodes.DELETE,
current_start_idx=current_cells[0],
current_final_idx=current_cells[-1] + 1,
# NOTE: This is intentionally the last index for both of these
updated_start_idx=op_action.updated_final_idx,
updated_final_idx=op_action.updated_final_idx,
),
current_notebook,
updated_notebook,
net_shift,
)
else:
raise NotImplementedError
return net_shift
def shutdown_from_signal(*args, **kwargs):
if ORIGINAL:
ORIGINAL(*args, **kwargs)
J_LOGGER.info("SERVER: Shutting down server")
server.shutdown()
