def start_watching_activity(self, kernel_id):
"""Start watching IOPub messages on a kernel for activity.
- update last_activity on every message
- record execution_state from status messages
"""
kernel = self._kernels[kernel_id]
# add busy/activity markers:
kernel.execution_state = 'starting'
kernel.last_activity = utcnow()
kernel._activity_stream = kernel.connect_iopub()
session = Session(
config=kernel.session.config,
key=kernel.session.key,
)
def record_activity(msg_list):
"""Record an IOPub message arriving from a kernel"""
self.last_kernel_activity = kernel.last_activity = utcnow()
idents, fed_msg_list = session.feed_identities(msg_list)
msg = session.deserialize(fed_msg_list)
msg_type = msg['header']['msg_type']
if msg_type == 'status':
kernel.execution_state = msg['content']['execution_state']
self.log.debug("activity on %s: %s (%s)", kernel_id, msg_type,
kernel.execution_state)
else:
self.log.debug("activity on %s: %s", kernel_id, msg_type)
kernel._activity_stream.on_recv(record_activity)
async def cull_kernel_if_idle(self, kernel_id):
kernel = self._kernels[kernel_id]
if hasattr(
kernel, "last_activity"
): # last_activity is monkey-patched, so ensure that has occurred
self.log.debug(
"kernel_id=%s, kernel_name=%s, last_activity=%s",
kernel_id,
kernel.kernel_name,
kernel.last_activity,
)
dt_now = utcnow()
dt_idle = dt_now - kernel.last_activity
# Compute idle properties
is_idle_time = dt_idle > timedelta(seconds=self.cull_idle_timeout)
is_idle_execute = self.cull_busy or (kernel.execution_state !=
"busy")
connections = self._kernel_connections.get(kernel_id, 0)
is_idle_connected = self.cull_connected or not connections
# Cull the kernel if all three criteria are met
if is_idle_time and is_idle_execute and is_idle_connected:
idle_duration = int(dt_idle.total_seconds())
self.log.warning(
"Culling '%s' kernel '%s' (%s) with %d connections due to %s seconds of inactivity.",
kernel.execution_state,
kernel.kernel_name,
kernel_id,
connections,
idle_duration,
)
await ensure_async(self.shutdown_kernel(kernel_id))
def update_api_activity(self):
"""Update last_activity of API requests"""
# record activity of authenticated requests
if (
self._track_activity
and getattr(self, '_user_cache', None)
and self.get_argument('no_track_activity', None) is None
):
self.settings['api_last_activity'] = utcnow()
def record_activity(msg_list):
"""Record an IOPub message arriving from a kernel"""
self.last_kernel_activity = kernel.last_activity = utcnow()
idents, fed_msg_list = session.feed_identities(msg_list)
msg = session.deserialize(fed_msg_list)
msg_type = msg['header']['msg_type']
self.log.debug("activity on %s: %s", kernel_id, msg_type)
if msg_type == 'status':
kernel.execution_state = msg['content']['execution_state']
def shutdown_kernel(self, kernel_id, now=False):
"""Shutdown a kernel by kernel_id"""
self._check_kernel_id(kernel_id)
kernel = self._kernels[kernel_id]
kernel._activity_stream.close()
kernel._activity_stream = None
self.stop_buffering(kernel_id)
self._kernel_connections.pop(kernel_id, None)
self.last_kernel_activity = utcnow()
return super(MappingKernelManager, self).shutdown_kernel(kernel_id,
now=now)
def create(self, **kwargs):
"""Create a new terminal."""
name, term = self.new_named_terminal(**kwargs)
# Monkey-patch last-activity, similar to kernels. Should we need
# more functionality per terminal, we can look into possible sub-
# classing or containment then.
term.last_activity = utcnow()
model = self.get_terminal_model(name)
# Increase the metric by one because a new terminal was created
TERMINAL_CURRENTLY_RUNNING_TOTAL.inc()
# Ensure culler is initialized
self._initialize_culler()
return model
def test_no_track_activity(self):
# initialize with old last api activity
old = utcnow() - timedelta(days=1)
settings = self.server.web_app.settings
settings['api_last_activity'] = old
# accessing status doesn't update activity
self.get('status')
assert settings['api_last_activity'] == old
# accessing with ?no_track_activity doesn't update activity
self.get('contents?no_track_activity=1')
assert settings['api_last_activity'] == old
# accessing without ?no_track_activity does update activity
self.get('contents')
assert settings['api_last_activity'] > old
def record_activity(msg_list):
"""Record an IOPub message arriving from a kernel"""
self.last_kernel_activity = kernel.last_activity = utcnow()
idents, fed_msg_list = session.feed_identities(msg_list)
msg = session.deserialize(fed_msg_list)
msg_type = msg["header"]["msg_type"]
if msg_type == "status":
kernel.execution_state = msg["content"]["execution_state"]
self.log.debug("activity on %s: %s (%s)", kernel_id, msg_type,
kernel.execution_state)
else:
self.log.debug("activity on %s: %s", kernel_id, msg_type)
async def start_kernel(self, kernel_id=None, path=None, **kwargs):
"""Start a kernel for a session and return its kernel_id.
Parameters
----------
kernel_id : uuid
The uuid to associate the new kernel with. If this
is not None, this kernel will be persistent whenever it is
requested.
path : API path
The API path (unicode, '/' delimited) for the cwd.
Will be transformed to an OS path relative to root_dir.
kernel_name : str
The name identifying which kernel spec to launch. This is ignored if
an existing kernel is returned, but it may be checked in the future.
"""
if kernel_id is None or kernel_id not in self:
if path is not None:
kwargs["cwd"] = self.cwd_for_path(path)
if kernel_id is not None:
kwargs["kernel_id"] = kernel_id
kernel_id = await ensure_async(self.pinned_superclass.start_kernel(self, **kwargs))
self._kernel_connections[kernel_id] = 0
task = asyncio.create_task(self._finish_kernel_start(kernel_id))
if not getattr(self, "use_pending_kernels", None):
await task
else:
self._pending_kernel_tasks[kernel_id] = task
# add busy/activity markers:
kernel = self.get_kernel(kernel_id)
kernel.execution_state = "starting"
kernel.reason = ""
kernel.last_activity = utcnow()
self.log.info("Kernel started: %s", kernel_id)
self.log.debug("Kernel args: %r", kwargs)
# Increase the metric of number of kernels running
# for the relevant kernel type by 1
KERNEL_CURRENTLY_RUNNING_TOTAL.labels(type=self._kernels[kernel_id].kernel_name).inc()
else:
self.log.info("Using existing kernel: %s", kernel_id)
# Initialize culling if not already
if not self._initialized_culler:
self.initialize_culler()
return kernel_id
async def cull_kernel_if_idle(self, kernel_id):
kernel = self._kernels[kernel_id]
self.log.debug("kernel_id=%s, kernel_name=%s, last_activity=%s", kernel_id, kernel.kernel_name, kernel.last_activity)
if kernel.last_activity is not None:
dt_now = utcnow()
dt_idle = dt_now - kernel.last_activity
# Compute idle properties
is_idle_time = dt_idle > timedelta(seconds=self.cull_idle_timeout)
is_idle_execute = self.cull_busy or (kernel.execution_state != 'busy')
connections = self._kernel_connections.get(kernel_id, 0)
is_idle_connected = self.cull_connected or not connections
# Cull the kernel if all three criteria are met
if (is_idle_time and is_idle_execute and is_idle_connected):
idle_duration = int(dt_idle.total_seconds())
self.log.warning("Culling '%s' kernel '%s' (%s) with %d connections due to %s seconds of inactivity.",
kernel.execution_state, kernel.kernel_name, kernel_id, connections, idle_duration)
await self.shutdown_kernel(kernel_id)
async def _cull_inactive_terminal(self, name):
try:
term = self.terminals[name]
except KeyError:
return # KeyErrors are somewhat expected since the terminal can be terminated as the culling check is made.
self.log.debug("name=%s, last_activity=%s", name, term.last_activity)
if hasattr(term, 'last_activity'):
dt_now = utcnow()
dt_inactive = dt_now - term.last_activity
# Compute idle properties
is_time = dt_inactive > timedelta(
seconds=self.cull_inactive_timeout)
# Cull the kernel if all three criteria are met
if (is_time):
inactivity = int(dt_inactive.total_seconds())
self.log.warning(
"Culling terminal '%s' due to %s seconds of inactivity.",
name, inactivity)
await self.terminate(name, force=True)
def __init__(self, **kwargs):
self.pinned_superclass = MultiKernelManager
self._pending_kernel_tasks = {}
self.pinned_superclass.__init__(self, **kwargs)
self.last_kernel_activity = utcnow()
def write_message(self, message, binary=False):
super(TermSocket, self).write_message(message, binary=binary)
self.application.settings['terminal_last_activity'] = utcnow()
def __init__(self, **kwargs):
super(MappingKernelManager, self).__init__(**kwargs)
self.last_kernel_activity = utcnow()
def _update_activity(self):
self.application.settings['terminal_last_activity'] = utcnow()
# terminal may not be around on deletion/cull
if self.term_name in self.terminal_manager.terminals:
self.terminal_manager.terminals[
self.term_name].last_activity = utcnow()
def __init__(self, **kwargs):
self.pinned_superclass = AsyncMultiKernelManager
self.pinned_superclass.__init__(self, **kwargs)
self.last_kernel_activity = utcnow()
import pytest
from tornado import web
from jupyter_server._tz import isoformat
from jupyter_server._tz import utcnow
from jupyter_server.services.contents.manager import ContentsManager
from jupyter_server.services.kernels.kernelmanager import MappingKernelManager
from jupyter_server.services.sessions.sessionmanager import SessionManager
class DummyKernel(object):
def __init__(self, kernel_name="python"):
self.kernel_name = kernel_name
dummy_date = utcnow()
dummy_date_s = isoformat(dummy_date)
class DummyMKM(MappingKernelManager):
"""MappingKernelManager interface that doesn't start kernels, for testing"""
def __init__(self, *args, **kwargs):
super(DummyMKM, self).__init__(*args, **kwargs)
self.id_letters = iter(u"ABCDEFGHIJK")
def _new_id(self):
return next(self.id_letters)
async def start_kernel(self, kernel_id=None, path=None, kernel_name="python", **kwargs):
kernel_id = kernel_id or self._new_id()
def on_message(self, message):
super(TermSocket, self).on_message(message)
self.application.settings['terminal_last_activity'] = utcnow()
