def __init__(self, **kwargs):
# Pin the superclass to better control the MRO. This is needed by
# AsyncMappingKernelManager so that it can give priority to methods
# on AsyncMultiKernelManager over this superclass.
self.pinned_superclass = MultiKernelManager
self.pinned_superclass.__init__(self, **kwargs)
self.last_kernel_activity = utcnow()
def start_watching_activity(self, kernel_id):
"""Start watching IOPub messages on a kernel for activity.
- update last_activity on every message
- record execution_state from status messages
"""
kernel = self._kernels[kernel_id]
# add busy/activity markers:
kernel.execution_state = 'starting'
kernel.last_activity = utcnow()
kernel._activity_stream = kernel.connect_iopub()
session = Session(
config=kernel.session.config,
key=kernel.session.key,
)
def record_activity(msg_list):
"""Record an IOPub message arriving from a kernel"""
kernel.last_activity = utcnow()
idents, fed_msg_list = session.feed_identities(msg_list)
msg = session.deserialize(fed_msg_list)
msg_type = msg['header']['msg_type']
self.log.debug("activity on %s: %s", kernel_id, msg_type)
if msg_type == 'status':
kernel.execution_state = msg['content']['execution_state']
kernel._activity_stream.on_recv(record_activity)
async def cull_kernel_if_idle(self, kernel_id):
try:
kernel = self._kernels[kernel_id]
except KeyError:
return # KeyErrors are somewhat expected since the kernel can be shutdown as the culling check is made.
if hasattr(
kernel, 'last_activity'
): # last_activity is monkey-patched, so ensure that has occurred
self.log.debug("kernel_id=%s, kernel_name=%s, last_activity=%s",
kernel_id, kernel.kernel_name, kernel.last_activity)
dt_now = utcnow()
dt_idle = dt_now - kernel.last_activity
# Compute idle properties
is_idle_time = dt_idle > timedelta(seconds=self.cull_idle_timeout)
is_idle_execute = self.cull_busy or (kernel.execution_state !=
'busy')
connections = self._kernel_connections.get(kernel_id, 0)
is_idle_connected = self.cull_connected or not connections
# Cull the kernel if all three criteria are met
if (is_idle_time and is_idle_execute and is_idle_connected):
idle_duration = int(dt_idle.total_seconds())
self.log.warning(
"Culling '%s' kernel '%s' (%s) with %d connections due to %s seconds of inactivity.",
kernel.execution_state, kernel.kernel_name, kernel_id,
connections, idle_duration)
await maybe_future(self.shutdown_kernel(kernel_id))
def start_watching_activity(self, kernel_id):
"""Start watching IOPub messages on a kernel for activity.
- update last_activity on every message
- record execution_state from status messages
"""
kernel = self._kernels[kernel_id]
# add busy/activity markers:
kernel.execution_state = 'starting'
kernel.last_activity = utcnow()
kernel._activity_stream = kernel.connect_iopub()
session = Session(
config=kernel.session.config,
key=kernel.session.key,
)
def record_activity(msg_list):
"""Record an IOPub message arriving from a kernel"""
kernel.last_activity = utcnow()
idents, fed_msg_list = session.feed_identities(msg_list)
msg = session.deserialize(fed_msg_list)
msg_type = msg['header']['msg_type']
self.log.debug("activity on %s: %s", kernel_id, msg_type)
if msg_type == 'status':
kernel.execution_state = msg['content']['execution_state']
kernel._activity_stream.on_recv(record_activity)
def update_api_activity(self):
"""Update last_activity of API requests"""
# record activity of authenticated requests
if (
self._track_activity
and getattr(self, '_user_cache', None)
and self.get_argument('no_track_activity', None) is None
):
self.settings['api_last_activity'] = utcnow()
def record_activity(msg_list):
"""Record an IOPub message arriving from a kernel"""
kernel.last_activity = utcnow()
idents, fed_msg_list = kernel.session.feed_identities(msg_list)
msg = kernel.session.deserialize(fed_msg_list)
msg_type = msg['header']['msg_type']
self.log.debug("activity on %s: %s", kernel_id, msg_type)
if msg_type == 'status':
kernel.execution_state = msg['content']['execution_state']
def shutdown_kernel(self, kernel_id, now=False):
"""Shutdown a kernel by kernel_id"""
self._check_kernel_id(kernel_id)
kernel = self._kernels[kernel_id]
kernel._activity_stream.close()
kernel._activity_stream = None
self.stop_buffering(kernel_id)
self._kernel_connections.pop(kernel_id, None)
self.last_kernel_activity = utcnow()
return super(MappingKernelManager, self).shutdown_kernel(kernel_id, now=now)
def shutdown_kernel(self, kernel_id, now=False):
"""Shutdown a kernel by kernel_id"""
self._check_kernel_id(kernel_id)
kernel = self._kernels[kernel_id]
kernel._activity_stream.close()
self.stop_buffering(kernel_id)
self._kernel_connections.pop(kernel_id, None)
self.last_kernel_activity = utcnow()
return super(MappingKernelManager, self).shutdown_kernel(kernel_id,
now=now)
def record_activity(msg_list):
"""Record an IOPub message arriving from a kernel"""
kernel.last_activity = utcnow()
idents, fed_msg_list = kernel.session.feed_identities(msg_list)
msg = kernel.session.deserialize(fed_msg_list)
msg_type = msg['header']['msg_type']
self.log.debug("activity on %s: %s", kernel_id, msg_type)
if msg_type == 'status':
kernel.execution_state = msg['content']['execution_state']
def _finish_create(self, name, term):
# Monkey-patch last-activity, similar to kernels. Should we need
# more functionality per terminal, we can look into possible sub-
# classing or containment then.
term.last_activity = utcnow()
model = self.get_terminal_model(name)
# Increase the metric by one because a new terminal was created
TERMINAL_CURRENTLY_RUNNING_TOTAL.inc()
# Ensure culler is initialized
self._initialize_culler()
return model
def test_no_track_activity(self):
# initialize with old last api activity
old = utcnow() - timedelta(days=1)
settings = self.notebook.web_app.settings
settings['api_last_activity'] = old
# accessing status doesn't update activity
self.get('status')
assert settings['api_last_activity'] == old
# accessing with ?no_track_activity doesn't update activity
self.get('contents?no_track_activity=1')
assert settings['api_last_activity'] == old
# accessing without ?no_track_activity does update activity
self.get('contents')
assert settings['api_last_activity'] > old
def cull_kernel_if_idle(self, kernel_id):
kernel = self._kernels[kernel_id]
self.log.debug("kernel_id=%s, kernel_name=%s, last_activity=%s",
kernel_id, kernel.kernel_name, kernel.last_activity)
if kernel.last_activity is not None:
dt_now = utcnow()
dt_idle = dt_now - kernel.last_activity
if dt_idle > timedelta(seconds=self.cull_idle_timeout
): # exceeds timeout, can be culled
idle_duration = int(dt_idle.total_seconds())
self.log.warning(
"Culling kernel '%s' (%s) due to %s seconds of inactivity.",
kernel.kernel_name, kernel_id, idle_duration)
self.shutdown_kernel(kernel_id)
def cull_kernel_if_idle(self, kernel_id):
kernel = self._kernels[kernel_id]
self.log.debug("kernel_id=%s, kernel_name=%s, last_activity=%s", kernel_id, kernel.kernel_name, kernel.last_activity)
if kernel.last_activity is not None:
dt_now = utcnow()
dt_idle = dt_now - kernel.last_activity
# Compute idle properties
is_idle_time = dt_idle > timedelta(seconds=self.cull_idle_timeout)
is_idle_execute = self.cull_busy or (kernel.execution_state != 'busy')
connections = self._kernel_connections.get(kernel_id, 0)
is_idle_connected = self.cull_connected or not connections
# Cull the kernel if all three criteria are met
if (is_idle_time and is_idle_execute and is_idle_connected):
idle_duration = int(dt_idle.total_seconds())
self.log.warning("Culling '%s' kernel '%s' (%s) with %d connections due to %s seconds of inactivity.",
kernel.execution_state, kernel.kernel_name, kernel_id, connections, idle_duration)
self.shutdown_kernel(kernel_id)
def cull_kernel_if_idle(self, kernel_id):
kernel = self._kernels[kernel_id]
self.log.debug("kernel_id=%s, kernel_name=%s, last_activity=%s", kernel_id, kernel.kernel_name, kernel.last_activity)
if kernel.last_activity is not None:
dt_now = utcnow()
dt_idle = dt_now - kernel.last_activity
# Compute idle properties
is_idle_time = dt_idle > timedelta(seconds=self.cull_idle_timeout)
is_idle_execute = self.cull_busy or (kernel.execution_state != 'busy')
connections = self._kernel_connections.get(kernel_id, 0)
is_idle_connected = self.cull_connected or not connections
# Cull the kernel if all three criteria are met
if (is_idle_time and is_idle_execute and is_idle_connected):
idle_duration = int(dt_idle.total_seconds())
self.log.warning("Culling '%s' kernel '%s' (%s) with %d connections due to %s seconds of inactivity.",
kernel.execution_state, kernel.kernel_name, kernel_id, connections, idle_duration)
self.shutdown_kernel(kernel_id)
def shutdown_kernel(self, kernel_id, now=False):
"""Shutdown a kernel by kernel_id"""
self._check_kernel_id(kernel_id)
kernel = self._kernels[kernel_id]
if kernel._activity_stream:
kernel._activity_stream.close()
kernel._activity_stream = None
self.stop_buffering(kernel_id)
self._kernel_connections.pop(kernel_id, None)
self.last_kernel_activity = utcnow()
# Decrease the metric of number of kernels
# running for the relevant kernel type by 1
KERNEL_CURRENTLY_RUNNING_TOTAL.labels(
type=self._kernels[kernel_id].kernel_name
).dec()
return super(MappingKernelManager, self).shutdown_kernel(kernel_id, now=now)
def shutdown_kernel(self, kernel_id, now=False):
"""Shutdown a kernel by kernel_id"""
self._check_kernel_id(kernel_id)
kernel = self._kernels[kernel_id]
if kernel._activity_stream:
kernel._activity_stream.close()
kernel._activity_stream = None
self.stop_buffering(kernel_id)
self._kernel_connections.pop(kernel_id, None)
self.last_kernel_activity = utcnow()
# Decrease the metric of number of kernels
# running for the relevant kernel type by 1
KERNEL_CURRENTLY_RUNNING_TOTAL.labels(
type=self._kernels[kernel_id].kernel_name
).dec()
return super(MappingKernelManager, self).shutdown_kernel(kernel_id, now=now)
def run_cell(self, cell, cell_index, *args, **kwargs):
before = datetime.utcnow()
exec_reply, outs = super(ExecuteTimePreprocessor, self).run_cell(
cell, cell_index, *args, **kwargs)
if exec_reply.get('msg_type', '') == 'execute_reply':
ets = cell.setdefault('metadata', {}).setdefault('ExecuteTime', {})
if 'started' in exec_reply.get('metadata', {}):
# started value should is already a string, so don't isoformat
ets['start_time'] = exec_reply['metadata']['started']
else:
# attempt to fallback to datetime obj for execution request msg
ets['start_time'] = exec_reply.get(
'parent_header', {}).get('date', before).isoformat()
ets['end_time'] = (
exec_reply.get('header', {}).get('date') or nbtz.utcnow()
).isoformat()
return exec_reply, outs
async def _cull_inactive_terminal(self, name):
try:
term = self.terminals[name]
except KeyError:
return # KeyErrors are somewhat expected since the terminal can be terminated as the culling check is made.
self.log.debug("name=%s, last_activity=%s", name, term.last_activity)
if hasattr(term, 'last_activity'):
dt_now = utcnow()
dt_inactive = dt_now - term.last_activity
# Compute idle properties
is_time = dt_inactive > timedelta(
seconds=self.cull_inactive_timeout)
# Cull the kernel if all three criteria are met
if (is_time):
inactivity = int(dt_inactive.total_seconds())
self.log.warning(
"Culling terminal '%s' due to %s seconds of inactivity.",
name, inactivity)
await self.terminate(name, force=True)
def run_cell(self, cell, cell_index, *args, **kwargs):
before = datetime.utcnow()
exec_reply, outs = super(ExecuteTimePreprocessor,
self).run_cell(cell, cell_index, *args,
**kwargs)
if exec_reply.get('msg_type', '') == 'execute_reply':
ets = cell.setdefault('metadata', {}).setdefault('ExecuteTime', {})
if 'started' in exec_reply.get('metadata', {}):
# started value should is already a string, so don't isoformat
ets['start_time'] = exec_reply['metadata']['started']
else:
# attempt to fallback to datetime obj for execution request msg
ets['start_time'] = exec_reply.get('parent_header',
{}).get('date',
before).isoformat()
ets['end_time'] = (exec_reply.get('header', {}).get('date')
or nbtz.utcnow()).isoformat()
return exec_reply, outs
def on_message(self, message):
super(TermSocket, self).on_message(message)
self.application.settings['terminal_last_activity'] = utcnow()
def get_current_time():
# Return the current time stamp in UTC time epoch format in milliseconds, e.g.
return timegm(_tz.utcnow().utctimetuple()) * 1000
def __init__(self, **kwargs):
super(MappingKernelManager, self).__init__(**kwargs)
self.last_kernel_activity = utcnow()
def write_message(self, message, binary=False):
super(TermSocket, self).write_message(message, binary=binary)
self.application.settings['terminal_last_activity'] = utcnow()
def get_current_time():
# Return the current time stamp in UTC time epoch format in milliseconds, e.g.
return timegm(_tz.utcnow().utctimetuple()) * 1000
def __init__(self, **kwargs):
# Pin the superclass to better control the MRO.
self.pinned_superclass = AsyncMultiKernelManager
self.pinned_superclass.__init__(self, **kwargs)
self.last_kernel_activity = utcnow()
def write_message(self, message, binary=False):
with open('/tmp/test_terminal_output.txt', 'a') as f:
f.write(json.dumps(message))
super(TermSocket, self).write_message(message, binary=binary)
self.application.settings['terminal_last_activity'] = utcnow()
from functools import partial
from unittest import TestCase
from tornado import gen, web
from tornado.ioloop import IOLoop
from ..sessionmanager import SessionManager
from notebook.services.kernels.kernelmanager import MappingKernelManager
from notebook.services.contents.manager import ContentsManager
from notebook._tz import utcnow, isoformat
class DummyKernel(object):
def __init__(self, kernel_name='python'):
self.kernel_name = kernel_name
dummy_date = utcnow()
dummy_date_s = isoformat(dummy_date)
class DummyMKM(MappingKernelManager):
"""MappingKernelManager interface that doesn't start kernels, for testing"""
def __init__(self, *args, **kwargs):
super(DummyMKM, self).__init__(*args, **kwargs)
self.id_letters = iter(u'ABCDEFGHIJK')
def _new_id(self):
return next(self.id_letters)
def start_kernel(self, kernel_id=None, path=None, kernel_name='python', **kwargs):
kernel_id = kernel_id or self._new_id()
k = self._kernels[kernel_id] = DummyKernel(kernel_name=kernel_name)
self._kernel_connections[kernel_id] = 0
def write_message(self, message, binary=False):
super(TermSocket, self).write_message(message, binary=binary)
self.application.settings['terminal_last_activity'] = utcnow()
def __init__(self, **kwargs):
super(MappingKernelManager, self).__init__(**kwargs)
self.last_kernel_activity = utcnow()
def create_checkpoint(self, contents_mgr, path):
"""Create a checkpoint."""
return dict(id=path, last_modified=tz.utcnow())
def update_api_activity(self):
"""Update last_activity of API requests"""
# record activity of authenticated requests
if self._track_activity and self.get_current_user():
self.settings['api_last_activity'] = utcnow()
def update_api_activity(self):
"""Update last_activity of API requests"""
# record activity of authenticated requests
if self._track_activity and self.get_current_user():
self.settings['api_last_activity'] = utcnow()
def update_api_activity(self):
"""Update last_activity of API requests"""
# record activity of authenticated requests
if self._track_activity and getattr(self, '_user_cache', None):
self.settings['api_last_activity'] = utcnow()
from tornado import gen, web
from tornado.ioloop import IOLoop
from ..sessionmanager import SessionManager
from notebook.services.kernels.kernelmanager import MappingKernelManager
from notebook.services.contents.manager import ContentsManager
from notebook._tz import utcnow, isoformat
class DummyKernel(object):
def __init__(self, kernel_name='python'):
self.kernel_name = kernel_name
dummy_date = utcnow()
dummy_date_s = isoformat(dummy_date)
class DummyMKM(MappingKernelManager):
"""MappingKernelManager interface that doesn't start kernels, for testing"""
def __init__(self, *args, **kwargs):
super(DummyMKM, self).__init__(*args, **kwargs)
self.id_letters = iter(u'ABCDEFGHIJK')
def _new_id(self):
return next(self.id_letters)
def start_kernel(self,
kernel_id=None,
path=None,
def on_message(self, message):
super(TermSocket, self).on_message(message)
self.application.settings['terminal_last_activity'] = utcnow()
def _update_activity(self):
self.application.settings['terminal_last_activity'] = utcnow()
# terminal may not be around on deletion/cull
if self.term_name in self.terminal_manager.terminals:
self.terminal_manager.terminals[
self.term_name].last_activity = utcnow()