def row_to_model(self, row, tolerate_culled=False):
"""Takes sqlite database session row and turns it into a dictionary"""
kernel_culled = yield maybe_future(self.kernel_culled(
row['kernel_id']))
if kernel_culled:
# The kernel was culled or died without deleting the session.
# We can't use delete_session here because that tries to find
# and shut down the kernel - so we'll delete the row directly.
#
# If caller wishes to tolerate culled kernels, log a warning
# and return None. Otherwise, raise KeyError with a similar
# message.
self.cursor.execute("DELETE FROM session WHERE session_id=?",
(row['session_id'], ))
msg = "Kernel '{kernel_id}' appears to have been culled or died unexpectedly, " \
"invalidating session '{session_id}'. The session has been removed.".\
format(kernel_id=row['kernel_id'],session_id=row['session_id'])
if tolerate_culled:
self.log.warning(msg + " Continuing...")
raise gen.Return(None)
raise KeyError(msg)
kernel_model = yield maybe_future(
self.kernel_manager.kernel_model(row['kernel_id']))
model = {
'id': row['session_id'],
'path': row['path'],
'name': row['name'],
'type': row['type'],
'kernel': kernel_model
}
if row['type'] == 'notebook':
# Provide the deprecated API.
model['notebook'] = {'path': row['path'], 'name': row['name']}
raise gen.Return(model)
def post(self, path=''):
"""Create a new file in the specified path.
POST creates new files. The server always decides on the name.
POST /api/contents/path
New untitled, empty file or directory.
POST /api/contents/path
with body {"copy_from" : "/path/to/OtherNotebook.ipynb"}
New copy of OtherNotebook in path
"""
cm = self.contents_manager
file_exists = yield maybe_future(cm.file_exists(path))
if file_exists:
raise web.HTTPError(400, "Cannot POST to files, use PUT instead.")
dir_exists = yield maybe_future(cm.dir_exists(path))
if not dir_exists:
raise web.HTTPError(404, "No such directory: %s" % path)
model = self.get_json_body()
if model is not None:
copy_from = model.get('copy_from')
ext = model.get('ext', '')
type = model.get('type', '')
if copy_from:
yield self._copy(copy_from, path)
else:
yield self._new_untitled(path, type=type, ext=ext)
else:
yield self._new_untitled(path)
def delete_session(self, session_id):
"""Deletes the row in the session database with given session_id"""
session = yield maybe_future(self.get_session(session_id=session_id))
yield maybe_future(
self.kernel_manager.shutdown_kernel(session["kernel"]["id"]))
self.cursor.execute("DELETE FROM session WHERE session_id=?",
(session_id, ))
def get(self, *args, **kwargs):
# pre_get can be a coroutine in subclasses
# assign and yield in two step to avoid tornado 3 issues
res = self.pre_get()
yield maybe_future(res)
res = super(AuthenticatedZMQStreamHandler, self).get(*args, **kwargs)
yield maybe_future(res)
def patch(self, session_id):
"""Patch updates sessions:
- path updates session to track renamed paths
- kernel.name starts a new kernel with a given kernelspec
"""
sm = self.session_manager
km = self.kernel_manager
model = self.get_json_body()
if model is None:
raise web.HTTPError(400, "No JSON data provided")
# get the previous session model
before = yield maybe_future(sm.get_session(session_id=session_id))
changes = {}
if "notebook" in model and "path" in model["notebook"]:
self.log.warning("Sessions API changed, see updated swagger docs")
model["path"] = model["notebook"]["path"]
model["type"] = "notebook"
if "path" in model:
changes["path"] = model["path"]
if "name" in model:
changes["name"] = model["name"]
if "type" in model:
changes["type"] = model["type"]
if "kernel" in model:
# Kernel id takes precedence over name.
if model["kernel"].get("id") is not None:
kernel_id = model["kernel"]["id"]
if kernel_id not in km:
raise web.HTTPError(400, "No such kernel: %s" % kernel_id)
changes["kernel_id"] = kernel_id
elif model["kernel"].get("name") is not None:
kernel_name = model["kernel"]["name"]
kernel_id = yield sm.start_kernel_for_session(
session_id,
kernel_name=kernel_name,
name=before["name"],
path=before["path"],
type=before["type"],
)
changes["kernel_id"] = kernel_id
yield maybe_future(sm.update_session(session_id, **changes))
model = yield maybe_future(sm.get_session(session_id=session_id))
if model["kernel"]["id"] != before["kernel"]["id"]:
# kernel_id changed because we got a new kernel
# shutdown the old one
yield maybe_future(km.shutdown_kernel(before["kernel"]["id"]))
self.finish(json.dumps(model, default=date_default))
def post(self):
km = self.kernel_manager
model = self.get_json_body()
if model is None:
model = {'name': km.default_kernel_name}
else:
model.setdefault('name', km.default_kernel_name)
kernel_id = yield maybe_future(
km.start_kernel(kernel_name=model['name']))
model = yield maybe_future(km.kernel_model(kernel_id))
location = url_path_join(self.base_url, 'api', 'kernels',
url_escape(kernel_id))
self.set_header('Location', location)
self.set_status(201)
self.finish(json.dumps(model, default=date_default))
def get(self, path=""):
"""Return a model for a file or directory.
A directory model contains a list of models (without content)
of the files and directories it contains.
"""
path = path or ""
type = self.get_query_argument("type", default=None)
if type not in {None, "directory", "file", "notebook"}:
raise web.HTTPError(400, u"Type %r is invalid" % type)
format = self.get_query_argument("format", default=None)
if format not in {None, "text", "base64"}:
raise web.HTTPError(400, u"Format %r is invalid" % format)
content = self.get_query_argument("content", default="1")
if content not in {"0", "1"}:
raise web.HTTPError(400, u"Content %r is invalid" % content)
content = int(content)
model = yield maybe_future(
self.contents_manager.get(
path=path,
type=type,
format=format,
content=content,
))
validate_model(model, expect_content=content)
self._finish_model(model, location=False)
def save_session(self,
session_id,
path=None,
name=None,
type=None,
kernel_id=None):
"""Saves the items for the session with the given session_id
Given a session_id (and any other of the arguments), this method
creates a row in the sqlite session database that holds the information
for a session.
Parameters
----------
session_id : str
uuid for the session; this method must be given a session_id
path : str
the path for the given session
name: str
the name of the session
type: string
the type of the session
kernel_id : str
a uuid for the kernel associated with this session
Returns
-------
model : dict
a dictionary of the session model
"""
self.cursor.execute("INSERT INTO session VALUES (?,?,?,?,?)",
(session_id, path, name, type, kernel_id))
result = yield maybe_future(self.get_session(session_id=session_id))
raise gen.Return(result)
def delete(self, path=''):
"""delete a file in the given path"""
cm = self.contents_manager
self.log.warning('delete %s', path)
yield maybe_future(cm.delete(path))
self.set_status(204)
self.finish()
def update_session(self, session_id, **kwargs):
"""Updates the values in the session database.
Changes the values of the session with the given session_id
with the values from the keyword arguments.
Parameters
----------
session_id : str
a uuid that identifies a session in the sqlite3 database
**kwargs : str
the key must correspond to a column title in session database,
and the value replaces the current value in the session
with session_id.
"""
yield maybe_future(self.get_session(session_id=session_id))
if not kwargs:
# no changes
return
sets = []
for column in kwargs.keys():
if column not in self._columns:
raise TypeError("No such column: %r" % column)
sets.append("%s=?" % column)
query = "UPDATE session SET %s WHERE session_id=?" % (", ".join(sets))
self.cursor.execute(query, list(kwargs.values()) + [session_id])
def get(self, path=''):
"""Return a model for a file or directory.
A directory model contains a list of models (without content)
of the files and directories it contains.
"""
path = path or ''
type = self.get_query_argument('type', default=None)
if type not in {None, 'directory', 'file', 'notebook'}:
raise web.HTTPError(400, u'Type %r is invalid' % type)
format = self.get_query_argument('format', default=None)
if format not in {None, 'text', 'base64'}:
raise web.HTTPError(400, u'Format %r is invalid' % format)
content = self.get_query_argument('content', default='1')
if content not in {'0', '1'}:
raise web.HTTPError(400, u'Content %r is invalid' % content)
content = int(content)
model = yield maybe_future(
self.contents_manager.get(
path=path,
type=type,
format=format,
content=content,
))
validate_model(model, expect_content=content)
self._finish_model(model, location=False)
def post(self):
km = self.kernel_manager
model = self.get_json_body()
if model is None:
model = {"name": km.default_kernel_name}
else:
model.setdefault("name", km.default_kernel_name)
kernel_id = yield maybe_future(
km.start_kernel(kernel_name=model["name"]))
model = yield maybe_future(km.kernel_model(kernel_id))
location = url_path_join(self.base_url, "api", "kernels",
url_escape(kernel_id))
self.set_header("Location", location)
self.set_status(201)
self.finish(json.dumps(model, default=date_default))
def post(self, kernel_id, action):
km = self.kernel_manager
if action == "interrupt":
km.interrupt_kernel(kernel_id)
self.set_status(204)
if action == "restart":
try:
yield maybe_future(km.restart_kernel(kernel_id))
except Exception as e:
self.log.error("Exception restarting kernel", exc_info=True)
self.set_status(500)
else:
model = yield maybe_future(km.kernel_model(kernel_id))
self.write(json.dumps(model, default=date_default))
self.finish()
def _upload(self, model, path):
"""Handle upload of a new file to path"""
self.log.info(u"Uploading file to %s", path)
model = yield maybe_future(self.contents_manager.new(model, path))
self.set_status(201)
validate_model(model, expect_content=False)
self._finish_model(model)
def _new_untitled(self, path, type='', ext=''):
"""Create a new, empty untitled entity"""
self.log.info(u"Creating new %s in %s", type or 'file', path)
model = yield maybe_future(
self.contents_manager.new_untitled(path=path, type=type, ext=ext))
self.set_status(201)
validate_model(model, expect_content=False)
self._finish_model(model)
def _save(self, model, path):
"""Save an existing file."""
chunk = model.get("chunk", None)
if not chunk or chunk == -1: # Avoid tedious log information
self.log.info(u"Saving file at %s", path)
model = yield maybe_future(self.contents_manager.save(model, path))
validate_model(model, expect_content=False)
self._finish_model(model)
def patch(self, path=''):
"""PATCH renames a file or directory without re-uploading content."""
cm = self.contents_manager
model = self.get_json_body()
if model is None:
raise web.HTTPError(400, u'JSON body missing')
model = yield maybe_future(cm.update(model, path))
validate_model(model, expect_content=False)
self._finish_model(model)
def delete(self, session_id):
# Deletes the session with given session_id
sm = self.session_manager
try:
yield maybe_future(sm.delete_session(session_id))
except KeyError:
# the kernel was deleted but the session wasn't!
raise web.HTTPError(410, "Kernel deleted before session")
self.set_status(204)
self.finish()
def start_kernel_for_session(self, session_id, path, name, type,
kernel_name):
"""Start a new kernel for a given session."""
# allow contents manager to specify kernels cwd
kernel_path = self.contents_manager.get_kernel_path(path=path)
kernel_id = yield maybe_future(
self.kernel_manager.start_kernel(path=kernel_path,
kernel_name=kernel_name))
# py2-compat
raise gen.Return(kernel_id)
def _copy(self, copy_from, copy_to=None):
"""Copy a file, optionally specifying a target directory."""
self.log.info(u"Copying {copy_from} to {copy_to}".format(
copy_from=copy_from,
copy_to=copy_to or '',
))
model = yield maybe_future(
self.contents_manager.copy(copy_from, copy_to))
self.set_status(201)
validate_model(model, expect_content=False)
self._finish_model(model)
def post(self, path=''):
"""post creates a new checkpoint"""
cm = self.contents_manager
checkpoint = yield maybe_future(cm.create_checkpoint(path))
data = json.dumps(checkpoint, default=date_default)
location = url_path_join(self.base_url, 'api/contents',
url_escape(path), 'checkpoints',
url_escape(checkpoint['id']))
self.set_header('Location', location)
self.set_status(201)
self.finish(data)
def list_sessions(self):
"""Returns a list of dictionaries containing all the information from
the session database"""
c = self.cursor.execute("SELECT * FROM session")
result = []
# We need to use fetchall() here, because row_to_model can delete rows,
# which messes up the cursor if we're iterating over rows.
for row in c.fetchall():
try:
model = yield maybe_future(self.row_to_model(row))
result.append(model)
except KeyError:
pass
raise gen.Return(result)
def put(self, path=''):
"""Saves the file in the location specified by name and path.
PUT is very similar to POST, but the requester specifies the name,
whereas with POST, the server picks the name.
PUT /api/contents/path/Name.ipynb
Save notebook at ``path/Name.ipynb``. Notebook structure is specified
in `content` key of JSON request body. If content is not specified,
create a new empty notebook.
"""
model = self.get_json_body()
if model:
if model.get('copy_from'):
raise web.HTTPError(400, "Cannot copy with PUT, only POST")
exists = yield maybe_future(
self.contents_manager.file_exists(path))
if exists:
yield maybe_future(self._save(model, path))
else:
yield maybe_future(self._upload(model, path))
else:
yield maybe_future(self._new_untitled(path))
def post(self, path=""):
"""post creates a new checkpoint"""
cm = self.contents_manager
checkpoint = yield maybe_future(cm.create_checkpoint(path))
data = json.dumps(checkpoint, default=date_default)
location = url_path_join(
self.base_url,
"api/contents",
url_escape(path),
"checkpoints",
url_escape(checkpoint["id"]),
)
self.set_header("Location", location)
self.set_status(201)
self.finish(data)
def get(self):
# if started was missing, use unix epoch
started = self.settings.get('started', utcfromtimestamp(0))
started = isoformat(started)
kernels = yield maybe_future(self.kernel_manager.list_kernels())
total_connections = sum(k['connections'] for k in kernels)
last_activity = isoformat(self.application.last_activity())
model = {
'started': started,
'last_activity': last_activity,
'kernels': len(kernels),
'connections': total_connections,
}
self.finish(json.dumps(model, sort_keys=True))
def start_kernel(self, kernel_id=None, path=None, **kwargs):
"""Start a kernel for a session and return its kernel_id.
Parameters
----------
kernel_id : uuid
The uuid to associate the new kernel with. If this
is not None, this kernel will be persistent whenever it is
requested.
path : API path
The API path (unicode, '/' delimited) for the cwd.
Will be transformed to an OS path relative to root_dir.
kernel_name : str
The name identifying which kernel spec to launch. This is ignored if
an existing kernel is returned, but it may be checked in the future.
"""
if kernel_id is None:
if path is not None:
kwargs['cwd'] = self.cwd_for_path(path)
kernel_id = yield maybe_future(
super(MappingKernelManager, self).start_kernel(**kwargs)
)
self._kernel_connections[kernel_id] = 0
self.start_watching_activity(kernel_id)
self.log.info("Kernel started: %s" % kernel_id)
self.log.debug("Kernel args: %r" % kwargs)
# register callback for failed auto-restart
self.add_restart_callback(kernel_id,
lambda : self._handle_kernel_died(kernel_id),
'dead',
)
# Increase the metric of number of kernels running
# for the relevant kernel type by 1
KERNEL_CURRENTLY_RUNNING_TOTAL.labels(
type=self._kernels[kernel_id].kernel_name
).inc()
else:
self._check_kernel_id(kernel_id)
self.log.info("Using existing kernel: %s" % kernel_id)
# Initialize culling if not already
if not self._initialized_culler:
self.initialize_culler()
# py2-compat
raise gen.Return(kernel_id)
def session_exists(self, path):
"""Check to see if the session of a given name exists"""
exists = False
self.cursor.execute("SELECT * FROM session WHERE path=?", (path, ))
row = self.cursor.fetchone()
if row is not None:
# Note, although we found a row for the session, the associated kernel may have
# been culled or died unexpectedly. If that's the case, we should delete the
# row, thereby terminating the session. This can be done via a call to
# row_to_model that tolerates that condition. If row_to_model returns None,
# we'll return false, since, at that point, the session doesn't exist anyway.
model = yield maybe_future(
self.row_to_model(row, tolerate_culled=True))
if model is not None:
exists = True
raise gen.Return(exists)
def get_session(self, **kwargs):
"""Returns the model for a particular session.
Takes a keyword argument and searches for the value in the session
database, then returns the rest of the session's info.
Parameters
----------
**kwargs : keyword argument
must be given one of the keywords and values from the session database
(i.e. session_id, path, name, type, kernel_id)
Returns
-------
model : dict
returns a dictionary that includes all the information from the
session described by the kwarg.
"""
if not kwargs:
raise TypeError("must specify a column to query")
conditions = []
for column in kwargs.keys():
if column not in self._columns:
raise TypeError("No such column: %r", column)
conditions.append("%s=?" % column)
query = "SELECT * FROM session WHERE %s" % (" AND ".join(conditions))
self.cursor.execute(query, list(kwargs.values()))
try:
row = self.cursor.fetchone()
except KeyError:
# The kernel is missing, so the session just got deleted.
row = None
if row is None:
q = []
for key, value in kwargs.items():
q.append("%s=%r" % (key, value))
raise web.HTTPError(404, u"Session not found: %s" % (", ".join(q)))
model = yield maybe_future(self.row_to_model(row))
raise gen.Return(model)
def get(self, path, include_body=True):
cm = self.contents_manager
if cm.is_hidden(path) and not cm.allow_hidden:
self.log.info("Refusing to serve hidden file, via 404 Error")
raise web.HTTPError(404)
path = path.strip('/')
if '/' in path:
_, name = path.rsplit('/', 1)
else:
name = path
model = yield maybe_future(
cm.get(path, type='file', content=include_body))
if self.get_argument("download", False):
self.set_attachment_header(name)
# get mimetype from filename
if name.lower().endswith('.ipynb'):
self.set_header('Content-Type', 'application/x-ipynb+json')
else:
cur_mime = mimetypes.guess_type(name)[0]
if cur_mime == 'text/plain':
self.set_header('Content-Type', 'text/plain; charset=UTF-8')
elif cur_mime is not None:
self.set_header('Content-Type', cur_mime)
else:
if model['format'] == 'base64':
self.set_header('Content-Type', 'application/octet-stream')
else:
self.set_header('Content-Type',
'text/plain; charset=UTF-8')
if include_body:
if model['format'] == 'base64':
b64_bytes = model['content'].encode('ascii')
self.write(decodebytes(b64_bytes))
elif model['format'] == 'json':
self.write(json.dumps(model['content']))
else:
self.write(model['content'])
self.flush()
def get(self, path, include_body=True):
cm = self.contents_manager
if cm.is_hidden(path) and not cm.allow_hidden:
self.log.info("Refusing to serve hidden file, via 404 Error")
raise web.HTTPError(404)
path = path.strip("/")
if "/" in path:
_, name = path.rsplit("/", 1)
else:
name = path
model = yield maybe_future(
cm.get(path, type="file", content=include_body))
if self.get_argument("download", False):
self.set_attachment_header(name)
# get mimetype from filename
if name.lower().endswith(".ipynb"):
self.set_header("Content-Type", "application/x-ipynb+json")
else:
cur_mime = mimetypes.guess_type(name)[0]
if cur_mime == "text/plain":
self.set_header("Content-Type", "text/plain; charset=UTF-8")
elif cur_mime is not None:
self.set_header("Content-Type", cur_mime)
else:
if model["format"] == "base64":
self.set_header("Content-Type", "application/octet-stream")
else:
self.set_header("Content-Type",
"text/plain; charset=UTF-8")
if include_body:
if model["format"] == "base64":
b64_bytes = model["content"].encode("ascii")
self.write(decodebytes(b64_bytes))
elif model["format"] == "json":
self.write(json.dumps(model["content"]))
else:
self.write(model["content"])
self.flush()