def get_client(cf):
connection_file = find_connection_file(cf)
km = KernelManager(connection_file=connection_file)
km.load_connection_file()
client = km.client()
return KernelClient(client)
def get_client(cf, profile=None):
"""
Usage:
>>> kc = get_client('kernel-143a2687-f294-42b1-bdcb-6f1cc2f4cc87.json', 'dale')
>>> data = kc.execute("'123'")
>>> data
{u'text/plain': u'123'}
"""
connection_file = find_connection_file(cf, profile=profile)
km = KernelManager(connection_file=connection_file)
km.load_connection_file()
client = km.client()
return KernelClient(client)
def get_client(cf, profile=None):
"""
Usage:
>>> kc = get_client('kernel-143a2687-f294-42b1-bdcb-6f1cc2f4cc87.json', 'dale')
>>> data = kc.execute("'123'")
>>> data
{u'text/plain': u'123'}
"""
connection_file = find_connection_file(cf, profile=profile)
km = KernelManager(connection_file=connection_file)
km.load_connection_file()
client = km.client()
return KernelClient(client)
def __enter__(self):
self.km = KernelManager()
self.km.start_kernel(
extra_arguments=self.extra_arguments,
stderr=open(os.devnull, 'w')
)
self.kc = self.km.client()
self.kc.start_channels()
self.iopub = self.kc.iopub_channel
self.shell = self.kc.shell_channel
# run %pylab inline, because some notebooks assume this
# even though they shouldn't
self.shell.send("pass")
self.shell.get_msg()
while True:
try:
self.iopub.get_msg(timeout=1)
except Empty:
break
return self
def preprocess(self, nb, resources):
kernel_name = nb.metadata.get('kernelspec', {}).get('name', 'python')
self.km = KernelManager(kernel_name=kernel_name)
self.km.start_kernel(startup_timeout=60, extra_arguments=self.extra_arguments)
self.kc = self.km.client()
self.kc.start_channels(stdin=False)
self.kc.wait_for_ready()
try:
self.log.info("Executing notebook with kernel: %s" % kernel_name)
nb, resources = super(ExecutePreprocessor, self).preprocess(nb, resources)
finally:
self.kc.stop_channels()
self.km.shutdown_kernel(now=True)
return nb, resources
def __enter__(self):
self.km = KernelManager()
self.km.start_kernel(extra_arguments=self.extra_arguments,
stderr=open(os.devnull, 'w'))
self.kc = self.km.client()
self.kc.start_channels()
self.iopub = self.kc.iopub_channel
self.shell = self.kc.shell_channel
# run %pylab inline, because some notebooks assume this
# even though they shouldn't
self.shell.send("pass")
self.shell.get_msg()
while True:
try:
self.iopub.get_msg(timeout=1)
except Empty:
break
return self
class IPyKernel(object):
"""
A simple wrapper class to run cells in an IPython Notebook.
Notes
-----
- Use `with` construct to properly instantiate
- IPython 3.0.0+ is assumed for this version
"""
def __init__(self, console=None, nb_version=4):
# default timeout time is 60 seconds
self.default_timeout = 60
self.extra_arguments = ['--pylab=inline']
self.nb_version = nb_version
def __enter__(self):
self.km = KernelManager()
self.km.start_kernel(extra_arguments=self.extra_arguments,
stderr=open(os.devnull, 'w'))
self.kc = self.km.client()
self.kc.start_channels()
self.iopub = self.kc.iopub_channel
self.shell = self.kc.shell_channel
# run %pylab inline, because some notebooks assume this
# even though they shouldn't
self.shell.send("pass")
self.shell.get_msg()
while True:
try:
self.iopub.get_msg(timeout=1)
except Empty:
break
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.kc.stop_channels()
self.km.shutdown_kernel()
del self.km
def run(self, cell, timeout=None):
"""
Run a notebook cell in the IPythonKernel
Parameters
----------
cell : IPython.notebook.Cell
the cell to be run
timeout : int or None (default)
the time in seconds after which a cell is stopped and assumed to
have timed out. If set to None the value in `default_timeout`
is used
Returns
-------
list of outs
a list of NotebookNodes of the returned types. This is
similar to the list of outputs generated when a cell is run
"""
use_timeout = self.default_timeout
if timeout is not None:
use_timeout = timeout
if hasattr(cell, 'input'):
self.kc.execute(cell.input)
elif hasattr(cell, 'source'):
self.kc.execute(cell.source)
else:
raise AttributeError('No source/input key')
self.shell.get_msg(timeout=use_timeout)
outs = []
while True:
try:
msg = self.iopub.get_msg(timeout=0.5)
except Empty:
break
msg_type = msg['msg_type']
if msg_type in ('status', 'pyin', 'execute_input'):
continue
elif msg_type == 'clear_output':
outs = []
continue
content = msg['content']
out = NotebookNode(output_type=msg_type)
if msg_type == 'stream':
out.name = content['name']
out.text = content['text']
elif msg_type in ('display_data', 'pyout', 'execute_result'):
if hasattr(content, 'execution_count'):
out['execution_count'] = content['execution_count']
else:
out['execution_count'] = None
out['data'] = content['data']
out['metadata'] = content['metadata']
elif msg_type == 'error':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
else:
print "unhandled iopub msg:", msg_type
outs.append(out)
return outs
def sanitize(self, s):
"""sanitize a string for comparison.
fix universal newlines, strip trailing newlines, and normalize likely
random values (memory addresses and UUIDs)
"""
if not isinstance(s, basestring):
return s
# normalize newline:
s = s.replace('\r\n', '\n')
# ignore trailing newlines (but not space)
s = s.rstrip('\n')
# normalize hex addresses:
s = re.sub(r'0x[a-f0-9]+', '0xFFFFFFFF', s)
# normalize UUIDs:
s = re.sub(r'[a-f0-9]{8}(\-[a-f0-9]{4}){3}\-[a-f0-9]{12}', 'U-U-I-D',
s)
# fix problem with
return s
def compare_outputs(self,
test,
ref,
skip_compare=('traceback', 'latex',
'execution_count')):
"""
Compare two lists of `NotebookNode`s
Parameters
----------
test : list of `NotebookNode`
the list of be tested generated by the kernel
ref : list of `NotebookNode`
the reference list read from the notebook
skip_compare : list of str
a list of strings that name node types that are not to be tested
Returns
-------
bool
is True if both lists are different
list of diff
a list of diff (str) the represent the differences
"""
diff = False
diff_list = []
if self.nb_version == 4:
for key in ref:
if key not in test:
return True, [
"missing key: %s != %s" % (test.keys(), ref.keys())
]
elif key not in skip_compare:
if key == 'data':
for data_key in test[key]:
my_diff = self.do_diff(data_key, test[key],
ref[key])
if my_diff is not None:
diff_list += my_diff
diff = True
else:
# can this happen?
my_diff = self.do_diff(key, test, ref)
if my_diff is not None:
diff_list += my_diff
diff = True
return diff, diff_list
def do_diff(self, key, test_cell, ref_cell):
"""
Compare the key of two dicts
Parameters
----------
key : string
the key to be compared
test_cell : dict
a dict with `key` as a key of string value
ref_cell : dict
a dict with `key` as a key of string value
Returns
-------
list of diff (str)
a list of diff representing the differences
"""
if hasattr(ref_cell, key):
s1 = self.sanitize(ref_cell[key])
else:
s1 = ''
if hasattr(test_cell, key):
s2 = self.sanitize(test_cell[key])
else:
s2 = ''
if key in ['image/png', 'image/svg', 'image/svg+xml']:
if s1 != s2:
return [
'>>> diff in %s (size new : %d vs size old : %d )' %
(key, len(s1), len(s2))
]
else:
if s1 != s2:
expected = s1.splitlines(1)
actual = s2.splitlines(1)
diff = difflib.ndiff(expected, actual)
return ['>>> diff in ' + key] + list(diff)
return None
def get_commands(self, cell):
"""
Extract potential commands from the first line of a cell
if a code cell starts with the hashbang `#!` it can be followed by
a comma separated list of commands. Each command can be
a single key `skip`
or
a key/value pair separated by a colon `timeout:[int]`
Parameters
----------
cell : a NotebookCell
the cell to be examined
Returns
-------
dict
a dict of key/value pairs. For a single command the value is `True`
"""
commands = {}
source = self.get_source(cell)
if source is not None:
lines = source.splitlines()
if len(lines) > 0:
first_line = lines[0]
if first_line.startswith('#!'):
txt = first_line[2:].strip()
parts = txt.split(',')
for part in parts:
subparts = part.split(':')
if len(subparts) == 1:
commands[subparts[0].strip().lower()] = True
elif len(subparts) == 2:
commands[subparts[0].strip().lower()] = subparts[1]
return commands
def get_source(self, cell):
"""
get the source code of a cell
Notes
-----
This is legacy of IPython 2/3 conversion.
Parameters
----------
cell : a NotebookCell
the cell to be examined
Returns
-------
string
the source code
"""
if cell.cell_type == 'code':
if hasattr(cell, 'input'):
return cell.input
elif hasattr(cell, 'source'):
return cell.source
else:
return None
def is_empty_cell(self, cell):
"""
Check if a cell has no code
Parameters
----------
cell : a NotebookCell
the cell to be examined
Returns
-------
bool
True if the cell has no code, False otherwise
"""
source = self.get_source(cell)
if source is None or source == '':
return True
else:
return False
__author__ = 'gilgamesh'
from tornado.ioloop import IOLoop
from tornado.web import RequestHandler, Application, url, StaticFileHandler
from IPython.kernel.manager import KernelManager
import Handlers
_man=KernelManager()
_man.start_kernel()
_cli = _man.client()
_cli.start_channels()
_chan = _cli.shell_channel
_chan.execute("i=0")
def make_app():
return Application([
(r"/static/(.*)", StaticFileHandler, {"path": "/home/lakmal/PycharmProjects/AnantaFramework/ananta_gui/html"}),
url(r"/", Handlers.HelloHandler),
url(r"/ws/load",Handlers.FileLoadHandler),
url(r"/ws/read_des",Handlers.DesReadHandler),
url(r"/ws/test/",Handlers.TestHandler)
])
def main():
app = make_app()
app.listen(8800)
IOLoop.current().start()
class IPyKernel(object):
"""
A simple wrapper class to run cells in an IPython Notebook.
Notes
-----
- Use `with` construct to properly instantiate
- IPython 3.0.0+ is assumed for this version
"""
def __init__(self, console = None, nb_version=4):
# default timeout time is 60 seconds
self.default_timeout = 60
self.extra_arguments = ['--pylab=inline']
self.nb_version = nb_version
def __enter__(self):
self.km = KernelManager()
self.km.start_kernel(
extra_arguments=self.extra_arguments,
stderr=open(os.devnull, 'w')
)
self.kc = self.km.client()
self.kc.start_channels()
self.iopub = self.kc.iopub_channel
self.shell = self.kc.shell_channel
# run %pylab inline, because some notebooks assume this
# even though they shouldn't
self.shell.send("pass")
self.shell.get_msg()
while True:
try:
self.iopub.get_msg(timeout=1)
except Empty:
break
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.kc.stop_channels()
self.km.shutdown_kernel()
del self.km
def run(self, cell, timeout = None):
"""
Run a notebook cell in the IPythonKernel
Parameters
----------
cell : IPython.notebook.Cell
the cell to be run
timeout : int or None (default)
the time in seconds after which a cell is stopped and assumed to
have timed out. If set to None the value in `default_timeout`
is used
Returns
-------
list of outs
a list of NotebookNodes of the returned types. This is
similar to the list of outputs generated when a cell is run
"""
use_timeout = self.default_timeout
if timeout is not None:
use_timeout = timeout
if hasattr(cell, 'input'):
self.kc.execute(cell.input)
elif hasattr(cell, 'source'):
self.kc.execute(cell.source)
else:
raise AttributeError('No source/input key')
self.shell.get_msg(timeout=use_timeout)
outs = []
while True:
try:
msg = self.iopub.get_msg(timeout=0.5)
except Empty:
break
msg_type = msg['msg_type']
if msg_type in ('status', 'pyin', 'execute_input'):
continue
elif msg_type == 'clear_output':
outs = []
continue
content = msg['content']
out = NotebookNode(output_type=msg_type)
if msg_type == 'stream':
out.name = content['name']
out.text = content['text']
elif msg_type in ('display_data', 'pyout', 'execute_result'):
if hasattr(content, 'execution_count'):
out['execution_count'] = content['execution_count']
else:
out['execution_count'] = None
out['data'] = content['data']
out['metadata'] = content['metadata']
elif msg_type == 'error':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
else:
print "unhandled iopub msg:", msg_type
outs.append(out)
return outs
def sanitize(self, s):
"""sanitize a string for comparison.
fix universal newlines, strip trailing newlines, and normalize likely
random values (memory addresses and UUIDs)
"""
if not isinstance(s, basestring):
return s
# normalize newline:
s = s.replace('\r\n', '\n')
# ignore trailing newlines (but not space)
s = s.rstrip('\n')
# normalize hex addresses:
s = re.sub(r'0x[a-f0-9]+', '0xFFFFFFFF', s)
# normalize UUIDs:
s = re.sub(r'[a-f0-9]{8}(\-[a-f0-9]{4}){3}\-[a-f0-9]{12}', 'U-U-I-D', s)
# fix problem with
return s
def compare_outputs(
self,
test,
ref,
skip_compare=('traceback', 'latex', 'execution_count')
):
"""
Compare two lists of `NotebookNode`s
Parameters
----------
test : list of `NotebookNode`
the list of be tested generated by the kernel
ref : list of `NotebookNode`
the reference list read from the notebook
skip_compare : list of str
a list of strings that name node types that are not to be tested
Returns
-------
bool
is True if both lists are different
list of diff
a list of diff (str) the represent the differences
"""
diff = False
diff_list = []
if self.nb_version == 4:
for key in ref:
if key not in test:
return True, [ "missing key: %s != %s" %
(test.keys(), ref.keys()) ]
elif key not in skip_compare:
if key == 'data':
for data_key in test[key]:
my_diff = self.do_diff(
data_key,
test[key],
ref[key])
if my_diff is not None:
diff_list += my_diff
diff = True
else:
# can this happen?
my_diff = self.do_diff(key, test, ref)
if my_diff is not None:
diff_list += my_diff
diff = True
return diff, diff_list
def do_diff(self, key, test_cell, ref_cell):
"""
Compare the key of two dicts
Parameters
----------
key : string
the key to be compared
test_cell : dict
a dict with `key` as a key of string value
ref_cell : dict
a dict with `key` as a key of string value
Returns
-------
list of diff (str)
a list of diff representing the differences
"""
if hasattr(ref_cell, key):
s1 = self.sanitize(ref_cell[key])
else:
s1 = ''
if hasattr(test_cell, key):
s2 = self.sanitize(test_cell[key])
else:
s2 = ''
if key in ['image/png', 'image/svg', 'image/svg+xml']:
if s1 != s2:
return ['>>> diff in %s (size new : %d vs size old : %d )' %
(key, len(s1), len(s2) )]
else:
if s1 != s2:
expected=s1.splitlines(1)
actual=s2.splitlines(1)
diff=difflib.ndiff(expected, actual)
return [ '>>> diff in ' + key ] + list(diff)
return None
def get_commands(self, cell):
"""
Extract potential commands from the first line of a cell
if a code cell starts with the hashbang `#!` it can be followed by
a comma separated list of commands. Each command can be
a single key `skip`
or
a key/value pair separated by a colon `timeout:[int]`
Parameters
----------
cell : a NotebookCell
the cell to be examined
Returns
-------
dict
a dict of key/value pairs. For a single command the value is `True`
"""
commands = {}
source = self.get_source(cell)
if source is not None:
lines = source.splitlines()
if len(lines) > 0:
first_line = lines[0]
if first_line.startswith('#!'):
txt = first_line[2:].strip()
parts = txt.split(',')
for part in parts:
subparts = part.split(':')
if len(subparts) == 1:
commands[subparts[0].strip().lower()] = True
elif len(subparts) == 2:
commands[subparts[0].strip().lower()] = subparts[1]
return commands
def get_source(self, cell):
"""
get the source code of a cell
Notes
-----
This is legacy of IPython 2/3 conversion.
Parameters
----------
cell : a NotebookCell
the cell to be examined
Returns
-------
string
the source code
"""
if cell.cell_type == 'code':
if hasattr(cell, 'input'):
return cell.input
elif hasattr(cell, 'source'):
return cell.source
else:
return None
def is_empty_cell(self, cell):
"""
Check if a cell has no code
Parameters
----------
cell : a NotebookCell
the cell to be examined
Returns
-------
bool
True if the cell has no code, False otherwise
"""
source = self.get_source(cell)
if source is None or source == '':
return True
else:
return False
class Execute(ExecutePreprocessor):
def preprocess(self, nb, resources):
kernel_name = nb.metadata.get('kernelspec', {}).get('name', 'python')
self.km = KernelManager(kernel_name=kernel_name)
self.km.start_kernel(startup_timeout=60, extra_arguments=self.extra_arguments)
self.kc = self.km.client()
self.kc.start_channels(stdin=False)
self.kc.wait_for_ready()
try:
self.log.info("Executing notebook with kernel: %s" % kernel_name)
nb, resources = super(ExecutePreprocessor, self).preprocess(nb, resources)
finally:
self.kc.stop_channels()
self.km.shutdown_kernel(now=True)
return nb, resources
def run_cell(self, cell):
msg_id = self.kc.execute(cell.source)
self.log.debug("Executing cell:\n%s", cell.source)
# wait for finish, with timeout
while True:
try:
msg = self.kc.shell_channel.get_msg(timeout=self.timeout)
except Empty:
self.log.error("Timeout waiting for execute reply")
self.log.error("Interrupting kernel")
self.km.interrupt_kernel()
break
if msg['parent_header'].get('msg_id') == msg_id:
break
else:
# not our reply
continue
outs = []
while True:
try:
msg = self.kc.iopub_channel.get_msg(timeout=self.timeout)
except Empty:
self.log.warn("Timeout waiting for IOPub output")
break
if msg['parent_header'].get('msg_id') != msg_id:
# not an output from our execution
continue
msg_type = msg['msg_type']
self.log.debug("output: %s", msg_type)
content = msg['content']
# set the prompt number for the input and the output
if 'execution_count' in content:
cell['execution_count'] = content['execution_count']
if msg_type == 'status':
self.log.debug(content)
if content['execution_state'] == 'idle':
break
else:
continue
elif msg_type == 'execute_input':
continue
elif msg_type == 'clear_output':
outs = []
continue
try:
out = output_from_msg(msg)
except ValueError:
self.log.error("unhandled iopub msg: " + msg_type)
else:
outs.append(out)
return outs
