def km_from_string(s=''): """create kernel manager from IPKernelApp string such as '--shell=47378 --iopub=39859 --stdin=36778 --hb=52668' for IPython 0.11 or just 'kernel-12345.json' for IPython 0.12 """ global km, kc, send from os.path import join as pjoin from IPython.config.loader import KeyValueConfigLoader try: from IPython.kernel import ( KernelManager, find_connection_file, ) except ImportError: from IPython.zmq.blockingkernelmanager import BlockingKernelManager as KernelManager from IPython.zmq.kernelapp import kernel_aliases from IPython.lib.kernel import find_connection_file s = s.replace('--existing', '') if '--profile' in s: k, p = s.split('--profile') k = k.lstrip().rstrip() # kernel part of the string p = p.lstrip().rstrip() # profile part of the string fullpath = find_connection_file(k, p) else: fullpath = find_connection_file(s.lstrip().rstrip()) km = KernelManager(connection_file=fullpath) km.load_connection_file() km.start_channels() send = km.shell_channel.execute respond(None, "python.client.error.ipython-version", None) return s = s.replace('--existing', '') if '--profile' in s: k, p = s.split('--profile') k = k.lstrip().rstrip() # kernel part of the string p = p.lstrip().rstrip() # profile part of the string fullpath = find_connection_file(k, p) else: fullpath = find_connection_file(s.lstrip().rstrip()) km = KernelManager(connection_file=fullpath) km.load_connection_file() kc = km.client() kc.start_channels() send = kc.shell_channel.execute return km
def __init__(self, nb_in, pylab=False, mpl_inline=False): self.km = KernelManager() if pylab: self.km.start_kernel(extra_arguments=['--pylab=inline']) elif mpl_inline: self.km.start_kernel(extra_arguments=['--matplotlib=inline']) else: self.km.start_kernel() if platform.system() == 'Darwin': # There is sometimes a race condition where the first # execute command hits the kernel before it's ready. # It appears to happen only on Darwin (Mac OS) and an # easy (but clumsy) way to mitigate it is to sleep # for a second. sleep(1) self.kc = self.km.client() self.kc.start_channels() self.shell = self.kc.shell_channel self.iopub = self.kc.iopub_channel logging.info('Reading notebook %s', nb_in) self.nb = read(open(nb_in), 'json')
def __init__(self, **kw): """Initializes the notebook runner. Requires config rcloud_python_lib_path -- or raises exception""" self.km = KernelManager() self.km.kernel_cmd = make_ipkernel_cmd( """ import sys; sys.path.extend("{RCPATH}".split(":")); from rcloud_kernel import main; main()""".format(RCPATH=kw["rcloud_python_lib_path"]), **kw) del kw["rcloud_python_lib_path"] self.km.client_factory = MyClient self.km.start_kernel(**kw) if platform.system() == 'Darwin': # There is sometimes a race condition where the first # execute command hits the kernel before it's ready. # It appears to happen only on Darwin (Mac OS) and an # easy (but clumsy) way to mitigate it is to sleep # for a second. sleep(1) self.kc = self.km.client() self.kc.start_channels() self.shell = self.kc.shell_channel self.iopub = self.kc.iopub_channel
def test_notebook(nb): km = KernelManager() km.start_kernel(extra_arguments=['--pylab=inline'], stderr=open(os.devnull, 'w')) try: kc = km.client() kc.start_channels() iopub = kc.iopub_channel except AttributeError: # IPython 0.13 kc = km kc.start_channels() iopub = kc.sub_channel shell = kc.shell_channel # run %pylab inline, because some notebooks assume this # even though they shouldn't shell.execute("pass") shell.get_msg() while True: try: iopub.get_msg(timeout=1) except Empty: break successes = 0 failures = 0 errors = 0 for ws in nb.worksheets: for cell in ws.cells: if cell.cell_type != 'code': continue try: outs = run_cell(shell, iopub, cell) except Exception as e: print "failed to run cell:", repr(e) print cell.input errors += 1 continue failed = False for out, ref in zip(outs, cell.outputs): if not compare_outputs(out, ref): failed = True if failed: failures += 1 else: successes += 1 sys.stdout.write('.') print print "tested notebook %s" % nb.metadata.name print " %3i cells successfully replicated" % successes if failures: print " %3i cells mismatched output" % failures if errors: print " %3i cells failed to complete" % errors kc.stop_channels() km.shutdown_kernel() del km
def __init__(self, notebook_dir, extra_args=None, profile=None, timeout=90): self.notebook_dir = os.path.abspath(notebook_dir) self.profile = profile self.timeout = timeout km = KernelManager() if extra_args is None: extra_args = [] if profile is not None: extra_args += ["--profile=%s" % profile] km.start_kernel(stderr=open(os.devnull, 'w'), extra_arguments=extra_args) try: kc = km.client() kc.start_channels() iopub = kc.iopub_channel except AttributeError: # still on 0.13 kc = km kc.start_channels() iopub = kc.sub_channel shell = kc.shell_channel # make sure it's working shell.execute("pass") shell.get_msg() # all of these should be run pylab inline shell.execute("%pylab inline") shell.get_msg() self.kc = kc self.km = km self.iopub = iopub
def __init__(self, **kw): """Initializes the notebook runner. Requires config rcloud_python_lib_path -- or raises exception""" self.km = KernelManager() self.km.kernel_cmd = make_ipkernel_cmd( """ import sys; import time; # to test for slow staring kernels, add a delay here sys.path.extend("{RCPATH}".split(":")); from rcloud_kernel import main; main()""".format(RCPATH=kw["rcloud_python_lib_path"]), **kw) del kw["rcloud_python_lib_path"] self.km.client_factory = MyClient self.km.start_kernel( **kw) # This is a non-blocking call to launch the process self.completer = completer.Completer() self.completer.limit_to__all__ = True # There is a possible race condition if the system is slow to load # the modules for the kernel and we issue commands to the kernel rightaway. # We saw these on CentOS and OSX. So, issue an empty command to the kernel and wait for return self.kc = self.km.client() self.kc.start_channels() self.shell = self.kc.shell_channel self.shell.execute("") _ = self.shell.get_msgs() self.iopub = self.kc.iopub_channel
def __enter__(self): self.km = KernelManager() self.km.start_kernel(extra_arguments=self.extra_arguments, stderr=open(os.devnull, 'w')) try: self.kc = self.km.client() self.kc.start_channels() self.iopub = self.kc.iopub_channel except AttributeError: # IPython 0.13 self.kc = self.km self.kc.start_channels() self.iopub = self.kc.sub_channel self.shell = self.kc.shell_channel # run %pylab inline, because some notebooks assume this # even though they shouldn't self.shell.execute("pass") self.shell.get_msg() while True: try: self.iopub.get_msg(timeout=1) except Empty: break return self
def __init__(self): self.km = KernelManager() self.km.start_kernel(stderr=open(os.devnull, 'w')) self.kc = self.km.client() self.kc.start_channels() self.shell = self.kc.shell_channel self.shell.execute("pass") self.shell.get_msg()
def run_notebook(nb, cell_filter=lambda cell: cell, extra_arguments=['--pylab=inline', '--profile=stats'], modify_outputs=True, run_cells=True): """ Take a notebook and send all its cells to a kernel. Takes an optional filter to modify the results of the cell after being run and having its output set by `run_cell` if modify_outputs is True. """ km = KernelManager() km.start_kernel(extra_arguments=extra_arguments, stderr=open(os.devnull, 'w')) try: kc = km.client() except AttributeError: # 0.13 kc = km kc.start_channels() shell = kc.shell_channel shell.execute("pass") shell.get_msg() successes = 0 failures = 0 errors = 0 prompt_number = 1 for ws in nb.worksheets: new_cells = [] for cell in ws.cells: cell.prompt_number = prompt_number if cell['cell_type'] != 'code': new_cells.append(cell) continue if run_cells: try: outs = run_cell(kc, cell, collect_outputs=modify_outputs) except Exception as e: sys.stdout.write("failed to run cell:" + repr(e)) errors += 1 continue sys.stdout.write('.') if modify_outputs: cell.outputs = outs new_cell = cell_filter(cell) if new_cell is not None: new_cells.append(new_cell) prompt_number += 1 sys.stdout.write('\n') ws.cells = new_cells km.shutdown_kernel() del km return nb
def run_notebook(nb): """Run IPython Notebook. Paramters: ---------- nb : IPython Notebook in JSON format. Returns: -------- ret : int Return value; 0 in case of no failure, 1 otherwise """ km = KernelManager() km.start_kernel(stderr=open(os.devnull, 'w')) try: kc = km.client() except AttributeError: # 0.13 kc = km kc.start_channels() shell = kc.shell_channel # simple ping: shell.execute("pass") reply = shell.get_msg() cells = 0 failures = 0 for ws in nb.worksheets: for cell in ws.cells: if cell.cell_type != 'code': continue shell.execute(cell.input) # wait for finish, maximum 20s reply = shell.get_msg(timeout=20)['content'] if reply['status'] == 'error': failures += 1 print "\nFAILURE:" print cell.input print '-----' print "raised:" print '\n'.join(reply['traceback']) cells += 1 sys.stdout.write('.') print print "ran notebook %s" % nb.metadata.name print " ran %3i cells" % cells if failures: print " %3i cells raised exceptions" % failures kc.stop_channels() km.shutdown_kernel() del km if failures: return 1 return 0
def run_notebook(nb): km = KernelManager() km.start_kernel(stderr=open(os.devnull, 'w')) kc = km.client() kc.start_channels() try: kc.wait_for_ready() except AttributeError: # IPython < 3 kc.kernel_info() while True: msg = kc.get_shell_msg(block=True, timeout=30) if msg['msg_type'] == 'kernel_info_reply': break # Flush IOPub channel while True: try: msg = kc.get_iopub_msg(block=True, timeout=0.2) except Empty: break # simple ping: kc.execute("pass") kc.get_shell_msg() cells = 0 failures = 0 if hasattr(nb, 'worksheets'): # nobody uses more than 1 worksheet ws = nb.worksheets[0] else: # no more worksheet level in new format ws = nb for cell in ws.cells: if cell.cell_type != 'code': continue outputs, failed = run_cell(kc, cell) cell.outputs = outputs cell['prompt_number'] = cells failures += failed cells += 1 sys.stdout.write('.') sys.stdout.flush() print() print("ran %3i cells" % cells) if failures: print(" %3i cells raised exceptions" % failures) kc.stop_channels() km.shutdown_kernel() del km
def setup(): global KM, KC KM = KernelManager() KM.start_kernel(stdout=PIPE, stderr=PIPE) KC = KM.client() KC.start_channels() # wait for kernel to be ready KC.execute("pass") KC.get_shell_msg(block=True, timeout=5) flush_channels()
def start_new_kernel(): """start a new kernel, and return its Manager and Client""" km = KernelManager() km.start_kernel(stdout=PIPE, stderr=PIPE) kc = km.client() kc.start_channels() msg_id = kc.kernel_info() kc.get_shell_msg(block=True, timeout=STARTUP_TIMEOUT) flush_channels(kc) return km, kc
def __init__( self, nb, pylab=False, mpl_inline=False, profile_dir=None, working_dir=None): self.km = KernelManager() args = [] if pylab: args.append('--pylab=inline') logging.warn( '--pylab is deprecated and will be removed in a future version' ) elif mpl_inline: args.append('--matplotlib=inline') logging.warn( '--matplotlib is deprecated and' + ' will be removed in a future version' ) if profile_dir: args.append('--profile-dir=%s' % os.path.abspath(profile_dir)) cwd = os.getcwd() if working_dir: os.chdir(working_dir) self.km.start_kernel(extra_arguments=args) os.chdir(cwd) if platform.system() == 'Darwin': # There is sometimes a race condition where the first # execute command hits the kernel before it's ready. # It appears to happen only on Darwin (Mac OS) and an # easy (but clumsy) way to mitigate it is to sleep # for a second. sleep(1) self.kc = self.km.client() self.kc.start_channels() try: self.kc.wait_for_ready() except AttributeError: # IPython < 3 self._wait_for_ready_backport() self.nb = nb
def start_new_kernel(argv=None): """start a new kernel, and return its Manager and Client""" km = KernelManager() kwargs = dict(stdout=PIPE, stderr=PIPE) if argv: kwargs['extra_arguments'] = argv km.start_kernel(**kwargs) kc = km.client() kc.start_channels() msg_id = kc.kernel_info() kc.get_shell_msg(block=True, timeout=STARTUP_TIMEOUT) flush_channels(kc) return km, kc
def run_nb_offline(nb_path): """ Read notebook from filepath and execute it; report errors in code cells. """ if not os.path.isfile(nb_path): raise Exception('Invalid path: %s' % nb_path) with open(nb_path) as f: nb = reads(f.read(), 'json') logging.info("Running notebook %s" % nb.metadata.name) km = KernelManager() km.start_kernel(stderr=open(os.devnull, 'w')) try: kc = km.client() except AttributeError: # 0.13 kc = km kc.start_channels() shell = kc.shell_channel # simple ping: shell.execute("pass") shell.get_msg() cells = 0 failures = 0 for ws in nb.worksheets: for cell in ws.cells: if cell.cell_type != 'code': #ONLY RUN CODE CELLS continue shell.execute(cell.input) # wait for finish, maximum TIMEOUT reply = shell.get_msg(timeout=MAX_TIMEOUT)['content'] if reply['status'] == 'error': failures += 1 logging.info("\nNotebook FAILURE:") logging.info(cell.input) logging.info('-----') logging.info('raised:') logging.info('\n'.join(reply['traceback'])) cells += 1 # sys.stdout.write('.') logging.info("Finished running notebook") logging.info(" ran %3i cells" % cells) if failures: logging.warning(" %3i cells raised exceptions" % failures) kc.stop_channels() km.shutdown_kernel() del km
def run_notebook(nb, pylab_inline=True, timeout=20): """ Run the notebook, populating the output cells with appropriate content. Params ------ nb : the contents of a notebook as a string pylab_inline : i.e. should the command be executed as if it was flagged with --paylab=inline timeout : the length of time in seconds to wait before the script is considered timed out. I set this to a big value for some data heavy scripts """ # Start the kernel. km = KernelManager() args = {} if pylab_inline: args['extra_arguments'] = ['--pylab=inline'] km.start_kernel(**args) # Get our client. try: kc = km.client() except AttributeError: kc = km kc.start_channels() shell = kc.shell_channel # Ping the kernel. shell.execute('pass') shell.get_msg() # Run all the cells. cells_executed, cells_failed = 0, 0 for ws in nb.worksheets: for cell in ws.cells: cell.prompt_number = cells_executed + 1 if cell.cell_type != 'code': continue cells_executed += 1 run_cell(kc, cell, timeout) # Clean up resources. (Hopefully?) kc.stop_channels() km.shutdown_kernel() del km return cells_failed
def new_ipy(s=''): """Create a new IPython kernel (optionally with extra arguments) XXX: Allow passing of profile information here Examples -------- new_ipy() """ from IPython.kernel import KernelManager km = KernelManager() km.start_kernel() return km_from_string(km.connection_file)
def setup(): global KM, KC KM = KernelManager() KM.start_kernel(stdout=PIPE, stderr=PIPE) KC = KM.client() KC.start_channels() # wait for kernel to be ready try: msg = KC.iopub_channel.get_msg(block=True, timeout=STARTUP_TIMEOUT) except Empty: pass msg_id = KC.kernel_info() KC.get_shell_msg(block=True, timeout=STARTUP_TIMEOUT) flush_channels()
def start_new_kernel(argv=None): """start a new kernel, and return its Manager and Client""" km = KernelManager() kwargs = dict(stdout=nose.ipy_stream_capturer.writefd, stderr=STDOUT) if argv: kwargs['extra_arguments'] = argv km.start_kernel(**kwargs) nose.ipy_stream_capturer.ensure_started() kc = km.client() kc.start_channels() msg_id = kc.kernel_info() kc.get_shell_msg(block=True, timeout=STARTUP_TIMEOUT) flush_channels(kc) return km, kc
def execute_kernel(nb, t, tshell): """ Load Kernel stuff iopub may be necessary to run through the cell """ km = KernelManager() km.start_kernel(extra_arguments=['--matplotlib=inline'], stderr=open(os.devnull, 'w')) try: kc = km.client() kc.start_channels() iopub = kc.iopub_channel except AttributeError: # IPython 0.13 kc = km kc.start_channels() iopub = kc.sub_channel shell = kc.shell_channel """ This part needs revision """ # run %pylab inline, because some notebooks assume this # even though they shouldn't shell.execute("pass") shell.get_msg() while True: try: iopub.get_msg(timeout=1) except Empty: break """ Try to print cell by cell """ # Only one worksheet in the current ipython nbs structure for cell in nb.worksheets[0].cells: # If the cell is code, move to next cell if cell.cell_type != 'code': continue # Otherwise the cell is an output cell, run it! try: outs = run_cell(shell, iopub, cell, t, tshell) print outs except Exception as e: print "failed to run cell:", repr(e) print cell.input
def run_notebook(nb): km = KernelManager() km.start_kernel(extra_arguments=['--profile', 'stats'])#, stderr=open(os.devnull, 'w')) try: kc = km.client() except AttributeError: # 0.13 kc = km kc.start_channels() shell = kc.shell_channel # simple ping: shell.execute("pass") shell.get_msg() shell.execute("datadir = '%s'" % os.path.abspath(os.path.join( \ os.path.abspath(os.path.dirname(__file__)), '..', 'data'))) cells = 0 failures = 0 for ws in nb.worksheets: for cell in ws.cells: if cell.cell_type != 'code': continue shell.execute(cell.input) # wait for finish, maximum 20s msg = shell.get_msg(timeout=20) reply = msg['content'] if reply['status'] == 'error': failures += 1 print "\nFAILURE:" print cell.input print '-----' print "raised:" print '\n'.join(reply['traceback']) cells += 1 sys.stdout.write('.') print "ran notebook %s" % nb.metadata.name print " ran %3i cells" % cells if failures: print " %3i cells raised exceptions" % failures kc.stop_channels() km.shutdown_kernel() del km
def __init__(self, **kw): self.km = KernelManager() self.km.kernel_cmd = make_ipkernel_cmd('import sys; sys.path.append("%s"); from rcloud_kernel import main; main()' % kw["rcloud_support_path"], **kw) del kw["rcloud_support_path"] self.km.client_factory = MyClient self.km.start_kernel(**kw) if platform.system() == 'Darwin': # There is sometimes a race condition where the first # execute command hits the kernel before it's ready. # It appears to happen only on Darwin (Mac OS) and an # easy (but clumsy) way to mitigate it is to sleep # for a second. sleep(1) self.kc = self.km.client() self.kc.start_channels() self.shell = self.kc.shell_channel self.iopub = self.kc.iopub_channel
def run_notebook(nb): km = KernelManager() km.start_kernel(stderr=open(os.devnull, 'w')) if hasattr(km, 'client'): kc = km.client() kc.start_channels() iopub = kc.iopub_channel else: # IPython 0.13 compat kc = km kc.start_channels() iopub = kc.sub_channel shell = kc.shell_channel # simple ping: shell.execute("pass") shell.get_msg() cells = 0 failures = 0 for ws in nb.worksheets: for cell in ws.cells: if cell.cell_type != 'code': continue outputs, failed = run_cell(shell, iopub, cell) cell.outputs = outputs cell['prompt_number'] = cells failures += failed cells += 1 sys.stdout.write('.') sys.stdout.flush() print() print("ran notebook %s" % nb.metadata.name) print(" ran %3i cells" % cells) if failures: print(" %3i cells raised exceptions" % failures) kc.stop_channels() km.shutdown_kernel() del km
def run_notebook(nb): km = KernelManager() km.start_kernel(stderr=open(os.devnull, 'w')) try: kc = km.client() except AttributeError: # 0.13 kc = km kc.start_channels() shell = kc.shell_channel # simple ping: shell.execute("pass") shell.get_msg() cells = 0 failures = 0 for ws in nb.worksheets: for cell in ws.cells: if cell.cell_type != 'code': continue shell.execute(cell.input) # wait for finish, maximum 20s reply = shell.get_msg(timeout=20)['content'] if reply['status'] == 'error': failures += 1 print("\nFAILURE:") print(cell.input) print('-----') print("raised:") print('\n'.join(reply['traceback'])) print(cell) cells += 1 sys.stdout.write('.') print() print("ran notebook %s" % nb.metadata.name) print(" ran %3i cells" % cells) if failures: print(" %3i cells raised exceptions" % failures) kc.stop_channels() km.shutdown_kernel() del km
def __init__(self, nb, pylab=False, mpl_inline=False, working_dir=None): self.km = KernelManager() args = [] if pylab: args.append('--pylab=inline') logging.warn( '--pylab is deprecated and will be removed in a future version' ) elif mpl_inline: args.append('--matplotlib=inline') logging.warn( '--matplotlib is deprecated and will be removed in a future version' ) cwd = os.getcwd() if working_dir: os.chdir(working_dir) self.km.start_kernel(extra_arguments=args) os.chdir(cwd) if platform.system() == 'Darwin': # There is sometimes a race condition where the first # execute command hits the kernel before it's ready. # It appears to happen only on Darwin (Mac OS) and an # easy (but clumsy) way to mitigate it is to sleep # for a second. sleep(1) self.kc = self.km.client() self.kc.start_channels() self.shell = self.kc.shell_channel self.iopub = self.kc.iopub_channel self.nb = nb
def new_kernel(): """start a kernel in a subprocess, and wait for it to be ready Returns ------- kernel_manager: connected KernelManager instance """ KM = KernelManager() KM.start_kernel(stdout=PIPE, stderr=PIPE) KC = KM.client() KC.start_channels() # wait for kernel to be ready KC.shell_channel.execute("import sys") KC.shell_channel.get_msg(block=True, timeout=STARTUP_TIMEOUT) flush_channels(KC) try: yield KC finally: KC.stop_channels() KM.shutdown_kernel()
def _create_client(self): from IPython.kernel import KernelManager self.km = KernelManager() self.km.write_connection_file() self.kc = self.km.client() self.kc.start_channels() self.km.start_kernel(extra_arguments=self.extra_arguments, stderr=open(os.devnull, 'w')) self.iopub = self.kc.iopub_channel self.shell = self.kc.shell_channel self.shell.kernel_info() try: self.shell.get_msg(timeout=self.timeout) except Empty: self.log.error("Timeout waiting for kernel_info reply") raise # flush IOPub while True: try: self.iopub.get_msg(block=True, timeout=0.25) except Empty: break
def test_notebook(nb): km = KernelManager() km.start_kernel(extra_arguments=[], stderr=open(os.devnull, 'w')) kc = km.client() kc.start_channels() iopub = kc.iopub_channel shell = kc.shell_channel shell.kernel_info() while True: try: kc.iopub_channel.get_msg(timeout=1) except Empty: break errors = 0 cells = 0 for ws in nb.worksheets: for cell in ws.cells: if cell.cell_type != 'code': continue cells += 1 try: outs = run_cell(shell, iopub, cell) except Exception as e: print("failed to run cell:", repr(e)) print(cell.input) errors += 1 continue cell.outputs = outs if errors: print(" %3i cells failed to complete" % errors) if cells: print("%i code cells from notebook %s" % (cells, nb.metadata.name)) kc.stop_channels() km.shutdown_kernel() del km
def run_notebook(nb): """ Run each code cell in a given notebook and update with the new output """ km = KernelManager() km.start_kernel(extra_arguments=['--pylab=inline']) try: kc = km.client() kc.start_channels() iopub = kc.iopub_channel except AttributeError: # IPython 0.13 kc = km kc.start_channels() iopub = kc.sub_channel shell = kc.shell_channel shell.execute("pass") shell.get_msg() while True: try: iopub.get_msg(timeout=1) except Empty: break for ws in nb.worksheets: for cell in ws.cells: if cell.cell_type != 'code': continue try: cell.outputs = run_cell(shell, iopub, cell) except Exception as e: return -1 kc.stop_channels() km.shutdown_kernel() del km return nb