def km_from_string(s=''): """ Create kernel manager. """ global km, kc, send s = s.replace('--existing', '') try: if '--profile' in s: k, p = s.split('--profile') k = k.lstrip().rstrip() # kernel part of the string p = p.lstrip().rstrip() # profile part of the string fullpath = find_connection_file(k, p) elif s == '': fullpath = find_connection_file() else: fullpath = find_connection_file(s.lstrip().rstrip()) except IOError: vim_echo(":IPython " + s + " failed", "Info") vim_echo("^-- failed '" + s + "' not found", "Error") return km = KernelManager(connection_file=fullpath) km.load_connection_file() kc = km.client() kc.start_channels() send = kc.execute vim_echo('Connected to {}'.format(fullpath))
def connect_kernel_from_file(kernel_type, connection_file): from jupyter_client import KernelManager # Create the kernel manager and connect a client # See: <http://jupyter-client.readthedocs.io/en/stable/api/client.html> km = KernelManager(connection_file=connection_file) km.load_connection_file() kc = km.client() kc.start_channels() # Alias execute function def _send(msg, **kwargs): """Send a message to the kernel client.""" # Include dedent of msg so we don't get odd indentation errors. return kc.execute(textwrap.dedent(msg), **kwargs) send = _send # Ping the kernel kc.kernel_info() try: reply = kc.get_shell_msg(timeout=1) except Empty: return {"connected": False} else: # pid = get_pid(kernel_type) # Ask kernel for its PID return {"connected": True, "kc": kc, "send": send}
def temporary_connect_query(connection_file, code, retvar): # connect from jupyter_client import KernelManager _km = KernelManager(connection_file=connection_file) _km.load_connection_file() _kc = _km.client() _kc.start_channels() def _send(msg, **kwargs): """Send a message to the kernel client.""" # Include dedent of msg so we don't get odd indentation errors. return _kc.execute(textwrap.dedent(msg), **kwargs) # send code msg_id = _send(code, silent=True, user_expressions={"_ret": retvar}) try: reply = get_reply_msg(msg_id, _kc=_kc) except Empty: vim_echom("no reply from jupyter kernel", "WarningMsg") return -1 # get result retval = None try: # Requires the fix for https://github.com/JuliaLang/IJulia.jl/issues/815 retval = reply["content"]["user_expressions"]["_ret"]["data"][ "text/plain"] except KeyError: vim_echom("Could not get return value, kernel not ready?") # disconnect _kc.stop_channels() return retval
def __init__(self, timeout: int = 100, template_type: TemplateType = TemplateType.FULL): """ Initializes Exporter with default timeout (100 seconds) and template (FULL) and handles instantiation of km and ep variables for usage when generating NotebookNodes and their outputs. Args: timeout (int): Amount of time in seconds that a visualization can run for before being stopped. template_type (TemplateType): Type of template to use when generating visualization output. """ self.timeout = timeout self.template_type = template_type # Create custom KernelManager. # This will circumvent issues where kernel is shutdown after # preprocessing. Due to the shutdown, latency would be introduced # because a kernel must be started per visualization. self.km = KernelManager() self.km.start_kernel() self.ep = ExecutePreprocessor(timeout=self.timeout, kernel_name='python3', allow_errors=True)
def test_cleanup_context(self): km = KernelManager() self.assertIsNotNone(km.context) km.cleanup_resources(restart=False) self.assertTrue(km.context.closed)
def __init__(self, use_exist=False): """ ## Description Initializes the `kernel_manager` and `client` objects and starts the kernel. Also initializes the pretty printer for displaying object properties and execution result payloads. ## Parameters None. """ if not use_exist: # Initialize kernel and client self.kernel_manager, self.client = start_new_kernel() self.send = self.client.execute else: self.kernel_manager = KernelManager( connection_file=find_connection_file()) self.kernel_manager.load_connection_file(find_connection_file()) self.client = self.kernel_manager.client() self.client.start_channels() self.send = self.client.execute # Initialize pretty printer self.pp = PrettyPrinter(indent=2)
def kernel_client(config): try: if config.kernel != 'existing': cnx_file = find_connection_file(filename=config.kernel) else: cnx_file = find_connection_file() km = KernelManager(connection_file=cnx_file) km.load_connection_file() kc = km.client() kc.start_channels() try: msg = kc.shell_channel.get_msg() except Empty: try: # try again :) sleep(1) msg = kc.shell_channel.get_msg() except Empty: raise RuntimeError( 'No message found in the channel. Is the kernel alive?') except Exception as e: print(format_text('PyExecError: ipython_exec was not able to connect to the desired jupyter kernel\n' + \ "Hint: execute 'ipython_start_kernel' or 'jupyter console' first\n" + \ format_exc(), config.to)) exit(1) return kc
def create_kernel_manager(self): """Create the kernel manager and connect a client. Returns ------- bool True if client connects successfully, False on failure. """ # Get client kernel_manager = KernelManager(connection_file=self.cfile) # The json may be badly encoding especially if autoconnecting try: kernel_manager.load_connection_file() # pylint: disable=broad-except except Exception: return False self.km_client = kernel_manager.client() # Open channel self.km_client.start_channels() # Ping the kernel self.km_client.kernel_info() try: self.km_client.get_shell_msg(timeout=1) return True except Empty: return False
class Exporter: """Handler for interaction with NotebookNodes, including output generation. Attributes: timeout (int): Amount of time in seconds that a visualization can run for before being stopped. template_type (TemplateType): Type of template to use when generating visualization output. km (KernelManager): Custom KernelManager that stays alive between visualizations. ep (ExecutePreprocessor): Process that is responsible for generating outputs from NotebookNodes. """ def __init__(self, timeout: int = 100, template_type: TemplateType = TemplateType.FULL): """ Initializes Exporter with default timeout (100 seconds) and template (FULL) and handles instantiation of km and ep variables for usage when generating NotebookNodes and their outputs. Args: timeout (int): Amount of time in seconds that a visualization can run for before being stopped. template_type (TemplateType): Type of template to use when generating visualization output. """ self.timeout = timeout self.template_type = template_type # Create custom KernelManager. # This will circumvent issues where kernel is shutdown after # preprocessing. Due to the shutdown, latency would be introduced # because a kernel must be started per visualization. self.km = KernelManager() self.km.start_kernel() self.ep = ExecutePreprocessor(timeout=self.timeout, kernel_name='python3', allow_errors=True) def generate_html_from_notebook(self, nb: NotebookNode) -> Text: """Converts a provided NotebookNode to HTML. Args: nb: NotebookNode that should be converted to HTML. Returns: HTML from converted NotebookNode as a string. """ # HTML generator and exporter object html_exporter = HTMLExporter() template_file = "templates/{}.tpl".format(self.template_type.value) html_exporter.template_file = str(Path.cwd() / template_file) # Output generator self.ep.preprocess(nb, {"metadata": {"path": Path.cwd()}}, self.km) # Export all html and outputs body, _ = html_exporter.from_notebook_node(nb) return body
def new_kernel(): """ Create a new IPython kernel (optionally with extra arguments) """ km = KernelManager() km.start_kernel() return km_from_string(km.connection_file)
def start(self): self.km = KernelManager(kernel_name='python', client_class='execute.Client') self.km.start_kernel() self.kc = self.km.client() self.kc.start_channels() time.sleep(2) atexit.register(self.shutdown_kernel)
def __init__(self): """ Construct an IPythonConnection object without connecting """ self.kmgr = KernelManager(kernel_name='python3') self.client = None self._status_ = 'disconnected' self._kernel_ = {}
def test_no_cleanup_shared_context(self, zmq_context): """kernel manager does not terminate shared context""" km = KernelManager(context=zmq_context) assert km.context == zmq_context assert km.context is not None km.cleanup_resources(restart=False) assert km.context.closed is False assert zmq_context.closed is False
def _run_signaltest_lifecycle(self, config=None): km = KernelManager(config=config, kernel_name="signaltest") kc = self._prepare_kernel(km, stdout=PIPE, stderr=PIPE) def execute(cmd): request_id = kc.execute(cmd) while True: reply = kc.get_shell_msg(TIMEOUT) if reply["parent_header"]["msg_id"] == request_id: break content = reply["content"] assert content["status"] == "ok" return content execute("start") assert km.is_alive() execute("check") assert km.is_alive() km.restart_kernel(now=True) assert km.is_alive() execute("check") km.shutdown_kernel() assert km.context.closed
def _run_signaltest_lifecycle(self, config=None): km = KernelManager(config=config, kernel_name='signaltest') kc = self._prepare_kernel(km, stdout=PIPE, stderr=PIPE) def execute(cmd): request_id = kc.execute(cmd) while True: reply = kc.get_shell_msg(TIMEOUT) if reply['parent_header']['msg_id'] == request_id: break content = reply['content'] assert content['status'] == 'ok' return content execute("start") assert km.is_alive() execute('check') assert km.is_alive() km.restart_kernel(now=True) assert km.is_alive() execute('check') km.shutdown_kernel() assert km.context.closed
def connect_to_kernel(self): """Create kernel manager from existing connection file.""" from jupyter_client import KernelManager, find_connection_file # Test if connection is alive connected = self.check_connection() attempt = 0 max_attempts = 3 while not connected and attempt < max_attempts: attempt += 1 try: cfile = find_connection_file( ) # default filename='kernel-*.json' except IOError: self.vim_echom( "kernel connection attempt {:d} failed - no kernel file". format(attempt), style="Error") continue # Create the kernel manager and connect a client # See: <http://jupyter-client.readthedocs.io/en/stable/api/client.html> km = KernelManager(connection_file=cfile) km.load_connection_file() self.kc = km.client() self.kc.start_channels() # Alias execute function def _send(msg, **kwargs): """Send a message to the kernel client.""" # Include dedent of msg so we don't get odd indentation errors. return self.kc.execute(textwrap.dedent(msg), **kwargs) self.send = _send # Ping the kernel self.kc.kernel_info() try: self.reply = self.kc.get_shell_msg(timeout=1) except Empty: continue else: connected = True if connected: # Send command so that monitor knows vim is commected # send('"_vim_client"', store_history=False) self.pid = self.set_pid() # Ask kernel for its PID self.vim_echom('kernel connection successful! pid = {}'.format( self.pid), style='Question') else: self.kc.stop_channels() self.vim_echom('kernel connection attempt timed out', style='Error')
def start(self, cwd=None, executable=None): self.km = KernelManager(kernel_name='python', client_class='nei.execute.Client') self.executable = executable if executable is not None: self.km.kernel_spec.argv[0] = executable self.km.start_kernel(**({} if cwd is None else {'cwd': cwd})) self.kc = self.km.client() self.kc.start_channels() time.sleep(2) atexit.register(self.shutdown_kernel)
def setUp(self): try: KernelSpecManager().get_kernel_spec(NATIVE_KERNEL_NAME) except NoSuchKernel: pytest.skip() self.km = KernelManager(kernel_name=NATIVE_KERNEL_NAME) self.km.start_kernel() self.kc = self.km.client() self.kc.start_channels() self.addCleanup(self.kc.stop_channels) self.addCleanup(self.km.shutdown_kernel)
def setUp(self): try: KernelSpecManager().get_kernel_spec('python3_clean') except NoSuchKernel: pytest.skip() self.km = KernelManager(kernel_name='python3_clean') self.km.start_kernel() self.kc = self.km.client() self.kc.start_channels() self.addCleanup(self.kc.stop_channels) self.addCleanup(self.km.shutdown_kernel)
def test_no_cleanup_shared_context(self): """kernel manager does not terminate shared context""" import zmq ctx = zmq.Context() km = KernelManager(context=ctx) self.assertEquals(km.context, ctx) self.assertIsNotNone(km.context) km.cleanup_resources(restart=False) self.assertFalse(km.context.closed) self.assertFalse(ctx.closed) ctx.term()
def start_kernel(): manager = KernelManager() manager.start_kernel() port_names = ["shell", "stdin", "iopub", "hb", "control"] ports = dict(list(zip(port_names, manager.ports))) kernel_info = { "key": manager.session.key.decode("utf-8"), "ports": ports, } return json.dumps(kernel_info)
def connect_to_kernel(): """Create kernel manager from existing connection file.""" from jupyter_client import KernelManager, find_connection_file global kc, pid, send # Test if connection is alive connected = check_connection() attempt = 0 max_attempts = 3 while not connected and attempt < max_attempts: attempt += 1 try: cfile = find_connection_file() # default filename='kernel-*.json' except IOError: vim_echom("kernel connection attempt {:d} failed - no kernel file"\ .format(attempt), style="Error") continue # Create the kernel manager and connect a client # See: <http://jupyter-client.readthedocs.io/en/stable/api/client.html> km = KernelManager(connection_file=cfile) km.load_connection_file() kc = km.client() kc.start_channels() # Alias execute function def _send(msg, **kwargs): """Send a message to the kernel client.""" # Include dedent of msg so we don't get odd indentation errors. return kc.execute(textwrap.dedent(msg), **kwargs) send = _send # Ping the kernel kc.kernel_info() try: reply = kc.get_shell_msg(timeout=1) except Empty: continue else: connected = True if connected: # Send command so that monitor knows vim is commected # send('"_vim_client"', store_history=False) pid = set_pid() # Ask kernel for its PID vim_echom('kernel connection successful! pid = {}'.format(pid), style='Question') else: kc.stop_channels() vim_echom('kernel connection attempt timed out', style='Error')
def __init__(self, working_directory, kernel='python3', embed_kernel=False): if embed_kernel: self._kernel_manager = InProcessKernelManager(kernel_name=kernel) else: self._kernel_manager = KernelManager(kernel_name=kernel) stderr = open(os.devnull, 'w') self._kernel_manager.start_kernel(cwd=working_directory, stderr=stderr) self._init_client()
def new_ipy(s=''): """Create a new IPython kernel (optionally with extra arguments) XXX: Allow passing of profile information here Examples -------- new_ipy() """ from jupyter_client import KernelManager km = KernelManager() km.start_kernel() return km_from_string(km.connection_file)
class Kernel(object): def __init__(self, kernel_name): self.kernel = KernelManager(kernel_name=kernel_name) def start(self): self.kernel.start_kernel() self.client = self.kernel.client() self.client.start_channels() try: self.client.wait_for_ready() except RuntimeError: print("[metys] Unable to start kernel.") self.shutdown() raise info = self.client.kernel_info(reply=True) if info and "content" in info and "language_info" in info["content"]: self.language_info = info["content"]["language_info"] def shutdown(self): self.client.shutdown() def is_complete(self, code): return self.client.is_complete(code) def execute_chunk(self, chunk): chunk.messages = [] if hasattr(self, "language_info"): chunk.language_info = self.language_info input = ( chunk.input.format(**chunk.options) if chunk.options["expand_options"] else chunk.input ) if "code_template" in chunk.options: input = chunk.options["code_template"].format(input, **chunk.options) def output_hook(msg): return ( chunk.messages.append(msg) if msg["msg_type"] in ("display_data", "execute_result", "stream") else None ) chunk.reply = self.client.execute_interactive( input, store_history=False, allow_stdin=False, output_hook=output_hook )
def __init__( self, parsed, kernel, source, mode, figdir, outdir, embed_kernel=False ): super(JupyterProcessor, self).__init__( parsed, kernel, source, mode, figdir, outdir ) self.extra_arguments = None self.timeout = -1 path = os.path.abspath(outdir) if embed_kernel: km = InProcessKernelManager(kernel_name=kernel) else: km = KernelManager(kernel_name=kernel) km.start_kernel(cwd=path, stderr=open(os.devnull, "w")) kc = km.client() kc.start_channels() try: kc.wait_for_ready() except RuntimeError: print( "Timeout from starting kernel\nTry restarting python session and running weave again" ) kc.stop_channels() km.shutdown_kernel() raise self.km = km self.kc = kc self.kc.allow_stdin = False
def run_notebook(nb): km = KernelManager() km.start_kernel(stderr=open(os.devnull, 'w')) kc = km.client() kc.start_channels() shell = kc.shell_channel # simple ping: kc.execute("pass") shell.get_msg() failures = 0 for cell in nb.cells: if cell.cell_type != 'code': continue kc.execute(cell.source) try: # wait for finish, w/ timeout reply = shell.get_msg(timeout=TIMEOUT)['content'] except Empty: raise Exception( 'Timeout (%.1f) when executing the following %s cell: "%s"' % (TIMEOUT, cell.cell_type, cell.source.strip())) if reply['status'] == 'error': failures += 1 print("\nFAILURE:", file=sys.stderr) print('\n'.join(reply['traceback']), file=sys.stderr) print(file=sys.stderr) kc.stop_channels() km.shutdown_kernel() del km if failures > 0: raise Exception()
def _check_ipynb(): kernel_manager = KernelManager() kernel_manager.start_kernel() kernel_client = kernel_manager.client() kernel_client.start_channels() try: # IPython 3.x kernel_client.wait_for_ready() iopub = kernel_client shell = kernel_client except AttributeError: # Ipython 2.x # Based on https://github.com/paulgb/runipy/pull/49/files iopub = kernel_client.iopub_channel shell = kernel_client.shell_channel shell.get_shell_msg = shell.get_msg iopub.get_iopub_msg = iopub.get_msg successes = 0 failures = 0 errors = 0 report = '' _execute_code("print('Hello World')", shell, iopub, timeout=1) kernel_client.stop_channels() kernel_manager.shutdown_kernel() passed = not (failures or errors) print(report)
def run_notebook(nb): km = KernelManager() km.start_kernel(stderr=open(os.devnull, 'w')) kc = km.client() kc.start_channels() # simple ping: kc.execute("pass") kc.get_shell_msg() cells = 0 failures = 0 for cell in nb.cells: if cell.cell_type != 'code': continue outputs, failed = run_cell(kc, cell) cell.outputs = outputs cells += 1 cell['execution_count'] = cells failures += failed sys.stdout.write('.') sys.stdout.flush() print() #print("ran notebook %s" % nb.metadata.name) print(" ran %3i cells" % cells) if failures: print(" %3i cells raised exceptions" % failures) kc.stop_channels() km.shutdown_kernel() del km
def run_notebook(nb): km = KernelManager() km.start_kernel(stderr=open(os.devnull, 'w')) kc = km.client() kc.start_channels() shell = kc.shell_channel # simple ping: kc.execute("pass") shell.get_msg() failures = 0 for cell in nb.cells: if cell.cell_type != 'code': continue kc.execute(cell.source) # wait for finish, maximum 20s reply = shell.get_msg(timeout=20)['content'] if reply['status'] == 'error': failures += 1 print("\nFAILURE:") print('\n'.join(reply['traceback'])) print() kc.stop_channels() km.shutdown_kernel() del km if failures > 0: raise Exception()
def __init__(self, *args, **kwargs): super(FormatManager, self).__init__(*args, **kwargs) if self._instance is not None: raise ValueError("FormatManager is a singleton, access with" " FormatManager.format_manager") self._formats = {} self._km = KernelManager()
def new_ipy(s=''): """Create a new IPython kernel (optionally with extra arguments) XXX: Allow passing of profile information here Examples -------- new_ipy() """ try: from jupyter_client import KernelManager except ImportError: from IPython.kernel import KernelManager km = KernelManager() km.start_kernel() return km_from_string(km.connection_file)
def start_kernel(self, name): """Start a new kernel""" base, ext = os.path.splitext(self.parent.connection_file) cf = '{base}-{name}{ext}'.format( base=base, name=name, ext=ext, ) manager = KernelManager( kernel_name=name, session=self.session, context=self.future_context, connection_file=cf, ) manager.start_kernel() self.kernels[name] = kernel = KernelProxy( manager=manager, shell_upstream=self.shell_stream) self.iosub.connect(kernel.iopub_url) return self.kernels[name]
def run_notebook(nb): km = KernelManager() km.start_kernel(stderr=open(os.devnull, 'w')) kc = km.client() kc.start_channels() shell = kc.shell_channel # simple ping: kc.execute("pass") shell.get_msg() failures = 0 for cell in nb.cells: if cell.cell_type != 'code': continue kc.execute(cell.source) # wait for finish, maximum 20s reply = shell.get_msg(timeout=60)['content'] if reply['status'] == 'error': failures += 1 print("\nFAILURE:") print('\n'.join(reply['traceback'])) print() kc.stop_channels() km.shutdown_kernel() del km if failures > 0: raise Exception()
def main(args): parser = argparse.ArgumentParser() parser.add_argument('--port', type=int) parser.add_argument('--kernel') parser.add_argument('--conn-file') args = parser.parse_args() if args.conn_file: if runtime_dir: conn_file = (args.conn_file if os.path.isabs(args.conn_file) else os.path.join(runtime_dir(), args.conn_file)) else: # IPython 3 pd = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), 'default') conn_file = os.path.join(pd.security_dir, args.conn_file) kwargs = {'connection_file': conn_file} if args.kernel: kwargs['kernel_name'] = args.kernel manager = KernelManager(**kwargs) semaphore = multiprocessing.Semaphore() semaphore.acquire() def onsignal(*args): semaphore.release() signal.signal(signal.SIGTERM, onsignal) import platform if platform.system() == 'Windows': signal.signal(signal.SIGBREAK, onsignal) else: signal.signal(signal.SIGQUIT, onsignal) # Emacs sends SIGHUP upon exit signal.signal(signal.SIGHUP, onsignal) manager.start_kernel() try: semaphore.acquire() except KeyboardInterrupt: pass manager.shutdown_kernel() else: app = make_app() app.listen(args.port) tornado.ioloop.IOLoop.current().start()
def test_notebook(ipynb): with open(ipynb, encoding='utf-8') as f: nb = nbformat.reads_json(f.read()) km = KernelManager() # Do not save the history to disk, as it can yield spurious lock errors. # See https://github.com/ipython/ipython/issues/2845 km.start_kernel(extra_arguments=['--HistoryManager.hist_file=:memory:']) kc = km.client() kc.start_channels() try: kc.wait_for_ready() except AttributeError: _wait_for_ready_backport(kc) successes = 0 failures = 0 errors = 0 for i, cell in enumerate(nb.cells): if cell.cell_type != 'code' or cell.source.startswith('%timeit'): continue try: outs = run_cell(kc, cell) except Exception as e: print("failed to run cell:", repr(e)) print(cell.input) errors += 1 continue failed = False if len(outs) != len(cell.outputs): print("output length mismatch (expected {}, got {})".format( len(cell.outputs), len(outs))) failed = True if not compare_outputs(outs, cell.outputs): failed = True print("cell %d: " % i, end="") if failed: print("FAIL") failures += 1 else: print("OK") successes += 1 print("tested notebook %s" % ipynb) print(" %3i cells successfully replicated" % successes) if failures: print(" %3i cells mismatched output" % failures) if errors: print(" %3i cells failed to complete" % errors) kc.stop_channels() km.shutdown_kernel() del km if failures | errors: sys.exit(1)
def run_notebook(nb): km = KernelManager() km.start_kernel(stderr=open(os.devnull, 'w')) if hasattr(km, 'client'): kc = km.client() kc.start_channels() iopub = kc.iopub_channel else: # IPython 0.13 compat kc = km kc.start_channels() iopub = kc.sub_channel shell = kc.shell_channel # simple ping: shell.send("pass") shell.get_msg() cells = 0 failures = 0 for ws in nb.worksheets: for cell in ws.cells: if cell.cell_type != 'code': continue outputs, failed = run_cell(shell, iopub, cell) cell.outputs = outputs cell['prompt_number'] = cells failures += failed cells += 1 sys.stdout.write('.') print() print("ran notebook %s" % nb.metadata.name) print(" ran %3i cells" % cells) if failures: print(" %3i cells raised exceptions" % failures) kc.stop_channels() km.shutdown_kernel() del km
def run_notebook(nb): """ Run each code cell in a given notebook and update with the new output """ km = KernelManager() km.start_kernel(extra_arguments=['--pylab=inline']) try: kc = km.client() kc.start_channels() iopub = kc.iopub_channel except AttributeError: # IPython 0.13 kc = km kc.start_channels() iopub = kc.sub_channel shell = kc.shell_channel shell.execute("pass") shell.get_msg() while True: try: iopub.get_msg(timeout=1) except Empty: break for ws in nb.worksheets: for cell in ws.cells: if cell.cell_type != 'code': continue try: cell.outputs = run_cell(shell, iopub, cell) except Exception as e: return -1 kc.stop_channels() km.shutdown_kernel() del km return nb
def __init__(self, nb, profile_dir=None, working_dir=None, comment="", fLOG=noLOG, theNotebook=None, code_init=None, kernel_name="python", log_level="30", extended_args=None, kernel=False, filename=None, replacements=None): """ constuctor @param nb notebook as JSON @param profile_dir profile directory @param working_dir working directory @param comment additional information added to error message @param theNotebook if not None, populate the variable *theNotebook* with this value in the notebook @param code_init to initialize the notebook with a python code as if it was a cell @param fLOG logging function @param log_level Choices: (0, 10, 20, 30=default, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL') @param kernel_name kernel name, it can be None @param extended_args others arguments to pass to the command line ('--KernelManager.autorestar=True' for example), see :ref:`l-ipython_notebook_args` for a full list @param kernel *kernel* is True by default, the notebook can be run, if False, the notebook can be read but not run @param filename to add the notebook file if there is one in error messages @param replacements replacements to make in every cell before running it, dictionary ``{ string: string }`` .. versionchanged:: 1.4 Parameter *replacements* was added. """ if kernel: try: from jupyter_client import KernelManager except ImportError: from ipykernel import KernelManager self.km = KernelManager( kernel_name=kernel_name) if kernel_name is not None else KernelManager() else: self.km = None self.fLOG = fLOG self.theNotebook = theNotebook self.code_init = code_init self._filename = filename if filename is not None else "memory" self.replacements = replacements self.init_args = dict(profile_dir=profile_dir, working_dir=working_dir, comment=comment, fLOG=fLOG, theNotebook=theNotebook, code_init=code_init, kernel_name="python", log_level="30", extended_args=None, kernel=kernel, filename=filename, replacements=replacements) args = [] if profile_dir: args.append('--profile-dir=%s' % os.path.abspath(profile_dir)) if log_level: args.append('--log-level=%s' % log_level) if extended_args is not None and len(extended_args) > 0: for opt in extended_args: if not opt.startswith("--"): raise SyntaxError( "every option should start with '--': " + opt) if "=" not in opt: raise SyntaxError( "every option should be assigned a value: " + opt) args.append(opt) if kernel: cwd = os.getcwd() if working_dir: os.chdir(working_dir) if self.km is not None: if sys.version_info[0] == 2 and args is not None: # I did not find a way to make it work args = None warnings.warn( "args is not None: {0}, unable to use it in Python 2.7".format(args)) self.km.start_kernel() else: try: self.km.start_kernel(extra_arguments=args) except Exception as e: raise Exception( "Failure with args: {0}\nand error:\n{1}".format(args, str(e))) from e if platform.system() == 'Darwin': # see http://www.pypedia.com/index.php/notebook_runner # There is sometimes a race condition where the first # execute command hits the kernel before it's ready. # It appears to happen only on Darwin (Mac OS) and an # easy (but clumsy) way to mitigate it is to sleep # for a second. sleep(1) os.chdir(cwd) self.kc = self.km.client() self.kc.start_channels(stdin=False) # if it does not work, it probably means IPython < 3 self.kc.wait_for_ready() else: self.km = None self.kc = None self.nb = nb self.comment = comment
class NotebookRunner(object): """ The kernel communicates with mime-types while the notebook uses short labels for different cell types. We'll use this to map from kernel types to notebook format types. This classes executes a notebook end to end. .. index:: kernel, notebook The class can use different kernels. The next links gives more information on how to create or test a kernel: * `jupyter_kernel_test <https://github.com/jupyter/jupyter_kernel_test>`_ * `simple_kernel <https://github.com/dsblank/simple_kernel>`_ .. faqref:: :title: Do I need to shutdown the kernel after running a notebook? .. index:: travis If the class is instantiated with *kernel=True*, a kernel will be started. It must be shutdown otherwise the program might be waiting for it for ever. That is one of the reasons why the travis build does not complete. The build finished but cannot temrinate until all kernels are shutdown. """ #. available output types MIME_MAP = { 'image/jpeg': 'jpeg', 'image/png': 'png', 'image/gif': 'gif', 'text/plain': 'text', 'text/html': 'html', 'text/latex': 'latex', 'application/javascript': 'html', 'image/svg+xml': 'svg', } def __init__(self, nb, profile_dir=None, working_dir=None, comment="", fLOG=noLOG, theNotebook=None, code_init=None, kernel_name="python", log_level="30", extended_args=None, kernel=False, filename=None, replacements=None): """ constuctor @param nb notebook as JSON @param profile_dir profile directory @param working_dir working directory @param comment additional information added to error message @param theNotebook if not None, populate the variable *theNotebook* with this value in the notebook @param code_init to initialize the notebook with a python code as if it was a cell @param fLOG logging function @param log_level Choices: (0, 10, 20, 30=default, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL') @param kernel_name kernel name, it can be None @param extended_args others arguments to pass to the command line ('--KernelManager.autorestar=True' for example), see :ref:`l-ipython_notebook_args` for a full list @param kernel *kernel* is True by default, the notebook can be run, if False, the notebook can be read but not run @param filename to add the notebook file if there is one in error messages @param replacements replacements to make in every cell before running it, dictionary ``{ string: string }`` .. versionchanged:: 1.4 Parameter *replacements* was added. """ if kernel: try: from jupyter_client import KernelManager except ImportError: from ipykernel import KernelManager self.km = KernelManager( kernel_name=kernel_name) if kernel_name is not None else KernelManager() else: self.km = None self.fLOG = fLOG self.theNotebook = theNotebook self.code_init = code_init self._filename = filename if filename is not None else "memory" self.replacements = replacements self.init_args = dict(profile_dir=profile_dir, working_dir=working_dir, comment=comment, fLOG=fLOG, theNotebook=theNotebook, code_init=code_init, kernel_name="python", log_level="30", extended_args=None, kernel=kernel, filename=filename, replacements=replacements) args = [] if profile_dir: args.append('--profile-dir=%s' % os.path.abspath(profile_dir)) if log_level: args.append('--log-level=%s' % log_level) if extended_args is not None and len(extended_args) > 0: for opt in extended_args: if not opt.startswith("--"): raise SyntaxError( "every option should start with '--': " + opt) if "=" not in opt: raise SyntaxError( "every option should be assigned a value: " + opt) args.append(opt) if kernel: cwd = os.getcwd() if working_dir: os.chdir(working_dir) if self.km is not None: if sys.version_info[0] == 2 and args is not None: # I did not find a way to make it work args = None warnings.warn( "args is not None: {0}, unable to use it in Python 2.7".format(args)) self.km.start_kernel() else: try: self.km.start_kernel(extra_arguments=args) except Exception as e: raise Exception( "Failure with args: {0}\nand error:\n{1}".format(args, str(e))) from e if platform.system() == 'Darwin': # see http://www.pypedia.com/index.php/notebook_runner # There is sometimes a race condition where the first # execute command hits the kernel before it's ready. # It appears to happen only on Darwin (Mac OS) and an # easy (but clumsy) way to mitigate it is to sleep # for a second. sleep(1) os.chdir(cwd) self.kc = self.km.client() self.kc.start_channels(stdin=False) # if it does not work, it probably means IPython < 3 self.kc.wait_for_ready() else: self.km = None self.kc = None self.nb = nb self.comment = comment def to_json(self, filename=None, encoding="utf8"): """ convert the notebook into json @param filename filename or stream @param encoding encoding @return Json string if filename is None, None otherwise .. versionchanged:: 1.4 The function now returns the json string if filename is None. """ if isinstance(filename, str # unicode# ): with open(filename, "w", encoding=encoding) as payload: self.to_json(payload) elif filename is None: st = StringIO() st.write(writes(self.nb)) return st.getvalue() else: filename.write(writes(self.nb)) @staticmethod def read_json(js, profile_dir=None, encoding="utf8", working_dir=None, comment="", fLOG=noLOG, code_init=None, kernel_name="python", log_level="30", extended_args=None, kernel=False, replacements=None): """ read a notebook from a JSON stream or string @param js string or stream @param profile_dir profile directory @param encoding encoding for the notebooks @param kernel to start a kernel or not when reading the notebook (to execute it) @param working_dir working directory @param comment additional information added to error message @param code_init to initialize the notebook with a python code as if it was a cell @param fLOG logging function @param log_level Choices: (0, 10, 20, 30=default, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL') @param kernel_name kernel name, it can be None @param extended_args others arguments to pass to the command line ('--KernelManager.autorestar=True' for example), see :ref:`l-ipython_notebook_args` for a full list @param kernel *kernel* is True by default, the notebook can be run, if False, the notebook can be read but not run @param replacements replacements to make in every cell before running it, dictionary ``{ string: string }`` @return instance of @see cl NotebookRunner .. versionchanged:: 1.5 Add constructor parameters. """ if isinstance(js, str # unicode# ): st = StringIO(js) else: st = js from .notebook_helper import read_nb return read_nb(st, encoding=encoding, kernel=kernel, profile_dir=profile_dir, working_dir=working_dir, comment=comment, fLOG=fLOG, code_init=code_init, kernel_name="python", log_level="30", extended_args=None, replacements=replacements) def copy(self): """ copy the notebook (just the content) @return instance of @see cl NotebookRunner .. versionadded:: 1.1 .. versionchanged:: 1.5 Add constructor parameters. """ st = StringIO() self.to_json(st) args = self.init_args.copy() for name in ["theNotebook", "filename"]: if name in args: del args[name] return NotebookRunner.read_json(st.getvalue(), **args) def __add__(self, nb): """ merges two notebooks together, returns a new none @param nb notebook @return new notebook """ c = self.copy() c.merge_notebook(nb) return c def shutdown_kernel(self): """ shut down kernel """ self.fLOG('-- shutdown kernel') if self.kc is None: raise ValueError( "No kernel was started, specify kernel=True when initializing the instance.") self.kc.stop_channels() self.km.shutdown_kernel(now=True) def clean_code(self, code): """ clean the code before running it, the function comment out instruction such as ``show()`` @param code code (string) @return cleaned code .. versionchanged:: 1.4 Do replacements. """ has_bokeh = "bokeh." in code or "from bokeh" in code or "import bokeh" in code if code is None: return code else: lines = [_.strip("\n\r").rstrip(" \t") for _ in code.split("\n")] res = [] show_is_last = False for line in lines: if line.replace(" ", "") == "show()": line = line.replace("show", "#show") show_is_last = True elif has_bokeh and line.replace(" ", "") == "output_notebook()": line = line.replace("output_notebook", "#output_notebook") else: show_is_last = False if self.replacements is not None: for k, v in self.replacements.items(): line = line.replace(k, v) res.append(line) if show_is_last: res.append('"nothing to show"') return "\n".join(res) @staticmethod def get_cell_code(cell): """ return the code of a cell @param cell a cell or a string @return boolean (=iscell), string """ if isinstance(cell, str # unicode# ): iscell = False return iscell, cell else: iscell = True try: return iscell, cell.source except AttributeError: return iscell, cell.input def run_cell(self, index_cell, cell, clean_function=None): ''' Run a notebook cell and update the output of that cell in-place. @param index_cell index of the cell @param cell cell to execute @param clean_function cleaning function to apply to the code before running it @return output of the cell ''' iscell, codei = NotebookRunner.get_cell_code(cell) self.fLOG('-- running cell:\n%s\n' % codei) code = self.clean_code(codei) if clean_function is not None: code = clean_function(code) if len(code) == 0: return "" if self.kc is None: raise ValueError( "No kernel was started, specify kernel=True when initializing the instance.") self.kc.execute(code) reply = self.kc.get_shell_msg() reason = None try: status = reply['content']['status'] except KeyError: status = 'error' reason = "no status key in reply['content']" if status == 'error': ansi_escape = re.compile(r'\x1b[^m]*m') try: tr = [ansi_escape.sub('', _) for _ in reply['content']['traceback']] except KeyError: tr = ["No traceback, available keys in reply['content']"] + \ [_ for _ in reply['content']] traceback_text = '\n'.join(tr) self.fLOG("ERR:\n", traceback_text) else: traceback_text = '' self.fLOG('-- cell returned') outs = list() nbissue = 0 while True: try: msg = self.kc.get_iopub_msg(timeout=1) if msg['msg_type'] == 'status': if msg['content']['execution_state'] == 'idle': break except Empty: # execution state should return to idle before the queue becomes empty, # if it doesn't, something bad has happened status = "error" reason = "exception Empty was raised" nbissue += 1 if nbissue > 10: # the notebook is empty return "" else: continue content = msg['content'] msg_type = msg['msg_type'] # IPython 3.0.0-dev writes pyerr/pyout in the notebook format but uses # error/execute_result in the message spec. This does the translation # needed for tests to pass with IPython 3.0.0-dev notebook3_format_conversions = { 'error': 'pyerr', 'execute_result': 'pyout' } msg_type = notebook3_format_conversions.get(msg_type, msg_type) out = NotebookNode(output_type=msg_type) if 'execution_count' in content: if iscell: cell['prompt_number'] = content['execution_count'] out.prompt_number = content['execution_count'] if msg_type in ('status', 'pyin', 'execute_input'): continue elif msg_type == 'stream': out.stream = content['name'] # in msgspec 5, this is name, text # in msgspec 4, this is name, data if 'text' in content: out.text = content['text'] else: out.data = content['data'] elif msg_type in ('display_data', 'pyout'): out.data = content['data'] elif msg_type == 'pyerr': out.ename = content['ename'] out.evalue = content['evalue'] out.traceback = content['traceback'] elif msg_type == 'clear_output': outs = list() continue elif msg_type == 'comm_open' or msg_type == 'comm_msg': # widgets in a notebook out.data = content["data"] out.comm_id = content["comm_id"] else: dcontent = "\n".join("{0}={1}".format(k, v) for k, v in sorted(content.items())) raise NotImplementedError( 'unhandled iopub message: %s' % msg_type + "\nCONTENT:\n" + dcontent) outs.append(out) if iscell: cell['outputs'] = outs raw = [] for _ in outs: try: t = _.data except AttributeError: continue # see MIMEMAP to see the available output type for k, v in t.items(): if k.startswith("text"): raw.append(v) sraw = "\n".join(raw) self.fLOG(sraw) def reply2string(reply): sreply = [] for k, v in sorted(reply.items()): if isinstance(v, dict): temp = [] for _, __ in sorted(v.items()): temp.append(" [{0}]={1}".format(_, str(__))) v = "\n".join(temp) sreply.append("reply['{0}']=dict\n{1}".format(k, v)) else: sreply.append("reply['{0}']={1}".format(k, str(v))) sreply = "\n".join(sreply) return sreply if status == 'error': sreply = reply2string(reply) if len(code) < 5: scode = [code] else: scode = "" mes = "FILENAME\n{10}:1:1\n{7}\nCELL status={8}, reason={9} -- {4} length={5} -- {6}:\n-----------------\n{0}" + \ "\n-----------------\nTRACE:\n{1}\nRAW:\n{2}REPLY:\n{3}" raise NotebookError(mes.format( code, traceback_text, sraw, sreply, index_cell, len( code), scode, self.comment, status, reason, self._filename)) return outs def iter_code_cells(self): ''' Iterate over the notebook cells containing code. ''' for cell in self.iter_cells(): if cell.cell_type == 'code': yield cell def iter_cells(self): ''' Iterate over the notebook cells. ''' if hasattr(self.nb, "worksheets"): for ws in self.nb.worksheets: for cell in ws.cells: yield cell else: for cell in self.nb.cells: yield cell def first_cell(self): """ Returns the first cell. """ for cell in self.iter_cells(): return cell def _cell_container(self): """ returns a cells container, it may change according to the format @return cell container """ if hasattr(self.nb, "worksheets"): last = None for ws in self.nb.worksheets: last = ws if last is None: raise NotebookError("no cell container") return last.cells else: return self.nb.cells def __len__(self): """ return the number of cells, it iterates on cells to get this information and does cache the information @return int .. versionadded:: 1.1 """ return sum(1 for _ in self.iter_cells()) def cell_type(self, cell): """ returns the cell type @param cell from @see me iter_cells @return type """ return cell.cell_type def cell_metadata(self, cell): """ returns the cell metadata @param cell cell @return metadata """ return cell.metadata def _check_thumbnail_tuple(self, b): """ checks types for a thumbnail @param b tuple image, format @return b The function raises an exception if the type is incorrect. """ if not isinstance(b, tuple): raise TypeError("tuple expected, not {0}".format(type(b))) if len(b) != 2: raise TypeError( "tuple expected of lengh 2, not {0}".format(len(b))) if b[1] == "svg": if not isinstance(b[0], str): raise TypeError( "str expected for svg, not {0}".format(type(b[0]))) else: if not isinstance(b[0], bytes): raise TypeError( "bytes expected for images, not {0}".format(type(b[0]))) return b def create_picture_from(self, text, format, asbytes=True, context=None): """ Creates a picture from text. @param text the text @param format text, json, ... @param context (str) indication on the content of text (error, ...) @param asbytes results as bytes or as an image @return tuple (picture, format) or PIL.Image (if asbytes is False) The picture will be bytes, the format png, bmp... The size of the picture will depend on the text. The longer, the bigger. The method relies on matplotlib and then convert the image into a PIL image. HTML could be rendered with QWebPage from PyQt (not implemented). """ if not isinstance(text, (str, bytes)): text = str(text) if "\n" not in text: rows = [] for i in range(0, len(text), 20): end = min(i + 20, len(text)) rows.append(text[i:end]) text = "\n".join(text) if len(text) > 200: text = text[:200] size = len(text) // 10 figsize = (3 + size, 3 + size) lines = text.replace("\t", " ").replace("\r", "").split("\n") import matplotlib.pyplot as plt from matplotlib.textpath import TextPath from matplotlib.font_manager import FontProperties fig = plt.figure(figsize=figsize) ax = fig.add_subplot(111) fp = FontProperties(size=200) dx = 0 dy = 0 for i, line in enumerate(lines): if len(line.strip()) > 0: ax.text(0, -dy, line, fontproperties=fp, va='top') tp = TextPath((0, -dy), line, prop=fp) bb = tp.get_extents() dy += bb.height dx = max(dx, bb.width) ratio = abs(dx) / max(abs(dy), 1) ratio = max(min(ratio, 3), 1) fig.set_size_inches(int((1 + size) * ratio), 1 + size) ax.set_xlim([0, dx]) ax.set_ylim([-dy, 0]) ax.set_axis_off() sio = BytesIO() fig.savefig(sio, format="png") plt.close() if asbytes: b = sio.getvalue(), "png" self._check_thumbnail_tuple(b) return b else: try: from PIL import Image except ImportError: import Image img = Image.open(sio) return img def cell_image(self, cell, image_from_text=False): """ returns the cell image or None if not found @param cell cell to examine @param image_from_text produce an image even if it is not one @return None for no image or a list of tuple (image as bytes, extension) for each output of the cell """ kind = self.cell_type(cell) if kind != "code": return None results = [] for output in cell.outputs: if output["output_type"] in {"execute_result", "display_data"}: data = output["data"] for k, v in data.items(): if k == "text/plain": if image_from_text: b = self.create_picture_from( v, "text", context=output["output_type"]) results.append(b) elif k == "application/javascript": if image_from_text: b = self.create_picture_from(v, "js") results.append(b) elif k == "application/json": if image_from_text: b = self.create_picture_from(v, "json") results.append(b) elif k == "image/svg+xml": if not isinstance(v, str): raise TypeError( "This should be str not '{0}' (=SVG).".format(type(v))) results.append((v, "svg")) elif k == "text/html": if image_from_text: b = self.create_picture_from(v, "html") results.append(b) elif k == "text/latex": if image_from_text: b = self.create_picture_from(v, "latex") results.append(b) elif k in {"image/png", "image/jpg", "image/jpeg", "image/gif"}: if not isinstance(v, bytes): v = base64.b64decode(v) if not isinstance(v, bytes): raise TypeError( "This should be bytes not '{0}' (=IMG:{1}).".format(type(v), k)) results.append((v, k.split("/")[-1])) else: raise NotImplementedError("cell type: {0}\nk={1}\nv={2}\nCELL:\n{3}".format(kind, k, v, cell)) elif output["output_type"] == "error": vl = output["traceback"] if image_from_text: for v in vl: b = self.create_picture_from( v, "text", context="error") results.append(b) elif output["output_type"] == "stream": v = output["text"] if image_from_text: b = self.create_picture_from(v, "text") results.append(b) else: raise NotImplementedError("cell type: {0}\noutput type: {1}\nOUT:\n{2}\nCELL:\n{3}" .format(kind, output["output_type"], output, cell)) if len(results) > 0: res = self._merge_images(results) self._check_thumbnail_tuple(res) return res else: return None def cell_height(self, cell): """ approximate the height of a cell by its number of lines it contains @param cell cell @return number of cell """ kind = self.cell_type(cell) if kind == "markdown": content = cell.source lines = content.split("\n") nbs = sum(1 + len(line) // 80 for line in lines) return nbs elif kind == "raw": content = cell.source lines = content.split("\n") nbs = sum(1 + len(line) // 80 for line in lines) return nbs elif kind == "code": content = cell.source lines = content.split("\n") nbl = len(lines) for output in cell.outputs: if output["output_type"] == "execute_result" or \ output["output_type"] == "display_data": data = output["data"] for k, v in data.items(): if k == "text/plain": nbl += len(v.split("\n")) elif k == "application/javascript": # rough estimation nbl += len(v.split("\n")) // 2 elif k == "application/json": # rough estimation try: nbl += len(v.split("{")) except AttributeError: nbl += len(v) // 5 + 1 elif k == "image/svg+xml": nbl += len(v) // 5 elif k == "text/html": nbl += len(v.split("\n")) elif k == "text/latex": nbl += len(v.split("\\\\")) * 2 elif k in {"image/png", "image/jpg", "image/jpeg", "image/gif"}: nbl += len(v) // 50 else: raise NotImplementedError("cell type: {0}\nk={1}\nv={2}\nCELL:\n{3}".format(kind, k, v, cell)) elif output["output_type"] == "stream": v = output["text"] nbl += len(v.split("\n")) elif output["output_type"] == "error": v = output["traceback"] nbl += len(v) else: raise NotImplementedError("cell type: {0}\noutput type: {1}\nOUT:\n{2}\nCELL:\n{3}" .format(kind, output["output_type"], output, cell)) return nbl else: raise NotImplementedError( "cell type: {0}\nCELL:\n{1}".format(kind, cell)) def add_tag_slide(self, max_nb_cell=4, max_nb_line=25): """ tries to add tags for a slide show when they are too few @param max_nb_cell maximum number of cells within a slide @param max_nb_line maximum number of lines within a slide @return list of modified cells { #slide: (kind, reason, cell) } """ res = {} nbline = 0 nbcell = 0 for i, cell in enumerate(self.iter_cells()): meta = cell.metadata if "slideshow" in meta: st = meta["slideshow"]["slide_type"] if st in ["slide", "subslide"]: nbline = 0 nbcell = 0 else: if cell.cell_type == "markdown": content = cell.source if content.startswith("# ") or \ content.startswith("## ") or \ content.startswith("### "): meta["slideshow"] = {'slide_type': 'slide'} nbline = 0 nbcell = 0 res[i] = ("slide", "section", cell) dh = self.cell_height(cell) dc = 1 new_nbline = nbline + dh new_cell = dc + nbcell if "slideshow" not in meta: if new_cell > max_nb_cell or \ new_nbline > max_nb_line: res[i] = ( "subslide", "{0}-{1} <-> {2}-{3}".format(nbcell, nbline, dc, dh), cell) nbline = 0 nbcell = 0 meta["slideshow"] = {'slide_type': 'subslide'} nbline += dh nbcell += dc return res def run_notebook(self, skip_exceptions=False, progress_callback=None, additional_path=None, valid=None, clean_function=None): ''' Run all the cells of a notebook in order and update the outputs in-place. If ``skip_exceptions`` is set, then if exceptions occur in a cell, the subsequent cells are run (by default, the notebook execution stops). @param skip_exceptions skip exception @param progress_callback call back function @param additional_path additional paths (as a list or None if none) @param valid if not None, valid is a function which returns whether or not the cell should be executed or not, if the function returns None, the execution of the notebooks and skip the execution of the other cells @param clean_function function which cleans a cell's code before executing it (None for None) @return dictionary with statistics .. versionchanged:: 1.1 The function adds the local variable ``theNotebook`` with the absolute file name of the notebook. .. versionchanged:: 1.4 Function *valid* can now return None to stop the execution of the notebook before this cell. ''' # additional path if additional_path is not None: if not isinstance(additional_path, list): raise TypeError( "additional_path should be a list not: " + str(additional_path)) code = ["import sys"] for p in additional_path: code.append("sys.path.append(r'{0}')".format(p)) cell = "\n".join(code) self.run_cell(-1, cell) # we add local variable theNotebook if self.theNotebook is not None: cell = "theNotebook = r'''{0}'''".format(self.theNotebook) self.run_cell(-1, cell) # initialisation with a code not inside the notebook if self.code_init is not None: self.run_cell(-1, self.code_init) # execution of the notebook nbcell = 0 nbrun = 0 nbnerr = 0 cl = time.clock() for i, cell in enumerate(self.iter_code_cells()): nbcell += 1 iscell, codei = NotebookRunner.get_cell_code(cell) if valid is not None: r = valid(codei) if r is None: break elif not r: continue try: nbrun += 1 self.run_cell(i, cell, clean_function=clean_function) nbnerr += 1 except Empty as er: raise Exception( "{0}\nissue when executing:\n{1}".format(self.comment, codei)) from er except NotebookError as e: if not skip_exceptions: raise else: raise Exception( "issue when executing:\n{0}".format(codei)) from e if progress_callback: progress_callback(i) etime = time.clock() - cl return dict(nbcell=nbcell, nbrun=nbrun, nbvalid=nbnerr, time=etime) def count_code_cells(self): ''' @return the number of code cells in the notebook .. versionadded:: 1.1 ''' return sum(1 for _ in self.iter_code_cells()) def merge_notebook(self, nb): """ append notebook *nb* to this one @param nb notebook or list of notebook (@see cl NotebookRunner) @return number of added cells .. faqref:: :title: How to merge notebook? The following code merges two notebooks into the first one and stores the result unto a file. @code from pyquickhelper.ipythonhelper import read_nb nb1 = read_nb("<file1>", kernel=False) nb2 = read_nb("<file2>", kernel=False) nb1.merge_notebook(nb2) nb1.to_json(outfile) @endcode .. versionadded:: 1.1 """ if isinstance(nb, list): s = 0 for n in nb: s += self.merge_notebook(n) return s else: last = self._cell_container() s = 0 for cell in nb.iter_cells(): last.append(cell) s += 1 return s def get_description(self): """ Get summary and description of this notebook. We expect the first cell to contain a title and a description of its content. @return header, description .. versionadded:: 1.5 """ def split_header(s, get_header=True): s = s.lstrip().rstrip() parts = s.splitlines() if parts[0].startswith('#'): if get_header: header = re.sub('#+\s*', '', parts.pop(0)) if not parts: return header, '' else: header = '' rest = '\n'.join(parts).lstrip().split('\n\n') desc = rest[0].replace('\n', ' ') return header, desc else: if get_header: if parts[0].startswith(('=', '-')): parts = parts[1:] header = parts.pop(0) if parts and parts[0].startswith(('=', '-')): parts.pop(0) if not parts: return header, '' else: header = '' rest = '\n'.join(parts).lstrip().split('\n\n') desc = rest[0].replace('\n', ' ') return header, desc first_cell = self.first_cell() if not first_cell['cell_type'] == 'markdown': raise ValueError("The first cell is not in markdown but '{0}'.".format( first_cell['cell_type'])) header, desc = split_header(first_cell['source']) if not desc and len(self.nb['cells']) > 1: second_cell = self.nb['cells'][1] if second_cell['cell_type'] == 'markdown': _, desc = split_header(second_cell['source'], False) reg_link = "(\\[(.*?)\\]\\(([^ ]*)\\))" reg = re.compile(reg_link) new_desc = reg.sub("\\2", desc) if "http://" in new_desc or "https://" in new_desc: raise ValueError( "Wrong regular expression:\n{0}\nMODIFIED:\n{1}".format(desc, new_desc)) return header, new_desc.replace('"', "") def get_thumbnail(self, max_width=200, max_height=200): """ Process the notebook and create one picture based on the outputs to illustrate a notebook. @param max_width maximum size of the thumbnail @param max_height maximum size of the thumbnail @return string (SVG) or Image (PIL) This functionality might not works with Python 2.7. .. versionadded:: 1.5 """ images = [] cells = list(self.iter_cells()) cells.reverse() for cell in cells: c = self.cell_image(cell, False) if c is not None and len(c) > 0 and len(c[0]) > 0: self._check_thumbnail_tuple(c) images.append(c) if len(images) == 0: for cell in cells: c = self.cell_image(cell, True) if c is not None and len(c) > 0 and len(c[0]) > 0: self._check_thumbnail_tuple(c) images.append(c) if len(c[0]) >= 1000: break if len(images) == 0: # no image, we need to consider the default one no_image = os.path.join( os.path.dirname(__file__), 'no_image_nb.png') with open(no_image, "rb") as f: c = (f.read(), "png") self._check_thumbnail_tuple(c) images.append(c) # select the image if len(images) == 0: raise ValueError("There should be at least one image.") elif len(images) == 1: image = images[0] else: # maybe later we'll implement a different logic # we pick the last one image = images[0] # zoom if image[1] != "svg": img = self._scale_image( image[0], image[1], max_width=max_width, max_height=max_height) return img else: return image[0] def _scale_image(self, in_bytes, format=None, max_width=200, max_height=200): """ Scales an image with the same aspect ratio centered in an image with a given max_width and max_height. @param in_bytes image as bytes @param format indication of the format (can be empty) @param max_width maximum size of the thumbnail @param max_height maximum size of the thumbnail @return Image (PIL) .. versionadded:: 1.5 """ # local import to avoid testing dependency on PIL: try: from PIL import Image except ImportError: import Image if isinstance(in_bytes, tuple): in_bytes = in_bytes[0] if not isinstance(in_bytes, bytes): raise TypeError("bytes expected, not {0}".format(type(in_bytes))) img = Image.open(BytesIO(in_bytes)) width_in, height_in = img.size scale_w = max_width / float(width_in) scale_h = max_height / float(height_in) if height_in * scale_w <= max_height: scale = scale_w else: scale = scale_h if scale >= 1.0: return img width_sc = int(round(scale * width_in)) height_sc = int(round(scale * height_in)) # resize the image and center img.thumbnail((width_sc, height_sc), Image.ANTIALIAS) thumb = Image.new('RGB', (max_width, max_height), (255, 255, 255)) pos_insert = ((max_width - width_sc) // 2, (max_height - height_sc) // 2) thumb.paste(img, pos_insert) return thumb def _merge_images(self, results): """ Merges images defined by (buffer, format). The method uses PIL to merge images when possible. @return [ (image, format) ] .. versionadded:: 1.5 """ if len(results) == 1: results = results[0] self._check_thumbnail_tuple(results) return results elif len(results) == 0: return None formats_counts = Counter(_[1] for _ in results) if len(formats_counts) == 1: format = results[0][1] else: items = sorted(((v, k) for k, v in formats_counts.items()), False) for it in items: format = it break results = [_ for _ in results if _[1] == format] if format == "svg": return ("\n".join(_[0] for _ in results), format) else: # local import to avoid testing dependency on PIL: try: from PIL import Image except ImportError: import Image dx = 0. dy = 0. over = 0.7 imgs = [] for in_bytes, f in results: img = Image.open(BytesIO(in_bytes)) imgs.append(img) dx = max(dx, img.size[0]) dy += img.size[1] * over new_im = Image.new('RGB', (int(dx), int(dy)), (220, 220, 220)) for img in imgs: dy -= img.size[1] * over new_im.paste(img, (0, max(int(dy), 0))) image_buffer = BytesIO() new_im.save(image_buffer, "PNG") b = image_buffer.getvalue(), "png" return b
def paths(): for fullpath in glob(os.path.join(os.path.dirname(filename), 'kernel*')): if not re.match('^(.*/)?kernel-[0-9]+.json', fullpath): continue yield fullpath connected = False while not connected: try: filename = find_connection_file('kernel*') except IOError: continue for fullpath in paths(): km = KernelManager(connection_file=fullpath) km.load_connection_file() kc = km.client() kc.start_channels() try: send = kc.execute except AttributeError: send = kc.shell_channel.execute if not hasattr(kc, 'iopub_channel'): kc.iopub_channel = kc.sub_channel send('', silent=True) try: msg = kc.shell_channel.get_msg(timeout=1) connected = True
def km_from_string(s=''): """create kernel manager from IPKernelApp string such as '--shell=47378 --iopub=39859 --stdin=36778 --hb=52668' for IPython 0.11 or just 'kernel-12345.json' for IPython 0.12 """ try: import IPython except ImportError: raise ImportError("Could not find IPython. " + _install_instructions) try: from traitlets.config.loader import KeyValueConfigLoader except ImportError: from IPython.config.loader import KeyValueConfigLoader try: from jupyter_client import KernelManager, find_connection_file except ImportError: try: from IPython.kernel import KernelManager, find_connection_file except ImportError: # IPython < 1.0 from IPython.zmq.blockingkernelmanager import BlockingKernelManager as KernelManager from IPython.zmq.kernelapp import kernel_aliases try: from IPython.lib.kernel import find_connection_file except ImportError: # < 0.12, no find_connection_file pass global km, kc, send, history, complete, object_info # Test if connection is still alive connected = False starttime = time.time() attempt = 0 s = s.replace('--existing', '') while not connected and (time.time() - starttime) < 5.0: if not attempt and os.path.isfile(s): fullpath = s else: try: s = fullpath = find_connection_file('kernel*') except IOError: echo("IPython connection attempt #%d failed - no kernel file" % attempt, "Warning") time.sleep(1) continue attempt += 1 km = KernelManager(connection_file=fullpath) km.load_connection_file() kc = km.client() kc.start_channels() if 'connection_file' in KernelManager.class_trait_names(): # 0.12 uses files instead of a collection of ports # include default IPython search path # filefind also allows for absolute paths, in which case the search # is ignored try: # XXX: the following approach will be brittle, depending on what # connection strings will end up looking like in the future, and # whether or not they are allowed to have spaces. I'll have to sync # up with the IPython team to address these issues -pi if '--profile' in s: k,p = s.split('--profile') k = k.lstrip().rstrip() # kernel part of the string p = p.lstrip().rstrip() # profile part of the string fullpath = find_connection_file(k,p) else: fullpath = find_connection_file(s.lstrip().rstrip()) except IOError as e: echo(":IPython " + s + " failed", "Info") echo("^-- failed '" + s + "' not found", "Error") return km = KernelManager(connection_file = fullpath) km.load_connection_file() else: if s == '': echo(":IPython 0.11 requires the full connection string") return loader = KeyValueConfigLoader(s.split(), aliases=kernel_aliases) cfg = loader.load_config()['KernelApp'] try: km = KernelManager( shell_address=(ip, cfg['shell_port']), sub_address=(ip, cfg['iopub_port']), stdin_address=(ip, cfg['stdin_port']), hb_address=(ip, cfg['hb_port'])) except KeyError as e: echo(":IPython " +s + " failed", "Info") echo("^-- failed --"+e.message.replace('_port','')+" not specified", "Error") return try: kc = km.client() except AttributeError: # 0.13 kc = km kc.start_channels() execute = kc.execute if hasattr(kc, 'execute') else kc.shell_channel.execute history = kc.history if hasattr(kc, 'history') else kc.shell_channel.history complete = kc.complete if hasattr(kc, 'complete') else kc.shell_channel.complete object_info = kc.inspect if hasattr(kc, 'inspect') else kc.shell_channel.object_info def send(msg, **kwargs): kwds = dict( store_history=vim_vars.get('ipython_store_history', True), ) kwds.update(kwargs) return execute(msg, **kwds) send('', silent=True) try: msg = kc.shell_channel.get_msg(timeout=1) connected = True except: echo("IPython connection attempt #%d failed - no messages" % attempt, "Warning") continue #XXX: backwards compatibility for IPython < 1.0 if not hasattr(kc, 'iopub_channel'): kc.iopub_channel = kc.sub_channel set_pid() if not connected: echo("IPython connection attempt timed out", "Error") return else: vim.command('redraw') echo("IPython connection successful") send('"_vim_client";_=_;__=__', store_history=False) #XXX: backwards compatibility for IPython < 0.13 sc = kc.shell_channel if hasattr(sc, 'object_info'): import inspect num_oinfo_args = len(inspect.getargspec(sc.object_info).args) if num_oinfo_args == 2: # patch the object_info method which used to only take one argument klass = sc.__class__ klass._oinfo_orig = klass.object_info klass.object_info = lambda s,x,y: s._oinfo_orig(x) #XXX: backwards compatibility for IPython < 1.0 if not hasattr(kc, 'iopub_channel'): kc.iopub_channel = kc.sub_channel # now that we're connect to an ipython kernel, activate completion # machinery, but do so only for the local buffer if the user added the # following line the vimrc: # let g:ipy_completefunc = 'local' vim.command(""" if g:ipy_completefunc == 'global' set completefunc=CompleteIPython elseif g:ipy_completefunc == 'local' setl completefunc=CompleteIPython elseif g:ipy_completefunc == 'omni' setl omnifunc=CompleteIPython endif """) # also activate GUI doc balloons if in gvim vim.command(""" if has('balloon_eval') set bexpr=IPythonBalloonExpr() endif """) return km
def run_notebook(notebook): f = open(notebook) if not f: return False nb = reads(f.read(), 3) km = KernelManager() km.start_kernel(extra_arguments=['--pylab=inline'], stderr=open('/tmp/km.stderr', 'w')) kc = km.client() kc.start_channels() shell = kc.shell_channel shell.get_msg() successes = 0 failures = 0 errors = 0 for ws in nb.worksheets: for cell in ws.cells: if cell.cell_type != 'code': continue try: status, outs = run_cell(kc, cell, 30) except Exception as e: # currently turned off to avoid jenkins hang # print "failed to run cell:", repr(e) # print cell.input # print dir(cell) errors += 1 continue failed = False # currently turned off to avoid jenkins hang # print "Count outs: %d" % len(outs) # print "Count cell_out: %d" % len(cell.outputs) #for out, ref in zip(outs, cell.outputs): # print "OUT[%s]" % outs # print "EXP[%s]" % ref # #if not compare_outputs(out, ref): # # failed = True # # break if status != "ok" or failed: failures += 1 else: successes += 1 print print "tested notebook %s" % nb.metadata.name print " %3i cells successfully replicated" % successes if failures: print " %3i cells mismatched output" % failures if errors: print " %3i cells failed to complete" % errors kc.stop_channels() km.shutdown_kernel() del km if failures > 0 or errors > 0: return False else: return True