Ejemplo n.º 1
0
def km_from_string(s=''):
    """
    Create kernel manager.
    """

    global km, kc, send

    s = s.replace('--existing', '')

    try:
        if '--profile' in s:
            k, p = s.split('--profile')
            k = k.lstrip().rstrip()  # kernel part of the string
            p = p.lstrip().rstrip()  # profile part of the string
            fullpath = find_connection_file(k, p)
        elif s == '':
            fullpath = find_connection_file()
        else:
            fullpath = find_connection_file(s.lstrip().rstrip())
    except IOError:
        vim_echo(":IPython " + s + " failed", "Info")
        vim_echo("^-- failed '" + s + "' not found", "Error")
        return

    km = KernelManager(connection_file=fullpath)
    km.load_connection_file()
    kc = km.client()
    kc.start_channels()
    send = kc.execute
    vim_echo('Connected to {}'.format(fullpath))
Ejemplo n.º 2
0
def connect_kernel_from_file(kernel_type, connection_file):
    from jupyter_client import KernelManager
    # Create the kernel manager and connect a client
    # See: <http://jupyter-client.readthedocs.io/en/stable/api/client.html>
    km = KernelManager(connection_file=connection_file)
    km.load_connection_file()
    kc = km.client()
    kc.start_channels()

    # Alias execute function
    def _send(msg, **kwargs):
        """Send a message to the kernel client."""
        # Include dedent of msg so we don't get odd indentation errors.
        return kc.execute(textwrap.dedent(msg), **kwargs)

    send = _send

    # Ping the kernel
    kc.kernel_info()
    try:
        reply = kc.get_shell_msg(timeout=1)
    except Empty:
        return {"connected": False}
    else:
        # pid = get_pid(kernel_type)  # Ask kernel for its PID
        return {"connected": True, "kc": kc, "send": send}
Ejemplo n.º 3
0
def temporary_connect_query(connection_file, code, retvar):
    # connect
    from jupyter_client import KernelManager
    _km = KernelManager(connection_file=connection_file)
    _km.load_connection_file()
    _kc = _km.client()
    _kc.start_channels()

    def _send(msg, **kwargs):
        """Send a message to the kernel client."""
        # Include dedent of msg so we don't get odd indentation errors.
        return _kc.execute(textwrap.dedent(msg), **kwargs)

    # send code
    msg_id = _send(code, silent=True, user_expressions={"_ret": retvar})

    try:
        reply = get_reply_msg(msg_id, _kc=_kc)
    except Empty:
        vim_echom("no reply from jupyter kernel", "WarningMsg")
        return -1

    # get result
    retval = None
    try:
        # Requires the fix for https://github.com/JuliaLang/IJulia.jl/issues/815
        retval = reply["content"]["user_expressions"]["_ret"]["data"][
            "text/plain"]
    except KeyError:
        vim_echom("Could not get return value, kernel not ready?")

    # disconnect
    _kc.stop_channels()

    return retval
Ejemplo n.º 4
0
    def __init__(self,
                 timeout: int = 100,
                 template_type: TemplateType = TemplateType.FULL):
        """
        Initializes Exporter with default timeout (100 seconds) and template
        (FULL) and handles instantiation of km and ep variables for usage when
        generating NotebookNodes and their outputs.

        Args:
            timeout (int): Amount of time in seconds that a visualization can
            run for before being stopped.
            template_type (TemplateType): Type of template to use when
            generating visualization output.
        """
        self.timeout = timeout
        self.template_type = template_type
        # Create custom KernelManager.
        # This will circumvent issues where kernel is shutdown after
        # preprocessing. Due to the shutdown, latency would be introduced
        # because a kernel must be started per visualization.
        self.km = KernelManager()
        self.km.start_kernel()
        self.ep = ExecutePreprocessor(timeout=self.timeout,
                                      kernel_name='python3',
                                      allow_errors=True)
Ejemplo n.º 5
0
    def test_cleanup_context(self):
        km = KernelManager()
        self.assertIsNotNone(km.context)

        km.cleanup_resources(restart=False)

        self.assertTrue(km.context.closed)
Ejemplo n.º 6
0
    def __init__(self, use_exist=False):
        """
        ## Description
        Initializes the `kernel_manager` and `client` objects
        and starts the kernel. Also initializes the pretty printer
        for displaying object properties and execution result
        payloads.

        ## Parameters
        None.
        """
        if not use_exist:
            # Initialize kernel and client
            self.kernel_manager, self.client = start_new_kernel()
            self.send = self.client.execute
        else:
            self.kernel_manager = KernelManager(
                connection_file=find_connection_file())
            self.kernel_manager.load_connection_file(find_connection_file())
            self.client = self.kernel_manager.client()
            self.client.start_channels()
            self.send = self.client.execute

        # Initialize pretty printer
        self.pp = PrettyPrinter(indent=2)
Ejemplo n.º 7
0
def kernel_client(config):
    try:
        if config.kernel != 'existing':
            cnx_file = find_connection_file(filename=config.kernel)
        else:
            cnx_file = find_connection_file()

        km = KernelManager(connection_file=cnx_file)
        km.load_connection_file()

        kc = km.client()
        kc.start_channels()

        try:
            msg = kc.shell_channel.get_msg()
        except Empty:
            try:
                # try again :)
                sleep(1)
                msg = kc.shell_channel.get_msg()
            except Empty:
                raise RuntimeError(
                    'No message found in the channel. Is the kernel alive?')

    except Exception as e:
        print(format_text('PyExecError: ipython_exec was not able to connect to the desired jupyter kernel\n' + \
                          "Hint: execute 'ipython_start_kernel' or 'jupyter console' first\n" + \
                          format_exc(), config.to))
        exit(1)

    return kc
Ejemplo n.º 8
0
    def create_kernel_manager(self):
        """Create the kernel manager and connect a client.

        Returns
        -------
        bool
            True if client connects successfully, False on failure.
        """
        # Get client
        kernel_manager = KernelManager(connection_file=self.cfile)
        # The json may be badly encoding especially if autoconnecting
        try:
            kernel_manager.load_connection_file()
        # pylint: disable=broad-except
        except Exception:
            return False
        self.km_client = kernel_manager.client()

        # Open channel
        self.km_client.start_channels()

        # Ping the kernel
        self.km_client.kernel_info()
        try:
            self.km_client.get_shell_msg(timeout=1)
            return True
        except Empty:
            return False
Ejemplo n.º 9
0
class Exporter:
    """Handler for interaction with NotebookNodes, including output generation.

    Attributes:
        timeout (int): Amount of time in seconds that a visualization can run
        for before being stopped.
        template_type (TemplateType): Type of template to use when generating
        visualization output.
        km (KernelManager): Custom KernelManager that stays alive between
        visualizations.
        ep (ExecutePreprocessor): Process that is responsible for generating
        outputs from NotebookNodes.

    """
    def __init__(self,
                 timeout: int = 100,
                 template_type: TemplateType = TemplateType.FULL):
        """
        Initializes Exporter with default timeout (100 seconds) and template
        (FULL) and handles instantiation of km and ep variables for usage when
        generating NotebookNodes and their outputs.

        Args:
            timeout (int): Amount of time in seconds that a visualization can
            run for before being stopped.
            template_type (TemplateType): Type of template to use when
            generating visualization output.
        """
        self.timeout = timeout
        self.template_type = template_type
        # Create custom KernelManager.
        # This will circumvent issues where kernel is shutdown after
        # preprocessing. Due to the shutdown, latency would be introduced
        # because a kernel must be started per visualization.
        self.km = KernelManager()
        self.km.start_kernel()
        self.ep = ExecutePreprocessor(timeout=self.timeout,
                                      kernel_name='python3',
                                      allow_errors=True)

    def generate_html_from_notebook(self, nb: NotebookNode) -> Text:
        """Converts a provided NotebookNode to HTML.

        Args:
            nb: NotebookNode that should be converted to HTML.

        Returns:
            HTML from converted NotebookNode as a string.

        """
        # HTML generator and exporter object
        html_exporter = HTMLExporter()
        template_file = "templates/{}.tpl".format(self.template_type.value)
        html_exporter.template_file = str(Path.cwd() / template_file)
        # Output generator
        self.ep.preprocess(nb, {"metadata": {"path": Path.cwd()}}, self.km)
        # Export all html and outputs
        body, _ = html_exporter.from_notebook_node(nb)
        return body
Ejemplo n.º 10
0
def new_kernel():
    """
    Create a new IPython kernel (optionally with extra arguments)
    """

    km = KernelManager()
    km.start_kernel()
    return km_from_string(km.connection_file)
Ejemplo n.º 11
0
 def start(self):
     self.km = KernelManager(kernel_name='python',
                             client_class='execute.Client')
     self.km.start_kernel()
     self.kc = self.km.client()
     self.kc.start_channels()
     time.sleep(2)
     atexit.register(self.shutdown_kernel)
Ejemplo n.º 12
0
 def __init__(self):
     """
     Construct an IPythonConnection object without connecting
     """
     self.kmgr = KernelManager(kernel_name='python3')
     self.client = None
     self._status_ = 'disconnected'
     self._kernel_ = {}
Ejemplo n.º 13
0
    def test_no_cleanup_shared_context(self, zmq_context):
        """kernel manager does not terminate shared context"""
        km = KernelManager(context=zmq_context)
        assert km.context == zmq_context
        assert km.context is not None

        km.cleanup_resources(restart=False)
        assert km.context.closed is False
        assert zmq_context.closed is False
Ejemplo n.º 14
0
    def _run_signaltest_lifecycle(self, config=None):
        km = KernelManager(config=config, kernel_name="signaltest")
        kc = self._prepare_kernel(km, stdout=PIPE, stderr=PIPE)

        def execute(cmd):
            request_id = kc.execute(cmd)
            while True:
                reply = kc.get_shell_msg(TIMEOUT)
                if reply["parent_header"]["msg_id"] == request_id:
                    break
            content = reply["content"]
            assert content["status"] == "ok"
            return content

        execute("start")
        assert km.is_alive()
        execute("check")
        assert km.is_alive()

        km.restart_kernel(now=True)
        assert km.is_alive()
        execute("check")

        km.shutdown_kernel()
        assert km.context.closed
Ejemplo n.º 15
0
    def _run_signaltest_lifecycle(self, config=None):
        km = KernelManager(config=config, kernel_name='signaltest')
        kc = self._prepare_kernel(km, stdout=PIPE, stderr=PIPE)

        def execute(cmd):
            request_id = kc.execute(cmd)
            while True:
                reply = kc.get_shell_msg(TIMEOUT)
                if reply['parent_header']['msg_id'] == request_id:
                    break
            content = reply['content']
            assert content['status'] == 'ok'
            return content

        execute("start")
        assert km.is_alive()
        execute('check')
        assert km.is_alive()

        km.restart_kernel(now=True)
        assert km.is_alive()
        execute('check')

        km.shutdown_kernel()
        assert km.context.closed
Ejemplo n.º 16
0
    def connect_to_kernel(self):
        """Create kernel manager from existing connection file."""
        from jupyter_client import KernelManager, find_connection_file

        # Test if connection is alive
        connected = self.check_connection()
        attempt = 0
        max_attempts = 3
        while not connected and attempt < max_attempts:
            attempt += 1
            try:
                cfile = find_connection_file(
                )  # default filename='kernel-*.json'
            except IOError:
                self.vim_echom(
                    "kernel connection attempt {:d} failed - no kernel file".
                    format(attempt),
                    style="Error")
                continue

            # Create the kernel manager and connect a client
            # See: <http://jupyter-client.readthedocs.io/en/stable/api/client.html>
            km = KernelManager(connection_file=cfile)
            km.load_connection_file()
            self.kc = km.client()
            self.kc.start_channels()

            # Alias execute function
            def _send(msg, **kwargs):
                """Send a message to the kernel client."""
                # Include dedent of msg so we don't get odd indentation errors.
                return self.kc.execute(textwrap.dedent(msg), **kwargs)

            self.send = _send

            # Ping the kernel
            self.kc.kernel_info()
            try:
                self.reply = self.kc.get_shell_msg(timeout=1)
            except Empty:
                continue
            else:
                connected = True

        if connected:
            # Send command so that monitor knows vim is commected
            # send('"_vim_client"', store_history=False)
            self.pid = self.set_pid()  # Ask kernel for its PID
            self.vim_echom('kernel connection successful! pid = {}'.format(
                self.pid),
                           style='Question')
        else:
            self.kc.stop_channels()
            self.vim_echom('kernel connection attempt timed out',
                           style='Error')
Ejemplo n.º 17
0
 def start(self, cwd=None, executable=None):
     self.km = KernelManager(kernel_name='python',
                             client_class='nei.execute.Client')
     self.executable = executable
     if executable is not None:
         self.km.kernel_spec.argv[0] = executable
     self.km.start_kernel(**({} if cwd is None else {'cwd': cwd}))
     self.kc = self.km.client()
     self.kc.start_channels()
     time.sleep(2)
     atexit.register(self.shutdown_kernel)
    def setUp(self):
        try:
            KernelSpecManager().get_kernel_spec(NATIVE_KERNEL_NAME)
        except NoSuchKernel:
            pytest.skip()

        self.km = KernelManager(kernel_name=NATIVE_KERNEL_NAME)
        self.km.start_kernel()
        self.kc = self.km.client()
        self.kc.start_channels()
        self.addCleanup(self.kc.stop_channels)
        self.addCleanup(self.km.shutdown_kernel)
    def setUp(self):
        try:
            KernelSpecManager().get_kernel_spec('python3_clean')
        except NoSuchKernel:
            pytest.skip()

        self.km = KernelManager(kernel_name='python3_clean')
        self.km.start_kernel()
        self.kc = self.km.client()
        self.kc.start_channels()
        self.addCleanup(self.kc.stop_channels)
        self.addCleanup(self.km.shutdown_kernel)
Ejemplo n.º 20
0
    def test_no_cleanup_shared_context(self):
        """kernel manager does not terminate shared context"""
        import zmq
        ctx = zmq.Context()
        km = KernelManager(context=ctx)
        self.assertEquals(km.context, ctx)
        self.assertIsNotNone(km.context)

        km.cleanup_resources(restart=False)
        self.assertFalse(km.context.closed)
        self.assertFalse(ctx.closed)

        ctx.term()
Ejemplo n.º 21
0
def start_kernel():
    manager = KernelManager()
    manager.start_kernel()

    port_names = ["shell", "stdin", "iopub", "hb", "control"]
    ports = dict(list(zip(port_names, manager.ports)))

    kernel_info = {
        "key": manager.session.key.decode("utf-8"),
        "ports": ports,
    }

    return json.dumps(kernel_info)
Ejemplo n.º 22
0
def connect_to_kernel():
    """Create kernel manager from existing connection file."""
    from jupyter_client import KernelManager, find_connection_file

    global kc, pid, send

    # Test if connection is alive
    connected = check_connection()
    attempt = 0
    max_attempts = 3
    while not connected and attempt < max_attempts:
        attempt += 1
        try:
            cfile = find_connection_file()  # default filename='kernel-*.json'
        except IOError:
            vim_echom("kernel connection attempt {:d} failed - no kernel file"\
                      .format(attempt), style="Error")
            continue

        # Create the kernel manager and connect a client
        # See: <http://jupyter-client.readthedocs.io/en/stable/api/client.html>
        km = KernelManager(connection_file=cfile)
        km.load_connection_file()
        kc = km.client()
        kc.start_channels()

        # Alias execute function
        def _send(msg, **kwargs):
            """Send a message to the kernel client."""
            # Include dedent of msg so we don't get odd indentation errors.
            return kc.execute(textwrap.dedent(msg), **kwargs)
        send = _send

        # Ping the kernel
        kc.kernel_info()
        try:
            reply = kc.get_shell_msg(timeout=1)
        except Empty:
            continue
        else:
            connected = True

    if connected:
        # Send command so that monitor knows vim is commected
        # send('"_vim_client"', store_history=False)
        pid = set_pid() # Ask kernel for its PID
        vim_echom('kernel connection successful! pid = {}'.format(pid),
                  style='Question')
    else:
        kc.stop_channels()
        vim_echom('kernel connection attempt timed out', style='Error')
Ejemplo n.º 23
0
    def __init__(self,
                 working_directory,
                 kernel='python3',
                 embed_kernel=False):

        if embed_kernel:
            self._kernel_manager = InProcessKernelManager(kernel_name=kernel)
        else:
            self._kernel_manager = KernelManager(kernel_name=kernel)

        stderr = open(os.devnull, 'w')
        self._kernel_manager.start_kernel(cwd=working_directory, stderr=stderr)

        self._init_client()
Ejemplo n.º 24
0
def new_ipy(s=''):
    """Create a new IPython kernel (optionally with extra arguments)

    XXX: Allow passing of profile information here

    Examples
    --------

        new_ipy()

    """
    from jupyter_client import KernelManager
    km = KernelManager()
    km.start_kernel()
    return km_from_string(km.connection_file)
Ejemplo n.º 25
0
class Kernel(object):
    def __init__(self, kernel_name):
        self.kernel = KernelManager(kernel_name=kernel_name)

    def start(self):
        self.kernel.start_kernel()
        self.client = self.kernel.client()
        self.client.start_channels()
        try:
            self.client.wait_for_ready()
        except RuntimeError:
            print("[metys] Unable to start kernel.")
            self.shutdown()
            raise
        info = self.client.kernel_info(reply=True)
        if info and "content" in info and "language_info" in info["content"]:
            self.language_info = info["content"]["language_info"]

    def shutdown(self):
        self.client.shutdown()

    def is_complete(self, code):
        return self.client.is_complete(code)

    def execute_chunk(self, chunk):
        chunk.messages = []
        if hasattr(self, "language_info"):
            chunk.language_info = self.language_info
        input = (
            chunk.input.format(**chunk.options)
            if chunk.options["expand_options"]
            else chunk.input
        )

        if "code_template" in chunk.options:
            input = chunk.options["code_template"].format(input, **chunk.options)

        def output_hook(msg):
            return (
                chunk.messages.append(msg)
                if msg["msg_type"] in ("display_data", "execute_result", "stream")
                else None
            )

        chunk.reply = self.client.execute_interactive(
            input, store_history=False, allow_stdin=False,
            output_hook=output_hook
        )
Ejemplo n.º 26
0
    def __init__(
        self, parsed, kernel, source, mode, figdir, outdir, embed_kernel=False
    ):
        super(JupyterProcessor, self).__init__(
            parsed, kernel, source, mode, figdir, outdir
        )

        self.extra_arguments = None
        self.timeout = -1
        path = os.path.abspath(outdir)

        if embed_kernel:
            km = InProcessKernelManager(kernel_name=kernel)
        else:
            km = KernelManager(kernel_name=kernel)

        km.start_kernel(cwd=path, stderr=open(os.devnull, "w"))
        kc = km.client()
        kc.start_channels()
        try:
            kc.wait_for_ready()
        except RuntimeError:
            print(
                "Timeout from starting kernel\nTry restarting python session and running weave again"
            )
            kc.stop_channels()
            km.shutdown_kernel()
            raise

        self.km = km
        self.kc = kc
        self.kc.allow_stdin = False
Ejemplo n.º 27
0
def run_notebook(nb):
    km = KernelManager()
    km.start_kernel(stderr=open(os.devnull, 'w'))
    kc = km.client()
    kc.start_channels()
    shell = kc.shell_channel
    # simple ping:
    kc.execute("pass")
    shell.get_msg()

    failures = 0
    for cell in nb.cells:
        if cell.cell_type != 'code':
            continue
        kc.execute(cell.source)
        try:
            # wait for finish, w/ timeout
            reply = shell.get_msg(timeout=TIMEOUT)['content']
        except Empty:
            raise Exception(
                'Timeout (%.1f) when executing the following %s cell: "%s"' %
                (TIMEOUT, cell.cell_type, cell.source.strip()))
        if reply['status'] == 'error':
            failures += 1
            print("\nFAILURE:", file=sys.stderr)
            print('\n'.join(reply['traceback']), file=sys.stderr)
            print(file=sys.stderr)

    kc.stop_channels()
    km.shutdown_kernel()
    del km
    if failures > 0:
        raise Exception()
Ejemplo n.º 28
0
def _check_ipynb():
    kernel_manager = KernelManager()
    kernel_manager.start_kernel()
    kernel_client = kernel_manager.client()
    kernel_client.start_channels()

    try:
        # IPython 3.x
        kernel_client.wait_for_ready()
        iopub = kernel_client
        shell = kernel_client
    except AttributeError:
        # Ipython 2.x
        # Based on https://github.com/paulgb/runipy/pull/49/files
        iopub = kernel_client.iopub_channel
        shell = kernel_client.shell_channel
        shell.get_shell_msg = shell.get_msg
        iopub.get_iopub_msg = iopub.get_msg

    successes = 0
    failures = 0
    errors = 0

    report = ''
    _execute_code("print('Hello World')", shell, iopub, timeout=1)

    kernel_client.stop_channels()
    kernel_manager.shutdown_kernel()

    passed = not (failures or errors)

    print(report)
Ejemplo n.º 29
0
def run_notebook(nb):
    km = KernelManager()
    km.start_kernel(stderr=open(os.devnull, 'w'))
    kc = km.client()
    kc.start_channels()

    # simple ping:
    kc.execute("pass")
    kc.get_shell_msg()

    cells = 0
    failures = 0
    for cell in nb.cells:
        if cell.cell_type != 'code':
            continue

        outputs, failed = run_cell(kc, cell)
        cell.outputs = outputs
        cells += 1
        cell['execution_count'] = cells
        failures += failed
        sys.stdout.write('.')
        sys.stdout.flush()

    print()
    #print("ran notebook %s" % nb.metadata.name)
    print("    ran %3i cells" % cells)
    if failures:
        print("    %3i cells raised exceptions" % failures)
    kc.stop_channels()
    km.shutdown_kernel()
    del km
Ejemplo n.º 30
0
def run_notebook(nb):
    km = KernelManager()
    km.start_kernel(stderr=open(os.devnull, 'w'))
    kc = km.client()
    kc.start_channels()
    shell = kc.shell_channel
    # simple ping:
    kc.execute("pass")
    shell.get_msg()

    failures = 0
    for cell in nb.cells:
        if cell.cell_type != 'code':
            continue
        kc.execute(cell.source)
        # wait for finish, maximum 20s
        reply = shell.get_msg(timeout=20)['content']
        if reply['status'] == 'error':
            failures += 1
            print("\nFAILURE:")
            print('\n'.join(reply['traceback']))
            print()

    kc.stop_channels()
    km.shutdown_kernel()
    del km
    if failures > 0:
        raise Exception()
Ejemplo n.º 31
0
    def __init__(self, *args, **kwargs):
        super(FormatManager, self).__init__(*args, **kwargs)

        if self._instance is not None:
            raise ValueError("FormatManager is a singleton, access with"
                             " FormatManager.format_manager")

        self._formats = {}
        self._km = KernelManager()
Ejemplo n.º 32
0
def new_ipy(s=''):
    """Create a new IPython kernel (optionally with extra arguments)

    XXX: Allow passing of profile information here

    Examples
    --------

        new_ipy()

    """
    try:
        from jupyter_client import KernelManager
    except ImportError:
        from IPython.kernel import KernelManager
    km = KernelManager()
    km.start_kernel()
    return km_from_string(km.connection_file)
Ejemplo n.º 33
0
 def start_kernel(self, name):
     """Start a new kernel"""
     base, ext = os.path.splitext(self.parent.connection_file)
     cf = '{base}-{name}{ext}'.format(
         base=base,
         name=name,
         ext=ext,
     )
     manager = KernelManager(
         kernel_name=name,
         session=self.session,
         context=self.future_context,
         connection_file=cf,
     )
     manager.start_kernel()
     self.kernels[name] = kernel = KernelProxy(
         manager=manager,
         shell_upstream=self.shell_stream)
     self.iosub.connect(kernel.iopub_url)
     return self.kernels[name]
Ejemplo n.º 34
0
def run_notebook(nb):
    km = KernelManager()
    km.start_kernel(stderr=open(os.devnull, 'w'))
    kc = km.client()
    kc.start_channels()
    shell = kc.shell_channel
    # simple ping:
    kc.execute("pass")
    shell.get_msg()
    
    failures = 0
    for cell in nb.cells:
        if cell.cell_type != 'code':
            continue
        kc.execute(cell.source)
        # wait for finish, maximum 20s
        reply = shell.get_msg(timeout=60)['content']
        if reply['status'] == 'error':
            failures += 1
            print("\nFAILURE:")
            print('\n'.join(reply['traceback']))
            print()

    kc.stop_channels()
    km.shutdown_kernel()
    del km
    if failures > 0:
        raise Exception()
Ejemplo n.º 35
0
def main(args):
    parser = argparse.ArgumentParser()
    parser.add_argument('--port', type=int)
    parser.add_argument('--kernel')
    parser.add_argument('--conn-file')
    args = parser.parse_args()
    if args.conn_file:
        if runtime_dir:
            conn_file = (args.conn_file if os.path.isabs(args.conn_file)
                         else os.path.join(runtime_dir(), args.conn_file))
        else: # IPython 3
            pd = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), 'default')
            conn_file = os.path.join(pd.security_dir, args.conn_file)
        kwargs = {'connection_file': conn_file}
        if args.kernel:
            kwargs['kernel_name'] = args.kernel
        manager = KernelManager(**kwargs)

        semaphore = multiprocessing.Semaphore()
        semaphore.acquire()
        def onsignal(*args):
            semaphore.release()
        signal.signal(signal.SIGTERM, onsignal)
        import platform
        if platform.system() == 'Windows':
            signal.signal(signal.SIGBREAK, onsignal)
        else:
            signal.signal(signal.SIGQUIT, onsignal)
            # Emacs sends SIGHUP upon exit
            signal.signal(signal.SIGHUP, onsignal)

        manager.start_kernel()
        try:
            semaphore.acquire()
        except KeyboardInterrupt: pass
        manager.shutdown_kernel()
    else:
        app = make_app()
        app.listen(args.port)
        tornado.ioloop.IOLoop.current().start()
Ejemplo n.º 36
0
def run_notebook(nb):
    km = KernelManager()
    km.start_kernel(stderr=open(os.devnull, 'w'))
    kc = km.client()
    kc.start_channels()

    # simple ping:
    kc.execute("pass")
    kc.get_shell_msg()

    cells = 0
    failures = 0
    for cell in nb.cells:
        if cell.cell_type != 'code':
            continue

        outputs, failed = run_cell(kc, cell)
        cell.outputs = outputs
        cells += 1
        cell['execution_count'] = cells
        failures += failed
        sys.stdout.write('.')
        sys.stdout.flush()

    print()
    #print("ran notebook %s" % nb.metadata.name)
    print("    ran %3i cells" % cells)
    if failures:
        print("    %3i cells raised exceptions" % failures)
    kc.stop_channels()
    km.shutdown_kernel()
    del km
Ejemplo n.º 37
0
def test_notebook(ipynb):
    with open(ipynb, encoding='utf-8') as f:
        nb = nbformat.reads_json(f.read())
    km = KernelManager()
    # Do not save the history to disk, as it can yield spurious lock errors.
    # See https://github.com/ipython/ipython/issues/2845
    km.start_kernel(extra_arguments=['--HistoryManager.hist_file=:memory:'])

    kc = km.client()
    kc.start_channels()

    try:
        kc.wait_for_ready()
    except AttributeError:
        _wait_for_ready_backport(kc)

    successes = 0
    failures = 0
    errors = 0
    for i, cell in enumerate(nb.cells):
        if cell.cell_type != 'code' or cell.source.startswith('%timeit'):
            continue
        try:
            outs = run_cell(kc, cell)
        except Exception as e:
            print("failed to run cell:", repr(e))
            print(cell.input)
            errors += 1
            continue

        failed = False
        if len(outs) != len(cell.outputs):
            print("output length mismatch (expected {}, got {})".format(
                  len(cell.outputs), len(outs)))
            failed = True
        if not compare_outputs(outs, cell.outputs):
            failed = True
        print("cell %d: " % i, end="")
        if failed:
            print("FAIL")
            failures += 1
        else:
            print("OK")
            successes += 1


    print("tested notebook %s" % ipynb)
    print("    %3i cells successfully replicated" % successes)
    if failures:
        print("    %3i cells mismatched output" % failures)
    if errors:
        print("    %3i cells failed to complete" % errors)
    kc.stop_channels()
    km.shutdown_kernel()
    del km
    if failures | errors:
        sys.exit(1)
Ejemplo n.º 38
0
def run_notebook(nb):
    km = KernelManager()
    km.start_kernel(stderr=open(os.devnull, 'w'))
    if hasattr(km, 'client'):
        kc = km.client()
        kc.start_channels()
        iopub = kc.iopub_channel
    else:
        # IPython 0.13 compat
        kc = km
        kc.start_channels()
        iopub = kc.sub_channel
    shell = kc.shell_channel

    # simple ping:
    shell.send("pass")
    shell.get_msg()

    cells = 0
    failures = 0
    for ws in nb.worksheets:
        for cell in ws.cells:
            if cell.cell_type != 'code':
                continue

            outputs, failed = run_cell(shell, iopub, cell)
            cell.outputs = outputs
            cell['prompt_number'] = cells
            failures += failed
            cells += 1
            sys.stdout.write('.')

    print()
    print("ran notebook %s" % nb.metadata.name)
    print("    ran %3i cells" % cells)
    if failures:
        print("    %3i cells raised exceptions" % failures)
    kc.stop_channels()
    km.shutdown_kernel()
    del km
Ejemplo n.º 39
0
def run_notebook(nb):
    """
    Run each code cell in a given notebook and update with the new output
    """
    km = KernelManager()
    km.start_kernel(extra_arguments=['--pylab=inline'])
    try:
        kc = km.client()
        kc.start_channels()
        iopub = kc.iopub_channel
    except AttributeError:
        # IPython 0.13
        kc = km
        kc.start_channels()
        iopub = kc.sub_channel
    shell = kc.shell_channel

    shell.execute("pass")
    shell.get_msg()
    while True:
        try:
            iopub.get_msg(timeout=1)
        except Empty:
            break

    for ws in nb.worksheets:
        for cell in ws.cells:
            if cell.cell_type != 'code':
                continue
            try:
                cell.outputs = run_cell(shell, iopub, cell)
            except Exception as e:
                return -1

    kc.stop_channels()
    km.shutdown_kernel()
    del km
    return nb
Ejemplo n.º 40
0
    def __init__(self, nb, profile_dir=None, working_dir=None,
                 comment="", fLOG=noLOG, theNotebook=None, code_init=None,
                 kernel_name="python", log_level="30", extended_args=None,
                 kernel=False, filename=None, replacements=None):
        """
        constuctor

        @param      nb              notebook as JSON
        @param      profile_dir     profile directory
        @param      working_dir     working directory
        @param      comment         additional information added to error message
        @param      theNotebook     if not None, populate the variable *theNotebook* with this value in the notebook
        @param      code_init       to initialize the notebook with a python code as if it was a cell
        @param      fLOG            logging function
        @param      log_level       Choices: (0, 10, 20, 30=default, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL')
        @param      kernel_name     kernel name, it can be None
        @param      extended_args   others arguments to pass to the command line ('--KernelManager.autorestar=True' for example),
                                    see :ref:`l-ipython_notebook_args` for a full list
        @param      kernel          *kernel* is True by default, the notebook can be run, if False,
                                    the notebook can be read but not run
        @param      filename        to add the notebook file if there is one in error messages
        @param      replacements    replacements to make in every cell before running it,
                                    dictionary ``{ string: string }``

        .. versionchanged:: 1.4
            Parameter *replacements* was added.
        """
        if kernel:
            try:
                from jupyter_client import KernelManager
            except ImportError:
                from ipykernel import KernelManager
            self.km = KernelManager(
                kernel_name=kernel_name) if kernel_name is not None else KernelManager()
        else:
            self.km = None
        self.fLOG = fLOG
        self.theNotebook = theNotebook
        self.code_init = code_init
        self._filename = filename if filename is not None else "memory"
        self.replacements = replacements
        self.init_args = dict(profile_dir=profile_dir, working_dir=working_dir,
                              comment=comment, fLOG=fLOG, theNotebook=theNotebook, code_init=code_init,
                              kernel_name="python", log_level="30", extended_args=None,
                              kernel=kernel, filename=filename, replacements=replacements)
        args = []

        if profile_dir:
            args.append('--profile-dir=%s' % os.path.abspath(profile_dir))
        if log_level:
            args.append('--log-level=%s' % log_level)

        if extended_args is not None and len(extended_args) > 0:
            for opt in extended_args:
                if not opt.startswith("--"):
                    raise SyntaxError(
                        "every option should start with '--': " + opt)
                if "=" not in opt:
                    raise SyntaxError(
                        "every option should be assigned a value: " + opt)
                args.append(opt)

        if kernel:
            cwd = os.getcwd()

            if working_dir:
                os.chdir(working_dir)

            if self.km is not None:
                if sys.version_info[0] == 2 and args is not None:
                    # I did not find a way to make it work
                    args = None
                    warnings.warn(
                        "args is not None: {0}, unable to use it in Python 2.7".format(args))
                    self.km.start_kernel()
                else:
                    try:
                        self.km.start_kernel(extra_arguments=args)
                    except Exception as e:
                        raise Exception(
                            "Failure with args: {0}\nand error:\n{1}".format(args, str(e))) from e

                if platform.system() == 'Darwin':
                    # see http://www.pypedia.com/index.php/notebook_runner
                    # There is sometimes a race condition where the first
                    # execute command hits the kernel before it's ready.
                    # It appears to happen only on Darwin (Mac OS) and an
                    # easy (but clumsy) way to mitigate it is to sleep
                    # for a second.
                    sleep(1)

            os.chdir(cwd)

            self.kc = self.km.client()
            self.kc.start_channels(stdin=False)
            # if it does not work, it probably means IPython < 3
            self.kc.wait_for_ready()
        else:
            self.km = None
            self.kc = None
        self.nb = nb
        self.comment = comment
Ejemplo n.º 41
0
class NotebookRunner(object):

    """
    The kernel communicates with mime-types while the notebook
    uses short labels for different cell types. We'll use this to
    map from kernel types to notebook format types.

    This classes executes a notebook end to end.

    .. index:: kernel, notebook

    The class can use different kernels. The next links gives more
    information on how to create or test a kernel:

    * `jupyter_kernel_test <https://github.com/jupyter/jupyter_kernel_test>`_
    * `simple_kernel <https://github.com/dsblank/simple_kernel>`_

    .. faqref::
        :title: Do I need to shutdown the kernel after running a notebook?

        .. index:: travis

        If the class is instantiated with *kernel=True*, a kernel will
        be started. It must be shutdown otherwise the program might
        be waiting for it for ever. That is one of the reasons why the
        travis build does not complete. The build finished but cannot temrinate
        until all kernels are shutdown.
    """

    #. available output types
    MIME_MAP = {
        'image/jpeg': 'jpeg',
        'image/png': 'png',
        'image/gif': 'gif',
        'text/plain': 'text',
        'text/html': 'html',
        'text/latex': 'latex',
        'application/javascript': 'html',
        'image/svg+xml': 'svg',
    }

    def __init__(self, nb, profile_dir=None, working_dir=None,
                 comment="", fLOG=noLOG, theNotebook=None, code_init=None,
                 kernel_name="python", log_level="30", extended_args=None,
                 kernel=False, filename=None, replacements=None):
        """
        constuctor

        @param      nb              notebook as JSON
        @param      profile_dir     profile directory
        @param      working_dir     working directory
        @param      comment         additional information added to error message
        @param      theNotebook     if not None, populate the variable *theNotebook* with this value in the notebook
        @param      code_init       to initialize the notebook with a python code as if it was a cell
        @param      fLOG            logging function
        @param      log_level       Choices: (0, 10, 20, 30=default, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL')
        @param      kernel_name     kernel name, it can be None
        @param      extended_args   others arguments to pass to the command line ('--KernelManager.autorestar=True' for example),
                                    see :ref:`l-ipython_notebook_args` for a full list
        @param      kernel          *kernel* is True by default, the notebook can be run, if False,
                                    the notebook can be read but not run
        @param      filename        to add the notebook file if there is one in error messages
        @param      replacements    replacements to make in every cell before running it,
                                    dictionary ``{ string: string }``

        .. versionchanged:: 1.4
            Parameter *replacements* was added.
        """
        if kernel:
            try:
                from jupyter_client import KernelManager
            except ImportError:
                from ipykernel import KernelManager
            self.km = KernelManager(
                kernel_name=kernel_name) if kernel_name is not None else KernelManager()
        else:
            self.km = None
        self.fLOG = fLOG
        self.theNotebook = theNotebook
        self.code_init = code_init
        self._filename = filename if filename is not None else "memory"
        self.replacements = replacements
        self.init_args = dict(profile_dir=profile_dir, working_dir=working_dir,
                              comment=comment, fLOG=fLOG, theNotebook=theNotebook, code_init=code_init,
                              kernel_name="python", log_level="30", extended_args=None,
                              kernel=kernel, filename=filename, replacements=replacements)
        args = []

        if profile_dir:
            args.append('--profile-dir=%s' % os.path.abspath(profile_dir))
        if log_level:
            args.append('--log-level=%s' % log_level)

        if extended_args is not None and len(extended_args) > 0:
            for opt in extended_args:
                if not opt.startswith("--"):
                    raise SyntaxError(
                        "every option should start with '--': " + opt)
                if "=" not in opt:
                    raise SyntaxError(
                        "every option should be assigned a value: " + opt)
                args.append(opt)

        if kernel:
            cwd = os.getcwd()

            if working_dir:
                os.chdir(working_dir)

            if self.km is not None:
                if sys.version_info[0] == 2 and args is not None:
                    # I did not find a way to make it work
                    args = None
                    warnings.warn(
                        "args is not None: {0}, unable to use it in Python 2.7".format(args))
                    self.km.start_kernel()
                else:
                    try:
                        self.km.start_kernel(extra_arguments=args)
                    except Exception as e:
                        raise Exception(
                            "Failure with args: {0}\nand error:\n{1}".format(args, str(e))) from e

                if platform.system() == 'Darwin':
                    # see http://www.pypedia.com/index.php/notebook_runner
                    # There is sometimes a race condition where the first
                    # execute command hits the kernel before it's ready.
                    # It appears to happen only on Darwin (Mac OS) and an
                    # easy (but clumsy) way to mitigate it is to sleep
                    # for a second.
                    sleep(1)

            os.chdir(cwd)

            self.kc = self.km.client()
            self.kc.start_channels(stdin=False)
            # if it does not work, it probably means IPython < 3
            self.kc.wait_for_ready()
        else:
            self.km = None
            self.kc = None
        self.nb = nb
        self.comment = comment

    def to_json(self, filename=None, encoding="utf8"):
        """
        convert the notebook into json

        @param      filename        filename or stream
        @param      encoding        encoding
        @return                     Json string if filename is None, None otherwise

        .. versionchanged:: 1.4
            The function now returns the json string if filename is None.
        """
        if isinstance(filename, str  # unicode#
                      ):
            with open(filename, "w", encoding=encoding) as payload:
                self.to_json(payload)
        elif filename is None:
            st = StringIO()
            st.write(writes(self.nb))
            return st.getvalue()
        else:
            filename.write(writes(self.nb))

    @staticmethod
    def read_json(js, profile_dir=None, encoding="utf8",
                  working_dir=None, comment="", fLOG=noLOG, code_init=None,
                  kernel_name="python", log_level="30", extended_args=None,
                  kernel=False, replacements=None):
        """
        read a notebook from a JSON stream or string

        @param      js              string or stream
        @param      profile_dir     profile directory
        @param      encoding        encoding for the notebooks
        @param      kernel          to start a kernel or not when reading the notebook (to execute it)
        @param      working_dir     working directory
        @param      comment         additional information added to error message
        @param      code_init       to initialize the notebook with a python code as if it was a cell
        @param      fLOG            logging function
        @param      log_level       Choices: (0, 10, 20, 30=default, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL')
        @param      kernel_name     kernel name, it can be None
        @param      extended_args   others arguments to pass to the command line ('--KernelManager.autorestar=True' for example),
                                    see :ref:`l-ipython_notebook_args` for a full list
        @param      kernel          *kernel* is True by default, the notebook can be run, if False,
                                    the notebook can be read but not run
        @param      replacements    replacements to make in every cell before running it,
                                    dictionary ``{ string: string }``
        @return                     instance of @see cl NotebookRunner

        .. versionchanged:: 1.5
            Add constructor parameters.
        """
        if isinstance(js, str  # unicode#
                      ):
            st = StringIO(js)
        else:
            st = js
        from .notebook_helper import read_nb
        return read_nb(st, encoding=encoding, kernel=kernel,
                       profile_dir=profile_dir, working_dir=working_dir,
                       comment=comment, fLOG=fLOG, code_init=code_init,
                       kernel_name="python", log_level="30", extended_args=None,
                       replacements=replacements)

    def copy(self):
        """
        copy the notebook (just the content)

        @return         instance of @see cl NotebookRunner

        .. versionadded:: 1.1

        .. versionchanged:: 1.5
            Add constructor parameters.
        """
        st = StringIO()
        self.to_json(st)
        args = self.init_args.copy()
        for name in ["theNotebook", "filename"]:
            if name in args:
                del args[name]
        return NotebookRunner.read_json(st.getvalue(), **args)

    def __add__(self, nb):
        """
        merges two notebooks together, returns a new none

        @param      nb      notebook
        @return             new notebook
        """
        c = self.copy()
        c.merge_notebook(nb)
        return c

    def shutdown_kernel(self):
        """
        shut down kernel
        """
        self.fLOG('-- shutdown kernel')
        if self.kc is None:
            raise ValueError(
                "No kernel was started, specify kernel=True when initializing the instance.")
        self.kc.stop_channels()
        self.km.shutdown_kernel(now=True)

    def clean_code(self, code):
        """
        clean the code before running it, the function comment out
        instruction such as ``show()``

        @param      code        code (string)
        @return                 cleaned code

        .. versionchanged:: 1.4
            Do replacements.
        """
        has_bokeh = "bokeh." in code or "from bokeh" in code or "import bokeh" in code
        if code is None:
            return code
        else:
            lines = [_.strip("\n\r").rstrip(" \t") for _ in code.split("\n")]
            res = []
            show_is_last = False
            for line in lines:
                if line.replace(" ", "") == "show()":
                    line = line.replace("show", "#show")
                    show_is_last = True
                elif has_bokeh and line.replace(" ", "") == "output_notebook()":
                    line = line.replace("output_notebook", "#output_notebook")
                else:
                    show_is_last = False
                if self.replacements is not None:
                    for k, v in self.replacements.items():
                        line = line.replace(k, v)
                res.append(line)
                if show_is_last:
                    res.append('"nothing to show"')
            return "\n".join(res)

    @staticmethod
    def get_cell_code(cell):
        """
        return the code of a cell

        @param      cell        a cell or a string
        @return                 boolean (=iscell), string
        """
        if isinstance(cell, str  # unicode#
                      ):
            iscell = False
            return iscell, cell
        else:
            iscell = True
            try:
                return iscell, cell.source
            except AttributeError:
                return iscell, cell.input

    def run_cell(self, index_cell, cell, clean_function=None):
        '''
        Run a notebook cell and update the output of that cell in-place.

        @param      index_cell          index of the cell
        @param      cell                cell to execute
        @param      clean_function      cleaning function to apply to the code before running it
        @return                         output of the cell
        '''
        iscell, codei = NotebookRunner.get_cell_code(cell)

        self.fLOG('-- running cell:\n%s\n' % codei)

        code = self.clean_code(codei)
        if clean_function is not None:
            code = clean_function(code)
        if len(code) == 0:
            return ""
        if self.kc is None:
            raise ValueError(
                "No kernel was started, specify kernel=True when initializing the instance.")
        self.kc.execute(code)

        reply = self.kc.get_shell_msg()
        reason = None
        try:
            status = reply['content']['status']
        except KeyError:
            status = 'error'
            reason = "no status key in reply['content']"

        if status == 'error':
            ansi_escape = re.compile(r'\x1b[^m]*m')
            try:
                tr = [ansi_escape.sub('', _)
                      for _ in reply['content']['traceback']]
            except KeyError:
                tr = ["No traceback, available keys in reply['content']"] + \
                    [_ for _ in reply['content']]
            traceback_text = '\n'.join(tr)
            self.fLOG("ERR:\n", traceback_text)
        else:
            traceback_text = ''
            self.fLOG('-- cell returned')

        outs = list()
        nbissue = 0
        while True:
            try:
                msg = self.kc.get_iopub_msg(timeout=1)
                if msg['msg_type'] == 'status':
                    if msg['content']['execution_state'] == 'idle':
                        break
            except Empty:
                # execution state should return to idle before the queue becomes empty,
                # if it doesn't, something bad has happened
                status = "error"
                reason = "exception Empty was raised"
                nbissue += 1
                if nbissue > 10:
                    # the notebook is empty
                    return ""
                else:
                    continue

            content = msg['content']
            msg_type = msg['msg_type']

            # IPython 3.0.0-dev writes pyerr/pyout in the notebook format but uses
            # error/execute_result in the message spec. This does the translation
            # needed for tests to pass with IPython 3.0.0-dev
            notebook3_format_conversions = {
                'error': 'pyerr',
                'execute_result': 'pyout'
            }
            msg_type = notebook3_format_conversions.get(msg_type, msg_type)

            out = NotebookNode(output_type=msg_type)

            if 'execution_count' in content:
                if iscell:
                    cell['prompt_number'] = content['execution_count']
                out.prompt_number = content['execution_count']

            if msg_type in ('status', 'pyin', 'execute_input'):
                continue

            elif msg_type == 'stream':
                out.stream = content['name']
                # in msgspec 5, this is name, text
                # in msgspec 4, this is name, data
                if 'text' in content:
                    out.text = content['text']
                else:
                    out.data = content['data']

            elif msg_type in ('display_data', 'pyout'):
                out.data = content['data']

            elif msg_type == 'pyerr':
                out.ename = content['ename']
                out.evalue = content['evalue']
                out.traceback = content['traceback']

            elif msg_type == 'clear_output':
                outs = list()
                continue

            elif msg_type == 'comm_open' or msg_type == 'comm_msg':
                # widgets in a notebook
                out.data = content["data"]
                out.comm_id = content["comm_id"]

            else:
                dcontent = "\n".join("{0}={1}".format(k, v)
                                     for k, v in sorted(content.items()))
                raise NotImplementedError(
                    'unhandled iopub message: %s' % msg_type + "\nCONTENT:\n" + dcontent)

            outs.append(out)

        if iscell:
            cell['outputs'] = outs

        raw = []
        for _ in outs:
            try:
                t = _.data
            except AttributeError:
                continue

            # see MIMEMAP to see the available output type
            for k, v in t.items():
                if k.startswith("text"):
                    raw.append(v)

        sraw = "\n".join(raw)
        self.fLOG(sraw)

        def reply2string(reply):
            sreply = []
            for k, v in sorted(reply.items()):
                if isinstance(v, dict):
                    temp = []
                    for _, __ in sorted(v.items()):
                        temp.append("    [{0}]={1}".format(_, str(__)))
                    v = "\n".join(temp)
                    sreply.append("reply['{0}']=dict\n{1}".format(k, v))
                else:
                    sreply.append("reply['{0}']={1}".format(k, str(v)))
            sreply = "\n".join(sreply)
            return sreply

        if status == 'error':
            sreply = reply2string(reply)
            if len(code) < 5:
                scode = [code]
            else:
                scode = ""
            mes = "FILENAME\n{10}:1:1\n{7}\nCELL status={8}, reason={9} -- {4} length={5} -- {6}:\n-----------------\n{0}" + \
                  "\n-----------------\nTRACE:\n{1}\nRAW:\n{2}REPLY:\n{3}"
            raise NotebookError(mes.format(
                code, traceback_text, sraw, sreply, index_cell, len(
                    code), scode, self.comment, status, reason,
                self._filename))
        return outs

    def iter_code_cells(self):
        '''
        Iterate over the notebook cells containing code.
        '''
        for cell in self.iter_cells():
            if cell.cell_type == 'code':
                yield cell

    def iter_cells(self):
        '''
        Iterate over the notebook cells.
        '''
        if hasattr(self.nb, "worksheets"):
            for ws in self.nb.worksheets:
                for cell in ws.cells:
                    yield cell
        else:
            for cell in self.nb.cells:
                yield cell

    def first_cell(self):
        """
        Returns the first cell.
        """
        for cell in self.iter_cells():
            return cell

    def _cell_container(self):
        """
        returns a cells container, it may change according to the format

        @return     cell container
        """
        if hasattr(self.nb, "worksheets"):
            last = None
            for ws in self.nb.worksheets:
                last = ws
            if last is None:
                raise NotebookError("no cell container")
            return last.cells
        else:
            return self.nb.cells

    def __len__(self):
        """
        return the number of cells, it iterates on cells
        to get this information and does cache the information

        @return         int

        .. versionadded:: 1.1
        """
        return sum(1 for _ in self.iter_cells())

    def cell_type(self, cell):
        """
        returns the cell type

        @param      cell        from @see me iter_cells
        @return                 type
        """
        return cell.cell_type

    def cell_metadata(self, cell):
        """
        returns the cell metadata

        @param      cell        cell
        @return                 metadata
        """
        return cell.metadata

    def _check_thumbnail_tuple(self, b):
        """
        checks types for a thumbnail

        @param      b       tuple   image, format
        @return             b

        The function raises an exception if the type is incorrect.
        """
        if not isinstance(b, tuple):
            raise TypeError("tuple expected, not {0}".format(type(b)))
        if len(b) != 2:
            raise TypeError(
                "tuple expected of lengh 2, not {0}".format(len(b)))
        if b[1] == "svg":
            if not isinstance(b[0], str):
                raise TypeError(
                    "str expected for svg, not {0}".format(type(b[0])))
        else:
            if not isinstance(b[0], bytes):
                raise TypeError(
                    "bytes expected for images, not {0}".format(type(b[0])))
        return b

    def create_picture_from(self, text, format, asbytes=True, context=None):
        """
        Creates a picture from text.

        @param      text        the text
        @param      format      text, json, ...
        @param      context     (str) indication on the content of text (error, ...)
        @param      asbytes     results as bytes or as an image
        @return                 tuple (picture, format) or PIL.Image (if asbytes is False)

        The picture will be bytes, the format png, bmp...
        The size of the picture will depend on the text.
        The longer, the bigger. The method relies on matplotlib
        and then convert the image into a PIL image.

        HTML could be rendered with QWebPage from PyQt (not implemented).
        """
        if not isinstance(text, (str, bytes)):
            text = str(text)
            if "\n" not in text:
                rows = []
                for i in range(0, len(text), 20):
                    end = min(i + 20, len(text))
                    rows.append(text[i:end])
                text = "\n".join(text)
        if len(text) > 200:
            text = text[:200]
        size = len(text) // 10
        figsize = (3 + size, 3 + size)
        lines = text.replace("\t", " ").replace("\r", "").split("\n")

        import matplotlib.pyplot as plt
        from matplotlib.textpath import TextPath
        from matplotlib.font_manager import FontProperties
        fig = plt.figure(figsize=figsize)
        ax = fig.add_subplot(111)
        fp = FontProperties(size=200)

        dx = 0
        dy = 0
        for i, line in enumerate(lines):
            if len(line.strip()) > 0:
                ax.text(0, -dy, line, fontproperties=fp, va='top')
                tp = TextPath((0, -dy), line, prop=fp)
                bb = tp.get_extents()
                dy += bb.height
                dx = max(dx, bb.width)

        ratio = abs(dx) / max(abs(dy), 1)
        ratio = max(min(ratio, 3), 1)
        fig.set_size_inches(int((1 + size) * ratio), 1 + size)
        ax.set_xlim([0, dx])
        ax.set_ylim([-dy, 0])
        ax.set_axis_off()
        sio = BytesIO()
        fig.savefig(sio, format="png")
        plt.close()

        if asbytes:
            b = sio.getvalue(), "png"
            self._check_thumbnail_tuple(b)
            return b
        else:
            try:
                from PIL import Image
            except ImportError:
                import Image
            img = Image.open(sio)
            return img

    def cell_image(self, cell, image_from_text=False):
        """
        returns the cell image or None if not found

        @param      cell            cell to examine
        @param      image_from_text produce an image even if it is not one
        @return                     None for no image or a list of tuple (image as bytes, extension)
                                    for each output of the cell
        """
        kind = self.cell_type(cell)
        if kind != "code":
            return None
        results = []
        for output in cell.outputs:
            if output["output_type"] in {"execute_result", "display_data"}:
                data = output["data"]
                for k, v in data.items():
                    if k == "text/plain":
                        if image_from_text:
                            b = self.create_picture_from(
                                v, "text", context=output["output_type"])
                            results.append(b)
                    elif k == "application/javascript":
                        if image_from_text:
                            b = self.create_picture_from(v, "js")
                            results.append(b)
                    elif k == "application/json":
                        if image_from_text:
                            b = self.create_picture_from(v, "json")
                            results.append(b)
                    elif k == "image/svg+xml":
                        if not isinstance(v, str):
                            raise TypeError(
                                "This should be str not '{0}' (=SVG).".format(type(v)))
                        results.append((v, "svg"))
                    elif k == "text/html":
                        if image_from_text:
                            b = self.create_picture_from(v, "html")
                            results.append(b)
                    elif k == "text/latex":
                        if image_from_text:
                            b = self.create_picture_from(v, "latex")
                            results.append(b)
                    elif k in {"image/png", "image/jpg", "image/jpeg", "image/gif"}:
                        if not isinstance(v, bytes):
                            v = base64.b64decode(v)
                        if not isinstance(v, bytes):
                            raise TypeError(
                                "This should be bytes not '{0}' (=IMG:{1}).".format(type(v), k))
                        results.append((v, k.split("/")[-1]))
                    else:
                        raise NotImplementedError("cell type: {0}\nk={1}\nv={2}\nCELL:\n{3}".format(kind,
                                                                                                    k, v, cell))
            elif output["output_type"] == "error":
                vl = output["traceback"]
                if image_from_text:
                    for v in vl:
                        b = self.create_picture_from(
                            v, "text", context="error")
                        results.append(b)
            elif output["output_type"] == "stream":
                v = output["text"]
                if image_from_text:
                    b = self.create_picture_from(v, "text")
                    results.append(b)
            else:
                raise NotImplementedError("cell type: {0}\noutput type: {1}\nOUT:\n{2}\nCELL:\n{3}"
                                          .format(kind, output["output_type"], output, cell))
        if len(results) > 0:
            res = self._merge_images(results)
            self._check_thumbnail_tuple(res)
            return res
        else:
            return None

    def cell_height(self, cell):
        """
        approximate the height of a cell by its number of lines it contains

        @param      cell        cell
        @return                 number of cell
        """
        kind = self.cell_type(cell)
        if kind == "markdown":
            content = cell.source
            lines = content.split("\n")
            nbs = sum(1 + len(line) // 80 for line in lines)
            return nbs
        elif kind == "raw":
            content = cell.source
            lines = content.split("\n")
            nbs = sum(1 + len(line) // 80 for line in lines)
            return nbs
        elif kind == "code":
            content = cell.source
            lines = content.split("\n")
            nbl = len(lines)

            for output in cell.outputs:
                if output["output_type"] == "execute_result" or \
                        output["output_type"] == "display_data":
                    data = output["data"]
                    for k, v in data.items():
                        if k == "text/plain":
                            nbl += len(v.split("\n"))
                        elif k == "application/javascript":
                            # rough estimation
                            nbl += len(v.split("\n")) // 2
                        elif k == "application/json":
                            # rough estimation
                            try:
                                nbl += len(v.split("{"))
                            except AttributeError:
                                nbl += len(v) // 5 + 1
                        elif k == "image/svg+xml":
                            nbl += len(v) // 5
                        elif k == "text/html":
                            nbl += len(v.split("\n"))
                        elif k == "text/latex":
                            nbl += len(v.split("\\\\")) * 2
                        elif k in {"image/png", "image/jpg", "image/jpeg", "image/gif"}:
                            nbl += len(v) // 50
                        else:
                            raise NotImplementedError("cell type: {0}\nk={1}\nv={2}\nCELL:\n{3}".format(kind,
                                                                                                        k, v, cell))
                elif output["output_type"] == "stream":
                    v = output["text"]
                    nbl += len(v.split("\n"))
                elif output["output_type"] == "error":
                    v = output["traceback"]
                    nbl += len(v)
                else:
                    raise NotImplementedError("cell type: {0}\noutput type: {1}\nOUT:\n{2}\nCELL:\n{3}"
                                              .format(kind, output["output_type"], output, cell))

            return nbl

        else:
            raise NotImplementedError(
                "cell type: {0}\nCELL:\n{1}".format(kind, cell))

    def add_tag_slide(self, max_nb_cell=4, max_nb_line=25):
        """
        tries to add tags for a slide show when they are too few

        @param      max_nb_cell     maximum number of cells within a slide
        @param      max_nb_line     maximum number of lines within a slide
        @return                     list of modified cells { #slide: (kind, reason, cell) }
        """
        res = {}
        nbline = 0
        nbcell = 0
        for i, cell in enumerate(self.iter_cells()):
            meta = cell.metadata
            if "slideshow" in meta:
                st = meta["slideshow"]["slide_type"]
                if st in ["slide", "subslide"]:
                    nbline = 0
                    nbcell = 0
            else:
                if cell.cell_type == "markdown":
                    content = cell.source
                    if content.startswith("# ") or \
                       content.startswith("## ") or \
                       content.startswith("### "):
                        meta["slideshow"] = {'slide_type': 'slide'}
                        nbline = 0
                        nbcell = 0
                        res[i] = ("slide", "section", cell)

            dh = self.cell_height(cell)
            dc = 1
            new_nbline = nbline + dh
            new_cell = dc + nbcell
            if "slideshow" not in meta:
                if new_cell > max_nb_cell or \
                   new_nbline > max_nb_line:
                    res[i] = (
                        "subslide", "{0}-{1} <-> {2}-{3}".format(nbcell, nbline, dc, dh), cell)
                    nbline = 0
                    nbcell = 0
                    meta["slideshow"] = {'slide_type': 'subslide'}

            nbline += dh
            nbcell += dc

        return res

    def run_notebook(self,
                     skip_exceptions=False,
                     progress_callback=None,
                     additional_path=None,
                     valid=None,
                     clean_function=None):
        '''
        Run all the cells of a notebook in order and update
        the outputs in-place.

        If ``skip_exceptions`` is set, then if exceptions occur in a cell, the
        subsequent cells are run (by default, the notebook execution stops).

        @param      skip_exceptions     skip exception
        @param      progress_callback   call back function
        @param      additional_path     additional paths (as a list or None if none)
        @param      valid               if not None, valid is a function which returns whether
                                        or not the cell should be executed or not, if the function
                                        returns None, the execution of the notebooks and skip the execution
                                        of the other cells
        @param      clean_function      function which cleans a cell's code before executing it (None for None)
        @return                         dictionary with statistics

        .. versionchanged:: 1.1
            The function adds the local variable ``theNotebook`` with
            the absolute file name of the notebook.

        .. versionchanged:: 1.4
            Function *valid* can now return None to stop the execution of the notebook
            before this cell.
        '''
        # additional path
        if additional_path is not None:
            if not isinstance(additional_path, list):
                raise TypeError(
                    "additional_path should be a list not: " + str(additional_path))
            code = ["import sys"]
            for p in additional_path:
                code.append("sys.path.append(r'{0}')".format(p))
            cell = "\n".join(code)
            self.run_cell(-1, cell)

        # we add local variable theNotebook
        if self.theNotebook is not None:
            cell = "theNotebook = r'''{0}'''".format(self.theNotebook)
            self.run_cell(-1, cell)

        # initialisation with a code not inside the notebook
        if self.code_init is not None:
            self.run_cell(-1, self.code_init)

        # execution of the notebook
        nbcell = 0
        nbrun = 0
        nbnerr = 0
        cl = time.clock()
        for i, cell in enumerate(self.iter_code_cells()):
            nbcell += 1
            iscell, codei = NotebookRunner.get_cell_code(cell)
            if valid is not None:
                r = valid(codei)
                if r is None:
                    break
                elif not r:
                    continue
            try:
                nbrun += 1
                self.run_cell(i, cell, clean_function=clean_function)
                nbnerr += 1
            except Empty as er:
                raise Exception(
                    "{0}\nissue when executing:\n{1}".format(self.comment, codei)) from er
            except NotebookError as e:
                if not skip_exceptions:
                    raise
                else:
                    raise Exception(
                        "issue when executing:\n{0}".format(codei)) from e
            if progress_callback:
                progress_callback(i)
        etime = time.clock() - cl
        return dict(nbcell=nbcell, nbrun=nbrun, nbvalid=nbnerr, time=etime)

    def count_code_cells(self):
        '''
        @return the number of code cells in the notebook

        .. versionadded:: 1.1
        '''
        return sum(1 for _ in self.iter_code_cells())

    def merge_notebook(self, nb):
        """
        append notebook *nb* to this one

        @param      nb      notebook or list of notebook (@see cl NotebookRunner)
        @return             number of added cells

        .. faqref::
            :title: How to merge notebook?

            The following code merges two notebooks into the first one
            and stores the result unto a file.

            @code
            from pyquickhelper.ipythonhelper import read_nb
            nb1 = read_nb("<file1>", kernel=False)
            nb2 = read_nb("<file2>", kernel=False)
            nb1.merge_notebook(nb2)
            nb1.to_json(outfile)
            @endcode

        .. versionadded:: 1.1
        """
        if isinstance(nb, list):
            s = 0
            for n in nb:
                s += self.merge_notebook(n)
            return s
        else:
            last = self._cell_container()
            s = 0
            for cell in nb.iter_cells():
                last.append(cell)
                s += 1
            return s

    def get_description(self):
        """
        Get summary and description of this notebook.
        We expect the first cell to contain a title and a description
        of its content.

        @return             header, description

        .. versionadded:: 1.5
        """
        def split_header(s, get_header=True):
            s = s.lstrip().rstrip()
            parts = s.splitlines()
            if parts[0].startswith('#'):
                if get_header:
                    header = re.sub('#+\s*', '', parts.pop(0))
                    if not parts:
                        return header, ''
                else:
                    header = ''
                rest = '\n'.join(parts).lstrip().split('\n\n')
                desc = rest[0].replace('\n', ' ')
                return header, desc
            else:
                if get_header:
                    if parts[0].startswith(('=', '-')):
                        parts = parts[1:]
                    header = parts.pop(0)
                    if parts and parts[0].startswith(('=', '-')):
                        parts.pop(0)
                    if not parts:
                        return header, ''
                else:
                    header = ''
                rest = '\n'.join(parts).lstrip().split('\n\n')
                desc = rest[0].replace('\n', ' ')
                return header, desc

        first_cell = self.first_cell()

        if not first_cell['cell_type'] == 'markdown':
            raise ValueError("The first cell is not in markdown but '{0}'.".format(
                first_cell['cell_type']))

        header, desc = split_header(first_cell['source'])
        if not desc and len(self.nb['cells']) > 1:
            second_cell = self.nb['cells'][1]
            if second_cell['cell_type'] == 'markdown':
                _, desc = split_header(second_cell['source'], False)

        reg_link = "(\\[(.*?)\\]\\(([^ ]*)\\))"
        reg = re.compile(reg_link)
        new_desc = reg.sub("\\2", desc)
        if "http://" in new_desc or "https://" in new_desc:
            raise ValueError(
                "Wrong regular expression:\n{0}\nMODIFIED:\n{1}".format(desc, new_desc))
        return header, new_desc.replace('"', "")

    def get_thumbnail(self, max_width=200, max_height=200):
        """
        Process the notebook and create one picture based on the outputs
        to illustrate a notebook.

        @param      max_width       maximum size of the thumbnail
        @param      max_height      maximum size of the thumbnail
        @return                     string (SVG) or Image (PIL)

        This functionality might not works with Python 2.7.

        .. versionadded:: 1.5
        """
        images = []
        cells = list(self.iter_cells())
        cells.reverse()
        for cell in cells:
            c = self.cell_image(cell, False)
            if c is not None and len(c) > 0 and len(c[0]) > 0:
                self._check_thumbnail_tuple(c)
                images.append(c)
        if len(images) == 0:
            for cell in cells:
                c = self.cell_image(cell, True)
                if c is not None and len(c) > 0 and len(c[0]) > 0:
                    self._check_thumbnail_tuple(c)
                    images.append(c)
                    if len(c[0]) >= 1000:
                        break
        if len(images) == 0:
            # no image, we need to consider the default one
            no_image = os.path.join(
                os.path.dirname(__file__), 'no_image_nb.png')
            with open(no_image, "rb") as f:
                c = (f.read(), "png")
                self._check_thumbnail_tuple(c)
                images.append(c)

        # select the image
        if len(images) == 0:
            raise ValueError("There should be at least one image.")
        elif len(images) == 1:
            image = images[0]
        else:
            # maybe later we'll implement a different logic
            # we pick the last one
            image = images[0]

        # zoom
        if image[1] != "svg":
            img = self._scale_image(
                image[0], image[1], max_width=max_width, max_height=max_height)
            return img
        else:
            return image[0]

    def _scale_image(self, in_bytes, format=None, max_width=200, max_height=200):
        """
        Scales an image with the same aspect ratio centered in an
        image with a given max_width and max_height.

        @param      in_bytes        image as bytes
        @param      format          indication of the format (can be empty)
        @param      max_width       maximum size of the thumbnail
        @param      max_height      maximum size of the thumbnail
        @return                     Image (PIL)

        .. versionadded:: 1.5
        """
        # local import to avoid testing dependency on PIL:
        try:
            from PIL import Image
        except ImportError:
            import Image

        if isinstance(in_bytes, tuple):
            in_bytes = in_bytes[0]
        if not isinstance(in_bytes, bytes):
            raise TypeError("bytes expected, not {0}".format(type(in_bytes)))
        img = Image.open(BytesIO(in_bytes))
        width_in, height_in = img.size
        scale_w = max_width / float(width_in)
        scale_h = max_height / float(height_in)

        if height_in * scale_w <= max_height:
            scale = scale_w
        else:
            scale = scale_h

        if scale >= 1.0:
            return img

        width_sc = int(round(scale * width_in))
        height_sc = int(round(scale * height_in))

        # resize the image and center
        img.thumbnail((width_sc, height_sc), Image.ANTIALIAS)
        thumb = Image.new('RGB', (max_width, max_height), (255, 255, 255))
        pos_insert = ((max_width - width_sc) // 2,
                      (max_height - height_sc) // 2)
        thumb.paste(img, pos_insert)
        return thumb

    def _merge_images(self, results):
        """
        Merges images defined by (buffer, format).
        The method uses PIL to merge images when possible.

        @return                     [ (image, format) ]

        .. versionadded:: 1.5
        """
        if len(results) == 1:
            results = results[0]
            self._check_thumbnail_tuple(results)
            return results
        elif len(results) == 0:
            return None
        formats_counts = Counter(_[1] for _ in results)
        if len(formats_counts) == 1:
            format = results[0][1]
        else:
            items = sorted(((v, k) for k, v in formats_counts.items()), False)
            for it in items:
                format = it
                break

        results = [_ for _ in results if _[1] == format]
        if format == "svg":
            return ("\n".join(_[0] for _ in results), format)
        else:
            # local import to avoid testing dependency on PIL:
            try:
                from PIL import Image
            except ImportError:
                import Image

            dx = 0.
            dy = 0.
            over = 0.7
            imgs = []
            for in_bytes, f in results:
                img = Image.open(BytesIO(in_bytes))
                imgs.append(img)
                dx = max(dx, img.size[0])
                dy += img.size[1] * over

            new_im = Image.new('RGB', (int(dx), int(dy)), (220, 220, 220))
            for img in imgs:
                dy -= img.size[1] * over
                new_im.paste(img, (0, max(int(dy), 0)))

            image_buffer = BytesIO()
            new_im.save(image_buffer, "PNG")
            b = image_buffer.getvalue(), "png"
            return b
Ejemplo n.º 42
0
def paths():
    for fullpath in glob(os.path.join(os.path.dirname(filename), 'kernel*')):
        if not re.match('^(.*/)?kernel-[0-9]+.json', fullpath):
            continue
        yield fullpath


connected = False
while not connected:
    try:
        filename = find_connection_file('kernel*')
    except IOError:
        continue

    for fullpath in paths():
        km = KernelManager(connection_file=fullpath)
        km.load_connection_file()

        kc = km.client()
        kc.start_channels()
        try:
            send = kc.execute
        except AttributeError:
            send = kc.shell_channel.execute
        if not hasattr(kc, 'iopub_channel'):
            kc.iopub_channel = kc.sub_channel

        send('', silent=True)
        try:
            msg = kc.shell_channel.get_msg(timeout=1)
            connected = True
Ejemplo n.º 43
0
def km_from_string(s=''):
    """create kernel manager from IPKernelApp string
    such as '--shell=47378 --iopub=39859 --stdin=36778 --hb=52668' for IPython 0.11
    or just 'kernel-12345.json' for IPython 0.12
    """
    try:
        import IPython
    except ImportError:
        raise ImportError("Could not find IPython. " + _install_instructions)
    try:
        from traitlets.config.loader import KeyValueConfigLoader
    except ImportError:
        from IPython.config.loader import KeyValueConfigLoader
    try:
        from jupyter_client import KernelManager, find_connection_file
    except ImportError:
        try:
            from IPython.kernel import  KernelManager, find_connection_file
        except ImportError:
            #  IPython < 1.0
            from IPython.zmq.blockingkernelmanager import BlockingKernelManager as KernelManager
            from IPython.zmq.kernelapp import kernel_aliases
            try:
                from IPython.lib.kernel import find_connection_file
            except ImportError:
                # < 0.12, no find_connection_file
                pass

    global km, kc, send, history, complete, object_info

    # Test if connection is still alive
    connected = False
    starttime = time.time()
    attempt = 0
    s = s.replace('--existing', '')
    while not connected and (time.time() - starttime) < 5.0:
        if not attempt and os.path.isfile(s):
            fullpath = s
        else:
            try:
                s = fullpath = find_connection_file('kernel*')
            except IOError:
                echo("IPython connection attempt #%d failed - no kernel file" % attempt, "Warning")
                time.sleep(1)
                continue
        attempt += 1

        km = KernelManager(connection_file=fullpath)
        km.load_connection_file()

        kc = km.client()
        kc.start_channels()

        if 'connection_file' in KernelManager.class_trait_names():
            # 0.12 uses files instead of a collection of ports
            # include default IPython search path
            # filefind also allows for absolute paths, in which case the search
            # is ignored
            try:
                # XXX: the following approach will be brittle, depending on what
                # connection strings will end up looking like in the future, and
                # whether or not they are allowed to have spaces. I'll have to sync
                # up with the IPython team to address these issues -pi
                if '--profile' in s:
                    k,p = s.split('--profile')
                    k = k.lstrip().rstrip() # kernel part of the string
                    p = p.lstrip().rstrip() # profile part of the string
                    fullpath = find_connection_file(k,p)
                else:
                    fullpath = find_connection_file(s.lstrip().rstrip())
            except IOError as e:
                echo(":IPython " + s + " failed", "Info")
                echo("^-- failed '" + s + "' not found", "Error")
                return
            km = KernelManager(connection_file = fullpath)
            km.load_connection_file()
        else:
            if s == '':
                echo(":IPython 0.11 requires the full connection string")
                return
            loader = KeyValueConfigLoader(s.split(), aliases=kernel_aliases)
            cfg = loader.load_config()['KernelApp']
            try:
                km = KernelManager(
                    shell_address=(ip, cfg['shell_port']),
                    sub_address=(ip, cfg['iopub_port']),
                    stdin_address=(ip, cfg['stdin_port']),
                    hb_address=(ip, cfg['hb_port']))
            except KeyError as e:
                echo(":IPython " +s + " failed", "Info")
                echo("^-- failed --"+e.message.replace('_port','')+" not specified", "Error")
                return

        try:
            kc = km.client()
        except AttributeError:
            # 0.13
            kc = km
        kc.start_channels()

        execute = kc.execute if hasattr(kc, 'execute') else kc.shell_channel.execute
        history = kc.history if hasattr(kc, 'history') else kc.shell_channel.history
        complete = kc.complete if hasattr(kc, 'complete') else kc.shell_channel.complete
        object_info = kc.inspect if hasattr(kc, 'inspect') else kc.shell_channel.object_info

        def send(msg, **kwargs):
            kwds = dict(
                store_history=vim_vars.get('ipython_store_history', True),
            )
            kwds.update(kwargs)
            return execute(msg, **kwds)

        send('', silent=True)
        try:
            msg = kc.shell_channel.get_msg(timeout=1)
            connected = True
        except:
            echo("IPython connection attempt #%d failed - no messages" % attempt, "Warning")
            continue

        #XXX: backwards compatibility for IPython < 1.0
        if not hasattr(kc, 'iopub_channel'):
            kc.iopub_channel = kc.sub_channel
        set_pid()

    if not connected:
        echo("IPython connection attempt timed out", "Error")
        return
    else:
        vim.command('redraw')
        echo("IPython connection successful")
        send('"_vim_client";_=_;__=__', store_history=False)

    #XXX: backwards compatibility for IPython < 0.13
    sc = kc.shell_channel
    if hasattr(sc, 'object_info'):
        import inspect
        num_oinfo_args = len(inspect.getargspec(sc.object_info).args)
        if num_oinfo_args == 2:
            # patch the object_info method which used to only take one argument
            klass = sc.__class__
            klass._oinfo_orig = klass.object_info
            klass.object_info = lambda s,x,y: s._oinfo_orig(x)

    #XXX: backwards compatibility for IPython < 1.0
    if not hasattr(kc, 'iopub_channel'):
        kc.iopub_channel = kc.sub_channel

    # now that we're connect to an ipython kernel, activate completion
    # machinery, but do so only for the local buffer if the user added the
    # following line the vimrc:
    #   let g:ipy_completefunc = 'local'
    vim.command("""
        if g:ipy_completefunc == 'global'
            set completefunc=CompleteIPython
        elseif g:ipy_completefunc == 'local'
            setl completefunc=CompleteIPython
        elseif g:ipy_completefunc == 'omni'
            setl omnifunc=CompleteIPython
        endif
        """)
    # also activate GUI doc balloons if in gvim
    vim.command("""
        if has('balloon_eval')
            set bexpr=IPythonBalloonExpr()
        endif
        """)
    return km
Ejemplo n.º 44
0
def run_notebook(notebook):
    f = open(notebook)
    if not f:
        return False
    nb = reads(f.read(), 3)

    km = KernelManager()
    km.start_kernel(extra_arguments=['--pylab=inline'],
                    stderr=open('/tmp/km.stderr', 'w'))
    kc = km.client()
    kc.start_channels()
    shell = kc.shell_channel

    shell.get_msg()

    successes = 0
    failures = 0
    errors = 0
    for ws in nb.worksheets:
        for cell in ws.cells:
            if cell.cell_type != 'code':
                continue
            try:
                status, outs = run_cell(kc, cell, 30)

            except Exception as e:
                # currently turned off to avoid jenkins hang
                # print "failed to run cell:", repr(e)
                # print cell.input
                # print dir(cell)
                errors += 1
                continue

            failed = False
            # currently turned off to avoid jenkins hang
            # print "Count outs: %d" % len(outs)
            # print "Count cell_out: %d" % len(cell.outputs)
            #for out, ref in zip(outs, cell.outputs):
            #    print "OUT[%s]" % outs
            #    print "EXP[%s]" % ref
            #    #if not compare_outputs(out, ref):
            #    #    failed = True
            #    #    break
            if status != "ok" or failed:
                failures += 1
            else:
                successes += 1

    print
    print "tested notebook %s" % nb.metadata.name
    print "    %3i cells successfully replicated" % successes
    if failures:
        print "    %3i cells mismatched output" % failures
    if errors:
        print "    %3i cells failed to complete" % errors
    kc.stop_channels()
    km.shutdown_kernel()
    del km

    if failures > 0 or errors > 0:
        return False
    else:
        return True