Esempio n. 1
1
def run_notebook(nb):
    km = KernelManager()
    km.start_kernel(stderr=open(os.devnull, 'w'))

    kc = km.client()
    kc.start_channels()
    try:
        kc.wait_for_ready()
    except AttributeError:
        # IPython < 3
        kc.kernel_info()
        while True:
            msg = kc.get_shell_msg(block=True, timeout=30)
            if msg['msg_type'] == 'kernel_info_reply':
                break

        # Flush IOPub channel
        while True:
            try:
                msg = kc.get_iopub_msg(block=True, timeout=0.2)
            except Empty:
                break

    # simple ping:
    kc.execute("pass")
    kc.get_shell_msg()

    cells = 0
    failures = 0
    if hasattr(nb, 'worksheets'):
        # nobody uses more than 1 worksheet
        ws = nb.worksheets[0]
    else:
        # no more worksheet level in new format
        ws = nb

    for cell in ws.cells:
        if cell.cell_type != 'code':
            continue

        outputs, failed = run_cell(kc, cell)
        cell.outputs = outputs
        cell['prompt_number'] = cells
        failures += failed
        cells += 1
        sys.stdout.write('.')
        sys.stdout.flush()

    print()
    print("ran %3i cells" % cells)
    if failures:
        print("    %3i cells raised exceptions" % failures)
    kc.stop_channels()
    km.shutdown_kernel()
    del km
Esempio n. 2
0
def test_connection_file_real_path():
    """ Verify realpath is used when formatting connection file """
    with mock.patch('os.path.realpath') as patched_realpath:
        patched_realpath.return_value = 'foobar'
        km = KernelManager(connection_file=os.path.join(
                tempfile.gettempdir(), "kernel-test.json"),
                kernel_name='test_kernel')

        # KernelSpec and launch args have to be mocked as we don't have an actual kernel on disk
        km._kernel_spec = KernelSpec(resource_dir='test', 
            **{
                "argv": [
                    "python.exe",
                    "-m",
                    "test_kernel",
                    "-f",
                    "{connection_file}"
                ],
                "env": {},
                "display_name": "test_kernel",
                "language": "python",
                "metadata": {}
            })
        km._launch_args = {}
        cmds = km.format_kernel_cmd()
        assert cmds[4] is 'foobar'
 def start(self):
     self.km = KernelManager()
     self.km.kernel_name = self.kernel_name
     self.km.start_kernel(extra_arguments=[self.context])
     self.kc = self.km.client()
     self.kc.start_channels()
     self.kc.wait_for_ready()
Esempio n. 4
0
 def start(self):
     self.km = KernelManager()
     self.km.kernel_name = self.kernel_name
     self.km.start_kernel()
     self.kc = self.km.client()
     self.kc.start_channels()
     self.kc.wait_for_ready()
Esempio n. 5
0
    def __enter__(self):
        self.km = KernelManager()
        self.km.start_kernel(extra_arguments=self.extra_arguments,
                             stderr=open(os.devnull, 'w'))

        self.kc = self.km.client()
        self.kc.start_channels()

        self.iopub = self.kc.iopub_channel
        self.shell = self.kc.shell_channel

        # run %pylab inline, because some notebooks assume this
        # even though they shouldn't

        self.shell.send("pass")
        self.shell.get_msg()
        while True:
            try:
                self.iopub.get_msg(timeout=0.05)
            except Exception as e:
                if repr(e) == 'Empty()':
                    break

                # we got a real error so raise it
                raise

        self.cmd_list = []
        self.msg_list = {}

        return self
Esempio n. 6
0
 def start_kernel(
     cls,
     kernelspec_name=None,
     connection_info=None,
     connection_name=None,
 ):
     """Start kernel and return a `Kernel` instance."""
     kernel_id = uuid.uuid4()
     if kernelspec_name:
         kernel_manager = KernelManager(kernel_name=kernelspec_name)
         kernel_manager.start_kernel()
     elif connection_info:
         kernel_manager = KernelManager()
         kernel_manager.load_connection_info(connection_info)
         # `KernelManager.kernel_name` is not automatically set from connection info.
         kernel_manager.kernel_name = connection_info.get("kernel_name", "")
     else:
         raise Exception(
             "You must specify any of {`kernelspec_name`, `connection_info`}."
         )
     kernel = KernelConnection(kernel_id,
                               kernel_manager,
                               cls,
                               connection_name=connection_name,
                               logger=cls.logger)
     cls.kernels[kernel_id] = kernel
     return kernel
Esempio n. 7
0
 def __init__(self, shell):
     super(GroovyMagics, self).__init__(shell)
     self.km = KernelManager()
     self.km.kernel_name = 'groovy'
     self.km.start_kernel()
     self.kc = self.km.client()
     self.kc.start_channels()
     try:
         self.kc.wait_for_ready()
         print("Groovy started successfully\n")
     except AttributeError:
         self._wait_for_ready_backport()
Esempio n. 8
0
 def start(self):
     self.km = KernelManager()
     self.km.kernel_name = 'groovy'
     self.km.start_kernel()
     atexit.register(self.stop_kernel)
     self.kc = self.km.client()
     self.kc.start_channels()
     try:
         self.kc.wait_for_ready()
         print("Groovy started successfully\n")
     except AttributeError:
         self._wait_for_ready_backport()
Esempio n. 9
0
 def start(self, kernel_name):
     self.km = KernelManager()
     self.km.kernel_name = kernel_name
     self.km.start_kernel(extra_arguments=[self._context_base64()])
     atexit.register(self.stop_kernel)
     self.kc = self.km.client()
     self.kc.start_channels()
     try:
         self.kc.wait_for_ready()
         print("{} started successfully\n".format(kernel_name.capitalize()))
     except AttributeError:
         self._wait_for_ready_backport()
Esempio n. 10
0
class GroovyMagics(Magics):
    _execution_count = 1

    def stop_kernel(self):
        self.kc.stop_channels()
        self.km.shutdown_kernel(now=True)

    def __init__(self, shell):
        super(GroovyMagics, self).__init__(shell)
        self.km = None

    def start(self):
        self.km = KernelManager()
        self.km.kernel_name = 'groovy'
        self.km.start_kernel()
        atexit.register(self.stop_kernel)
        self.kc = self.km.client()
        self.kc.start_channels()
        try:
            self.kc.wait_for_ready()
            print("Groovy started successfully\n")
        except AttributeError:
            self._wait_for_ready_backport()

    def run_cell(self, line, code):
        if not self.km:
            self.start()
        self.kc.execute(code, allow_stdin=True)
        reply = self.kc.get_shell_msg()

        while True:
            try:
                msg = self.kc.get_iopub_msg(timeout=1)
                if msg['msg_type'] == 'status':
                    if msg['content']['execution_state'] == 'idle':
                        break
            except Empty:
                print("empty ?!")
                raise
            self.shell.kernel.session.send(
                self.shell.kernel.iopub_socket,
                msg['msg_type'],
                msg['content'],
                metadata=msg['metadata'],
                parent=self.shell.kernel._parent_header,
                ident=msg.get('comm_id'),
                buffers=msg['buffers'],
            )

    @cell_magic
    def groovy(self, line, cell):
        return self.run_cell(line, cell)
Esempio n. 11
0
 def start():
     self.manager = KernelManager(kernel_name=self.name)
     self.manager.start_kernel()
     self.client = self.manager.blocking_client()
     self.client.start_channels()
     try:
         self.client.wait_for_ready(timeout=timeout)
     except TimeoutError:  # pragma: no cover
         self.manager.shutdown_kernel()
         return False
     else:
         self.client.execute_interactive(self.init_code)
         return self.client
Esempio n. 12
0
def get_client(cf, profile=None):
    """
    Usage:
        >>> kc = get_client('kernel-143a2687-f294-42b1-bdcb-6f1cc2f4cc87.json', 'dale')
        >>> data = kc.execute("'123'")
        >>> data
        {u'text/plain': u'123'}
    """
    connection_file = find_connection_file(cf, profile=profile)
    km = KernelManager(connection_file=connection_file)
    km.load_connection_file()

    client = km.client()
    return KernelClient(client)
Esempio n. 13
0
    def start_ipython_kernel(self):
        kernel_name = self._get_wrapped_kernel_name()
        self.km = KernelManager(
            kernel_name=kernel_name,
            client_class='jupyter_client.blocking.BlockingKernelClient')
        self.log.debug('kernel_manager: %s', str(self.km))

        self.log.info('start wrapped kernel: %s', kernel_name)
        self.km.start_kernel()
        self.kc = self.km.client()
        self.log.debug('kernel_client: %s', str(self.kc))

        self.log.debug('start_channels')
        self.kc.start_channels()

        try:
            self.log.debug('wait for ready of wrapped kernel')
            self.kc.wait_for_ready(timeout=None)
        except RuntimeError:
            self.kc.stop_channels()
            self.km.shutdown_kernel()
            raise

        for channel in self.proxy_channles:
            stream = getattr(self, channel + '_socket')
            thread = ChannelReaderThread(self, self.kc, stream, self.session,
                                         channel)
            thread.start()
            self.threads[channel] = thread

        for log_dir in self.log_dirs:
            if self._is_writable_dir(log_dir):
                self.log_path = log_dir
                break
        self.log.debug('log output directory: %s', self.log_path)

        if self._find_default_keyword_pattern_file() is None:
            self.log.info('default keyword pattern file "%s" not found',
                          IPYTHON_DEFAULT_PATTERN_FILE)
            try:
                self._generate_default_keyword_pattern_file()
            except Exception as e:
                self.log.exception(
                    "failed to generate default keyword pattern file: %s", e)

        self.exec_info = None

        self.notebook_path = self.get_notebook_path()
        self.log.debug('notebook_path: %s', self.notebook_path)
Esempio n. 14
0
def new_ipy(s=''):
    """Create a new IPython kernel (optionally with extra arguments)

    XXX: Allow passing of profile information here

    Examples
    --------

        new_ipy()

    """
    from jupyter_client.manager import KernelManager
    km = KernelManager()
    km.start_kernel()
    return km_from_string(km.connection_file)
Esempio n. 15
0
def new_ipy(s=''):
    """Create a new IPython kernel (optionally with extra arguments)

    XXX: Allow passing of profile information here

    Examples
    --------

        new_ipy()

    """
    from jupyter_client.manager import KernelManager
    km = KernelManager()
    km.start_kernel()
    return km_from_string(km.connection_file)
Esempio n. 16
0
    def __enter__(self):
        self.km = KernelManager()
        self.km.start_kernel(
            extra_arguments=self.extra_arguments,
            stderr=open(os.devnull, 'w')
        )

        self.kc = self.km.client()
        self.kc.start_channels()

        self.iopub = self.kc.iopub_channel
        self.shell = self.kc.shell_channel

        # run %pylab inline, because some notebooks assume this
        # even though they shouldn't

        self.shell.send("pass")
        self.shell.get_msg()
        while True:
            try:
                self.iopub.get_msg(timeout=0.05)
            except Empty:
                break

        return self
Esempio n. 17
0
    def __enter__(self):
        self.km = KernelManager()
        self.km.start_kernel(
            extra_arguments=self.extra_arguments,
            stderr=open(os.devnull, 'w')
        )

        self.kc = self.km.client()
        self.kc.start_channels()

        self.iopub = self.kc.iopub_channel
        self.shell = self.kc.shell_channel

        # run %pylab inline, because some notebooks assume this
        # even though they shouldn't

        self.shell.send("pass")
        self.shell.get_msg()
        while True:
            try:
                self.iopub.get_msg(timeout=0.05)
            except Exception as e:
                if repr(e) == 'Empty()':
                    break

                # we got a real error so raise it
                raise

        self.cmd_list = []
        self.msg_list = {}

        return self
Esempio n. 18
0
 def start(self):
     self.km = KernelManager()
     self.km.kernel_name = self.kernel_name
     self.km.start_kernel(extra_arguments=[self.context])
     self.kc = self.km.client()
     self.kc.start_channels()
     self.kc.wait_for_ready()
    def _make_kernel_manager(self, kernel_name, group_id):
        """Creates a new kernel manager for given kernel and group id if none exists, and returns it.

        Args:
            kernel_name (str): the kernel
            group_id (str): item group that will execute using this kernel

        Returns:
            KernelManager
        """
        if group_id is None:
            # Execute in isolation
            return KernelManager(kernel_name=kernel_name)
        key = (kernel_name, group_id)
        if key not in self._kernel_managers:
            self._kernel_managers[key] = KernelManager(kernel_name=kernel_name)
        return self._kernel_managers[key]
Esempio n. 20
0
def new_ipy(s=''):
    """Create a new IPython kernel (optionally with extra arguments)

    XXX: Allow passing of profile information here

    Examples
    --------

        new_ipy()

    """
    # Modified by Bo Peng.
    # This package has been deprecated. Need to import from ipykernel
    from jupyter_client.manager import KernelManager
    km = KernelManager()
    km.start_kernel()
    return km_from_string(km.connection_file)
Esempio n. 21
0
def new_ipy(s=''):
    """Create a new IPython kernel (optionally with extra arguments)

    XXX: Allow passing of profile information here

    Examples
    --------

        new_ipy()

    """
    # Modified by Bo Peng.
    # This package has been deprecated. Need to import from ipykernel
    from jupyter_client.manager import KernelManager
    km = KernelManager()
    km.start_kernel()
    return km_from_string(km.connection_file)
Esempio n. 22
0
 def set_contents(self, contents):
     self.contents = json.loads(contents)
     self.contents['notebook']['metadata']['pixiedust'] = {}
     kernel_spec = self.contents['notebook']['metadata']['kernelspec']
     km = KernelManager(kernel_name=kernel_spec['name'])
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
         self.kernel_spec = json.dumps(km.kernel_spec.to_dict(), indent=4, sort_keys=True)
Esempio n. 23
0
    def start_ipython_kernel(self):
        kernel_name = self._get_wrapped_kernel_name()
        self.km = KernelManager(
            kernel_name=kernel_name,
            client_class='jupyter_client.blocking.BlockingKernelClient')
        self.log.debug('kernel_manager: %s', str(self.km))

        self.log.info('start wrapped kernel: %s', kernel_name)
        self.km.start_kernel()
        self.kc = self.km.client()
        self.log.debug('kernel_client: %s', str(self.kc))

        self.log.debug('start_channels')
        self.kc.start_channels()

        self.flush_stream_event.set()

        try:
            self.log.debug('wait for ready of wrapped kernel')
            self.kc.wait_for_ready(timeout=None)
        except RuntimeError:
            self.kc.stop_channels()
            self.km.shutdown_kernel()
            raise

        for channel in self.proxy_channles:
            stream = getattr(self, channel + '_socket')
            thread = ChannelReaderThread(self, self.kc, stream, self.session,
                                         channel)
            thread.start()
            self.threads[channel] = thread

        self.notebook_path = self.get_notebook_path(self.kc)
        self.log_path = os.path.join(self.notebook_path, u'.log')
        if not os.path.exists(
                os.path.join(self.notebook_path,
                             IPYTHON_DEFAULT_PATTERN_FILE)):
            with open(
                    os.path.join(self.notebook_path,
                                 IPYTHON_DEFAULT_PATTERN_FILE), 'w') as f:
                f.write(IPYTHON_DEFAULT_PATTERN)
        self.exec_info = None
        self._init_log()

        self.log.debug('notebook_path: %s', self.notebook_path)
Esempio n. 24
0
def run_notebook(nb):
    km = KernelManager()
    km.start_kernel(stderr=open(os.devnull, 'w'))

    kc = km.client()
    kc.start_channels()

    cells = failures = 0
    for cell in nb.cells:
        if cell.cell_type != 'code':
            continue

        kc.execute(cell.source)

        # Wait to finish
        try:
            reply = kc.get_shell_msg()['content']
        except Empty:
            reply = {'status': 'error', 'traceback': ["Cell execution timed out!"]}
        if reply['status'] == 'error':
            failures += 1
            print("\nFAILURE:")
            print(cell.source)
            print('-----')
            print("Raised:")
            print('\n'.join(reply['traceback']))
        cells += 1
        sys.stdout.write('.')

    kc.stop_channels()
    km.shutdown_kernel()

    return cells, failures
Esempio n. 25
0
class JVMKernelMagic:

    def __init__(self, kernel_name, context):
        self.km = None
        self.kc = None
        self.comms = []
        self.kernel_name = kernel_name
        self.context = context
        self.start()

    def start(self):
        self.km = KernelManager()
        self.km.kernel_name = self.kernel_name
        self.km.start_kernel(extra_arguments=[self.context])
        self.kc = self.km.client()
        self.kc.start_channels()
        self.kc.wait_for_ready()

    def stop_kernel(self):
        self.kc.stop_channels()
        self.km.shutdown_kernel(now=True)

    def run_cell(self, code):
        if not self.km:
            self.start()
        self.kc.execute(code, allow_stdin=True)

    def get_shell_msg(self):
        return self.kc.get_shell_msg()

    def get_iopub_msg(self):
        try:
            msg = self.kc.get_iopub_msg(timeout=1)
            return msg
        except Empty:
            return None

    def pass_msg(self, msg_raw):
        msg_json = json.loads(msg_raw)
        content = msg_json['content']
        msg_type = msg_json['header']['msg_type']
        msg = self.kc.session.msg(msg_type, content)
        self.kc.shell_channel.send(msg)
        return None
Esempio n. 26
0
class JVMKernelMagic:

    def __init__(self, kernel_name, context):
        self.km = None
        self.kc = None
        self.comms = []
        self.kernel_name = kernel_name
        self.context = context
        self.start()

    def start(self):
        self.km = KernelManager()
        self.km.kernel_name = self.kernel_name
        self.km.start_kernel(extra_arguments=[self.context])
        self.kc = self.km.client()
        self.kc.start_channels()
        self.kc.wait_for_ready()

    def stop_kernel(self):
        self.kc.stop_channels()
        self.km.shutdown_kernel(now=True)

    def run_cell(self, code):
        if not self.km:
            self.start()
        self.kc.execute(code, allow_stdin=True)

    def get_shell_msg(self):
        return self.kc.get_shell_msg()

    def get_iopub_msg(self):
        try:
            msg = self.kc.get_iopub_msg(timeout=1)
            return msg
        except Empty:
            return None

    def pass_msg(self, msg_raw):
        msg_json = json.loads(msg_raw)
        content = msg_json['content']
        msg_type = msg_json['header']['msg_type']
        msg = self.kc.session.msg(msg_type, content)
        self.kc.shell_channel.send(msg)
        return None
Esempio n. 27
0
def test_feature(kernel_name, feature):
    print("...{}".format(feature))
    manager = KernelManager(kernel_name=kernel_name)
    manager.start_kernel()
    client = manager.client()

    with open(join(feature, "provided.json")) as fp:
        provided = json.load(fp)

    with open(join(feature, "expected.schema.json")) as fp:
        expected = json.load(fp)

    msg = client.session.msg(provided['header']['msg_type'],
                             provided['content'])
    client.shell_channel.send(msg)

    response = client.shell_channel.get_msg()

    jsonschema.validate(instance=response, schema=expected)
Esempio n. 28
0
def new_ipy(s=''):
    """Create a new IPython kernel (optionally with extra arguments)

    XXX: Allow passing of profile information here

    Examples
    --------

        new_ipy()

    """

    try:
        from jupyter_client.manager import KernelManager
    except ImportError:  # For compatibility with IPython 3 or lower
        from IPython.kernel import KernelManager

    km = KernelManager()
    km.start_kernel()
    return km_from_string(km.connection_file)
Esempio n. 29
0
 def start(self, kernel_name):
     self.km = KernelManager()
     self.km.kernel_name = kernel_name
     self.km.start_kernel(extra_arguments=[self._context_base64()])
     atexit.register(self.stop_kernel)
     self.kc = self.km.client()
     self.kc.start_channels()
     try:
         self.kc.wait_for_ready()
         print("{} started successfully\n".format(kernel_name.capitalize()))
     except AttributeError:
         self._wait_for_ready_backport()
Esempio n. 30
0
def run_notebook(nb):
    """Run notebook."""
    km = KernelManager()
    km.start_kernel(stderr=open(os.devnull, 'w'))
    kc = km.client()
    kc.start_channels()
    kc.execute("pass")
    kc.get_shell_msg()

    cells = 0
    failures = 0
    for ws in nb.worksheets:
        for cell in ws.cells:
            if cell.cell_type != 'code':
                continue
            kc.execute(cell.input)
            # wait for finish, maximum 20s
            reply = kc.get_shell_msg(timeout=20)['content']
            if reply['status'] == 'error':
                failures += 1
                print("\nFAILURE:")
                print(cell.input)
                print('-----')
                print("raised:")
                print('\n'.join(reply['traceback']))
            cells += 1
            sys.stdout.write('.')
    # print "ran notebook %s" % nb.metadata.name
    print("    ran %3i cells" % cells)
    if failures:
        print("    %3i cells raised exceptions" % failures)
    kc.stop_channels()
    km.shutdown_kernel()
    del km
    return failures
Esempio n. 31
0
    def __init__(self, shell):
        super(GroovyMagics, self).__init__(shell)
        self.km = KernelManager()
        self.km.kernel_name = 'groovy'
        self.km.start_kernel()
        self.kc = self.km.client()
        self.kc.start_channels()

        try:
            self.kc.wait_for_ready()
            print("Groovy started successfully\n")
        except AttributeError:
            self._wait_for_ready_backport()
Esempio n. 32
0
def createKernelSpecIfNeeded(kernelName=__TEST_KERNEL_NAME__, useSpark=False):
    try:
        km = KernelManager(kernel_name=kernelName)
        km.kernel_spec
        print("Kernal already exists!")
        return None
    except NoSuchKernel:
        print("Creating new Kernel {} target: {}".format(kernelName, useSpark))
        if useSpark:
            sparkHome = os.environ["SPARK_HOME"]
            overrides = {
                "argv":
                ["python", "-m", "ipykernel", "-f", "{connection_file}"],
                "env": {
                    "SCALA_HOME":
                    "{0}".format(os.environ["SCALA_HOME"]),
                    "SPARK_HOME":
                    "{0}".format(sparkHome),
                    "PYTHONPATH":
                    "{0}/python/:{0}/python/lib/py4j-0.9-src.zip".format(
                        sparkHome),
                    "PYTHONSTARTUP":
                    "{0}/python/pyspark/shell.py".format(sparkHome),
                    "PYSPARK_SUBMIT_ARGS":
                    "--driver-class-path {0}/data/libs/* --master local[10] pyspark-shell"
                    .format(
                        os.environ.get("PIXIEDUST_HOME",
                                       os.path.expanduser('~'))),
                    "SPARK_DRIVER_MEMORY":
                    "10G",
                    "SPARK_LOCAL_IP":
                    "127.0.0.1"
                }
            }
        else:
            overrides = {
                "argv":
                ["python", "-m", "ipykernel", "-f", "{connection_file}"],
                "env": {
                    "PIXIEDUST_HOME":
                    os.environ.get("PIXIEDUST_HOME", os.path.expanduser('~'))
                }
            }

        path = write_kernel_spec(overrides=overrides)
        dest = KernelSpecManager().install_kernel_spec(path,
                                                       kernel_name=kernelName,
                                                       user=True)
        # cleanup afterward
        shutil.rmtree(path)
        return dest
Esempio n. 33
0
def start_new_kernel(startup_timeout=60, kernel_name='python', spykernel=False,
                     **kwargs):
    """Start a new kernel, and return its Manager and Client"""
    km = KernelManager(kernel_name=kernel_name)
    if spykernel:
        km._kernel_spec = SpyderKernelSpec()
    km.start_kernel(**kwargs)
    kc = km.client()
    kc.start_channels()
    try:
        kc.wait_for_ready(timeout=startup_timeout)
    except RuntimeError:
        kc.stop_channels()
        km.shutdown_kernel()
        raise

    return km, kc
Esempio n. 34
0
def start_new_kernel(startup_timeout=60, kernel_name='python', **kwargs):
    """Start a new kernel, and return its Manager and Client"""
    logger.debug('Starting new kernel: "%s"' % kernel_name)
    km = KernelManager(kernel_name=kernel_name,
                       kernel_spec_manager=NbvalKernelspecManager())
    km.start_kernel(**kwargs)
    kc = km.client()
    kc.start_channels()
    try:
        kc.wait_for_ready(timeout=startup_timeout)
    except RuntimeError:
        logger.exception('Failure starting kernel "%s"', kernel_name)
        kc.stop_channels()
        km.shutdown_kernel()
        raise

    return km, kc
Esempio n. 35
0
def start_new_kernel(startup_timeout=60, kernel_name='python', spykernel=False,
                     **kwargs):
    """Start a new kernel, and return its Manager and Client"""
    km = KernelManager(kernel_name=kernel_name)
    if spykernel:
        km._kernel_spec = SpyderKernelSpec()
    km.start_kernel(**kwargs)
    kc = km.client()
    kc.start_channels()
    try:
        kc.wait_for_ready(timeout=startup_timeout)
    except RuntimeError:
        kc.stop_channels()
        km.shutdown_kernel()
        raise

    return km, kc
Esempio n. 36
0
def start_new_kernel(startup_timeout=60, kernel_name='python', **kwargs):
    """Start a new kernel, and return its Manager and Client"""
    km = KernelManager(kernel_name=kernel_name,
                       kernel_spec_manager=NbvalKernelspecManager())
    km.start_kernel(**kwargs)
    kc = km.client()
    kc.start_channels()
    try:
        kc.wait_for_ready(timeout=startup_timeout)
    except RuntimeError:
        kc.stop_channels()
        km.shutdown_kernel()
        raise

    return km, kc
Esempio n. 37
0
def run_notebook(notebook):
    """Run the notebook"""
    kernel_manager = KernelManager()
    kernel_manager.start_kernel(stderr=open(os.devnull, 'w'))
    kernel_client = kernel_manager.client()
    kernel_client.start_channels()

    for sheet in notebook.worksheets:
        for (prompt_number, cell) in enumerate(sheet.cells, 1):
            if cell.cell_type != "code":
                continue

            cell.outputs = run_cell(kernel_client, cell)

            cell.prompt_number = prompt_number
            if cell.outputs and cell.outputs[0]['output_type'] == 'pyout':
                cell.outputs[0]["prompt_number"] = prompt_number

    kernel_manager.shutdown_kernel()
Esempio n. 38
0
    def __init__(self, nb, pylab=False, mpl_inline=False, working_dir=None):
        self.km = KernelManager()

        args = []

        if pylab:
            args.append('--pylab=inline')
            logging.warn('--pylab is deprecated and will be removed in a '
                         'future version')
        elif mpl_inline:
            args.append('--matplotlib=inline')
            logging.warn('--matplotlib is deprecated and will be removed in a '
                         'future version')

        cwd = os.getcwd()

        if working_dir:
            os.chdir(working_dir)

        self.km.start_kernel(extra_arguments=args)

        os.chdir(cwd)

        if platform.system() == 'Darwin':
            # There is sometimes a race condition where the first
            # execute command hits the kernel before it's ready.
            # It appears to happen only on Darwin (Mac OS) and an
            # easy (but clumsy) way to mitigate it is to sleep
            # for a second.
            sleep(1)

        self.kc = self.km.client()
        self.kc.start_channels()
        try:
            self.kc.wait_for_ready()
        except AttributeError:
            # IPython < 3
            self._wait_for_ready_backport()

        self.nb = nb
Esempio n. 39
0
def run_notebook(notebook):
    """Run the notebook"""
    kernel_manager = KernelManager()
    kernel_manager.start_kernel(stderr=open(os.devnull, 'w'))
    kernel_client = kernel_manager.client()
    kernel_client.start_channels()

    for sheet in notebook.worksheets:
        for (prompt_number, cell) in enumerate(sheet.cells, 1):
            if cell.cell_type != "code":
                continue

            cell.outputs = run_cell(kernel_client, cell)

            cell.prompt_number = prompt_number
            if cell.outputs and cell.outputs[0]['output_type'] == 'pyout':
                cell.outputs[0]["prompt_number"] = prompt_number

    kernel_manager.shutdown_kernel()
Esempio n. 40
0
class IPyKernel(object):
    """
    A simple wrapper class to run cells in an IPython Notebook.

    Notes
    -----
    - Use `with` construct to properly instantiate
    - IPython 3.0.0+ is assumed for this version

    """
    def __init__(self, nb_version=4, extra_arguments=None):
        # default timeout time is 60 seconds
        self.default_timeout = 60

        if extra_arguments is None:
            extra_arguments = []
        self.extra_arguments = extra_arguments
        self.nb_version = nb_version

    def __enter__(self):
        self.km = KernelManager()
        self.km.start_kernel(extra_arguments=self.extra_arguments,
                             stderr=open(os.devnull, 'w'))

        self.kc = self.km.client()
        self.kc.start_channels()

        self.iopub = self.kc.iopub_channel
        self.shell = self.kc.shell_channel

        # run %pylab inline, because some notebooks assume this
        # even though they shouldn't

        self.shell.send("pass")
        self.shell.get_msg()
        while True:
            try:
                self.iopub.get_msg(timeout=0.05)
            except Exception as e:
                if repr(e) == 'Empty()':
                    break

                # we got a real error so raise it
                raise

        self.cmd_list = []
        self.msg_list = {}

        return self

    def clear(self):
        self.iopub.get_msgs()

    def execute(self, cmd):
        uid = self.kc.execute(cmd)
        self.cmd_list.append((uid, cmd))
        return uid

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.kc.stop_channels()
        self.km.shutdown_kernel()
        del self.msg_list
        del self.cmd_list
        del self.km

    def listen(self, uid, use_timeout=None):
        if use_timeout is None:
            use_timeout = self.default_timeout

        while True:
            if uid in self.msg_list and len(self.msg_list[uid]) > 0:
                return self.msg_list[uid].pop(0)

            msg = self.iopub.get_msg(timeout=use_timeout)
            if 'msg_id' in msg['parent_header']:
                msg_uid = msg['parent_header']['msg_id']

                if msg_uid not in self.msg_list:
                    self.msg_list[msg_uid] = []

                self.msg_list[msg_uid].append(msg)

    def run(self, cell, use_timeout=None):
        """
        Run a notebook cell in the IPythonKernel

        Parameters
        ----------
        cell : IPython.notebook.Cell
            the cell to be run
        use_timeout : int or None (default)
            the time in seconds after which a cell is stopped and assumed to
            have timed out. If set to None the value in `default_timeout`
            is used

        Returns
        -------
        list of ex_cell_outputs
            a list of NotebookNodes of the returned types. This is
            similar to the list of outputs generated when a cell is run
        """

        if timeout is not None:
            use_timeout = use_timeout
        else:
            use_timeout = self.default_timeout

        if hasattr(cell, 'source'):
            uid = self.execute(cell.source)
        else:
            raise AttributeError('No source/input key')

        outs = []
        stdout_cells = {}

        while True:
            msg = self.listen(uid, use_timeout)

            msg_type = msg['msg_type']

            if msg_type == 'execute_input':
                continue
            elif msg_type == 'clear_output':
                outs = []
                continue
            elif msg_type == 'status':
                if msg['content']['execution_state'] == 'idle':
                    # we are done with the cell, let's compare
                    break

                continue

            out_cell = nbformat.NotebookNode(output_type=msg_type)

            content = msg['content']

            if msg_type == 'stream':
                name = content['name']
                if name not in stdout_cells:
                    out_cell.name = name
                    out_cell.text = content['text']
                    stdout_cells[name] = out_cell
                    outs.append(out_cell)
                else:
                    # we already have a stdout cell, so append to it
                    stdout_cells[name].text += content['text']

            elif msg_type in ('display_data', 'execute_result'):
                if hasattr(content, 'execution_count'):
                    out_cell['execution_count'] = content['execution_count']
                else:
                    out_cell['execution_count'] = None

                out_cell['data'] = content['data']
                out_cell['metadata'] = content['metadata']

                outs.append(out_cell)

            elif msg_type == 'error':
                out_cell.ename = content['ename']
                out_cell.evalue = content['evalue']
                out_cell.traceback = content['traceback']

                outs.append(out_cell)

            elif msg_type.startswith('comm_'):
                # messages used to initialize, close and unpdate widgets
                # we will ignore these and hope for the best
                pass

            else:
                tv.warning("Unhandled iopub msg of type `%s`" % msg_type)

        return outs

    def get_commands(self, cell):
        """
        Extract potential commands from the first line of a cell

        if a code cell starts with the hashbang `#!` it can be followed by
        a comma separated list of commands. Each command can be
        1. a single key `skip`, or
        2. a key/value pair separated by a colon `timeout:[int]`

        Parameters
        ----------
        cell : a NotebookCell
            the cell to be examined

        Returns
        -------
        dict
            a dict of key/value pairs. For a single command the value is `True`
        """
        commands = {}
        source = cell.source
        if source is not None:
            lines = source.splitlines()
            if len(lines) > 0:
                n_line = 0
                line = lines[n_line].strip()
                while line.startswith('#!') or len(line) == 0:
                    txt = line[2:].strip()

                    parts = txt.split(',')
                    for part in parts:
                        subparts = part.split(':')
                        if len(subparts) == 1:
                            commands[subparts[0].strip().lower()] = True
                        elif len(subparts) == 2:
                            commands[subparts[0].strip().lower()] = subparts[1]

                    n_line += 1
                    line = lines[n_line]

        return commands

    def is_empty_cell(self, cell):
        """
        Check if a cell has no code

        Parameters
        ----------
        cell : a NotebookCell
            the cell to be examined

        Returns
        -------
        bool
            True if the cell has no code, False otherwise
        """
        return not bool(cell.source)
import pathlib
from jupyter_client.manager import KernelManager
from jupyter_client.connect import find_connection_file

# forkする?
m = KernelManager()
m.connection_file = "kernel-me.json"
m.write_connection_file()
info = m.get_connection_info()
print(info)
print(m.connection_file)

print(find_connection_file())
print(find_connection_file(pathlib.Path(m.connection_file).name))
# m.load_connection_file()
m.blocking_client()
Esempio n. 42
0
class IPyKernel(object):
    """
    A simple wrapper class to run cells in an IPython Notebook.

    Notes
    -----
    - Use `with` construct to properly instantiate
    - IPython 3.0.0+ is assumed for this version

    """

    def __init__(self, nb_version=4, extra_arguments=None):
        # default timeout time is 60 seconds
        self.default_timeout = 60

        if extra_arguments is None:
            extra_arguments = []
        self.extra_arguments = extra_arguments
        self.nb_version = nb_version

    def __enter__(self):
        self.km = KernelManager()
        self.km.start_kernel(
            extra_arguments=self.extra_arguments,
            stderr=open(os.devnull, 'w')
        )

        self.kc = self.km.client()
        self.kc.start_channels()

        self.iopub = self.kc.iopub_channel
        self.shell = self.kc.shell_channel

        # run %pylab inline, because some notebooks assume this
        # even though they shouldn't

        self.shell.send("pass")
        self.shell.get_msg()
        while True:
            try:
                self.iopub.get_msg(timeout=0.05)
            except Empty:
                break

        return self

    def execute(self, source):
        self.kc.execute(source + '\n')
        self.shell.get_msg(timeout=0.05)

        while True:
            try:
                msg = self.iopub.get_msg(timeout=0.05)
            except Empty:
                break

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.kc.stop_channels()
        self.km.shutdown_kernel()
        del self.km

    def run(self, cell, timeout=None):
        """
        Run a notebook cell in the IPythonKernel

        Parameters
        ----------
        cell : IPython.notebook.Cell
            the cell to be run
        timeout : int or None (default)
            the time in seconds after which a cell is stopped and assumed to
            have timed out. If set to None the value in `default_timeout`
            is used

        Returns
        -------
        list of outs
            a list of NotebookNodes of the returned types. This is
            similar to the list of outputs generated when a cell is run
        """

        use_timeout = self.default_timeout

        if timeout is not None:
            use_timeout = timeout

        if hasattr(cell, 'input'):
            self.kc.execute(cell.input)
        elif hasattr(cell, 'source'):
            self.kc.execute(cell.source)
        else:
            raise AttributeError('No source/input key')

        self.shell.get_msg(timeout=use_timeout)

        outs = []
        stdout_cells = {}

        while True:
            try:
                msg = self.iopub.get_msg(timeout=1.00)
            except Empty:
                break
            msg_type = msg['msg_type']
            if msg_type in ('pyin', 'execute_input'):
                continue
            elif msg_type == 'clear_output':
                outs = []
                continue
            elif msg_type == 'status':
                if msg['content']['execution_state'] == 'idle':
                    # we are done with the cell, let's compare
                    break

                continue

            content = msg['content']
            out = nbformat.NotebookNode(output_type=msg_type)

            if msg_type == 'stream':
                name = content['name']
                if name not in stdout_cells:
                    out.name = name
                    out.text = content['text']
                    stdout_cells[name] = out
                    outs.append(out)
                else:
                    # we already have a stdout cell, so append to it
                    stdout_cells[name].text += content['text']

            elif msg_type in ('display_data', 'pyout', 'execute_result'):
                if hasattr(content, 'execution_count'):
                    out['execution_count'] = content['execution_count']
                else:
                    out['execution_count'] = None
                out['data'] = content['data']
                out['metadata'] = content['metadata']

                outs.append(out)

            elif msg_type == 'error':
                out.ename = content['ename']
                out.evalue = content['evalue']
                out.traceback = content['traceback']

                outs.append(out)

            elif msg_type.startswith('comm_'):
                # messages used to initialize, close and unpdate widgets
                # we will ignore these and hope for the best
                pass

            else:
                print "unhandled iopub msg:", msg_type

        return outs

    @staticmethod
    def sanitize(s):
        """sanitize a string for comparison.

        fix universal newlines, strip trailing newlines, and normalize likely
        random values (memory addresses and UUIDs)

        Parameters
        ----------
        s : str
            string to be sanitized, i.e. remove UUIDs, Hex-addresses, unnecessary newlines
        """
        if not isinstance(s, basestring):
            return s
        # normalize newline:
        s = s.replace('\r\n', '\n')

        # ignore trailing newlines (but not space)
        s = s.rstrip('\n')

        # normalize hex addresses:
        s = re.sub(r'0x[a-f0-9]+', '0xFFFFFFFF', s)

        # normalize UUIDs:
        s = re.sub(r'[a-f0-9]{8}(\-[a-f0-9]{4}){3}\-[a-f0-9]{12}', 'U-U-I-D', s)

        # fix problem with

        return s

    def compare_outputs(
            self,
            test,
            ref,
            skip_compare=('traceback', 'latex', 'execution_count')
    ):
        """
        Compare two lists of `NotebookNode`s

        Parameters
        ----------
        test : list of `NotebookNode`
            the list of be tested generated by the kernel
        ref : list of `NotebookNode`
            the reference list read from the notebook
        skip_compare : list of str
            a list of strings that name node types that are not to be tested

        Returns
        -------
        bool
            is True if both lists are different
        list of diff
            a list of diff (str) the represent the differences
        """
        diff = False
        diff_list = []

        #        print ref.keys(), test.keys()

        if self.nb_version == 4:
            for key in ref:
                if key not in test:
                    return True, ["missing key: %s != %s" % (test.keys(), ref.keys())]

                elif key not in skip_compare:
                    if key == 'data':
                        for data_key in test[key]:
                            my_diff = self.do_diff(
                                data_key,
                                test[key],
                                ref[key])

                            if my_diff is not None:
                                diff_list += my_diff
                                diff = True

                    else:
                        # can this happen?
                        my_diff = self.do_diff(key, test, ref)
                        if my_diff is not None:
                            diff_list += my_diff
                            diff = True

        return diff, diff_list

    def do_diff(self, key, test_cell, ref_cell):
        """
        Compare the key of two dicts

        Parameters
        ----------
        key : string
            the key to be compared
        test_cell : dict
            a dict with `key` as a key of string value
        ref_cell : dict
            a dict with `key` as a key of string value

        Returns
        -------
        list of diff (str)
            a list of diff representing the differences
        """

        if hasattr(ref_cell, key):
            s1 = self.sanitize(ref_cell[key])
        else:
            s1 = ''

        if hasattr(test_cell, key):
            s2 = self.sanitize(test_cell[key])
        else:
            s2 = ''

        if key in ['image/png', 'image/svg', 'image/svg+xml']:
            if s1 != s2:
                return ['>>> diff in %s (size new : %d vs size old : %d )' %
                        (key, len(s1), len(s2))]
        else:
            if s1 != s2:
                expected = s1.splitlines(1)
                actual = s2.splitlines(1)
                diff = difflib.ndiff(expected, actual)

                return ['>>> diff in ' + key] + list(diff)

        return None

    def get_commands(self, cell):
        """
        Extract potential commands from the first line of a cell

        if a code cell starts with the hashbang `#!` it can be followed by
        a comma separated list of commands. Each command can be
        1. a single key `skip`, or
        2. a key/value pair separated by a colon `timeout:[int]`

        Parameters
        ----------
        cell : a NotebookCell
            the cell to be examined

        Returns
        -------
        dict
            a dict of key/value pairs. For a single command the value is `True`
        """
        commands = {}
        source = self.get_source(cell)
        if source is not None:
            lines = source.splitlines()
            if len(lines) > 0:
                n_line = 0
                line = lines[n_line].strip()
                while line.startswith('#!') or len(line) == 0:
                    txt = line[2:].strip()

                    parts = txt.split(',')
                    for part in parts:
                        subparts = part.split(':')
                        if len(subparts) == 1:
                            commands[subparts[0].strip().lower()] = True
                        elif len(subparts) == 2:
                            commands[subparts[0].strip().lower()] = subparts[1]

                    n_line += 1
                    line = lines[n_line]

        return commands

    def get_source(self, cell):
        """
        get the source code of a cell

        Notes
        -----
        This is legacy of IPython 2/3 conversion.

        Parameters
        ----------
        cell : a NotebookCell
            the cell to be examined

        Returns
        -------
        string
            the source code

        """
        if cell.cell_type == 'code':
            if hasattr(cell, 'input'):
                return cell.input
            elif hasattr(cell, 'source'):
                return cell.source
            else:
                return None

    def is_empty_cell(self, cell):
        """
        Check if a cell has no code

        Parameters
        ----------
        cell : a NotebookCell
            the cell to be examined

        Returns
        -------
        bool
            True if the cell has no code, False otherwise
        """
        source = self.get_source(cell)

        return source is None or source == ''
Esempio n. 43
0
class KernelMagics(Magics):
    _execution_count = 1

    def stop_kernel(self):
        self.kc.stop_channels()
        self.km.shutdown_kernel(now=True)

    def __init__(self, shell):
        super(KernelMagics, self).__init__(shell)
        self.km = None
        self.kc = None
        self.comms = []

    def start(self, kernel_name):
        self.km = KernelManager()
        self.km.kernel_name = kernel_name
        self.km.start_kernel(extra_arguments=[self._context_base64()])
        atexit.register(self.stop_kernel)
        self.kc = self.km.client()
        self.kc.start_channels()
        try:
            self.kc.wait_for_ready()
            print("{} started successfully\n".format(kernel_name.capitalize()))
        except AttributeError:
            self._wait_for_ready_backport()

    def run_cell(self, line, code):
        if not self.km:
            self.start()
        self.kc.execute(code, allow_stdin=True)
        reply = self.kc.get_shell_msg()
        self._handle_iopub_messages()

    def _handle_iopub_messages(self):
        while True:
            try:
                msg = self.kc.get_iopub_msg(timeout=1)
            except Empty:
                break
            comm_id = msg['content'].get('comm_id')
            if comm_id and comm_id not in self.comms:
                self.comms.append(comm_id)
            self.shell.kernel.session.send(self.shell.kernel.iopub_socket, msg['msg_type'],
                                           msg['content'],
                                           metadata=msg['metadata'],
                                           parent=self.shell.kernel._parent_header,
                                           ident=msg.get('comm_id'),
                                           buffers=msg['buffers'],
                                           )

    def pass_message(self, msg_raw):
        comm_id = msg_raw['content'].get('comm_id')
        if comm_id in self.comms:
            content = msg_raw['content']
            msg = self.kc.session.msg(msg_raw['msg_type'], content)
            self.kc.shell_channel.send(msg)
            self._handle_iopub_messages()
        else:
            self.log.warn("No such comm: %s", comm_id)
            if self.log.isEnabledFor(logging.DEBUG):
                # don't create the list of keys if debug messages aren't enabled
                self.log.debug("Current comms: %s", list(self.comms.keys()))

    def _context_base64(self):
        context_json = json.dumps({
            'port': os.environ["BEAKERX_AUTOTRANSLATION_PORT"],
            'contextId': get_ipython().kernel.session.session,
        })
        return base64.b64encode(context_json.encode('utf-8')).decode()
Esempio n. 44
0
def km_from_string(s=''):
    """create kernel manager from IPKernelApp string
    such as '--shell=47378 --iopub=39859 --stdin=36778 --hb=52668' for IPython 0.11
    or just 'kernel-12345.json' for IPython 0.12
    """
    try:
        import IPython
    except ImportError:
        raise ImportError("Could not find IPython. " + _install_instructions)
    # Modified by BoPeng
    #
    #from IPython.config.loader import KeyValueConfigLoader
    from traitlets.config.loader import KeyValueConfigLoader
    try:
        # Updated by Bo Peng for module names
        from jupyter_client.manager import  KernelManager
        from jupyter_client import find_connection_file
    except ImportError:
        #  IPython < 1.0
        from IPython.zmq.blockingkernelmanager import BlockingKernelManager as KernelManager
        from IPython.zmq.kernelapp import kernel_aliases
        try:
            from IPython.lib.kernel import find_connection_file
        except ImportError:
            # < 0.12, no find_connection_file
            pass

    global km, kc, send

    s = s.replace('--existing', '')
    if 'connection_file' in KernelManager.class_trait_names():
        # 0.12 uses files instead of a collection of ports
        # include default IPython search path
        # filefind also allows for absolute paths, in which case the search
        # is ignored
        try:
            # XXX: the following approach will be brittle, depending on what
            # connection strings will end up looking like in the future, and
            # whether or not they are allowed to have spaces. I'll have to sync
            # up with the IPython team to address these issues -pi
            if '--profile' in s:
                k,p = s.split('--profile')
                k = k.lstrip().rstrip() # kernel part of the string
                p = p.lstrip().rstrip() # profile part of the string
                fullpath = find_connection_file(k,p)
            else:
                fullpath = find_connection_file(s.lstrip().rstrip())
        except IOError as e:
            echo(":IPython " + s + " failed", "Info")
            echo("^-- failed '" + s + "' not found", "Error")
            return
        km = KernelManager(connection_file = fullpath)
        km.load_connection_file()
    else:
        if s == '':
            echo(":IPython 0.11 requires the full connection string")
            return
        loader = KeyValueConfigLoader(s.split(), aliases=kernel_aliases)
        cfg = loader.load_config()['KernelApp']
        try:
            km = KernelManager(
                shell_address=(ip, cfg['shell_port']),
                sub_address=(ip, cfg['iopub_port']),
                stdin_address=(ip, cfg['stdin_port']),
                hb_address=(ip, cfg['hb_port']))
        except KeyError as e:
            echo(":IPython " +s + " failed", "Info")
            echo("^-- failed --"+e.message.replace('_port','')+" not specified", "Error")
            return

    try:
        kc = km.client()
    except AttributeError:
        # 0.13
        kc = km
    kc.start_channels()

    try:
        send = kc.execute
    except AttributeError:
        # < 3.0
        send = kc.shell_channel.execute

    #XXX: backwards compatibility for IPython < 0.13
    try:
        import inspect
        sc = kc.shell_channel
        num_oinfo_args = len(inspect.getargspec(sc.object_info).args)
        if num_oinfo_args == 2:
            # patch the object_info method which used to only take one argument
            klass = sc.__class__
            klass._oinfo_orig = klass.object_info
            klass.object_info = lambda s,x,y: s._oinfo_orig(x)
    except Exception:
        pass

    #XXX: backwards compatibility for IPython < 1.0
    if not hasattr(kc, 'iopub_channel'):
        kc.iopub_channel = kc.sub_channel

    # now that we're connect to an ipython kernel, activate completion
    # machinery, but do so only for the local buffer if the user added the
    # following line the vimrc:
    #   let g:ipy_completefunc = 'local'
    vim.command("""
        if g:ipy_completefunc == 'global'
            set completefunc=CompleteIPython
        elseif g:ipy_completefunc == 'local'
            setl completefunc=CompleteIPython
        endif
        """)
    # also activate GUI doc balloons if in gvim
    vim.command("""
        if has('balloon_eval')
            set bexpr=IPythonBalloonExpr()
        endif
        """)
    set_pid()
    return km
Esempio n. 45
0
def km_from_string(s=''):
    """create kernel manager from IPKernelApp string
    such as '--shell=47378 --iopub=39859 --stdin=36778 --hb=52668' for IPython 0.11
    or just 'kernel-12345.json' for IPython 0.12
    """
    try:
        import IPython
    except ImportError:
        raise ImportError("Could not find IPython. " + _install_instructions)

    try:
        from traitlets.config.loader import KeyValueConfigLoader
    except ImportError:  # IPython <= 3.0
        from IPython.config.loader import KeyValueConfigLoader

    try:
        from jupyter_client.manager import KernelManager
        from jupyter_client.connect import find_connection_file

    except ImportError:  # IPython <= 3.0
        try:
            from IPython.kernel import (
                KernelManager,
                find_connection_file,
            )
        except ImportError:
            #  IPython < 1.0
            from IPython.zmq.blockingkernelmanager import BlockingKernelManager as KernelManager
            from IPython.zmq.kernelapp import kernel_aliases
            try:
                from IPython.lib.kernel import find_connection_file
            except ImportError:
                # < 0.12, no find_connection_file
                pass

    global km, kc, send

    s = s.replace('--existing', '')
    if 'connection_file' in KernelManager.class_trait_names():
        # 0.12 uses files instead of a collection of ports
        # include default IPython search path
        # filefind also allows for absolute paths, in which case the search
        # is ignored
        try:
            # XXX: the following approach will be brittle, depending on what
            # connection strings will end up looking like in the future, and
            # whether or not they are allowed to have spaces. I'll have to sync
            # up with the IPython team to address these issues -pi
            if '--profile' in s:
                k,p = s.split('--profile')
                k = k.lstrip().rstrip() # kernel part of the string
                p = p.lstrip().rstrip() # profile part of the string
                fullpath = find_connection_file(k,p)
            else:
                s = s.lstrip().rstrip();
                if(len(s) == 0):
                    fullpath = find_connection_file()
                else:
                    fullpath = find_connection_file(s.lstrip().rstrip())
        except IOError as e:
            echo(":IPython " + s + " failed", "Info")
            echo("^-- failed '" + s + "' not found", "Error")
            return
        km = KernelManager(connection_file = fullpath)
        km.load_connection_file()
    else:
        if s == '':
            echo(":IPython 0.11 requires the full connection string")
            return
        loader = KeyValueConfigLoader(s.split(), aliases=kernel_aliases)
        cfg = loader.load_config()['KernelApp']
        try:
            km = KernelManager(
                shell_address=(ip, cfg['shell_port']),
                sub_address=(ip, cfg['iopub_port']),
                stdin_address=(ip, cfg['stdin_port']),
                hb_address=(ip, cfg['hb_port']))
        except KeyError as e:
            echo(":IPython " +s + " failed", "Info")
            echo("^-- failed --"+e.message.replace('_port','')+" not specified", "Error")
            return

    try:
        kc = km.client()
    except AttributeError:
        # 0.13
        kc = km
    kc.start_channels()

    try:
        send = kc.execute
    except AttributeError:
        # < 3.0
        send = kc.shell_channel.execute

    #XXX: backwards compatibility for IPython < 0.13
    try:
        import inspect
        sc = kc.shell_channel
        num_oinfo_args = len(inspect.getargspec(sc.object_info).args)
        if num_oinfo_args == 2:
            # patch the object_info method which used to only take one argument
            klass = sc.__class__
            klass._oinfo_orig = klass.object_info
            klass.object_info = lambda s,x,y: s._oinfo_orig(x)
    except:
        pass

    #XXX: backwards compatibility for IPython < 1.0
    if not hasattr(kc, 'iopub_channel'):
        kc.iopub_channel = kc.sub_channel

    # now that we're connect to an ipython kernel, activate completion
    # machinery, but do so only for the local buffer if the user added the
    # following line the vimrc:
    #   let g:ipy_completefunc = 'local'
    vim.command("""
        if g:ipy_completefunc == 'global'
            set completefunc=CompleteIPython
        elseif g:ipy_completefunc == 'local'
            setl completefunc=CompleteIPython
        endif
        """)
    # also activate GUI doc balloons if in gvim
    vim.command("""
        if has('balloon_eval')
            set bexpr=IPythonBalloonExpr()
        endif
        """)
    set_pid()
    return km
Esempio n. 46
0
class GroovyMagics(Magics):
    _execution_count = 1

    def __init__(self, shell):
        super(GroovyMagics, self).__init__(shell)
        self.km = KernelManager()
        self.km.kernel_name = 'groovy'
        self.km.start_kernel()
        self.kc = self.km.client()
        self.kc.start_channels()

        try:
            self.kc.wait_for_ready()
            print("Groovy started successfully\n")
        except AttributeError:
            self._wait_for_ready_backport()

    def __del__(self):
        self.kc.stop_channels()
        self.km.shutdown_kernel(now=True)

    def run_cell(self, line, code):
        self.kc.execute(code, allow_stdin=True)
        reply = self.kc.get_shell_msg()
        status = reply['content']['status']

        outs = list()
        while True:
            try:
                msg = self.kc.get_iopub_msg(timeout=1)
                if msg['msg_type'] == 'status':
                    if msg['content']['execution_state'] == 'idle':
                        break
            except Empty:
                print("empty ?!")
                raise

            content = msg['content']
            msg_type = msg['msg_type']

            notebook3_format_conversions = {
                'error': 'pyerr',
                'execute_result': 'pyout'
            }
            msg_type = notebook3_format_conversions.get(msg_type, msg_type)

            out = NotebookNode(output_type=msg_type)

            if msg_type == 'pyout':
                print(content['data']['text/plain'])
                continue
            if msg_type in ('status', 'pyin', 'execute_input'):
                continue
            elif msg_type in ('comm_open', 'comm_msg', 'comm_close'):
                # TODO handle this msg ?!?!?!
                continue
            elif msg_type == 'stream':
                out.stream = content['name']
                if 'text' in content:
                    out.text = content['text']
                else:
                    out.text = content['data']
            elif msg_type in ('display_data', 'pyout'):
                for mime, data in content['data'].items():
                    try:
                        attr = self.MIME_MAP[mime]
                    except KeyError:
                        print("unhandled mime")
                        raise NotImplementedError('unhandled mime type: %s' %
                                                  mime)

                    setattr(out, attr, data)
            elif msg_type == 'pyerr':
                out.ename = content['ename']
                out.evalue = content['evalue']
                out.traceback = content['traceback']
            elif msg_type == 'clear_output':
                outs = list()
                continue
            else:
                print("unhandled " + msg_type)
                raise NotImplementedError('unhandled iopub message: %s' %
                                          msg_type)
            outs.append(out)
            # NOTE: Ver 4 format still have 'pyout', Why?
            # upgrade_outputs(outs)

            print(str(outs))
            print("status: {}".format(status))

    @cell_magic
    def groovy(self, line, cell):
        return self.run_cell(line, cell)
Esempio n. 47
0
class Kernel:
    name: str
    init_code: str = ""
    language: str = ""
    manager: Optional[KernelManager] = field(default=None, init=False)
    client: Optional[KernelClient] = field(default=None, init=False)
    report: Dict[str, Any] = field(default_factory=dict, init=False)

    def __post_init__(self):
        self.report["total"] = datetime.timedelta(0)
        kernel_spec = get_kernel_spec(self.name)
        self.language = kernel_spec.language
        if self.language == "python":
            self.init_code = (
                "from pheasant.renderers.jupyter.ipython import register_formatters\n"
                "register_formatters()")

        def shutdown():  # pragma: no cover
            if self.manager:
                print(f"Shutting down kernel [{self.name}]...", end="")
                self.manager.shutdown_kernel()
                print("Done.")

        atexit.register(shutdown)

    def start(self, timeout=10, silent=True) -> KernelClient:
        if self.manager:
            if self.manager.is_alive():
                return self.client
            else:  # pragma: no cover
                raise RuntimeError(f"Kernel {self.name} is not alive.")

        def start():
            self.manager = KernelManager(kernel_name=self.name)
            self.manager.start_kernel()
            self.client = self.manager.blocking_client()
            self.client.start_channels()
            try:
                self.client.wait_for_ready(timeout=timeout)
            except TimeoutError:  # pragma: no cover
                self.manager.shutdown_kernel()
                return False
            else:
                self.client.execute_interactive(self.init_code)
                return self.client

        init = f"Starting kernel [{self.name}]"
        progress_bar = progress_bar_factory(total=3, init=init)
        now = datetime.datetime.now()

        def message(result):
            dt = format_timedelta_human(datetime.datetime.now() - now)
            return f"Kernel [{self.name}] started ({dt})" if result else "Retrying..."

        for k in range(progress_bar.total):
            if silent and start():
                break
            elif silent:
                continue
            elif progress_bar.progress(start, message):
                progress_bar.finish()
                break
        else:
            raise TimeoutError  # pragma: no cover
        return self.client

    def shutdown(self) -> None:
        if self.manager:
            self.manager.shutdown_kernel()
            del self.client
            self.client = None
            del self.manager
            self.manager = None

    def restart(self) -> None:
        if self.manager:
            self.manager.restart_kernel()
            if self.client and self.init_code:
                self.client.execute_interactive(self.init_code)

    def execute(self, code: str, output_hook=None) -> List:
        client = self.client or self.start()
        outputs = []

        def _output_hook(msg):
            if output_hook:
                output_hook(msg)
            output = output_from_msg(msg)
            if output:
                outputs.append(output)

        msg = client.execute_interactive(code, output_hook=_output_hook)
        update_report(self.report, msg)
        return list(stream_joiner(outputs))

    def inspect(self,
                code: str,
                func: str = "getsource",
                output_hook=None) -> List:
        self.execute("import inspect")
        self.execute(code, output_hook=output_hook)
        outputs = self.execute(code_for_inspect(func))
        if len(outputs) == 1 and outputs[0]["type"] == "execute_result":
            source = ast.literal_eval(outputs[0]["data"]["text/plain"])
            return [dict(type="stream", name="source", text=source)]
        else:
            return outputs
Esempio n. 48
0
class IPyKernel(object):
    """
    A simple wrapper class to run cells in an IPython Notebook.

    Notes
    -----
    - Use `with` construct to properly instantiate
    - IPython 3.0.0+ is assumed for this version

    """

    def __init__(self, nb_version=4, extra_arguments=None):
        # default timeout time is 60 seconds
        self.default_timeout = 60

        if extra_arguments is None:
            extra_arguments = []
        self.extra_arguments = extra_arguments
        self.nb_version = nb_version

    def __enter__(self):
        self.km = KernelManager()
        self.km.start_kernel(
            extra_arguments=self.extra_arguments,
            stderr=open(os.devnull, 'w')
        )

        self.kc = self.km.client()
        self.kc.start_channels()

        self.iopub = self.kc.iopub_channel
        self.shell = self.kc.shell_channel

        # run %pylab inline, because some notebooks assume this
        # even though they shouldn't

        self.shell.send("pass")
        self.shell.get_msg()
        while True:
            try:
                self.iopub.get_msg(timeout=0.05)
            except Exception as e:
                if repr(e) == 'Empty()':
                    break

                # we got a real error so raise it
                raise

        self.cmd_list = []
        self.msg_list = {}

        return self

    def clear(self):
        self.iopub.get_msgs()

    def execute(self, cmd):
        uid = self.kc.execute(cmd)
        self.cmd_list.append((uid, cmd))
        return uid

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.kc.stop_channels()
        self.km.shutdown_kernel()
        del self.msg_list
        del self.cmd_list
        del self.km

    def listen(self, uid, use_timeout=None):
        if use_timeout is None:
            use_timeout = self.default_timeout

        while True:
            if uid in self.msg_list and len(self.msg_list[uid]) > 0:
                return self.msg_list[uid].pop(0)

            msg = self.iopub.get_msg(timeout=use_timeout)
            if 'msg_id' in msg['parent_header']:
                msg_uid = msg['parent_header']['msg_id']

                if msg_uid not in self.msg_list:
                    self.msg_list[msg_uid] = []

                self.msg_list[msg_uid].append(msg)

    def run(self, cell, use_timeout=None):
        """
        Run a notebook cell in the IPythonKernel

        Parameters
        ----------
        cell : IPython.notebook.Cell
            the cell to be run
        use_timeout : int or None (default)
            the time in seconds after which a cell is stopped and assumed to
            have timed out. If set to None the value in `default_timeout`
            is used

        Returns
        -------
        list of ex_cell_outputs
            a list of NotebookNodes of the returned types. This is
            similar to the list of outputs generated when a cell is run
        """

        if timeout is not None:
            use_timeout = use_timeout
        else:
            use_timeout = self.default_timeout

        if hasattr(cell, 'source'):
            uid = self.execute(cell.source)
        else:
            raise AttributeError('No source/input key')

        outs = []
        stdout_cells = {}

        while True:
            msg = self.listen(uid, use_timeout)

            msg_type = msg['msg_type']

            if msg_type == 'execute_input':
                continue
            elif msg_type == 'clear_output':
                outs = []
                continue
            elif msg_type == 'status':
                if msg['content']['execution_state'] == 'idle':
                    # we are done with the cell, let's compare
                    break

                continue

            out_cell = nbformat.NotebookNode(output_type=msg_type)

            content = msg['content']

            if msg_type == 'stream':
                name = content['name']
                if name not in stdout_cells:
                    out_cell.name = name
                    out_cell.text = content['text']
                    stdout_cells[name] = out_cell
                    outs.append(out_cell)
                else:
                    # we already have a stdout cell, so append to it
                    stdout_cells[name].text += content['text']

            elif msg_type in ('display_data', 'execute_result'):
                if hasattr(content, 'execution_count'):
                    out_cell['execution_count'] = content['execution_count']
                else:
                    out_cell['execution_count'] = None

                out_cell['data'] = content['data']
                out_cell['metadata'] = content['metadata']

                outs.append(out_cell)

            elif msg_type == 'error':
                out_cell.ename = content['ename']
                out_cell.evalue = content['evalue']
                out_cell.traceback = content['traceback']

                outs.append(out_cell)

            elif msg_type.startswith('comm_'):
                # messages used to initialize, close and unpdate widgets
                # we will ignore these and hope for the best
                pass

            else:
                tv.warning("Unhandled iopub msg of type `%s`" % msg_type)

        return outs

    def get_commands(self, cell):
        """
        Extract potential commands from the first line of a cell

        if a code cell starts with the hashbang `#!` it can be followed by
        a comma separated list of commands. Each command can be
        1. a single key `skip`, or
        2. a key/value pair separated by a colon `timeout:[int]`

        Parameters
        ----------
        cell : a NotebookCell
            the cell to be examined

        Returns
        -------
        dict
            a dict of key/value pairs. For a single command the value is `True`
        """
        commands = {}
        source = cell.source
        if source is not None:
            lines = source.splitlines()
            if len(lines) > 0:
                n_line = 0
                line = lines[n_line].strip()
                while line.startswith('#!') or len(line) == 0:
                    txt = line[2:].strip()

                    parts = txt.split(',')
                    for part in parts:
                        subparts = part.split(':')
                        if len(subparts) == 1:
                            commands[subparts[0].strip().lower()] = True
                        elif len(subparts) == 2:
                            commands[subparts[0].strip().lower()] = subparts[1]

                    n_line += 1
                    line = lines[n_line]

        return commands

    def is_empty_cell(self, cell):
        """
        Check if a cell has no code

        Parameters
        ----------
        cell : a NotebookCell
            the cell to be examined

        Returns
        -------
        bool
            True if the cell has no code, False otherwise
        """
        return not bool(cell.source)
Esempio n. 49
0
    def parse_command_line(self, argv):
        silent = "--silent" in argv
        silent_spark_version = None
        if silent:
            argv.remove("--silent")
            arg_str = None
            for i, arg in enumerate(argv):
                if arg.startswith("--spark"):
                    arg_str = arg
                    silent_spark_version = arg_str[len("--spark") + 1:]
                    if silent_spark_version not in self.spark_download_versions:
                        print("Invalid Spark version {}".format(
                            silent_spark_version))
                        self.exit(1)
                    break
            if arg_str:
                argv.remove(arg_str)
        super(InstallKernelSpec, self).parse_command_line(argv)

        self.pixiedust_home = os.environ.get(
            "PIXIEDUST_HOME", "{}{}pixiedust".format(os.path.expanduser('~'),
                                                     os.sep))
        if silent:
            answer = 'y'
        else:
            answer = self.confirm("Step 1: PIXIEDUST_HOME: {0}".format(
                self.pixiedust_home))

        if answer != 'y':
            self.pixiedust_home = input(
                self.hilite("Please enter a PIXIEDUST_HOME location: "))
            if not os.path.exists(self.pixiedust_home):
                create = self.confirm(
                    "Directory {0} does not exist".format(self.pixiedust_home),
                    "Create")
                if create != 'y':
                    self.exit(1)
                else:
                    os.makedirs(self.pixiedust_home)

        self.pixiedust_bin = os.path.join(self.pixiedust_home, "bin")
        download_spark = False
        if silent:
            self.spark_home = os.environ.get("SPARK_HOME", None)
            if not self.spark_home:
                download_spark = True
                self.spark_home = "{}{}spark".format(os.path.expanduser('~'),
                                                     os.sep)
        else:
            first_prompt = True
            while True:
                self.spark_home = os.environ.get(
                    "SPARK_HOME",
                    self.ensureDir(os.path.join(self.pixiedust_bin, "spark")))
                if self.spark_home:
                    answer = self.confirm("Step 2: SPARK_HOME: {0}".format(
                        self.spark_home))
                    if answer != 'y':
                        self.spark_home = input(
                            self.hilite(
                                "Step 2: Please enter a SPARK_HOME location: ")
                        )
                else:
                    if first_prompt:
                        first_prompt = False
                        prompt = "Step 2: Please enter a SPARK_HOME location: "
                    else:
                        prompt = "Please enter a SPARK_HOME location: "
                    self.spark_home = input(self.hilite(prompt))
                while self.spark_home.rfind(
                        os.sep) == len(self.spark_home) - 1:
                    self.spark_home = self.spark_home[0:len(self.spark_home) -
                                                      1]
                if not os.path.exists(self.spark_home):
                    create = self.confirm(
                        "Directory {0} does not exist".format(self.spark_home),
                        "Create")
                    if create != 'y':
                        continue
                    else:
                        os.makedirs(self.spark_home)
                        download_spark = True
                        break
                elif not os.path.exists('{}{}bin{}pyspark'.format(
                        self.spark_home, os.sep, os.sep)):
                    existingInstalls = [
                        d for d in os.listdir(self.spark_home)
                        if os.path.exists(
                            os.path.join(self.spark_home, d, 'bin', 'pyspark'))
                    ]
                    if len(existingInstalls) == 0:
                        download = self.confirm(
                            "Directory {0} does not contain a valid SPARK install"
                            .format(self.spark_home), "Download Spark")
                        if download == 'y':
                            download_spark = True
                            break
                    else:
                        existingInstalls.append("Create a new spark Install")
                        print(
                            "Select an existing spark install or create a new one"
                        )
                        message = "\n".join([
                            "{}. {}".format(k + 1, v)
                            for k, v in enumerate(existingInstalls)
                        ])
                        while True:
                            try:
                                answer = int(
                                    input(
                                        self.hilite(message) +
                                        "\n\tEnter your selection: ")) - 1
                                if answer < 0 or answer >= len(
                                        existingInstalls):
                                    raise Exception(
                                        "Please pick a number within the specified values"
                                    )
                                break
                            except Exception as e:
                                print("Invalid selection: {}".format(e))

                        download_spark = (answer == len(existingInstalls) - 1)
                        if not download_spark:
                            self.spark_home = os.path.join(
                                self.spark_home, existingInstalls[answer])
                        break
                else:
                    break

        if download_spark:
            self.download_spark(silent, silent_spark_version)

        scala_version = None
        spark_version = self.get_spark_version()
        if spark_version is None:
            print("Unable to obtain Spark version")
            self.exit(1)
        elif spark_version[0] == 1:
            scala_version = '2.10'  # Spark 1.x = Scala 2.10
        elif spark_version[0] == 2:
            scala_version = '2.11'  # Spark 2.x = Scala 2.11
        else:
            print("Invalid Spark version {}".format(spark_version))
            self.exit(1)

        #download spark-cloudant
        sparkCloudantUrl = "https://github.com/cloudant-labs/spark-cloudant/releases/download/v1.6.4/cloudant-spark-v1.6.4-167.jar"
        if spark_version[0] == 2:
            sparkCloudantUrl = "https://github.com/cloudant-labs/spark-cloudant/releases/download/v2.0.0/cloudant-spark-v2.0.0-185.jar"

        try:
            self.sparkCloudantPath = self.downloadFileToDir(
                sparkCloudantUrl, targetDir=self.pixiedust_bin)
            print("downloaded spark cloudant jar: {0}".format(
                self.sparkCloudantPath))
        except Exception as e:
            print(
                "Error downloading Cloudant Jar: {}. Install will continue without the spark Cloudant connector"
                .format(e))

        #download spark csv connector if in spark 1.6
        self.sparkCSVPath = None
        self.commonsCSVPath = None
        if spark_version[0] == 1:
            try:
                self.sparkCSVPath = self.downloadPackage(
                    "com.databricks:spark-csv_2.10:1.5.0")
                self.commonsCSVPath = self.downloadPackage(
                    "org.apache.commons:commons-csv:0")
            except Exception as e:
                print(
                    "Error downloading csv connector Jar: {}. Install will continue without it"
                    .format(e))

        download_scala = False
        if silent:
            self.scala_home = os.environ.get("SCALA_HOME", None)
            if not self.scala_home:
                download_scala = True
                self.scala_home = "{}{}scala".format(os.path.expanduser('~'),
                                                     os.sep)
        else:
            first_prompt = True
            while True:
                self.scala_home = os.environ.get(
                    "SCALA_HOME",
                    self.ensureDir(os.path.join(self.pixiedust_bin, "scala")))
                if self.scala_home:
                    answer = self.confirm("Step 3: SCALA_HOME: {0}".format(
                        self.scala_home))
                    if answer != 'y':
                        self.scala_home = input(
                            self.hilite(
                                "Step 3: Please enter a SCALA_HOME location: ")
                        )
                else:
                    if first_prompt:
                        first_prompt = False
                        prompt = "Step 3: Please enter a SCALA_HOME location: "
                    else:
                        prompt = "Please enter a SCALA_HOME location: "
                    self.scala_home = input(self.hilite(prompt))
                while self.scala_home.rfind(
                        os.sep) == len(self.scala_home) - 1:
                    self.scala_home = self.scala_home[0:len(self.scala_home) -
                                                      1]
                if not os.path.exists(self.scala_home):
                    create = self.confirm(
                        "Directory {0} does not exist".format(self.scala_home),
                        "Create")
                    if create != 'y':
                        continue
                    else:
                        os.makedirs(self.scala_home)
                        download_scala = True
                        break
                elif not os.path.exists('{}{}bin{}scala'.format(
                        self.scala_home, os.sep, os.sep)):

                    def acceptScalaVersion(dir):
                        version = self.get_scala_version_from_dir(
                            os.path.join(self.scala_home, dir))
                        return version is not None and str(
                            version[0]) + '.' + str(
                                version[1]) == scala_version

                    existingInstalls = [
                        d for d in os.listdir(self.scala_home)
                        if acceptScalaVersion(d)
                    ]
                    if len(existingInstalls) == 0:
                        download = self.confirm(
                            "Directory {0} does not contain a valid scala install"
                            .format(self.scala_home), "Download Scala")
                        if download == 'y':
                            download_scala = True
                            break
                    else:
                        existingInstalls.append("Create a new scala Install")
                        print(
                            "Select an existing scala install or create a new one"
                        )
                        message = "\n".join([
                            "{}. {}".format(k + 1, v)
                            for k, v in enumerate(existingInstalls)
                        ])
                        while True:
                            try:
                                answer = int(
                                    input(
                                        self.hilite(message) +
                                        "\n\tEnter your selection: ")) - 1
                                if answer < 0 or answer >= len(
                                        existingInstalls):
                                    raise Exception(
                                        "Please pick a number within the specified values"
                                    )
                                break
                            except Exception as e:
                                print("Invalid selection: {}".format(e))

                        download_scala = (answer == len(existingInstalls) - 1)
                        if not download_scala:
                            self.scala_home = os.path.join(
                                self.scala_home, existingInstalls[answer])
                        break
                else:
                    installed_scala_version = self.get_scala_version()
                    if not installed_scala_version or (
                            str(installed_scala_version[0]) + '.' +
                            str(installed_scala_version[1])) != scala_version:
                        print(
                            "A different version of Scala {0} is already installed in this directory."
                            .format(installed_scala_version))
                        continue
                    else:
                        break

        if download_scala:
            self.download_scala(scala_version)

        self.kernelName = "Python with Pixiedust (Spark {}.{})".format(
            spark_version[0], spark_version[1])
        if silent:
            answer = 'y'
        else:
            answer = self.confirm("Step 4: Kernel Name: {0}".format(
                self.kernelName))
        if answer != 'y':
            self.kernelName = input(
                self.hilite("Please enter a Kernel Name: "))

        try:
            km = KernelManager(kernel_name=self.kernelName)
            km.kernel_spec
            answer = self.confirm(
                "Kernel '{0}' already exists".format(self.kernelName),
                "Override")
            if answer != 'y':
                self.exit(1)
        except NoSuchKernel:
            pass

        dest = self.createKernelSpec()

        self.pixiedust_notebooks_dir = os.path.join(self.pixiedust_home,
                                                    "notebooks")
        if not os.path.isdir(self.pixiedust_notebooks_dir):
            os.makedirs(self.pixiedust_notebooks_dir)
        print("Downloading intro notebooks into {}".format(
            self.pixiedust_notebooks_dir))
        self.downloadIntroNotebooks()

        print("\n\n{}".format("#" * 100))
        print("#\tCongratulations: Kernel {0} was successfully created in {1}".
              format(self.kernelName, dest))
        print(
            "#\tYou can start the Notebook server with the following command:")
        print("#\t\t{}".format(
            self.hilite("jupyter notebook {}".format(
                self.pixiedust_notebooks_dir))))
        print("{}".format("#" * 100))
Esempio n. 50
0
class NotebookRunner(object):
    # The kernel communicates with mime-types while the notebook
    # uses short labels for different cell types. We'll use this to
    # map from kernel types to notebook format types.

    MIME_MAP = {
        'image/jpeg': 'jpeg',
        'image/png': 'png',
        'text/plain': 'text',
        'text/html': 'html',
        'text/latex': 'latex',
        'application/javascript': 'html',
        'image/svg+xml': 'svg',
    }

    def __init__(self, nb, pylab=False, mpl_inline=False, working_dir=None):
        self.km = KernelManager()

        args = []

        if pylab:
            args.append('--pylab=inline')
            logging.warn('--pylab is deprecated and will be removed in a '
                         'future version')
        elif mpl_inline:
            args.append('--matplotlib=inline')
            logging.warn('--matplotlib is deprecated and will be removed in a '
                         'future version')

        cwd = os.getcwd()

        if working_dir:
            os.chdir(working_dir)

        self.km.start_kernel(extra_arguments=args)

        os.chdir(cwd)

        if platform.system() == 'Darwin':
            # There is sometimes a race condition where the first
            # execute command hits the kernel before it's ready.
            # It appears to happen only on Darwin (Mac OS) and an
            # easy (but clumsy) way to mitigate it is to sleep
            # for a second.
            sleep(1)

        self.kc = self.km.client()
        self.kc.start_channels()
        try:
            self.kc.wait_for_ready()
        except AttributeError:
            # IPython < 3
            self._wait_for_ready_backport()

        self.nb = nb

    def __del__(self):
        self.kc.stop_channels()
        self.km.shutdown_kernel(now=True)

    def run_cell(self, cell, cidx):
        '''
        Run a notebook cell and update the output of that cell in-place.
        '''
        logging.debug('running cell {}'.format(cidx))
        # logging.debug(u'cell.input {}'.format(cell.input))
        self.kc.execute(cell.source)
        reply = self.kc.get_shell_msg()
        status = reply['content']['status']
        max_mem = system_memory_used()
        logging.info('  memory used: {}'.format(sizeof_fmt(max_mem)))
        if status == 'error':
            traceback_text = 'Cell raised uncaught exception: \n' + \
                '\n'.join(reply['content']['traceback'])
            traceback_text = remove_ansicolor(traceback_text)
            if 'NoDataFound' not in traceback_text:
                logging.error(traceback_text)
        else:
            logging.debug('run_cell ok')

        outs = list()
        while True:
            try:
                msg = self.kc.get_iopub_msg(timeout=1)
                if msg['msg_type'] == 'status':
                    if msg['content']['execution_state'] == 'idle':
                        break
            except Empty:
                # execution state should return to idle before the queue
                # becomes empty,
                # if it doesn't, something bad has happened
                logging.error("empty exception")
                raise

            content = msg['content']
            msg_type = msg['msg_type']

            # IPython 3.0.0-dev writes pyerr/pyout in the notebook format but
            # uses error/execute_result in the message spec. This does the
            # translation needed for tests to pass with IPython 3.0.0-dev
            notebook3_format_conversions = {
                'error': 'pyerr',
                'execute_result': 'pyout'
            }
            msg_type = notebook3_format_conversions.get(msg_type, msg_type)

            out = NotebookNode(output_type=msg_type)

            #if 'execution_count' in content:
                #cell['prompt_number'] = content['execution_count']
                #out.prompt_number = content['execution_count']

            if msg_type in ('status', 'pyin', 'execute_input'):
                continue
            elif msg_type == 'stream':
                out.stream = content['name']
                if 'text' in content:
                    out.text = content['text']
                else:
                    out.text = content['data']
                # print(out.text, end='')
            elif msg_type in ('display_data', 'pyout'):
                for mime, data in content['data'].items():
                    try:
                        attr = self.MIME_MAP[mime]
                    except KeyError:
                        logging.error("unhandled mime")
                        raise NotImplementedError('unhandled mime type: %s' %
                                                  mime)

                    setattr(out, attr, data)
            elif msg_type == 'pyerr':
                out.ename = content['ename']
                out.evalue = content['evalue']
                out.traceback = content['traceback']
            elif msg_type == 'clear_output':
                outs = list()
                continue
            else:
                logging.error("unhandled iopub")
                raise NotImplementedError('unhandled iopub message: %s' %
                                          msg_type)
            outs.append(out)
        # NOTE: Ver 4 format still have 'pyout', Why?
        cell['outputs'] = upgrade_outputs(outs)

        logging.debug("status: {}".format(status))
        if status == 'error':
            if 'NoDataFound' in traceback_text:
                raise NoDataFound(traceback_text.split('\n')[-1])
            else:
                logging.debug(u"NotebookError raised")
                raise NotebookError(traceback_text)

    def iter_code_cells(self):
        '''
        Iterate over the notebook cells containing code.
        '''
        for cell in self.nb['cells']:
            if cell.cell_type == 'code':
                yield cell

    def iter_cells(self):
        '''
        Iterate over the notebook cells.
        '''
        for cell in self.nb['cells']:
            yield cell

    def clear_outputs(self):
        for cell in self.iter_cells():
            if 'outputs' in cell:
                cell['outputs'] = []

    @property
    def cellcnt(self):
        return len(self.nb['cells'])

    def run_notebook(self, memory_used=None, progress_cb=None,
                     skip_exceptions=False):
        '''
        Run all the cells of a notebook in order and update
        the outputs in-place.

        If ``skip_exceptions`` is set, then if exceptions occur in a cell, the
        subsequent cells are run (by default, the notebook execution stops).
        '''
        cur = 0
        if memory_used is not None:
            memory_used.append(system_memory_used())
        for cell in self.iter_code_cells():
            cur += 1
            try:
                if progress_cb is not None:
                    progress_cb(cur)
                self.run_cell(cell, cur)
                if memory_used is not None:
                    memory_used.append(system_memory_used())
            except NotebookError:
                if not skip_exceptions:
                    raise
Esempio n. 51
0
class KernelMagics(Magics):
    _execution_count = 1

    def stop_kernel(self):
        self.kc.stop_channels()
        self.km.shutdown_kernel(now=True)

    def __init__(self, shell):
        super(KernelMagics, self).__init__(shell)
        self.km = None
        self.kc = None
        self.comms = []

    def start(self, kernel_name):
        self.km = KernelManager()
        self.km.kernel_name = kernel_name
        self.km.start_kernel(extra_arguments=[self._context_base64()])
        atexit.register(self.stop_kernel)
        self.kc = self.km.client()
        self.kc.start_channels()
        try:
            self.kc.wait_for_ready()
            print("{} started successfully\n".format(kernel_name.capitalize()))
        except AttributeError:
            self._wait_for_ready_backport()

    def run_cell(self, line, code):
        if not self.km:
            self.start()
        self.kc.execute(code, allow_stdin=True)
        reply = self.kc.get_shell_msg()
        self._handle_iopub_messages()

    def _handle_iopub_messages(self):
        while True:
            try:
                msg = self.kc.get_iopub_msg(timeout=1)
            except Empty:
                break
            comm_id = msg['content'].get('comm_id')
            if comm_id and comm_id not in self.comms:
                self.comms.append(comm_id)
            self.shell.kernel.session.send(
                self.shell.kernel.iopub_socket,
                msg['msg_type'],
                msg['content'],
                metadata=msg['metadata'],
                parent=self.shell.kernel._parent_header,
                ident=msg.get('comm_id'),
                buffers=msg['buffers'],
            )

    def pass_message(self, msg_raw):
        comm_id = msg_raw['content'].get('comm_id')
        if comm_id in self.comms:
            content = msg_raw['content']
            msg = self.kc.session.msg(msg_raw['msg_type'], content)
            self.kc.shell_channel.send(msg)
            self._handle_iopub_messages()
        else:
            self.log.warn("No such comm: %s", comm_id)
            if self.log.isEnabledFor(logging.DEBUG):
                # don't create the list of keys if debug messages aren't enabled
                self.log.debug("Current comms: %s", list(self.comms.keys()))

    def _context_base64(self):
        context_json = json.dumps({
            'port':
            os.environ["BEAKERX_AUTOTRANSLATION_PORT"],
            'contextId':
            get_ipython().kernel.session.session,
        })
        return base64.b64encode(context_json.encode('utf-8')).decode()