def run_notebook(nb): km = KernelManager() km.start_kernel(stderr=open(os.devnull, 'w')) kc = km.client() kc.start_channels() try: kc.wait_for_ready() except AttributeError: # IPython < 3 kc.kernel_info() while True: msg = kc.get_shell_msg(block=True, timeout=30) if msg['msg_type'] == 'kernel_info_reply': break # Flush IOPub channel while True: try: msg = kc.get_iopub_msg(block=True, timeout=0.2) except Empty: break # simple ping: kc.execute("pass") kc.get_shell_msg() cells = 0 failures = 0 if hasattr(nb, 'worksheets'): # nobody uses more than 1 worksheet ws = nb.worksheets[0] else: # no more worksheet level in new format ws = nb for cell in ws.cells: if cell.cell_type != 'code': continue outputs, failed = run_cell(kc, cell) cell.outputs = outputs cell['prompt_number'] = cells failures += failed cells += 1 sys.stdout.write('.') sys.stdout.flush() print() print("ran %3i cells" % cells) if failures: print(" %3i cells raised exceptions" % failures) kc.stop_channels() km.shutdown_kernel() del km
def test_connection_file_real_path(): """ Verify realpath is used when formatting connection file """ with mock.patch('os.path.realpath') as patched_realpath: patched_realpath.return_value = 'foobar' km = KernelManager(connection_file=os.path.join( tempfile.gettempdir(), "kernel-test.json"), kernel_name='test_kernel') # KernelSpec and launch args have to be mocked as we don't have an actual kernel on disk km._kernel_spec = KernelSpec(resource_dir='test', **{ "argv": [ "python.exe", "-m", "test_kernel", "-f", "{connection_file}" ], "env": {}, "display_name": "test_kernel", "language": "python", "metadata": {} }) km._launch_args = {} cmds = km.format_kernel_cmd() assert cmds[4] is 'foobar'
def start(self): self.km = KernelManager() self.km.kernel_name = self.kernel_name self.km.start_kernel(extra_arguments=[self.context]) self.kc = self.km.client() self.kc.start_channels() self.kc.wait_for_ready()
def start(self): self.km = KernelManager() self.km.kernel_name = self.kernel_name self.km.start_kernel() self.kc = self.km.client() self.kc.start_channels() self.kc.wait_for_ready()
def __enter__(self): self.km = KernelManager() self.km.start_kernel(extra_arguments=self.extra_arguments, stderr=open(os.devnull, 'w')) self.kc = self.km.client() self.kc.start_channels() self.iopub = self.kc.iopub_channel self.shell = self.kc.shell_channel # run %pylab inline, because some notebooks assume this # even though they shouldn't self.shell.send("pass") self.shell.get_msg() while True: try: self.iopub.get_msg(timeout=0.05) except Exception as e: if repr(e) == 'Empty()': break # we got a real error so raise it raise self.cmd_list = [] self.msg_list = {} return self
def start_kernel( cls, kernelspec_name=None, connection_info=None, connection_name=None, ): """Start kernel and return a `Kernel` instance.""" kernel_id = uuid.uuid4() if kernelspec_name: kernel_manager = KernelManager(kernel_name=kernelspec_name) kernel_manager.start_kernel() elif connection_info: kernel_manager = KernelManager() kernel_manager.load_connection_info(connection_info) # `KernelManager.kernel_name` is not automatically set from connection info. kernel_manager.kernel_name = connection_info.get("kernel_name", "") else: raise Exception( "You must specify any of {`kernelspec_name`, `connection_info`}." ) kernel = KernelConnection(kernel_id, kernel_manager, cls, connection_name=connection_name, logger=cls.logger) cls.kernels[kernel_id] = kernel return kernel
def __init__(self, shell): super(GroovyMagics, self).__init__(shell) self.km = KernelManager() self.km.kernel_name = 'groovy' self.km.start_kernel() self.kc = self.km.client() self.kc.start_channels() try: self.kc.wait_for_ready() print("Groovy started successfully\n") except AttributeError: self._wait_for_ready_backport()
def start(self): self.km = KernelManager() self.km.kernel_name = 'groovy' self.km.start_kernel() atexit.register(self.stop_kernel) self.kc = self.km.client() self.kc.start_channels() try: self.kc.wait_for_ready() print("Groovy started successfully\n") except AttributeError: self._wait_for_ready_backport()
def start(self, kernel_name): self.km = KernelManager() self.km.kernel_name = kernel_name self.km.start_kernel(extra_arguments=[self._context_base64()]) atexit.register(self.stop_kernel) self.kc = self.km.client() self.kc.start_channels() try: self.kc.wait_for_ready() print("{} started successfully\n".format(kernel_name.capitalize())) except AttributeError: self._wait_for_ready_backport()
class GroovyMagics(Magics): _execution_count = 1 def stop_kernel(self): self.kc.stop_channels() self.km.shutdown_kernel(now=True) def __init__(self, shell): super(GroovyMagics, self).__init__(shell) self.km = None def start(self): self.km = KernelManager() self.km.kernel_name = 'groovy' self.km.start_kernel() atexit.register(self.stop_kernel) self.kc = self.km.client() self.kc.start_channels() try: self.kc.wait_for_ready() print("Groovy started successfully\n") except AttributeError: self._wait_for_ready_backport() def run_cell(self, line, code): if not self.km: self.start() self.kc.execute(code, allow_stdin=True) reply = self.kc.get_shell_msg() while True: try: msg = self.kc.get_iopub_msg(timeout=1) if msg['msg_type'] == 'status': if msg['content']['execution_state'] == 'idle': break except Empty: print("empty ?!") raise self.shell.kernel.session.send( self.shell.kernel.iopub_socket, msg['msg_type'], msg['content'], metadata=msg['metadata'], parent=self.shell.kernel._parent_header, ident=msg.get('comm_id'), buffers=msg['buffers'], ) @cell_magic def groovy(self, line, cell): return self.run_cell(line, cell)
def start(): self.manager = KernelManager(kernel_name=self.name) self.manager.start_kernel() self.client = self.manager.blocking_client() self.client.start_channels() try: self.client.wait_for_ready(timeout=timeout) except TimeoutError: # pragma: no cover self.manager.shutdown_kernel() return False else: self.client.execute_interactive(self.init_code) return self.client
def get_client(cf, profile=None): """ Usage: >>> kc = get_client('kernel-143a2687-f294-42b1-bdcb-6f1cc2f4cc87.json', 'dale') >>> data = kc.execute("'123'") >>> data {u'text/plain': u'123'} """ connection_file = find_connection_file(cf, profile=profile) km = KernelManager(connection_file=connection_file) km.load_connection_file() client = km.client() return KernelClient(client)
def start_ipython_kernel(self): kernel_name = self._get_wrapped_kernel_name() self.km = KernelManager( kernel_name=kernel_name, client_class='jupyter_client.blocking.BlockingKernelClient') self.log.debug('kernel_manager: %s', str(self.km)) self.log.info('start wrapped kernel: %s', kernel_name) self.km.start_kernel() self.kc = self.km.client() self.log.debug('kernel_client: %s', str(self.kc)) self.log.debug('start_channels') self.kc.start_channels() try: self.log.debug('wait for ready of wrapped kernel') self.kc.wait_for_ready(timeout=None) except RuntimeError: self.kc.stop_channels() self.km.shutdown_kernel() raise for channel in self.proxy_channles: stream = getattr(self, channel + '_socket') thread = ChannelReaderThread(self, self.kc, stream, self.session, channel) thread.start() self.threads[channel] = thread for log_dir in self.log_dirs: if self._is_writable_dir(log_dir): self.log_path = log_dir break self.log.debug('log output directory: %s', self.log_path) if self._find_default_keyword_pattern_file() is None: self.log.info('default keyword pattern file "%s" not found', IPYTHON_DEFAULT_PATTERN_FILE) try: self._generate_default_keyword_pattern_file() except Exception as e: self.log.exception( "failed to generate default keyword pattern file: %s", e) self.exec_info = None self.notebook_path = self.get_notebook_path() self.log.debug('notebook_path: %s', self.notebook_path)
def new_ipy(s=''): """Create a new IPython kernel (optionally with extra arguments) XXX: Allow passing of profile information here Examples -------- new_ipy() """ from jupyter_client.manager import KernelManager km = KernelManager() km.start_kernel() return km_from_string(km.connection_file)
def __enter__(self): self.km = KernelManager() self.km.start_kernel( extra_arguments=self.extra_arguments, stderr=open(os.devnull, 'w') ) self.kc = self.km.client() self.kc.start_channels() self.iopub = self.kc.iopub_channel self.shell = self.kc.shell_channel # run %pylab inline, because some notebooks assume this # even though they shouldn't self.shell.send("pass") self.shell.get_msg() while True: try: self.iopub.get_msg(timeout=0.05) except Empty: break return self
def __enter__(self): self.km = KernelManager() self.km.start_kernel( extra_arguments=self.extra_arguments, stderr=open(os.devnull, 'w') ) self.kc = self.km.client() self.kc.start_channels() self.iopub = self.kc.iopub_channel self.shell = self.kc.shell_channel # run %pylab inline, because some notebooks assume this # even though they shouldn't self.shell.send("pass") self.shell.get_msg() while True: try: self.iopub.get_msg(timeout=0.05) except Exception as e: if repr(e) == 'Empty()': break # we got a real error so raise it raise self.cmd_list = [] self.msg_list = {} return self
def _make_kernel_manager(self, kernel_name, group_id): """Creates a new kernel manager for given kernel and group id if none exists, and returns it. Args: kernel_name (str): the kernel group_id (str): item group that will execute using this kernel Returns: KernelManager """ if group_id is None: # Execute in isolation return KernelManager(kernel_name=kernel_name) key = (kernel_name, group_id) if key not in self._kernel_managers: self._kernel_managers[key] = KernelManager(kernel_name=kernel_name) return self._kernel_managers[key]
def new_ipy(s=''): """Create a new IPython kernel (optionally with extra arguments) XXX: Allow passing of profile information here Examples -------- new_ipy() """ # Modified by Bo Peng. # This package has been deprecated. Need to import from ipykernel from jupyter_client.manager import KernelManager km = KernelManager() km.start_kernel() return km_from_string(km.connection_file)
def set_contents(self, contents): self.contents = json.loads(contents) self.contents['notebook']['metadata']['pixiedust'] = {} kernel_spec = self.contents['notebook']['metadata']['kernelspec'] km = KernelManager(kernel_name=kernel_spec['name']) with warnings.catch_warnings(): warnings.simplefilter("ignore") self.kernel_spec = json.dumps(km.kernel_spec.to_dict(), indent=4, sort_keys=True)
def start_ipython_kernel(self): kernel_name = self._get_wrapped_kernel_name() self.km = KernelManager( kernel_name=kernel_name, client_class='jupyter_client.blocking.BlockingKernelClient') self.log.debug('kernel_manager: %s', str(self.km)) self.log.info('start wrapped kernel: %s', kernel_name) self.km.start_kernel() self.kc = self.km.client() self.log.debug('kernel_client: %s', str(self.kc)) self.log.debug('start_channels') self.kc.start_channels() self.flush_stream_event.set() try: self.log.debug('wait for ready of wrapped kernel') self.kc.wait_for_ready(timeout=None) except RuntimeError: self.kc.stop_channels() self.km.shutdown_kernel() raise for channel in self.proxy_channles: stream = getattr(self, channel + '_socket') thread = ChannelReaderThread(self, self.kc, stream, self.session, channel) thread.start() self.threads[channel] = thread self.notebook_path = self.get_notebook_path(self.kc) self.log_path = os.path.join(self.notebook_path, u'.log') if not os.path.exists( os.path.join(self.notebook_path, IPYTHON_DEFAULT_PATTERN_FILE)): with open( os.path.join(self.notebook_path, IPYTHON_DEFAULT_PATTERN_FILE), 'w') as f: f.write(IPYTHON_DEFAULT_PATTERN) self.exec_info = None self._init_log() self.log.debug('notebook_path: %s', self.notebook_path)
def run_notebook(nb): km = KernelManager() km.start_kernel(stderr=open(os.devnull, 'w')) kc = km.client() kc.start_channels() cells = failures = 0 for cell in nb.cells: if cell.cell_type != 'code': continue kc.execute(cell.source) # Wait to finish try: reply = kc.get_shell_msg()['content'] except Empty: reply = {'status': 'error', 'traceback': ["Cell execution timed out!"]} if reply['status'] == 'error': failures += 1 print("\nFAILURE:") print(cell.source) print('-----') print("Raised:") print('\n'.join(reply['traceback'])) cells += 1 sys.stdout.write('.') kc.stop_channels() km.shutdown_kernel() return cells, failures
class JVMKernelMagic: def __init__(self, kernel_name, context): self.km = None self.kc = None self.comms = [] self.kernel_name = kernel_name self.context = context self.start() def start(self): self.km = KernelManager() self.km.kernel_name = self.kernel_name self.km.start_kernel(extra_arguments=[self.context]) self.kc = self.km.client() self.kc.start_channels() self.kc.wait_for_ready() def stop_kernel(self): self.kc.stop_channels() self.km.shutdown_kernel(now=True) def run_cell(self, code): if not self.km: self.start() self.kc.execute(code, allow_stdin=True) def get_shell_msg(self): return self.kc.get_shell_msg() def get_iopub_msg(self): try: msg = self.kc.get_iopub_msg(timeout=1) return msg except Empty: return None def pass_msg(self, msg_raw): msg_json = json.loads(msg_raw) content = msg_json['content'] msg_type = msg_json['header']['msg_type'] msg = self.kc.session.msg(msg_type, content) self.kc.shell_channel.send(msg) return None
def test_feature(kernel_name, feature): print("...{}".format(feature)) manager = KernelManager(kernel_name=kernel_name) manager.start_kernel() client = manager.client() with open(join(feature, "provided.json")) as fp: provided = json.load(fp) with open(join(feature, "expected.schema.json")) as fp: expected = json.load(fp) msg = client.session.msg(provided['header']['msg_type'], provided['content']) client.shell_channel.send(msg) response = client.shell_channel.get_msg() jsonschema.validate(instance=response, schema=expected)
def new_ipy(s=''): """Create a new IPython kernel (optionally with extra arguments) XXX: Allow passing of profile information here Examples -------- new_ipy() """ try: from jupyter_client.manager import KernelManager except ImportError: # For compatibility with IPython 3 or lower from IPython.kernel import KernelManager km = KernelManager() km.start_kernel() return km_from_string(km.connection_file)
def run_notebook(nb): """Run notebook.""" km = KernelManager() km.start_kernel(stderr=open(os.devnull, 'w')) kc = km.client() kc.start_channels() kc.execute("pass") kc.get_shell_msg() cells = 0 failures = 0 for ws in nb.worksheets: for cell in ws.cells: if cell.cell_type != 'code': continue kc.execute(cell.input) # wait for finish, maximum 20s reply = kc.get_shell_msg(timeout=20)['content'] if reply['status'] == 'error': failures += 1 print("\nFAILURE:") print(cell.input) print('-----') print("raised:") print('\n'.join(reply['traceback'])) cells += 1 sys.stdout.write('.') # print "ran notebook %s" % nb.metadata.name print(" ran %3i cells" % cells) if failures: print(" %3i cells raised exceptions" % failures) kc.stop_channels() km.shutdown_kernel() del km return failures
def createKernelSpecIfNeeded(kernelName=__TEST_KERNEL_NAME__, useSpark=False): try: km = KernelManager(kernel_name=kernelName) km.kernel_spec print("Kernal already exists!") return None except NoSuchKernel: print("Creating new Kernel {} target: {}".format(kernelName, useSpark)) if useSpark: sparkHome = os.environ["SPARK_HOME"] overrides = { "argv": ["python", "-m", "ipykernel", "-f", "{connection_file}"], "env": { "SCALA_HOME": "{0}".format(os.environ["SCALA_HOME"]), "SPARK_HOME": "{0}".format(sparkHome), "PYTHONPATH": "{0}/python/:{0}/python/lib/py4j-0.9-src.zip".format( sparkHome), "PYTHONSTARTUP": "{0}/python/pyspark/shell.py".format(sparkHome), "PYSPARK_SUBMIT_ARGS": "--driver-class-path {0}/data/libs/* --master local[10] pyspark-shell" .format( os.environ.get("PIXIEDUST_HOME", os.path.expanduser('~'))), "SPARK_DRIVER_MEMORY": "10G", "SPARK_LOCAL_IP": "127.0.0.1" } } else: overrides = { "argv": ["python", "-m", "ipykernel", "-f", "{connection_file}"], "env": { "PIXIEDUST_HOME": os.environ.get("PIXIEDUST_HOME", os.path.expanduser('~')) } } path = write_kernel_spec(overrides=overrides) dest = KernelSpecManager().install_kernel_spec(path, kernel_name=kernelName, user=True) # cleanup afterward shutil.rmtree(path) return dest
def start_new_kernel(startup_timeout=60, kernel_name='python', spykernel=False, **kwargs): """Start a new kernel, and return its Manager and Client""" km = KernelManager(kernel_name=kernel_name) if spykernel: km._kernel_spec = SpyderKernelSpec() km.start_kernel(**kwargs) kc = km.client() kc.start_channels() try: kc.wait_for_ready(timeout=startup_timeout) except RuntimeError: kc.stop_channels() km.shutdown_kernel() raise return km, kc
def start_new_kernel(startup_timeout=60, kernel_name='python', **kwargs): """Start a new kernel, and return its Manager and Client""" logger.debug('Starting new kernel: "%s"' % kernel_name) km = KernelManager(kernel_name=kernel_name, kernel_spec_manager=NbvalKernelspecManager()) km.start_kernel(**kwargs) kc = km.client() kc.start_channels() try: kc.wait_for_ready(timeout=startup_timeout) except RuntimeError: logger.exception('Failure starting kernel "%s"', kernel_name) kc.stop_channels() km.shutdown_kernel() raise return km, kc
def start_new_kernel(startup_timeout=60, kernel_name='python', **kwargs): """Start a new kernel, and return its Manager and Client""" km = KernelManager(kernel_name=kernel_name, kernel_spec_manager=NbvalKernelspecManager()) km.start_kernel(**kwargs) kc = km.client() kc.start_channels() try: kc.wait_for_ready(timeout=startup_timeout) except RuntimeError: kc.stop_channels() km.shutdown_kernel() raise return km, kc
def run_notebook(notebook): """Run the notebook""" kernel_manager = KernelManager() kernel_manager.start_kernel(stderr=open(os.devnull, 'w')) kernel_client = kernel_manager.client() kernel_client.start_channels() for sheet in notebook.worksheets: for (prompt_number, cell) in enumerate(sheet.cells, 1): if cell.cell_type != "code": continue cell.outputs = run_cell(kernel_client, cell) cell.prompt_number = prompt_number if cell.outputs and cell.outputs[0]['output_type'] == 'pyout': cell.outputs[0]["prompt_number"] = prompt_number kernel_manager.shutdown_kernel()
def __init__(self, nb, pylab=False, mpl_inline=False, working_dir=None): self.km = KernelManager() args = [] if pylab: args.append('--pylab=inline') logging.warn('--pylab is deprecated and will be removed in a ' 'future version') elif mpl_inline: args.append('--matplotlib=inline') logging.warn('--matplotlib is deprecated and will be removed in a ' 'future version') cwd = os.getcwd() if working_dir: os.chdir(working_dir) self.km.start_kernel(extra_arguments=args) os.chdir(cwd) if platform.system() == 'Darwin': # There is sometimes a race condition where the first # execute command hits the kernel before it's ready. # It appears to happen only on Darwin (Mac OS) and an # easy (but clumsy) way to mitigate it is to sleep # for a second. sleep(1) self.kc = self.km.client() self.kc.start_channels() try: self.kc.wait_for_ready() except AttributeError: # IPython < 3 self._wait_for_ready_backport() self.nb = nb
class IPyKernel(object): """ A simple wrapper class to run cells in an IPython Notebook. Notes ----- - Use `with` construct to properly instantiate - IPython 3.0.0+ is assumed for this version """ def __init__(self, nb_version=4, extra_arguments=None): # default timeout time is 60 seconds self.default_timeout = 60 if extra_arguments is None: extra_arguments = [] self.extra_arguments = extra_arguments self.nb_version = nb_version def __enter__(self): self.km = KernelManager() self.km.start_kernel(extra_arguments=self.extra_arguments, stderr=open(os.devnull, 'w')) self.kc = self.km.client() self.kc.start_channels() self.iopub = self.kc.iopub_channel self.shell = self.kc.shell_channel # run %pylab inline, because some notebooks assume this # even though they shouldn't self.shell.send("pass") self.shell.get_msg() while True: try: self.iopub.get_msg(timeout=0.05) except Exception as e: if repr(e) == 'Empty()': break # we got a real error so raise it raise self.cmd_list = [] self.msg_list = {} return self def clear(self): self.iopub.get_msgs() def execute(self, cmd): uid = self.kc.execute(cmd) self.cmd_list.append((uid, cmd)) return uid def __exit__(self, exc_type, exc_val, exc_tb): self.kc.stop_channels() self.km.shutdown_kernel() del self.msg_list del self.cmd_list del self.km def listen(self, uid, use_timeout=None): if use_timeout is None: use_timeout = self.default_timeout while True: if uid in self.msg_list and len(self.msg_list[uid]) > 0: return self.msg_list[uid].pop(0) msg = self.iopub.get_msg(timeout=use_timeout) if 'msg_id' in msg['parent_header']: msg_uid = msg['parent_header']['msg_id'] if msg_uid not in self.msg_list: self.msg_list[msg_uid] = [] self.msg_list[msg_uid].append(msg) def run(self, cell, use_timeout=None): """ Run a notebook cell in the IPythonKernel Parameters ---------- cell : IPython.notebook.Cell the cell to be run use_timeout : int or None (default) the time in seconds after which a cell is stopped and assumed to have timed out. If set to None the value in `default_timeout` is used Returns ------- list of ex_cell_outputs a list of NotebookNodes of the returned types. This is similar to the list of outputs generated when a cell is run """ if timeout is not None: use_timeout = use_timeout else: use_timeout = self.default_timeout if hasattr(cell, 'source'): uid = self.execute(cell.source) else: raise AttributeError('No source/input key') outs = [] stdout_cells = {} while True: msg = self.listen(uid, use_timeout) msg_type = msg['msg_type'] if msg_type == 'execute_input': continue elif msg_type == 'clear_output': outs = [] continue elif msg_type == 'status': if msg['content']['execution_state'] == 'idle': # we are done with the cell, let's compare break continue out_cell = nbformat.NotebookNode(output_type=msg_type) content = msg['content'] if msg_type == 'stream': name = content['name'] if name not in stdout_cells: out_cell.name = name out_cell.text = content['text'] stdout_cells[name] = out_cell outs.append(out_cell) else: # we already have a stdout cell, so append to it stdout_cells[name].text += content['text'] elif msg_type in ('display_data', 'execute_result'): if hasattr(content, 'execution_count'): out_cell['execution_count'] = content['execution_count'] else: out_cell['execution_count'] = None out_cell['data'] = content['data'] out_cell['metadata'] = content['metadata'] outs.append(out_cell) elif msg_type == 'error': out_cell.ename = content['ename'] out_cell.evalue = content['evalue'] out_cell.traceback = content['traceback'] outs.append(out_cell) elif msg_type.startswith('comm_'): # messages used to initialize, close and unpdate widgets # we will ignore these and hope for the best pass else: tv.warning("Unhandled iopub msg of type `%s`" % msg_type) return outs def get_commands(self, cell): """ Extract potential commands from the first line of a cell if a code cell starts with the hashbang `#!` it can be followed by a comma separated list of commands. Each command can be 1. a single key `skip`, or 2. a key/value pair separated by a colon `timeout:[int]` Parameters ---------- cell : a NotebookCell the cell to be examined Returns ------- dict a dict of key/value pairs. For a single command the value is `True` """ commands = {} source = cell.source if source is not None: lines = source.splitlines() if len(lines) > 0: n_line = 0 line = lines[n_line].strip() while line.startswith('#!') or len(line) == 0: txt = line[2:].strip() parts = txt.split(',') for part in parts: subparts = part.split(':') if len(subparts) == 1: commands[subparts[0].strip().lower()] = True elif len(subparts) == 2: commands[subparts[0].strip().lower()] = subparts[1] n_line += 1 line = lines[n_line] return commands def is_empty_cell(self, cell): """ Check if a cell has no code Parameters ---------- cell : a NotebookCell the cell to be examined Returns ------- bool True if the cell has no code, False otherwise """ return not bool(cell.source)
import pathlib from jupyter_client.manager import KernelManager from jupyter_client.connect import find_connection_file # forkする? m = KernelManager() m.connection_file = "kernel-me.json" m.write_connection_file() info = m.get_connection_info() print(info) print(m.connection_file) print(find_connection_file()) print(find_connection_file(pathlib.Path(m.connection_file).name)) # m.load_connection_file() m.blocking_client()
class IPyKernel(object): """ A simple wrapper class to run cells in an IPython Notebook. Notes ----- - Use `with` construct to properly instantiate - IPython 3.0.0+ is assumed for this version """ def __init__(self, nb_version=4, extra_arguments=None): # default timeout time is 60 seconds self.default_timeout = 60 if extra_arguments is None: extra_arguments = [] self.extra_arguments = extra_arguments self.nb_version = nb_version def __enter__(self): self.km = KernelManager() self.km.start_kernel( extra_arguments=self.extra_arguments, stderr=open(os.devnull, 'w') ) self.kc = self.km.client() self.kc.start_channels() self.iopub = self.kc.iopub_channel self.shell = self.kc.shell_channel # run %pylab inline, because some notebooks assume this # even though they shouldn't self.shell.send("pass") self.shell.get_msg() while True: try: self.iopub.get_msg(timeout=0.05) except Empty: break return self def execute(self, source): self.kc.execute(source + '\n') self.shell.get_msg(timeout=0.05) while True: try: msg = self.iopub.get_msg(timeout=0.05) except Empty: break def __exit__(self, exc_type, exc_val, exc_tb): self.kc.stop_channels() self.km.shutdown_kernel() del self.km def run(self, cell, timeout=None): """ Run a notebook cell in the IPythonKernel Parameters ---------- cell : IPython.notebook.Cell the cell to be run timeout : int or None (default) the time in seconds after which a cell is stopped and assumed to have timed out. If set to None the value in `default_timeout` is used Returns ------- list of outs a list of NotebookNodes of the returned types. This is similar to the list of outputs generated when a cell is run """ use_timeout = self.default_timeout if timeout is not None: use_timeout = timeout if hasattr(cell, 'input'): self.kc.execute(cell.input) elif hasattr(cell, 'source'): self.kc.execute(cell.source) else: raise AttributeError('No source/input key') self.shell.get_msg(timeout=use_timeout) outs = [] stdout_cells = {} while True: try: msg = self.iopub.get_msg(timeout=1.00) except Empty: break msg_type = msg['msg_type'] if msg_type in ('pyin', 'execute_input'): continue elif msg_type == 'clear_output': outs = [] continue elif msg_type == 'status': if msg['content']['execution_state'] == 'idle': # we are done with the cell, let's compare break continue content = msg['content'] out = nbformat.NotebookNode(output_type=msg_type) if msg_type == 'stream': name = content['name'] if name not in stdout_cells: out.name = name out.text = content['text'] stdout_cells[name] = out outs.append(out) else: # we already have a stdout cell, so append to it stdout_cells[name].text += content['text'] elif msg_type in ('display_data', 'pyout', 'execute_result'): if hasattr(content, 'execution_count'): out['execution_count'] = content['execution_count'] else: out['execution_count'] = None out['data'] = content['data'] out['metadata'] = content['metadata'] outs.append(out) elif msg_type == 'error': out.ename = content['ename'] out.evalue = content['evalue'] out.traceback = content['traceback'] outs.append(out) elif msg_type.startswith('comm_'): # messages used to initialize, close and unpdate widgets # we will ignore these and hope for the best pass else: print "unhandled iopub msg:", msg_type return outs @staticmethod def sanitize(s): """sanitize a string for comparison. fix universal newlines, strip trailing newlines, and normalize likely random values (memory addresses and UUIDs) Parameters ---------- s : str string to be sanitized, i.e. remove UUIDs, Hex-addresses, unnecessary newlines """ if not isinstance(s, basestring): return s # normalize newline: s = s.replace('\r\n', '\n') # ignore trailing newlines (but not space) s = s.rstrip('\n') # normalize hex addresses: s = re.sub(r'0x[a-f0-9]+', '0xFFFFFFFF', s) # normalize UUIDs: s = re.sub(r'[a-f0-9]{8}(\-[a-f0-9]{4}){3}\-[a-f0-9]{12}', 'U-U-I-D', s) # fix problem with return s def compare_outputs( self, test, ref, skip_compare=('traceback', 'latex', 'execution_count') ): """ Compare two lists of `NotebookNode`s Parameters ---------- test : list of `NotebookNode` the list of be tested generated by the kernel ref : list of `NotebookNode` the reference list read from the notebook skip_compare : list of str a list of strings that name node types that are not to be tested Returns ------- bool is True if both lists are different list of diff a list of diff (str) the represent the differences """ diff = False diff_list = [] # print ref.keys(), test.keys() if self.nb_version == 4: for key in ref: if key not in test: return True, ["missing key: %s != %s" % (test.keys(), ref.keys())] elif key not in skip_compare: if key == 'data': for data_key in test[key]: my_diff = self.do_diff( data_key, test[key], ref[key]) if my_diff is not None: diff_list += my_diff diff = True else: # can this happen? my_diff = self.do_diff(key, test, ref) if my_diff is not None: diff_list += my_diff diff = True return diff, diff_list def do_diff(self, key, test_cell, ref_cell): """ Compare the key of two dicts Parameters ---------- key : string the key to be compared test_cell : dict a dict with `key` as a key of string value ref_cell : dict a dict with `key` as a key of string value Returns ------- list of diff (str) a list of diff representing the differences """ if hasattr(ref_cell, key): s1 = self.sanitize(ref_cell[key]) else: s1 = '' if hasattr(test_cell, key): s2 = self.sanitize(test_cell[key]) else: s2 = '' if key in ['image/png', 'image/svg', 'image/svg+xml']: if s1 != s2: return ['>>> diff in %s (size new : %d vs size old : %d )' % (key, len(s1), len(s2))] else: if s1 != s2: expected = s1.splitlines(1) actual = s2.splitlines(1) diff = difflib.ndiff(expected, actual) return ['>>> diff in ' + key] + list(diff) return None def get_commands(self, cell): """ Extract potential commands from the first line of a cell if a code cell starts with the hashbang `#!` it can be followed by a comma separated list of commands. Each command can be 1. a single key `skip`, or 2. a key/value pair separated by a colon `timeout:[int]` Parameters ---------- cell : a NotebookCell the cell to be examined Returns ------- dict a dict of key/value pairs. For a single command the value is `True` """ commands = {} source = self.get_source(cell) if source is not None: lines = source.splitlines() if len(lines) > 0: n_line = 0 line = lines[n_line].strip() while line.startswith('#!') or len(line) == 0: txt = line[2:].strip() parts = txt.split(',') for part in parts: subparts = part.split(':') if len(subparts) == 1: commands[subparts[0].strip().lower()] = True elif len(subparts) == 2: commands[subparts[0].strip().lower()] = subparts[1] n_line += 1 line = lines[n_line] return commands def get_source(self, cell): """ get the source code of a cell Notes ----- This is legacy of IPython 2/3 conversion. Parameters ---------- cell : a NotebookCell the cell to be examined Returns ------- string the source code """ if cell.cell_type == 'code': if hasattr(cell, 'input'): return cell.input elif hasattr(cell, 'source'): return cell.source else: return None def is_empty_cell(self, cell): """ Check if a cell has no code Parameters ---------- cell : a NotebookCell the cell to be examined Returns ------- bool True if the cell has no code, False otherwise """ source = self.get_source(cell) return source is None or source == ''
class KernelMagics(Magics): _execution_count = 1 def stop_kernel(self): self.kc.stop_channels() self.km.shutdown_kernel(now=True) def __init__(self, shell): super(KernelMagics, self).__init__(shell) self.km = None self.kc = None self.comms = [] def start(self, kernel_name): self.km = KernelManager() self.km.kernel_name = kernel_name self.km.start_kernel(extra_arguments=[self._context_base64()]) atexit.register(self.stop_kernel) self.kc = self.km.client() self.kc.start_channels() try: self.kc.wait_for_ready() print("{} started successfully\n".format(kernel_name.capitalize())) except AttributeError: self._wait_for_ready_backport() def run_cell(self, line, code): if not self.km: self.start() self.kc.execute(code, allow_stdin=True) reply = self.kc.get_shell_msg() self._handle_iopub_messages() def _handle_iopub_messages(self): while True: try: msg = self.kc.get_iopub_msg(timeout=1) except Empty: break comm_id = msg['content'].get('comm_id') if comm_id and comm_id not in self.comms: self.comms.append(comm_id) self.shell.kernel.session.send(self.shell.kernel.iopub_socket, msg['msg_type'], msg['content'], metadata=msg['metadata'], parent=self.shell.kernel._parent_header, ident=msg.get('comm_id'), buffers=msg['buffers'], ) def pass_message(self, msg_raw): comm_id = msg_raw['content'].get('comm_id') if comm_id in self.comms: content = msg_raw['content'] msg = self.kc.session.msg(msg_raw['msg_type'], content) self.kc.shell_channel.send(msg) self._handle_iopub_messages() else: self.log.warn("No such comm: %s", comm_id) if self.log.isEnabledFor(logging.DEBUG): # don't create the list of keys if debug messages aren't enabled self.log.debug("Current comms: %s", list(self.comms.keys())) def _context_base64(self): context_json = json.dumps({ 'port': os.environ["BEAKERX_AUTOTRANSLATION_PORT"], 'contextId': get_ipython().kernel.session.session, }) return base64.b64encode(context_json.encode('utf-8')).decode()
def km_from_string(s=''): """create kernel manager from IPKernelApp string such as '--shell=47378 --iopub=39859 --stdin=36778 --hb=52668' for IPython 0.11 or just 'kernel-12345.json' for IPython 0.12 """ try: import IPython except ImportError: raise ImportError("Could not find IPython. " + _install_instructions) # Modified by BoPeng # #from IPython.config.loader import KeyValueConfigLoader from traitlets.config.loader import KeyValueConfigLoader try: # Updated by Bo Peng for module names from jupyter_client.manager import KernelManager from jupyter_client import find_connection_file except ImportError: # IPython < 1.0 from IPython.zmq.blockingkernelmanager import BlockingKernelManager as KernelManager from IPython.zmq.kernelapp import kernel_aliases try: from IPython.lib.kernel import find_connection_file except ImportError: # < 0.12, no find_connection_file pass global km, kc, send s = s.replace('--existing', '') if 'connection_file' in KernelManager.class_trait_names(): # 0.12 uses files instead of a collection of ports # include default IPython search path # filefind also allows for absolute paths, in which case the search # is ignored try: # XXX: the following approach will be brittle, depending on what # connection strings will end up looking like in the future, and # whether or not they are allowed to have spaces. I'll have to sync # up with the IPython team to address these issues -pi if '--profile' in s: k,p = s.split('--profile') k = k.lstrip().rstrip() # kernel part of the string p = p.lstrip().rstrip() # profile part of the string fullpath = find_connection_file(k,p) else: fullpath = find_connection_file(s.lstrip().rstrip()) except IOError as e: echo(":IPython " + s + " failed", "Info") echo("^-- failed '" + s + "' not found", "Error") return km = KernelManager(connection_file = fullpath) km.load_connection_file() else: if s == '': echo(":IPython 0.11 requires the full connection string") return loader = KeyValueConfigLoader(s.split(), aliases=kernel_aliases) cfg = loader.load_config()['KernelApp'] try: km = KernelManager( shell_address=(ip, cfg['shell_port']), sub_address=(ip, cfg['iopub_port']), stdin_address=(ip, cfg['stdin_port']), hb_address=(ip, cfg['hb_port'])) except KeyError as e: echo(":IPython " +s + " failed", "Info") echo("^-- failed --"+e.message.replace('_port','')+" not specified", "Error") return try: kc = km.client() except AttributeError: # 0.13 kc = km kc.start_channels() try: send = kc.execute except AttributeError: # < 3.0 send = kc.shell_channel.execute #XXX: backwards compatibility for IPython < 0.13 try: import inspect sc = kc.shell_channel num_oinfo_args = len(inspect.getargspec(sc.object_info).args) if num_oinfo_args == 2: # patch the object_info method which used to only take one argument klass = sc.__class__ klass._oinfo_orig = klass.object_info klass.object_info = lambda s,x,y: s._oinfo_orig(x) except Exception: pass #XXX: backwards compatibility for IPython < 1.0 if not hasattr(kc, 'iopub_channel'): kc.iopub_channel = kc.sub_channel # now that we're connect to an ipython kernel, activate completion # machinery, but do so only for the local buffer if the user added the # following line the vimrc: # let g:ipy_completefunc = 'local' vim.command(""" if g:ipy_completefunc == 'global' set completefunc=CompleteIPython elseif g:ipy_completefunc == 'local' setl completefunc=CompleteIPython endif """) # also activate GUI doc balloons if in gvim vim.command(""" if has('balloon_eval') set bexpr=IPythonBalloonExpr() endif """) set_pid() return km
def km_from_string(s=''): """create kernel manager from IPKernelApp string such as '--shell=47378 --iopub=39859 --stdin=36778 --hb=52668' for IPython 0.11 or just 'kernel-12345.json' for IPython 0.12 """ try: import IPython except ImportError: raise ImportError("Could not find IPython. " + _install_instructions) try: from traitlets.config.loader import KeyValueConfigLoader except ImportError: # IPython <= 3.0 from IPython.config.loader import KeyValueConfigLoader try: from jupyter_client.manager import KernelManager from jupyter_client.connect import find_connection_file except ImportError: # IPython <= 3.0 try: from IPython.kernel import ( KernelManager, find_connection_file, ) except ImportError: # IPython < 1.0 from IPython.zmq.blockingkernelmanager import BlockingKernelManager as KernelManager from IPython.zmq.kernelapp import kernel_aliases try: from IPython.lib.kernel import find_connection_file except ImportError: # < 0.12, no find_connection_file pass global km, kc, send s = s.replace('--existing', '') if 'connection_file' in KernelManager.class_trait_names(): # 0.12 uses files instead of a collection of ports # include default IPython search path # filefind also allows for absolute paths, in which case the search # is ignored try: # XXX: the following approach will be brittle, depending on what # connection strings will end up looking like in the future, and # whether or not they are allowed to have spaces. I'll have to sync # up with the IPython team to address these issues -pi if '--profile' in s: k,p = s.split('--profile') k = k.lstrip().rstrip() # kernel part of the string p = p.lstrip().rstrip() # profile part of the string fullpath = find_connection_file(k,p) else: s = s.lstrip().rstrip(); if(len(s) == 0): fullpath = find_connection_file() else: fullpath = find_connection_file(s.lstrip().rstrip()) except IOError as e: echo(":IPython " + s + " failed", "Info") echo("^-- failed '" + s + "' not found", "Error") return km = KernelManager(connection_file = fullpath) km.load_connection_file() else: if s == '': echo(":IPython 0.11 requires the full connection string") return loader = KeyValueConfigLoader(s.split(), aliases=kernel_aliases) cfg = loader.load_config()['KernelApp'] try: km = KernelManager( shell_address=(ip, cfg['shell_port']), sub_address=(ip, cfg['iopub_port']), stdin_address=(ip, cfg['stdin_port']), hb_address=(ip, cfg['hb_port'])) except KeyError as e: echo(":IPython " +s + " failed", "Info") echo("^-- failed --"+e.message.replace('_port','')+" not specified", "Error") return try: kc = km.client() except AttributeError: # 0.13 kc = km kc.start_channels() try: send = kc.execute except AttributeError: # < 3.0 send = kc.shell_channel.execute #XXX: backwards compatibility for IPython < 0.13 try: import inspect sc = kc.shell_channel num_oinfo_args = len(inspect.getargspec(sc.object_info).args) if num_oinfo_args == 2: # patch the object_info method which used to only take one argument klass = sc.__class__ klass._oinfo_orig = klass.object_info klass.object_info = lambda s,x,y: s._oinfo_orig(x) except: pass #XXX: backwards compatibility for IPython < 1.0 if not hasattr(kc, 'iopub_channel'): kc.iopub_channel = kc.sub_channel # now that we're connect to an ipython kernel, activate completion # machinery, but do so only for the local buffer if the user added the # following line the vimrc: # let g:ipy_completefunc = 'local' vim.command(""" if g:ipy_completefunc == 'global' set completefunc=CompleteIPython elseif g:ipy_completefunc == 'local' setl completefunc=CompleteIPython endif """) # also activate GUI doc balloons if in gvim vim.command(""" if has('balloon_eval') set bexpr=IPythonBalloonExpr() endif """) set_pid() return km
class GroovyMagics(Magics): _execution_count = 1 def __init__(self, shell): super(GroovyMagics, self).__init__(shell) self.km = KernelManager() self.km.kernel_name = 'groovy' self.km.start_kernel() self.kc = self.km.client() self.kc.start_channels() try: self.kc.wait_for_ready() print("Groovy started successfully\n") except AttributeError: self._wait_for_ready_backport() def __del__(self): self.kc.stop_channels() self.km.shutdown_kernel(now=True) def run_cell(self, line, code): self.kc.execute(code, allow_stdin=True) reply = self.kc.get_shell_msg() status = reply['content']['status'] outs = list() while True: try: msg = self.kc.get_iopub_msg(timeout=1) if msg['msg_type'] == 'status': if msg['content']['execution_state'] == 'idle': break except Empty: print("empty ?!") raise content = msg['content'] msg_type = msg['msg_type'] notebook3_format_conversions = { 'error': 'pyerr', 'execute_result': 'pyout' } msg_type = notebook3_format_conversions.get(msg_type, msg_type) out = NotebookNode(output_type=msg_type) if msg_type == 'pyout': print(content['data']['text/plain']) continue if msg_type in ('status', 'pyin', 'execute_input'): continue elif msg_type in ('comm_open', 'comm_msg', 'comm_close'): # TODO handle this msg ?!?!?! continue elif msg_type == 'stream': out.stream = content['name'] if 'text' in content: out.text = content['text'] else: out.text = content['data'] elif msg_type in ('display_data', 'pyout'): for mime, data in content['data'].items(): try: attr = self.MIME_MAP[mime] except KeyError: print("unhandled mime") raise NotImplementedError('unhandled mime type: %s' % mime) setattr(out, attr, data) elif msg_type == 'pyerr': out.ename = content['ename'] out.evalue = content['evalue'] out.traceback = content['traceback'] elif msg_type == 'clear_output': outs = list() continue else: print("unhandled " + msg_type) raise NotImplementedError('unhandled iopub message: %s' % msg_type) outs.append(out) # NOTE: Ver 4 format still have 'pyout', Why? # upgrade_outputs(outs) print(str(outs)) print("status: {}".format(status)) @cell_magic def groovy(self, line, cell): return self.run_cell(line, cell)
class Kernel: name: str init_code: str = "" language: str = "" manager: Optional[KernelManager] = field(default=None, init=False) client: Optional[KernelClient] = field(default=None, init=False) report: Dict[str, Any] = field(default_factory=dict, init=False) def __post_init__(self): self.report["total"] = datetime.timedelta(0) kernel_spec = get_kernel_spec(self.name) self.language = kernel_spec.language if self.language == "python": self.init_code = ( "from pheasant.renderers.jupyter.ipython import register_formatters\n" "register_formatters()") def shutdown(): # pragma: no cover if self.manager: print(f"Shutting down kernel [{self.name}]...", end="") self.manager.shutdown_kernel() print("Done.") atexit.register(shutdown) def start(self, timeout=10, silent=True) -> KernelClient: if self.manager: if self.manager.is_alive(): return self.client else: # pragma: no cover raise RuntimeError(f"Kernel {self.name} is not alive.") def start(): self.manager = KernelManager(kernel_name=self.name) self.manager.start_kernel() self.client = self.manager.blocking_client() self.client.start_channels() try: self.client.wait_for_ready(timeout=timeout) except TimeoutError: # pragma: no cover self.manager.shutdown_kernel() return False else: self.client.execute_interactive(self.init_code) return self.client init = f"Starting kernel [{self.name}]" progress_bar = progress_bar_factory(total=3, init=init) now = datetime.datetime.now() def message(result): dt = format_timedelta_human(datetime.datetime.now() - now) return f"Kernel [{self.name}] started ({dt})" if result else "Retrying..." for k in range(progress_bar.total): if silent and start(): break elif silent: continue elif progress_bar.progress(start, message): progress_bar.finish() break else: raise TimeoutError # pragma: no cover return self.client def shutdown(self) -> None: if self.manager: self.manager.shutdown_kernel() del self.client self.client = None del self.manager self.manager = None def restart(self) -> None: if self.manager: self.manager.restart_kernel() if self.client and self.init_code: self.client.execute_interactive(self.init_code) def execute(self, code: str, output_hook=None) -> List: client = self.client or self.start() outputs = [] def _output_hook(msg): if output_hook: output_hook(msg) output = output_from_msg(msg) if output: outputs.append(output) msg = client.execute_interactive(code, output_hook=_output_hook) update_report(self.report, msg) return list(stream_joiner(outputs)) def inspect(self, code: str, func: str = "getsource", output_hook=None) -> List: self.execute("import inspect") self.execute(code, output_hook=output_hook) outputs = self.execute(code_for_inspect(func)) if len(outputs) == 1 and outputs[0]["type"] == "execute_result": source = ast.literal_eval(outputs[0]["data"]["text/plain"]) return [dict(type="stream", name="source", text=source)] else: return outputs
class IPyKernel(object): """ A simple wrapper class to run cells in an IPython Notebook. Notes ----- - Use `with` construct to properly instantiate - IPython 3.0.0+ is assumed for this version """ def __init__(self, nb_version=4, extra_arguments=None): # default timeout time is 60 seconds self.default_timeout = 60 if extra_arguments is None: extra_arguments = [] self.extra_arguments = extra_arguments self.nb_version = nb_version def __enter__(self): self.km = KernelManager() self.km.start_kernel( extra_arguments=self.extra_arguments, stderr=open(os.devnull, 'w') ) self.kc = self.km.client() self.kc.start_channels() self.iopub = self.kc.iopub_channel self.shell = self.kc.shell_channel # run %pylab inline, because some notebooks assume this # even though they shouldn't self.shell.send("pass") self.shell.get_msg() while True: try: self.iopub.get_msg(timeout=0.05) except Exception as e: if repr(e) == 'Empty()': break # we got a real error so raise it raise self.cmd_list = [] self.msg_list = {} return self def clear(self): self.iopub.get_msgs() def execute(self, cmd): uid = self.kc.execute(cmd) self.cmd_list.append((uid, cmd)) return uid def __exit__(self, exc_type, exc_val, exc_tb): self.kc.stop_channels() self.km.shutdown_kernel() del self.msg_list del self.cmd_list del self.km def listen(self, uid, use_timeout=None): if use_timeout is None: use_timeout = self.default_timeout while True: if uid in self.msg_list and len(self.msg_list[uid]) > 0: return self.msg_list[uid].pop(0) msg = self.iopub.get_msg(timeout=use_timeout) if 'msg_id' in msg['parent_header']: msg_uid = msg['parent_header']['msg_id'] if msg_uid not in self.msg_list: self.msg_list[msg_uid] = [] self.msg_list[msg_uid].append(msg) def run(self, cell, use_timeout=None): """ Run a notebook cell in the IPythonKernel Parameters ---------- cell : IPython.notebook.Cell the cell to be run use_timeout : int or None (default) the time in seconds after which a cell is stopped and assumed to have timed out. If set to None the value in `default_timeout` is used Returns ------- list of ex_cell_outputs a list of NotebookNodes of the returned types. This is similar to the list of outputs generated when a cell is run """ if timeout is not None: use_timeout = use_timeout else: use_timeout = self.default_timeout if hasattr(cell, 'source'): uid = self.execute(cell.source) else: raise AttributeError('No source/input key') outs = [] stdout_cells = {} while True: msg = self.listen(uid, use_timeout) msg_type = msg['msg_type'] if msg_type == 'execute_input': continue elif msg_type == 'clear_output': outs = [] continue elif msg_type == 'status': if msg['content']['execution_state'] == 'idle': # we are done with the cell, let's compare break continue out_cell = nbformat.NotebookNode(output_type=msg_type) content = msg['content'] if msg_type == 'stream': name = content['name'] if name not in stdout_cells: out_cell.name = name out_cell.text = content['text'] stdout_cells[name] = out_cell outs.append(out_cell) else: # we already have a stdout cell, so append to it stdout_cells[name].text += content['text'] elif msg_type in ('display_data', 'execute_result'): if hasattr(content, 'execution_count'): out_cell['execution_count'] = content['execution_count'] else: out_cell['execution_count'] = None out_cell['data'] = content['data'] out_cell['metadata'] = content['metadata'] outs.append(out_cell) elif msg_type == 'error': out_cell.ename = content['ename'] out_cell.evalue = content['evalue'] out_cell.traceback = content['traceback'] outs.append(out_cell) elif msg_type.startswith('comm_'): # messages used to initialize, close and unpdate widgets # we will ignore these and hope for the best pass else: tv.warning("Unhandled iopub msg of type `%s`" % msg_type) return outs def get_commands(self, cell): """ Extract potential commands from the first line of a cell if a code cell starts with the hashbang `#!` it can be followed by a comma separated list of commands. Each command can be 1. a single key `skip`, or 2. a key/value pair separated by a colon `timeout:[int]` Parameters ---------- cell : a NotebookCell the cell to be examined Returns ------- dict a dict of key/value pairs. For a single command the value is `True` """ commands = {} source = cell.source if source is not None: lines = source.splitlines() if len(lines) > 0: n_line = 0 line = lines[n_line].strip() while line.startswith('#!') or len(line) == 0: txt = line[2:].strip() parts = txt.split(',') for part in parts: subparts = part.split(':') if len(subparts) == 1: commands[subparts[0].strip().lower()] = True elif len(subparts) == 2: commands[subparts[0].strip().lower()] = subparts[1] n_line += 1 line = lines[n_line] return commands def is_empty_cell(self, cell): """ Check if a cell has no code Parameters ---------- cell : a NotebookCell the cell to be examined Returns ------- bool True if the cell has no code, False otherwise """ return not bool(cell.source)
def parse_command_line(self, argv): silent = "--silent" in argv silent_spark_version = None if silent: argv.remove("--silent") arg_str = None for i, arg in enumerate(argv): if arg.startswith("--spark"): arg_str = arg silent_spark_version = arg_str[len("--spark") + 1:] if silent_spark_version not in self.spark_download_versions: print("Invalid Spark version {}".format( silent_spark_version)) self.exit(1) break if arg_str: argv.remove(arg_str) super(InstallKernelSpec, self).parse_command_line(argv) self.pixiedust_home = os.environ.get( "PIXIEDUST_HOME", "{}{}pixiedust".format(os.path.expanduser('~'), os.sep)) if silent: answer = 'y' else: answer = self.confirm("Step 1: PIXIEDUST_HOME: {0}".format( self.pixiedust_home)) if answer != 'y': self.pixiedust_home = input( self.hilite("Please enter a PIXIEDUST_HOME location: ")) if not os.path.exists(self.pixiedust_home): create = self.confirm( "Directory {0} does not exist".format(self.pixiedust_home), "Create") if create != 'y': self.exit(1) else: os.makedirs(self.pixiedust_home) self.pixiedust_bin = os.path.join(self.pixiedust_home, "bin") download_spark = False if silent: self.spark_home = os.environ.get("SPARK_HOME", None) if not self.spark_home: download_spark = True self.spark_home = "{}{}spark".format(os.path.expanduser('~'), os.sep) else: first_prompt = True while True: self.spark_home = os.environ.get( "SPARK_HOME", self.ensureDir(os.path.join(self.pixiedust_bin, "spark"))) if self.spark_home: answer = self.confirm("Step 2: SPARK_HOME: {0}".format( self.spark_home)) if answer != 'y': self.spark_home = input( self.hilite( "Step 2: Please enter a SPARK_HOME location: ") ) else: if first_prompt: first_prompt = False prompt = "Step 2: Please enter a SPARK_HOME location: " else: prompt = "Please enter a SPARK_HOME location: " self.spark_home = input(self.hilite(prompt)) while self.spark_home.rfind( os.sep) == len(self.spark_home) - 1: self.spark_home = self.spark_home[0:len(self.spark_home) - 1] if not os.path.exists(self.spark_home): create = self.confirm( "Directory {0} does not exist".format(self.spark_home), "Create") if create != 'y': continue else: os.makedirs(self.spark_home) download_spark = True break elif not os.path.exists('{}{}bin{}pyspark'.format( self.spark_home, os.sep, os.sep)): existingInstalls = [ d for d in os.listdir(self.spark_home) if os.path.exists( os.path.join(self.spark_home, d, 'bin', 'pyspark')) ] if len(existingInstalls) == 0: download = self.confirm( "Directory {0} does not contain a valid SPARK install" .format(self.spark_home), "Download Spark") if download == 'y': download_spark = True break else: existingInstalls.append("Create a new spark Install") print( "Select an existing spark install or create a new one" ) message = "\n".join([ "{}. {}".format(k + 1, v) for k, v in enumerate(existingInstalls) ]) while True: try: answer = int( input( self.hilite(message) + "\n\tEnter your selection: ")) - 1 if answer < 0 or answer >= len( existingInstalls): raise Exception( "Please pick a number within the specified values" ) break except Exception as e: print("Invalid selection: {}".format(e)) download_spark = (answer == len(existingInstalls) - 1) if not download_spark: self.spark_home = os.path.join( self.spark_home, existingInstalls[answer]) break else: break if download_spark: self.download_spark(silent, silent_spark_version) scala_version = None spark_version = self.get_spark_version() if spark_version is None: print("Unable to obtain Spark version") self.exit(1) elif spark_version[0] == 1: scala_version = '2.10' # Spark 1.x = Scala 2.10 elif spark_version[0] == 2: scala_version = '2.11' # Spark 2.x = Scala 2.11 else: print("Invalid Spark version {}".format(spark_version)) self.exit(1) #download spark-cloudant sparkCloudantUrl = "https://github.com/cloudant-labs/spark-cloudant/releases/download/v1.6.4/cloudant-spark-v1.6.4-167.jar" if spark_version[0] == 2: sparkCloudantUrl = "https://github.com/cloudant-labs/spark-cloudant/releases/download/v2.0.0/cloudant-spark-v2.0.0-185.jar" try: self.sparkCloudantPath = self.downloadFileToDir( sparkCloudantUrl, targetDir=self.pixiedust_bin) print("downloaded spark cloudant jar: {0}".format( self.sparkCloudantPath)) except Exception as e: print( "Error downloading Cloudant Jar: {}. Install will continue without the spark Cloudant connector" .format(e)) #download spark csv connector if in spark 1.6 self.sparkCSVPath = None self.commonsCSVPath = None if spark_version[0] == 1: try: self.sparkCSVPath = self.downloadPackage( "com.databricks:spark-csv_2.10:1.5.0") self.commonsCSVPath = self.downloadPackage( "org.apache.commons:commons-csv:0") except Exception as e: print( "Error downloading csv connector Jar: {}. Install will continue without it" .format(e)) download_scala = False if silent: self.scala_home = os.environ.get("SCALA_HOME", None) if not self.scala_home: download_scala = True self.scala_home = "{}{}scala".format(os.path.expanduser('~'), os.sep) else: first_prompt = True while True: self.scala_home = os.environ.get( "SCALA_HOME", self.ensureDir(os.path.join(self.pixiedust_bin, "scala"))) if self.scala_home: answer = self.confirm("Step 3: SCALA_HOME: {0}".format( self.scala_home)) if answer != 'y': self.scala_home = input( self.hilite( "Step 3: Please enter a SCALA_HOME location: ") ) else: if first_prompt: first_prompt = False prompt = "Step 3: Please enter a SCALA_HOME location: " else: prompt = "Please enter a SCALA_HOME location: " self.scala_home = input(self.hilite(prompt)) while self.scala_home.rfind( os.sep) == len(self.scala_home) - 1: self.scala_home = self.scala_home[0:len(self.scala_home) - 1] if not os.path.exists(self.scala_home): create = self.confirm( "Directory {0} does not exist".format(self.scala_home), "Create") if create != 'y': continue else: os.makedirs(self.scala_home) download_scala = True break elif not os.path.exists('{}{}bin{}scala'.format( self.scala_home, os.sep, os.sep)): def acceptScalaVersion(dir): version = self.get_scala_version_from_dir( os.path.join(self.scala_home, dir)) return version is not None and str( version[0]) + '.' + str( version[1]) == scala_version existingInstalls = [ d for d in os.listdir(self.scala_home) if acceptScalaVersion(d) ] if len(existingInstalls) == 0: download = self.confirm( "Directory {0} does not contain a valid scala install" .format(self.scala_home), "Download Scala") if download == 'y': download_scala = True break else: existingInstalls.append("Create a new scala Install") print( "Select an existing scala install or create a new one" ) message = "\n".join([ "{}. {}".format(k + 1, v) for k, v in enumerate(existingInstalls) ]) while True: try: answer = int( input( self.hilite(message) + "\n\tEnter your selection: ")) - 1 if answer < 0 or answer >= len( existingInstalls): raise Exception( "Please pick a number within the specified values" ) break except Exception as e: print("Invalid selection: {}".format(e)) download_scala = (answer == len(existingInstalls) - 1) if not download_scala: self.scala_home = os.path.join( self.scala_home, existingInstalls[answer]) break else: installed_scala_version = self.get_scala_version() if not installed_scala_version or ( str(installed_scala_version[0]) + '.' + str(installed_scala_version[1])) != scala_version: print( "A different version of Scala {0} is already installed in this directory." .format(installed_scala_version)) continue else: break if download_scala: self.download_scala(scala_version) self.kernelName = "Python with Pixiedust (Spark {}.{})".format( spark_version[0], spark_version[1]) if silent: answer = 'y' else: answer = self.confirm("Step 4: Kernel Name: {0}".format( self.kernelName)) if answer != 'y': self.kernelName = input( self.hilite("Please enter a Kernel Name: ")) try: km = KernelManager(kernel_name=self.kernelName) km.kernel_spec answer = self.confirm( "Kernel '{0}' already exists".format(self.kernelName), "Override") if answer != 'y': self.exit(1) except NoSuchKernel: pass dest = self.createKernelSpec() self.pixiedust_notebooks_dir = os.path.join(self.pixiedust_home, "notebooks") if not os.path.isdir(self.pixiedust_notebooks_dir): os.makedirs(self.pixiedust_notebooks_dir) print("Downloading intro notebooks into {}".format( self.pixiedust_notebooks_dir)) self.downloadIntroNotebooks() print("\n\n{}".format("#" * 100)) print("#\tCongratulations: Kernel {0} was successfully created in {1}". format(self.kernelName, dest)) print( "#\tYou can start the Notebook server with the following command:") print("#\t\t{}".format( self.hilite("jupyter notebook {}".format( self.pixiedust_notebooks_dir)))) print("{}".format("#" * 100))
class NotebookRunner(object): # The kernel communicates with mime-types while the notebook # uses short labels for different cell types. We'll use this to # map from kernel types to notebook format types. MIME_MAP = { 'image/jpeg': 'jpeg', 'image/png': 'png', 'text/plain': 'text', 'text/html': 'html', 'text/latex': 'latex', 'application/javascript': 'html', 'image/svg+xml': 'svg', } def __init__(self, nb, pylab=False, mpl_inline=False, working_dir=None): self.km = KernelManager() args = [] if pylab: args.append('--pylab=inline') logging.warn('--pylab is deprecated and will be removed in a ' 'future version') elif mpl_inline: args.append('--matplotlib=inline') logging.warn('--matplotlib is deprecated and will be removed in a ' 'future version') cwd = os.getcwd() if working_dir: os.chdir(working_dir) self.km.start_kernel(extra_arguments=args) os.chdir(cwd) if platform.system() == 'Darwin': # There is sometimes a race condition where the first # execute command hits the kernel before it's ready. # It appears to happen only on Darwin (Mac OS) and an # easy (but clumsy) way to mitigate it is to sleep # for a second. sleep(1) self.kc = self.km.client() self.kc.start_channels() try: self.kc.wait_for_ready() except AttributeError: # IPython < 3 self._wait_for_ready_backport() self.nb = nb def __del__(self): self.kc.stop_channels() self.km.shutdown_kernel(now=True) def run_cell(self, cell, cidx): ''' Run a notebook cell and update the output of that cell in-place. ''' logging.debug('running cell {}'.format(cidx)) # logging.debug(u'cell.input {}'.format(cell.input)) self.kc.execute(cell.source) reply = self.kc.get_shell_msg() status = reply['content']['status'] max_mem = system_memory_used() logging.info(' memory used: {}'.format(sizeof_fmt(max_mem))) if status == 'error': traceback_text = 'Cell raised uncaught exception: \n' + \ '\n'.join(reply['content']['traceback']) traceback_text = remove_ansicolor(traceback_text) if 'NoDataFound' not in traceback_text: logging.error(traceback_text) else: logging.debug('run_cell ok') outs = list() while True: try: msg = self.kc.get_iopub_msg(timeout=1) if msg['msg_type'] == 'status': if msg['content']['execution_state'] == 'idle': break except Empty: # execution state should return to idle before the queue # becomes empty, # if it doesn't, something bad has happened logging.error("empty exception") raise content = msg['content'] msg_type = msg['msg_type'] # IPython 3.0.0-dev writes pyerr/pyout in the notebook format but # uses error/execute_result in the message spec. This does the # translation needed for tests to pass with IPython 3.0.0-dev notebook3_format_conversions = { 'error': 'pyerr', 'execute_result': 'pyout' } msg_type = notebook3_format_conversions.get(msg_type, msg_type) out = NotebookNode(output_type=msg_type) #if 'execution_count' in content: #cell['prompt_number'] = content['execution_count'] #out.prompt_number = content['execution_count'] if msg_type in ('status', 'pyin', 'execute_input'): continue elif msg_type == 'stream': out.stream = content['name'] if 'text' in content: out.text = content['text'] else: out.text = content['data'] # print(out.text, end='') elif msg_type in ('display_data', 'pyout'): for mime, data in content['data'].items(): try: attr = self.MIME_MAP[mime] except KeyError: logging.error("unhandled mime") raise NotImplementedError('unhandled mime type: %s' % mime) setattr(out, attr, data) elif msg_type == 'pyerr': out.ename = content['ename'] out.evalue = content['evalue'] out.traceback = content['traceback'] elif msg_type == 'clear_output': outs = list() continue else: logging.error("unhandled iopub") raise NotImplementedError('unhandled iopub message: %s' % msg_type) outs.append(out) # NOTE: Ver 4 format still have 'pyout', Why? cell['outputs'] = upgrade_outputs(outs) logging.debug("status: {}".format(status)) if status == 'error': if 'NoDataFound' in traceback_text: raise NoDataFound(traceback_text.split('\n')[-1]) else: logging.debug(u"NotebookError raised") raise NotebookError(traceback_text) def iter_code_cells(self): ''' Iterate over the notebook cells containing code. ''' for cell in self.nb['cells']: if cell.cell_type == 'code': yield cell def iter_cells(self): ''' Iterate over the notebook cells. ''' for cell in self.nb['cells']: yield cell def clear_outputs(self): for cell in self.iter_cells(): if 'outputs' in cell: cell['outputs'] = [] @property def cellcnt(self): return len(self.nb['cells']) def run_notebook(self, memory_used=None, progress_cb=None, skip_exceptions=False): ''' Run all the cells of a notebook in order and update the outputs in-place. If ``skip_exceptions`` is set, then if exceptions occur in a cell, the subsequent cells are run (by default, the notebook execution stops). ''' cur = 0 if memory_used is not None: memory_used.append(system_memory_used()) for cell in self.iter_code_cells(): cur += 1 try: if progress_cb is not None: progress_cb(cur) self.run_cell(cell, cur) if memory_used is not None: memory_used.append(system_memory_used()) except NotebookError: if not skip_exceptions: raise
class KernelMagics(Magics): _execution_count = 1 def stop_kernel(self): self.kc.stop_channels() self.km.shutdown_kernel(now=True) def __init__(self, shell): super(KernelMagics, self).__init__(shell) self.km = None self.kc = None self.comms = [] def start(self, kernel_name): self.km = KernelManager() self.km.kernel_name = kernel_name self.km.start_kernel(extra_arguments=[self._context_base64()]) atexit.register(self.stop_kernel) self.kc = self.km.client() self.kc.start_channels() try: self.kc.wait_for_ready() print("{} started successfully\n".format(kernel_name.capitalize())) except AttributeError: self._wait_for_ready_backport() def run_cell(self, line, code): if not self.km: self.start() self.kc.execute(code, allow_stdin=True) reply = self.kc.get_shell_msg() self._handle_iopub_messages() def _handle_iopub_messages(self): while True: try: msg = self.kc.get_iopub_msg(timeout=1) except Empty: break comm_id = msg['content'].get('comm_id') if comm_id and comm_id not in self.comms: self.comms.append(comm_id) self.shell.kernel.session.send( self.shell.kernel.iopub_socket, msg['msg_type'], msg['content'], metadata=msg['metadata'], parent=self.shell.kernel._parent_header, ident=msg.get('comm_id'), buffers=msg['buffers'], ) def pass_message(self, msg_raw): comm_id = msg_raw['content'].get('comm_id') if comm_id in self.comms: content = msg_raw['content'] msg = self.kc.session.msg(msg_raw['msg_type'], content) self.kc.shell_channel.send(msg) self._handle_iopub_messages() else: self.log.warn("No such comm: %s", comm_id) if self.log.isEnabledFor(logging.DEBUG): # don't create the list of keys if debug messages aren't enabled self.log.debug("Current comms: %s", list(self.comms.keys())) def _context_base64(self): context_json = json.dumps({ 'port': os.environ["BEAKERX_AUTOTRANSLATION_PORT"], 'contextId': get_ipython().kernel.session.session, }) return base64.b64encode(context_json.encode('utf-8')).decode()