class RMessage(Reference): msg_id = Unicode() msg_type = Unicode() header = Dict() parent_header = Dict() content = Dict()
class Kernel(Configurable): #--------------------------------------------------------------------------- # Kernel interface #--------------------------------------------------------------------------- # attribute to override with a GUI eventloop = Any(None) def _eventloop_changed(self, name, old, new): """schedule call to eventloop from IOLoop""" loop = ioloop.IOLoop.instance() loop.add_timeout(time.time() + 0.1, self.enter_eventloop) shell = Instance('IPython.core.interactiveshell.InteractiveShellABC') shell_class = Type(ZMQInteractiveShell) session = Instance(Session) profile_dir = Instance('IPython.core.profiledir.ProfileDir') shell_streams = List() control_stream = Instance(ZMQStream) iopub_socket = Instance(zmq.Socket) stdin_socket = Instance(zmq.Socket) log = Instance(logging.Logger) user_module = Any() def _user_module_changed(self, name, old, new): if self.shell is not None: self.shell.user_module = new user_ns = Instance(dict, args=None, allow_none=True) def _user_ns_changed(self, name, old, new): if self.shell is not None: self.shell.user_ns = new self.shell.init_user_ns() # identities: int_id = Integer(-1) ident = Unicode() def _ident_default(self): return unicode_type(uuid.uuid4()) # Private interface _darwin_app_nap = Bool( True, config=True, help="""Whether to use appnope for compatiblity with OS X App Nap. Only affects OS X >= 10.9. """) # Time to sleep after flushing the stdout/err buffers in each execute # cycle. While this introduces a hard limit on the minimal latency of the # execute cycle, it helps prevent output synchronization problems for # clients. # Units are in seconds. The minimum zmq latency on local host is probably # ~150 microseconds, set this to 500us for now. We may need to increase it # a little if it's not enough after more interactive testing. _execute_sleep = Float(0.0005, config=True) # Frequency of the kernel's event loop. # Units are in seconds, kernel subclasses for GUI toolkits may need to # adapt to milliseconds. _poll_interval = Float(0.05, config=True) # If the shutdown was requested over the network, we leave here the # necessary reply message so it can be sent by our registered atexit # handler. This ensures that the reply is only sent to clients truly at # the end of our shutdown process (which happens after the underlying # IPython shell's own shutdown). _shutdown_message = None # This is a dict of port number that the kernel is listening on. It is set # by record_ports and used by connect_request. _recorded_ports = Dict() # A reference to the Python builtin 'raw_input' function. # (i.e., __builtin__.raw_input for Python 2.7, builtins.input for Python 3) _sys_raw_input = Any() _sys_eval_input = Any() # set of aborted msg_ids aborted = Set() def __init__(self, **kwargs): super(Kernel, self).__init__(**kwargs) # Initialize the InteractiveShell subclass self.shell = self.shell_class.instance( parent=self, profile_dir=self.profile_dir, user_module=self.user_module, user_ns=self.user_ns, kernel=self, ) self.shell.displayhook.session = self.session self.shell.displayhook.pub_socket = self.iopub_socket self.shell.displayhook.topic = self._topic('pyout') self.shell.display_pub.session = self.session self.shell.display_pub.pub_socket = self.iopub_socket self.shell.data_pub.session = self.session self.shell.data_pub.pub_socket = self.iopub_socket # TMP - hack while developing self.shell._reply_content = None # Build dict of handlers for message types msg_types = [ 'execute_request', 'complete_request', 'object_info_request', 'history_request', 'kernel_info_request', 'connect_request', 'shutdown_request', 'apply_request', ] self.shell_handlers = {} for msg_type in msg_types: self.shell_handlers[msg_type] = getattr(self, msg_type) comm_msg_types = ['comm_open', 'comm_msg', 'comm_close'] comm_manager = self.shell.comm_manager for msg_type in comm_msg_types: self.shell_handlers[msg_type] = getattr(comm_manager, msg_type) control_msg_types = msg_types + ['clear_request', 'abort_request'] self.control_handlers = {} for msg_type in control_msg_types: self.control_handlers[msg_type] = getattr(self, msg_type) def dispatch_control(self, msg): """dispatch control requests""" idents, msg = self.session.feed_identities(msg, copy=False) try: msg = self.session.unserialize(msg, content=True, copy=False) except: self.log.error("Invalid Control Message", exc_info=True) return self.log.debug("Control received: %s", msg) header = msg['header'] msg_id = header['msg_id'] msg_type = header['msg_type'] handler = self.control_handlers.get(msg_type, None) if handler is None: self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r", msg_type) else: try: handler(self.control_stream, idents, msg) except Exception: self.log.error("Exception in control handler:", exc_info=True) def dispatch_shell(self, stream, msg): """dispatch shell requests""" # flush control requests first if self.control_stream: self.control_stream.flush() idents, msg = self.session.feed_identities(msg, copy=False) try: msg = self.session.unserialize(msg, content=True, copy=False) except: self.log.error("Invalid Message", exc_info=True) return header = msg['header'] msg_id = header['msg_id'] msg_type = msg['header']['msg_type'] # Print some info about this message and leave a '--->' marker, so it's # easier to trace visually the message chain when debugging. Each # handler prints its message at the end. self.log.debug('\n*** MESSAGE TYPE:%s***', msg_type) self.log.debug(' Content: %s\n --->\n ', msg['content']) if msg_id in self.aborted: self.aborted.remove(msg_id) # is it safe to assume a msg_id will not be resubmitted? reply_type = msg_type.split('_')[0] + '_reply' status = {'status': 'aborted'} md = {'engine': self.ident} md.update(status) reply_msg = self.session.send(stream, reply_type, metadata=md, content=status, parent=msg, ident=idents) return handler = self.shell_handlers.get(msg_type, None) if handler is None: self.log.error("UNKNOWN MESSAGE TYPE: %r", msg_type) else: # ensure default_int_handler during handler call sig = signal(SIGINT, default_int_handler) try: handler(stream, idents, msg) except Exception: self.log.error("Exception in message handler:", exc_info=True) finally: signal(SIGINT, sig) def enter_eventloop(self): """enter eventloop""" self.log.info("entering eventloop") # restore default_int_handler signal(SIGINT, default_int_handler) while self.eventloop is not None: try: self.eventloop(self) except KeyboardInterrupt: # Ctrl-C shouldn't crash the kernel self.log.error("KeyboardInterrupt caught in kernel") continue else: # eventloop exited cleanly, this means we should stop (right?) self.eventloop = None break self.log.info("exiting eventloop") def start(self): """register dispatchers for streams""" self.shell.exit_now = False if self.control_stream: self.control_stream.on_recv(self.dispatch_control, copy=False) def make_dispatcher(stream): def dispatcher(msg): return self.dispatch_shell(stream, msg) return dispatcher for s in self.shell_streams: s.on_recv(make_dispatcher(s), copy=False) # publish idle status self._publish_status('starting') def do_one_iteration(self): """step eventloop just once""" if self.control_stream: self.control_stream.flush() for stream in self.shell_streams: # handle at most one request per iteration stream.flush(zmq.POLLIN, 1) stream.flush(zmq.POLLOUT) def record_ports(self, ports): """Record the ports that this kernel is using. The creator of the Kernel instance must call this methods if they want the :meth:`connect_request` method to return the port numbers. """ self._recorded_ports = ports #--------------------------------------------------------------------------- # Kernel request handlers #--------------------------------------------------------------------------- def _make_metadata(self, other=None): """init metadata dict, for execute/apply_reply""" new_md = { 'dependencies_met': True, 'engine': self.ident, 'started': datetime.now(), } if other: new_md.update(other) return new_md def _publish_pyin(self, code, parent, execution_count): """Publish the code request on the pyin stream.""" self.session.send(self.iopub_socket, u'pyin', { u'code': code, u'execution_count': execution_count }, parent=parent, ident=self._topic('pyin')) def _publish_status(self, status, parent=None): """send status (busy/idle) on IOPub""" self.session.send( self.iopub_socket, u'status', {u'execution_state': status}, parent=parent, ident=self._topic('status'), ) def execute_request(self, stream, ident, parent): """handle an execute_request""" self._publish_status(u'busy', parent) try: content = parent[u'content'] code = py3compat.cast_unicode_py2(content[u'code']) silent = content[u'silent'] store_history = content.get(u'store_history', not silent) except: self.log.error("Got bad msg: ") self.log.error("%s", parent) return md = self._make_metadata(parent['metadata']) shell = self.shell # we'll need this a lot here # Replace raw_input. Note that is not sufficient to replace # raw_input in the user namespace. if content.get('allow_stdin', False): raw_input = lambda prompt='': self._raw_input( prompt, ident, parent) input = lambda prompt='': eval(raw_input(prompt)) else: raw_input = input = lambda prompt='': self._no_raw_input() if py3compat.PY3: self._sys_raw_input = builtin_mod.input builtin_mod.input = raw_input else: self._sys_raw_input = builtin_mod.raw_input self._sys_eval_input = builtin_mod.input builtin_mod.raw_input = raw_input builtin_mod.input = input # Set the parent message of the display hook and out streams. shell.set_parent(parent) # Re-broadcast our input for the benefit of listening clients, and # start computing output if not silent: self._publish_pyin(code, parent, shell.execution_count) reply_content = {} try: # FIXME: the shell calls the exception handler itself. shell.run_cell(code, store_history=store_history, silent=silent) except: status = u'error' # FIXME: this code right now isn't being used yet by default, # because the run_cell() call above directly fires off exception # reporting. This code, therefore, is only active in the scenario # where runlines itself has an unhandled exception. We need to # uniformize this, for all exception construction to come from a # single location in the codbase. etype, evalue, tb = sys.exc_info() tb_list = traceback.format_exception(etype, evalue, tb) reply_content.update(shell._showtraceback(etype, evalue, tb_list)) else: status = u'ok' finally: # Restore raw_input. if py3compat.PY3: builtin_mod.input = self._sys_raw_input else: builtin_mod.raw_input = self._sys_raw_input builtin_mod.input = self._sys_eval_input reply_content[u'status'] = status # Return the execution counter so clients can display prompts reply_content['execution_count'] = shell.execution_count - 1 # FIXME - fish exception info out of shell, possibly left there by # runlines. We'll need to clean up this logic later. if shell._reply_content is not None: reply_content.update(shell._reply_content) e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='execute') reply_content['engine_info'] = e_info # reset after use shell._reply_content = None if 'traceback' in reply_content: self.log.info("Exception in execute request:\n%s", '\n'.join(reply_content['traceback'])) # At this point, we can tell whether the main code execution succeeded # or not. If it did, we proceed to evaluate user_variables/expressions if reply_content['status'] == 'ok': reply_content[u'user_variables'] = \ shell.user_variables(content.get(u'user_variables', [])) reply_content[u'user_expressions'] = \ shell.user_expressions(content.get(u'user_expressions', {})) else: # If there was an error, don't even try to compute variables or # expressions reply_content[u'user_variables'] = {} reply_content[u'user_expressions'] = {} # Payloads should be retrieved regardless of outcome, so we can both # recover partial output (that could have been generated early in a # block, before an error) and clear the payload system always. reply_content[u'payload'] = shell.payload_manager.read_payload() # Be agressive about clearing the payload because we don't want # it to sit in memory until the next execute_request comes in. shell.payload_manager.clear_payload() # Flush output before sending the reply. sys.stdout.flush() sys.stderr.flush() # FIXME: on rare occasions, the flush doesn't seem to make it to the # clients... This seems to mitigate the problem, but we definitely need # to better understand what's going on. if self._execute_sleep: time.sleep(self._execute_sleep) # Send the reply. reply_content = json_clean(reply_content) md['status'] = reply_content['status'] if reply_content['status'] == 'error' and \ reply_content['ename'] == 'UnmetDependency': md['dependencies_met'] = False reply_msg = self.session.send(stream, u'execute_reply', reply_content, parent, metadata=md, ident=ident) self.log.debug("%s", reply_msg) if not silent and reply_msg['content']['status'] == u'error': self._abort_queues() self._publish_status(u'idle', parent) def complete_request(self, stream, ident, parent): txt, matches = self._complete(parent) matches = {'matches': matches, 'matched_text': txt, 'status': 'ok'} matches = json_clean(matches) completion_msg = self.session.send(stream, 'complete_reply', matches, parent, ident) self.log.debug("%s", completion_msg) def object_info_request(self, stream, ident, parent): content = parent['content'] object_info = self.shell.object_inspect(content['oname'], detail_level=content.get( 'detail_level', 0)) # Before we send this object over, we scrub it for JSON usage oinfo = json_clean(object_info) msg = self.session.send(stream, 'object_info_reply', oinfo, parent, ident) self.log.debug("%s", msg) def history_request(self, stream, ident, parent): # We need to pull these out, as passing **kwargs doesn't work with # unicode keys before Python 2.6.5. hist_access_type = parent['content']['hist_access_type'] raw = parent['content']['raw'] output = parent['content']['output'] if hist_access_type == 'tail': n = parent['content']['n'] hist = self.shell.history_manager.get_tail(n, raw=raw, output=output, include_latest=True) elif hist_access_type == 'range': session = parent['content']['session'] start = parent['content']['start'] stop = parent['content']['stop'] hist = self.shell.history_manager.get_range(session, start, stop, raw=raw, output=output) elif hist_access_type == 'search': n = parent['content'].get('n') unique = parent['content'].get('unique', False) pattern = parent['content']['pattern'] hist = self.shell.history_manager.search(pattern, raw=raw, output=output, n=n, unique=unique) else: hist = [] hist = list(hist) content = {'history': hist} content = json_clean(content) msg = self.session.send(stream, 'history_reply', content, parent, ident) self.log.debug("Sending history reply with %i entries", len(hist)) def connect_request(self, stream, ident, parent): if self._recorded_ports is not None: content = self._recorded_ports.copy() else: content = {} msg = self.session.send(stream, 'connect_reply', content, parent, ident) self.log.debug("%s", msg) def kernel_info_request(self, stream, ident, parent): vinfo = { 'protocol_version': protocol_version, 'ipython_version': ipython_version, 'language_version': language_version, 'language': 'python', } msg = self.session.send(stream, 'kernel_info_reply', vinfo, parent, ident) self.log.debug("%s", msg) def shutdown_request(self, stream, ident, parent): self.shell.exit_now = True content = dict(status='ok') content.update(parent['content']) self.session.send(stream, u'shutdown_reply', content, parent, ident=ident) # same content, but different msg_id for broadcasting on IOPub self._shutdown_message = self.session.msg(u'shutdown_reply', content, parent) self._at_shutdown() # call sys.exit after a short delay loop = ioloop.IOLoop.instance() loop.add_timeout(time.time() + 0.1, loop.stop) #--------------------------------------------------------------------------- # Engine methods #--------------------------------------------------------------------------- def apply_request(self, stream, ident, parent): try: content = parent[u'content'] bufs = parent[u'buffers'] msg_id = parent['header']['msg_id'] except: self.log.error("Got bad msg: %s", parent, exc_info=True) return self._publish_status(u'busy', parent) # Set the parent message of the display hook and out streams. shell = self.shell shell.set_parent(parent) # pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent) # self.iopub_socket.send(pyin_msg) # self.session.send(self.iopub_socket, u'pyin', {u'code':code},parent=parent) md = self._make_metadata(parent['metadata']) try: working = shell.user_ns prefix = "_" + str(msg_id).replace("-", "") + "_" f, args, kwargs = unpack_apply_message(bufs, working, copy=False) fname = getattr(f, '__name__', 'f') fname = prefix + "f" argname = prefix + "args" kwargname = prefix + "kwargs" resultname = prefix + "result" ns = {fname: f, argname: args, kwargname: kwargs, resultname: None} # print ns working.update(ns) code = "%s = %s(*%s,**%s)" % (resultname, fname, argname, kwargname) try: exec(code, shell.user_global_ns, shell.user_ns) result = working.get(resultname) finally: for key in ns: working.pop(key) result_buf = serialize_object( result, buffer_threshold=self.session.buffer_threshold, item_threshold=self.session.item_threshold, ) except: # invoke IPython traceback formatting shell.showtraceback() # FIXME - fish exception info out of shell, possibly left there by # run_code. We'll need to clean up this logic later. reply_content = {} if shell._reply_content is not None: reply_content.update(shell._reply_content) e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='apply') reply_content['engine_info'] = e_info # reset after use shell._reply_content = None self.session.send(self.iopub_socket, u'pyerr', reply_content, parent=parent, ident=self._topic('pyerr')) self.log.info("Exception in apply request:\n%s", '\n'.join(reply_content['traceback'])) result_buf = [] if reply_content['ename'] == 'UnmetDependency': md['dependencies_met'] = False else: reply_content = {'status': 'ok'} # put 'ok'/'error' status in header, for scheduler introspection: md['status'] = reply_content['status'] # flush i/o sys.stdout.flush() sys.stderr.flush() reply_msg = self.session.send(stream, u'apply_reply', reply_content, parent=parent, ident=ident, buffers=result_buf, metadata=md) self._publish_status(u'idle', parent) #--------------------------------------------------------------------------- # Control messages #--------------------------------------------------------------------------- def abort_request(self, stream, ident, parent): """abort a specifig msg by id""" msg_ids = parent['content'].get('msg_ids', None) if isinstance(msg_ids, string_types): msg_ids = [msg_ids] if not msg_ids: self.abort_queues() for mid in msg_ids: self.aborted.add(str(mid)) content = dict(status='ok') reply_msg = self.session.send(stream, 'abort_reply', content=content, parent=parent, ident=ident) self.log.debug("%s", reply_msg) def clear_request(self, stream, idents, parent): """Clear our namespace.""" self.shell.reset(False) msg = self.session.send(stream, 'clear_reply', ident=idents, parent=parent, content=dict(status='ok')) #--------------------------------------------------------------------------- # Protected interface #--------------------------------------------------------------------------- def _wrap_exception(self, method=None): # import here, because _wrap_exception is only used in parallel, # and parallel has higher min pyzmq version from IPython.parallel.error import wrap_exception e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method=method) content = wrap_exception(e_info) return content def _topic(self, topic): """prefixed topic for IOPub messages""" if self.int_id >= 0: base = "engine.%i" % self.int_id else: base = "kernel.%s" % self.ident return py3compat.cast_bytes("%s.%s" % (base, topic)) def _abort_queues(self): for stream in self.shell_streams: if stream: self._abort_queue(stream) def _abort_queue(self, stream): poller = zmq.Poller() poller.register(stream.socket, zmq.POLLIN) while True: idents, msg = self.session.recv(stream, zmq.NOBLOCK, content=True) if msg is None: return self.log.info("Aborting:") self.log.info("%s", msg) msg_type = msg['header']['msg_type'] reply_type = msg_type.split('_')[0] + '_reply' status = {'status': 'aborted'} md = {'engine': self.ident} md.update(status) reply_msg = self.session.send(stream, reply_type, metadata=md, content=status, parent=msg, ident=idents) self.log.debug("%s", reply_msg) # We need to wait a bit for requests to come in. This can probably # be set shorter for true asynchronous clients. poller.poll(50) def _no_raw_input(self): """Raise StdinNotImplentedError if active frontend doesn't support stdin.""" raise StdinNotImplementedError("raw_input was called, but this " "frontend does not support stdin.") def _raw_input(self, prompt, ident, parent): # Flush output before making the request. sys.stderr.flush() sys.stdout.flush() # flush the stdin socket, to purge stale replies while True: try: self.stdin_socket.recv_multipart(zmq.NOBLOCK) except zmq.ZMQError as e: if e.errno == zmq.EAGAIN: break else: raise # Send the input request. content = json_clean(dict(prompt=prompt)) self.session.send(self.stdin_socket, u'input_request', content, parent, ident=ident) # Await a response. while True: try: ident, reply = self.session.recv(self.stdin_socket, 0) except Exception: self.log.warn("Invalid Message:", exc_info=True) except KeyboardInterrupt: # re-raise KeyboardInterrupt, to truncate traceback raise KeyboardInterrupt else: break try: value = py3compat.unicode_to_str(reply['content']['value']) except: self.log.error("Got bad raw_input reply: ") self.log.error("%s", parent) value = '' if value == '\x04': # EOF raise EOFError return value def _complete(self, msg): c = msg['content'] try: cpos = int(c['cursor_pos']) except: # If we don't get something that we can convert to an integer, at # least attempt the completion guessing the cursor is at the end of # the text, if there's any, and otherwise of the line cpos = len(c['text']) if cpos == 0: cpos = len(c['line']) return self.shell.complete(c['text'], c['line'], cpos) def _at_shutdown(self): """Actions taken at shutdown by the kernel, called by python's atexit. """ # io.rprint("Kernel at_shutdown") # dbg if self._shutdown_message is not None: self.session.send(self.iopub_socket, self._shutdown_message, ident=self._topic('shutdown')) self.log.debug("%s", self._shutdown_message) [s.flush(zmq.POLLOUT) for s in self.shell_streams]
class ScriptMagics(Magics): """Magics for talking to scripts This defines a base `%%script` cell magic for running a cell with a program in a subprocess, and registers a few top-level magics that call %%script with common interpreters. """ script_magics = List( config=True, help="""Extra script cell magics to define This generates simple wrappers of `%%script foo` as `%%foo`. If you want to add script magics that aren't on your path, specify them in script_paths """, ) def _script_magics_default(self): """default to a common list of programs""" defaults = [ 'sh', 'bash', 'perl', 'ruby', 'python', 'python2', 'python3', 'pypy', ] if os.name == 'nt': defaults.extend([ 'cmd', 'powershell', ]) return defaults script_paths = Dict( config=True, help= """Dict mapping short 'ruby' names to full paths, such as '/opt/secret/bin/ruby' Only necessary for items in script_magics where the default path will not find the right interpreter. """) def __init__(self, shell=None): super(ScriptMagics, self).__init__(shell=shell) self._generate_script_magics() self.job_manager = BackgroundJobManager() self.bg_processes = [] atexit.register(self.kill_bg_processes) def __del__(self): self.kill_bg_processes() def _generate_script_magics(self): cell_magics = self.magics['cell'] for name in self.script_magics: cell_magics[name] = self._make_script_magic(name) def _make_script_magic(self, name): """make a named magic, that calls %%script with a particular program""" # expand to explicit path if necessary: script = self.script_paths.get(name, name) @magic_arguments.magic_arguments() @script_args def named_script_magic(line, cell): # if line, add it as cl-flags if line: line = "%s %s" % (script, line) else: line = script return self.shebang(line, cell) # write a basic docstring: named_script_magic.__doc__ = \ """%%{name} script magic Run cells with {script} in a subprocess. This is a shortcut for `%%script {script}` """.format(**locals()) return named_script_magic @magic_arguments.magic_arguments() @script_args @cell_magic("script") def shebang(self, line, cell): """Run a cell via a shell command The `%%script` line is like the #! line of script, specifying a program (bash, perl, ruby, etc.) with which to run. The rest of the cell is run by that program. Examples -------- :: In [1]: %%script bash ...: for i in 1 2 3; do ...: echo $i ...: done 1 2 3 """ argv = arg_split(line, posix=not sys.platform.startswith('win')) args, cmd = self.shebang.parser.parse_known_args(argv) try: p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE) except OSError as e: if e.errno == errno.ENOENT: print("Couldn't find program: %r" % cmd[0]) return else: raise cell = cell.encode('utf8', 'replace') if args.bg: self.bg_processes.append(p) self._gc_bg_processes() if args.out: self.shell.user_ns[args.out] = p.stdout if args.err: self.shell.user_ns[args.err] = p.stderr self.job_manager.new(self._run_script, p, cell, daemon=True) if args.proc: self.shell.user_ns[args.proc] = p return try: out, err = p.communicate(cell) except KeyboardInterrupt: try: p.send_signal(signal.SIGINT) time.sleep(0.1) if p.poll() is not None: print("Process is interrupted.") return p.terminate() time.sleep(0.1) if p.poll() is not None: print("Process is terminated.") return p.kill() print("Process is killed.") except OSError: pass except Exception as e: print("Error while terminating subprocess (pid=%i): %s" \ % (p.pid, e)) return out = py3compat.bytes_to_str(out) err = py3compat.bytes_to_str(err) if args.out: self.shell.user_ns[args.out] = out else: sys.stdout.write(out) sys.stdout.flush() if args.err: self.shell.user_ns[args.err] = err else: sys.stderr.write(err) sys.stderr.flush() def _run_script(self, p, cell): """callback for running the script in the background""" p.stdin.write(cell) p.stdin.close() p.wait() @line_magic("killbgscripts") def killbgscripts(self, _nouse_=''): """Kill all BG processes started by %%script and its family.""" self.kill_bg_processes() print("All background processes were killed.") def kill_bg_processes(self): """Kill all BG processes which are still running.""" for p in self.bg_processes: if p.poll() is None: try: p.send_signal(signal.SIGINT) except: pass time.sleep(0.1) for p in self.bg_processes: if p.poll() is None: try: p.terminate() except: pass time.sleep(0.1) for p in self.bg_processes: if p.poll() is None: try: p.kill() except: pass self._gc_bg_processes() def _gc_bg_processes(self): self.bg_processes = [p for p in self.bg_processes if p.poll() is None]
class MultiKernelManager(LoggingConfigurable): """A class for managing multiple kernels.""" ipython_kernel_argv = List(Unicode) default_kernel_name = Unicode( NATIVE_KERNEL_NAME, config=True, help="The name of the default kernel to start") kernel_manager_class = DottedObjectName( "IPython.kernel.ioloop.IOLoopKernelManager", config=True, help="""The kernel manager class. This is configurable to allow subclassing of the KernelManager for customized behavior. """) def _kernel_manager_class_changed(self, name, old, new): self.kernel_manager_factory = import_item(new) kernel_manager_factory = Any( help="this is kernel_manager_class after import") def _kernel_manager_factory_default(self): return import_item(self.kernel_manager_class) context = Instance('zmq.Context') def _context_default(self): return zmq.Context.instance() connection_dir = Unicode('') _kernels = Dict() def list_kernel_ids(self): """Return a list of the kernel ids of the active kernels.""" # Create a copy so we can iterate over kernels in operations # that delete keys. return list(self._kernels.keys()) def __len__(self): """Return the number of running kernels.""" return len(self.list_kernel_ids()) def __contains__(self, kernel_id): return kernel_id in self._kernels def start_kernel(self, kernel_name=None, **kwargs): """Start a new kernel. The caller can pick a kernel_id by passing one in as a keyword arg, otherwise one will be picked using a uuid. To silence the kernel's stdout/stderr, call this using:: km.start_kernel(stdout=PIPE, stderr=PIPE) """ kernel_id = kwargs.pop('kernel_id', unicode_type(uuid.uuid4())) if kernel_id in self: raise DuplicateKernelError('Kernel already exists: %s' % kernel_id) if kernel_name is None: kernel_name = self.default_kernel_name # kernel_manager_factory is the constructor for the KernelManager # subclass we are using. It can be configured as any Configurable, # including things like its transport and ip. km = self.kernel_manager_factory( connection_file=os.path.join(self.connection_dir, "kernel-%s.json" % kernel_id), parent=self, autorestart=True, log=self.log, kernel_name=kernel_name, ) # FIXME: remove special treatment of IPython kernels if km.ipython_kernel: kwargs.setdefault('extra_arguments', self.ipython_kernel_argv) km.start_kernel(**kwargs) self._kernels[kernel_id] = km return kernel_id @kernel_method def shutdown_kernel(self, kernel_id, now=False, restart=False): """Shutdown a kernel by its kernel uuid. Parameters ========== kernel_id : uuid The id of the kernel to shutdown. now : bool Should the kernel be shutdown forcibly using a signal. restart : bool Will the kernel be restarted? """ self.log.info("Kernel shutdown: %s" % kernel_id) self.remove_kernel(kernel_id) @kernel_method def request_shutdown(self, kernel_id, restart=False): """Ask a kernel to shut down by its kernel uuid""" @kernel_method def finish_shutdown(self, kernel_id, waittime=1, pollinterval=0.1): """Wait for a kernel to finish shutting down, and kill it if it doesn't """ self.log.info("Kernel shutdown: %s" % kernel_id) @kernel_method def cleanup(self, kernel_id, connection_file=True): """Clean up a kernel's resources""" def remove_kernel(self, kernel_id): """remove a kernel from our mapping. Mainly so that a kernel can be removed if it is already dead, without having to call shutdown_kernel. The kernel object is returned. """ return self._kernels.pop(kernel_id) def shutdown_all(self, now=False): """Shutdown all kernels.""" kids = self.list_kernel_ids() for kid in kids: self.request_shutdown(kid) for kid in kids: self.finish_shutdown(kid) self.cleanup(kid) self.remove_kernel(kid) @kernel_method def interrupt_kernel(self, kernel_id): """Interrupt (SIGINT) the kernel by its uuid. Parameters ========== kernel_id : uuid The id of the kernel to interrupt. """ self.log.info("Kernel interrupted: %s" % kernel_id) @kernel_method def signal_kernel(self, kernel_id, signum): """Sends a signal to the kernel by its uuid. Note that since only SIGTERM is supported on Windows, this function is only useful on Unix systems. Parameters ========== kernel_id : uuid The id of the kernel to signal. """ self.log.info("Signaled Kernel %s with %s" % (kernel_id, signum)) @kernel_method def restart_kernel(self, kernel_id, now=False): """Restart a kernel by its uuid, keeping the same ports. Parameters ========== kernel_id : uuid The id of the kernel to interrupt. """ self.log.info("Kernel restarted: %s" % kernel_id) @kernel_method def is_alive(self, kernel_id): """Is the kernel alive. This calls KernelManager.is_alive() which calls Popen.poll on the actual kernel subprocess. Parameters ========== kernel_id : uuid The id of the kernel. """ def _check_kernel_id(self, kernel_id): """check that a kernel id is valid""" if kernel_id not in self: raise KeyError("Kernel with id not found: %s" % kernel_id) def get_kernel(self, kernel_id): """Get the single KernelManager object for a kernel by its uuid. Parameters ========== kernel_id : uuid The id of the kernel. """ self._check_kernel_id(kernel_id) return self._kernels[kernel_id] @kernel_method def add_restart_callback(self, kernel_id, callback, event='restart'): """add a callback for the KernelRestarter""" @kernel_method def remove_restart_callback(self, kernel_id, callback, event='restart'): """remove a callback for the KernelRestarter""" @kernel_method def get_connection_info(self, kernel_id): """Return a dictionary of connection data for a kernel. Parameters ========== kernel_id : uuid The id of the kernel. Returns ======= connection_dict : dict A dict of the information needed to connect to a kernel. This includes the ip address and the integer port numbers of the different channels (stdin_port, iopub_port, shell_port, hb_port). """ @kernel_method def connect_iopub(self, kernel_id, identity=None): """Return a zmq Socket connected to the iopub channel. Parameters ========== kernel_id : uuid The id of the kernel identity : bytes (optional) The zmq identity of the socket Returns ======= stream : zmq Socket or ZMQStream """ @kernel_method def connect_shell(self, kernel_id, identity=None): """Return a zmq Socket connected to the shell channel. Parameters ========== kernel_id : uuid The id of the kernel identity : bytes (optional) The zmq identity of the socket Returns ======= stream : zmq Socket or ZMQStream """ @kernel_method def connect_stdin(self, kernel_id, identity=None): """Return a zmq Socket connected to the stdin channel. Parameters ========== kernel_id : uuid The id of the kernel identity : bytes (optional) The zmq identity of the socket Returns ======= stream : zmq Socket or ZMQStream """ @kernel_method def connect_hb(self, kernel_id, identity=None): """Return a zmq Socket connected to the hb channel.
class EngineFactory(RegistrationFactory): """IPython engine""" # configurables: out_stream_factory = Type('IPython.zmq.iostream.OutStream', config=True, help="""The OutStream for handling stdout/err. Typically 'IPython.zmq.iostream.OutStream'""") display_hook_factory = Type('IPython.zmq.displayhook.ZMQDisplayHook', config=True, help="""The class for handling displayhook. Typically 'IPython.zmq.displayhook.ZMQDisplayHook'""") location = Unicode( config=True, help="""The location (an IP address) of the controller. This is used for disambiguating URLs, to determine whether loopback should be used to connect or the public address.""") timeout = CFloat( 5, config=True, help="""The time (in seconds) to wait for the Controller to respond to registration requests before giving up.""") sshserver = Unicode( config=True, help= """The SSH server to use for tunneling connections to the Controller.""" ) sshkey = Unicode( config=True, help= """The SSH private key file to use when tunneling connections to the Controller.""" ) paramiko = Bool( sys.platform == 'win32', config=True, help="""Whether to use paramiko instead of openssh for tunnels.""") # not configurable: connection_info = Dict() user_ns = Dict() id = Integer(allow_none=True) registrar = Instance('zmq.eventloop.zmqstream.ZMQStream') kernel = Instance(Kernel) bident = CBytes() ident = Unicode() def _ident_changed(self, name, old, new): self.bident = cast_bytes(new) using_ssh = Bool(False) def __init__(self, **kwargs): super(EngineFactory, self).__init__(**kwargs) self.ident = self.session.session def init_connector(self): """construct connection function, which handles tunnels.""" self.using_ssh = bool(self.sshkey or self.sshserver) if self.sshkey and not self.sshserver: # We are using ssh directly to the controller, tunneling localhost to localhost self.sshserver = self.url.split('://')[1].split(':')[0] if self.using_ssh: if tunnel.try_passwordless_ssh(self.sshserver, self.sshkey, self.paramiko): password = False else: password = getpass("SSH Password for %s: " % self.sshserver) else: password = False def connect(s, url): url = disambiguate_url(url, self.location) if self.using_ssh: self.log.debug("Tunneling connection to %s via %s", url, self.sshserver) return tunnel.tunnel_connection( s, url, self.sshserver, keyfile=self.sshkey, paramiko=self.paramiko, password=password, ) else: return s.connect(url) def maybe_tunnel(url): """like connect, but don't complete the connection (for use by heartbeat)""" url = disambiguate_url(url, self.location) if self.using_ssh: self.log.debug("Tunneling connection to %s via %s", url, self.sshserver) url, tunnelobj = tunnel.open_tunnel( url, self.sshserver, keyfile=self.sshkey, paramiko=self.paramiko, password=password, ) return str(url) return connect, maybe_tunnel def register(self): """send the registration_request""" self.log.info("Registering with controller at %s" % self.url) ctx = self.context connect, maybe_tunnel = self.init_connector() reg = ctx.socket(zmq.DEALER) reg.setsockopt(zmq.IDENTITY, self.bident) connect(reg, self.url) self.registrar = zmqstream.ZMQStream(reg, self.loop) content = dict(uuid=self.ident) self.registrar.on_recv( lambda msg: self.complete_registration(msg, connect, maybe_tunnel)) # print (self.session.key) self.session.send(self.registrar, "registration_request", content=content) def complete_registration(self, msg, connect, maybe_tunnel): # print msg self._abort_dc.stop() ctx = self.context loop = self.loop identity = self.bident idents, msg = self.session.feed_identities(msg) msg = self.session.unserialize(msg) content = msg['content'] info = self.connection_info def url(key): """get zmq url for given channel""" return str(info["interface"] + ":%i" % info[key]) if content['status'] == 'ok': self.id = int(content['id']) # launch heartbeat # possibly forward hb ports with tunnels hb_ping = maybe_tunnel(url('hb_ping')) hb_pong = maybe_tunnel(url('hb_pong')) heart = Heart(hb_ping, hb_pong, heart_id=identity) heart.start() # create Shell Connections (MUX, Task, etc.): shell_addrs = url('mux'), url('task') # Use only one shell stream for mux and tasks stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) stream.setsockopt(zmq.IDENTITY, identity) shell_streams = [stream] for addr in shell_addrs: connect(stream, addr) # control stream: control_addr = url('control') control_stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) control_stream.setsockopt(zmq.IDENTITY, identity) connect(control_stream, control_addr) # create iopub stream: iopub_addr = url('iopub') iopub_socket = ctx.socket(zmq.PUB) iopub_socket.setsockopt(zmq.IDENTITY, identity) connect(iopub_socket, iopub_addr) # disable history: self.config.HistoryManager.hist_file = ':memory:' # Redirect input streams and set a display hook. if self.out_stream_factory: sys.stdout = self.out_stream_factory(self.session, iopub_socket, u'stdout') sys.stdout.topic = cast_bytes('engine.%i.stdout' % self.id) sys.stderr = self.out_stream_factory(self.session, iopub_socket, u'stderr') sys.stderr.topic = cast_bytes('engine.%i.stderr' % self.id) if self.display_hook_factory: sys.displayhook = self.display_hook_factory( self.session, iopub_socket) sys.displayhook.topic = cast_bytes('engine.%i.pyout' % self.id) self.kernel = Kernel(config=self.config, int_id=self.id, ident=self.ident, session=self.session, control_stream=control_stream, shell_streams=shell_streams, iopub_socket=iopub_socket, loop=loop, user_ns=self.user_ns, log=self.log) self.kernel.shell.display_pub.topic = cast_bytes( 'engine.%i.displaypub' % self.id) # FIXME: This is a hack until IPKernelApp and IPEngineApp can be fully merged app = IPKernelApp(config=self.config, shell=self.kernel.shell, kernel=self.kernel, log=self.log) app.init_profile_dir() app.init_code() self.kernel.start() else: self.log.fatal("Registration Failed: %s" % msg) raise Exception("Registration Failed: %s" % msg) self.log.info("Completed registration with id %i" % self.id) def abort(self): self.log.fatal("Registration timed out after %.1f seconds" % self.timeout) if self.url.startswith('127.'): self.log.fatal(""" If the controller and engines are not on the same machine, you will have to instruct the controller to listen on an external IP (in ipcontroller_config.py): c.HubFactory.ip='*' # for all interfaces, internal and external c.HubFactory.ip='192.168.1.101' # or any interface that the engines can see or tunnel connections via ssh. """) self.session.send(self.registrar, "unregistration_request", content=dict(id=self.id)) time.sleep(1) sys.exit(255) def start(self): dc = ioloop.DelayedCallback(self.register, 0, self.loop) dc.start() self._abort_dc = ioloop.DelayedCallback(self.abort, self.timeout * 1000, self.loop) self._abort_dc.start()
class _Selection(DOMWidget): """Base class for Selection widgets ``values`` can be specified as a list or dict. If given as a list, it will be transformed to a dict of the form ``{str(value):value}``. """ value = Any(help="Selected value") values = Dict(help="""Dictionary of {name: value} the user can select. The keys of this dictionary are the strings that will be displayed in the UI, representing the actual Python choices. The keys of this dictionary are also available as value_names. """) value_name = Unicode(help="The name of the selected value", sync=True) value_names = List(Unicode, help="""Read-only list of names for each value. If values is specified as a list, this is the string representation of each element. Otherwise, it is the keys of the values dictionary. These strings are used to display the choices in the front-end.""", sync=True) disabled = Bool(False, help="Enable or disable user changes", sync=True) description = Unicode(help="Description of the value this widget represents", sync=True) def __init__(self, *args, **kwargs): self.value_lock = Lock() self._in_values_changed = False if 'values' in kwargs: values = kwargs['values'] # convert list values to an dict of {str(v):v} if isinstance(values, list): # preserve list order with an OrderedDict kwargs['values'] = OrderedDict((unicode_type(v), v) for v in values) # python3.3 turned on hash randomization by default - this means that sometimes, randomly # we try to set value before setting values, due to dictionary ordering. To fix this, force # the setting of self.values right now, before anything else runs self.values = kwargs.pop('values') DOMWidget.__init__(self, *args, **kwargs) self._value_in_values() def _values_changed(self, name, old, new): """Handles when the values dict has been changed. Setting values implies setting value names from the keys of the dict. """ self._in_values_changed = True try: self.value_names = list(new.keys()) finally: self._in_values_changed = False self._value_in_values() def _value_in_values(self): # ensure that the chosen value is one of the choices if self.values: if self.value not in self.values.values(): self.value = next(iter(self.values.values())) def _value_names_changed(self, name, old, new): if not self._in_values_changed: raise TraitError("value_names is a read-only proxy to values.keys(). Use the values dict instead.") def _value_changed(self, name, old, new): """Called when value has been changed""" if self.value_lock.acquire(False): try: # Reverse dictionary lookup for the value name for k,v in self.values.items(): if new == v: # set the selected value name self.value_name = k return # undo the change, and raise KeyError self.value = old raise KeyError(new) finally: self.value_lock.release() def _value_name_changed(self, name, old, new): """Called when the value name has been changed (typically by the frontend).""" if self.value_lock.acquire(False): try: self.value = self.values[new] finally: self.value_lock.release()
class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp): name = u'ipython' description = usage.cl_usage crash_handler_class = IPAppCrashHandler examples = _examples flags = Dict(flags) aliases = Dict(aliases) classes = List() def _classes_default(self): """This has to be in a method, for TerminalIPythonApp to be available.""" return [ InteractiveShellApp, # ShellApp comes before TerminalApp, because self.__class__, # it will also affect subclasses (e.g. QtConsole) TerminalInteractiveShell, PromptManager, HistoryManager, ProfileDir, PlainTextFormatter, IPCompleter, ScriptMagics, StoreMagics, ] subcommands = dict( qtconsole=('IPython.qt.console.qtconsoleapp.IPythonQtConsoleApp', """Launch the IPython Qt Console."""), notebook=('IPython.html.notebookapp.NotebookApp', """Launch the IPython HTML Notebook Server."""), profile=("IPython.core.profileapp.ProfileApp", "Create and manage IPython profiles."), kernel=("IPython.kernel.zmq.kernelapp.IPKernelApp", "Start a kernel without an attached frontend."), console=('IPython.terminal.console.app.ZMQTerminalIPythonApp', """Launch the IPython terminal-based Console."""), locate=('IPython.terminal.ipapp.LocateIPythonApp', LocateIPythonApp.description), history=('IPython.core.historyapp.HistoryApp', "Manage the IPython history database."), nbconvert=('IPython.nbconvert.nbconvertapp.NbConvertApp', "Convert notebooks to/from other formats."), trust=( 'IPython.nbformat.sign.TrustNotebookApp', "Sign notebooks to trust their potentially unsafe contents at load." ), kernelspec=('IPython.kernel.kernelspecapp.KernelSpecApp', "Manage IPython kernel specifications."), ) subcommands['install-nbextension'] = ( "IPython.html.nbextensions.NBExtensionApp", "Install IPython notebook extension files") # *do* autocreate requested profile, but don't create the config file. auto_create = Bool(True) # configurables ignore_old_config = Bool( False, config=True, help="Suppress warning messages about legacy config files") quick = Bool( False, config=True, help= """Start IPython quickly by skipping the loading of config files.""") def _quick_changed(self, name, old, new): if new: self.load_config_file = lambda *a, **kw: None self.ignore_old_config = True display_banner = Bool( True, config=True, help="Whether to display a banner upon starting IPython.") # if there is code of files to run from the cmd line, don't interact # unless the --i flag (App.force_interact) is true. force_interact = Bool( False, config=True, help="""If a command or file is given via the command-line, e.g. 'ipython foo.py', start an interactive shell after executing the file or command.""") def _force_interact_changed(self, name, old, new): if new: self.interact = True def _file_to_run_changed(self, name, old, new): if new: self.something_to_run = True if new and not self.force_interact: self.interact = False _code_to_run_changed = _file_to_run_changed _module_to_run_changed = _file_to_run_changed # internal, not-configurable interact = Bool(True) something_to_run = Bool(False) def parse_command_line(self, argv=None): """override to allow old '-pylab' flag with deprecation warning""" argv = sys.argv[1:] if argv is None else argv if '-pylab' in argv: # deprecated `-pylab` given, # warn and transform into current syntax argv = argv[:] # copy, don't clobber idx = argv.index('-pylab') warn.warn( "`-pylab` flag has been deprecated.\n" " Use `--matplotlib <backend>` and import pylab manually.") argv[idx] = '--pylab' return super(TerminalIPythonApp, self).parse_command_line(argv) @catch_config_error def initialize(self, argv=None): """Do actions after construct, but before starting the app.""" super(TerminalIPythonApp, self).initialize(argv) if self.subapp is not None: # don't bother initializing further, starting subapp return if not self.ignore_old_config: check_for_old_config(self.ipython_dir) # print self.extra_args if self.extra_args and not self.something_to_run: self.file_to_run = self.extra_args[0] self.init_path() # create the shell self.init_shell() # and draw the banner self.init_banner() # Now a variety of things that happen after the banner is printed. self.init_gui_pylab() self.init_extensions() self.init_code() def init_shell(self): """initialize the InteractiveShell instance""" # Create an InteractiveShell instance. # shell.display_banner should always be False for the terminal # based app, because we call shell.show_banner() by hand below # so the banner shows *before* all extension loading stuff. self.shell = TerminalInteractiveShell.instance( parent=self, display_banner=False, profile_dir=self.profile_dir, ipython_dir=self.ipython_dir, user_ns=self.user_ns) self.shell.configurables.append(self) def init_banner(self): """optionally display the banner""" if self.display_banner and self.interact: self.shell.show_banner() # Make sure there is a space below the banner. if self.log_level <= logging.INFO: print() def _pylab_changed(self, name, old, new): """Replace --pylab='inline' with --pylab='auto'""" if new == 'inline': warn.warn("'inline' not available as pylab backend, " "using 'auto' instead.") self.pylab = 'auto' def start(self): if self.subapp is not None: return self.subapp.start() # perform any prexec steps: if self.interact: self.log.debug("Starting IPython's mainloop...") self.shell.mainloop() else: self.log.debug("IPython not interactive...")
class NotebookApp(BaseIPythonApplication): name = 'ipython-notebook' description = """ The IPython HTML Notebook. This launches a Tornado based HTML Notebook Server that serves up an HTML5/Javascript Notebook client. """ examples = _examples classes = IPythonConsoleApp.classes + [ MappingKernelManager, NotebookManager, FileNotebookManager ] flags = Dict(flags) aliases = Dict(aliases) kernel_argv = List(Unicode) def _log_level_default(self): return logging.INFO def _log_format_default(self): """override default log format to include time""" return u"%(asctime)s.%(msecs).03d [%(name)s]%(highlevel)s %(message)s" # create requested profiles by default, if they don't exist: auto_create = Bool(True) # file to be opened in the notebook server file_to_run = Unicode('') # Network related information. ip = Unicode(LOCALHOST, config=True, help="The IP address the notebook server will listen on.") def _ip_changed(self, name, old, new): if new == u'*': self.ip = u'' port = Integer(8888, config=True, help="The port the notebook server will listen on.") port_retries = Integer( 50, config=True, help= "The number of additional ports to try if the specified port is not available." ) certfile = Unicode( u'', config=True, help="""The full path to an SSL/TLS certificate file.""") keyfile = Unicode( u'', config=True, help="""The full path to a private key file for usage with SSL/TLS.""") cookie_secret = Bytes(b'', config=True, help="""The random bytes used to secure cookies. By default this is a new random number every time you start the Notebook. Set it to a value in a config file to enable logins to persist across server sessions. Note: Cookie secrets should be kept private, do not share config files with cookie_secret stored in plaintext (you can read the value from a file). """) def _cookie_secret_default(self): return os.urandom(1024) password = Unicode(u'', config=True, help="""Hashed password to use for web authentication. To generate, type in a python/IPython shell: from IPython.lib import passwd; passwd() The string should be of the form type:salt:hashed-password. """) open_browser = Bool(True, config=True, help="""Whether to open in a browser after starting. The specific browser used is platform dependent and determined by the python standard library `webbrowser` module, unless it is overridden using the --browser (NotebookApp.browser) configuration option. """) browser = Unicode(u'', config=True, help="""Specify what command to use to invoke a web browser when opening the notebook. If not specified, the default browser will be determined by the `webbrowser` standard library module, which allows setting of the BROWSER environment variable to override it. """) use_less = Bool(False, config=True, help="""Wether to use Browser Side less-css parsing instead of compiled css version in templates that allows it. This is mainly convenient when working on the less file to avoid a build step, or if user want to overwrite some of the less variables without having to recompile everything. You will need to install the less.js component in the static directory either in the source tree or in your profile folder. """) webapp_settings = Dict( config=True, help="Supply overrides for the tornado.web.Application that the " "IPython notebook uses.") enable_mathjax = Bool( True, config=True, help="""Whether to enable MathJax for typesetting math/TeX MathJax is the javascript library IPython uses to render math/LaTeX. It is very large, so you may want to disable it if you have a slow internet connection, or for offline use of the notebook. When disabled, equations etc. will appear as their untransformed TeX source. """) def _enable_mathjax_changed(self, name, old, new): """set mathjax url to empty if mathjax is disabled""" if not new: self.mathjax_url = u'' base_project_url = Unicode('/', config=True, help='''The base URL for the notebook server. Leading and trailing slashes can be omitted, and will automatically be added. ''') def _base_project_url_changed(self, name, old, new): if not new.startswith('/'): self.base_project_url = '/' + new elif not new.endswith('/'): self.base_project_url = new + '/' base_kernel_url = Unicode('/', config=True, help='''The base URL for the kernel server Leading and trailing slashes can be omitted, and will automatically be added. ''') def _base_kernel_url_changed(self, name, old, new): if not new.startswith('/'): self.base_kernel_url = '/' + new elif not new.endswith('/'): self.base_kernel_url = new + '/' websocket_url = Unicode("", config=True, help="""The base URL for the websocket server, if it differs from the HTTP server (hint: it almost certainly doesn't). Should be in the form of an HTTP origin: ws[s]://hostname[:port] """) extra_static_paths = List( Unicode, config=True, help="""Extra paths to search for serving static files. This allows adding javascript/css to be available from the notebook server machine, or overriding individual files in the IPython""") def _extra_static_paths_default(self): return [os.path.join(self.profile_dir.location, 'static')] @property def static_file_path(self): """return extra paths + the default location""" return self.extra_static_paths + [DEFAULT_STATIC_FILES_PATH] mathjax_url = Unicode("", config=True, help="""The url for MathJax.js.""") def _mathjax_url_default(self): if not self.enable_mathjax: return u'' static_url_prefix = self.webapp_settings.get( "static_url_prefix", url_path_join(self.base_project_url, "static")) try: mathjax = filefind(os.path.join('mathjax', 'MathJax.js'), self.static_file_path) except IOError: if self.certfile: # HTTPS: load from Rackspace CDN, because SSL certificate requires it base = u"https://c328740.ssl.cf1.rackcdn.com" else: base = u"http://cdn.mathjax.org" url = base + u"/mathjax/latest/MathJax.js" self.log.info("Using MathJax from CDN: %s", url) return url else: self.log.info("Using local MathJax from %s" % mathjax) return url_path_join(static_url_prefix, u"mathjax/MathJax.js") def _mathjax_url_changed(self, name, old, new): if new and not self.enable_mathjax: # enable_mathjax=False overrides mathjax_url self.mathjax_url = u'' else: self.log.info("Using MathJax: %s", new) notebook_manager_class = DottedObjectName( 'IPython.html.services.notebooks.filenbmanager.FileNotebookManager', config=True, help='The notebook manager class to use.') trust_xheaders = Bool( False, config=True, help= ("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers" "sent by the upstream reverse proxy. Neccesary if the proxy handles SSL" )) def parse_command_line(self, argv=None): super(NotebookApp, self).parse_command_line(argv) if self.extra_args: f = os.path.abspath(self.extra_args[0]) if os.path.isdir(f): nbdir = f else: self.file_to_run = f nbdir = os.path.dirname(f) self.config.NotebookManager.notebook_dir = nbdir def init_kernel_argv(self): """construct the kernel arguments""" # Scrub frontend-specific flags self.kernel_argv = swallow_argv(self.argv, notebook_aliases, notebook_flags) # Kernel should inherit default config file from frontend self.kernel_argv.append("--IPKernelApp.parent_appname='%s'" % self.name) # Kernel should get *absolute* path to profile directory self.kernel_argv.extend(["--profile-dir", self.profile_dir.location]) def init_configurables(self): # force Session default to be secure default_secure(self.config) self.kernel_manager = MappingKernelManager( parent=self, log=self.log, kernel_argv=self.kernel_argv, connection_dir=self.profile_dir.security_dir, ) kls = import_item(self.notebook_manager_class) self.notebook_manager = kls(parent=self, log=self.log) self.notebook_manager.load_notebook_names() self.cluster_manager = ClusterManager(parent=self, log=self.log) self.cluster_manager.update_profiles() def init_logging(self): # This prevents double log messages because tornado use a root logger that # self.log is a child of. The logging module dipatches log messages to a log # and all of its ancenstors until propagate is set to False. self.log.propagate = False # hook up tornado 3's loggers to our app handlers for name in ('access', 'application', 'general'): logging.getLogger('tornado.%s' % name).handlers = self.log.handlers def init_webapp(self): """initialize tornado webapp and httpserver""" self.web_app = NotebookWebApplication(self, self.kernel_manager, self.notebook_manager, self.cluster_manager, self.log, self.base_project_url, self.webapp_settings) if self.certfile: ssl_options = dict(certfile=self.certfile) if self.keyfile: ssl_options['keyfile'] = self.keyfile else: ssl_options = None self.web_app.password = self.password self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options, xheaders=self.trust_xheaders) if not self.ip: warning = "WARNING: The notebook server is listening on all IP addresses" if ssl_options is None: self.log.critical(warning + " and not using encryption. This " "is not recommended.") if not self.password: self.log.critical( warning + " and not using authentication. " "This is highly insecure and not recommended.") success = None for port in random_ports(self.port, self.port_retries + 1): try: self.http_server.listen(port, self.ip) except socket.error as e: # XXX: remove the e.errno == -9 block when we require # tornado >= 3.0 if e.errno == -9 and tornado.version_info[0] < 3: # The flags passed to socket.getaddrinfo from # tornado.netutils.bind_sockets can cause "gaierror: # [Errno -9] Address family for hostname not supported" # when the interface is not associated, for example. # Changing the flags to exclude socket.AI_ADDRCONFIG does # not cause this error, but the only way to do this is to # monkeypatch socket to remove the AI_ADDRCONFIG attribute saved_AI_ADDRCONFIG = socket.AI_ADDRCONFIG self.log.warn('Monkeypatching socket to fix tornado bug') del (socket.AI_ADDRCONFIG) try: # retry the tornado call without AI_ADDRCONFIG flags self.http_server.listen(port, self.ip) except socket.error as e2: e = e2 else: self.port = port success = True break # restore the monekypatch socket.AI_ADDRCONFIG = saved_AI_ADDRCONFIG if e.errno != errno.EADDRINUSE: raise self.log.info( 'The port %i is already in use, trying another random port.' % port) else: self.port = port success = True break if not success: self.log.critical( 'ERROR: the notebook server could not be started because ' 'no available port could be found.') self.exit(1) def init_signal(self): if not sys.platform.startswith('win'): signal.signal(signal.SIGINT, self._handle_sigint) signal.signal(signal.SIGTERM, self._signal_stop) if hasattr(signal, 'SIGUSR1'): # Windows doesn't support SIGUSR1 signal.signal(signal.SIGUSR1, self._signal_info) if hasattr(signal, 'SIGINFO'): # only on BSD-based systems signal.signal(signal.SIGINFO, self._signal_info) def _handle_sigint(self, sig, frame): """SIGINT handler spawns confirmation dialog""" # register more forceful signal handler for ^C^C case signal.signal(signal.SIGINT, self._signal_stop) # request confirmation dialog in bg thread, to avoid # blocking the App thread = threading.Thread(target=self._confirm_exit) thread.daemon = True thread.start() def _restore_sigint_handler(self): """callback for restoring original SIGINT handler""" signal.signal(signal.SIGINT, self._handle_sigint) def _confirm_exit(self): """confirm shutdown on ^C A second ^C, or answering 'y' within 5s will cause shutdown, otherwise original SIGINT handler will be restored. This doesn't work on Windows. """ # FIXME: remove this delay when pyzmq dependency is >= 2.1.11 time.sleep(0.1) info = self.log.info info('interrupted') print self.notebook_info() sys.stdout.write("Shutdown this notebook server (y/[n])? ") sys.stdout.flush() r, w, x = select.select([sys.stdin], [], [], 5) if r: line = sys.stdin.readline() if line.lower().startswith('y'): self.log.critical("Shutdown confirmed") ioloop.IOLoop.instance().stop() return else: print "No answer for 5s:", print "resuming operation..." # no answer, or answer is no: # set it back to original SIGINT handler # use IOLoop.add_callback because signal.signal must be called # from main thread ioloop.IOLoop.instance().add_callback(self._restore_sigint_handler) def _signal_stop(self, sig, frame): self.log.critical("received signal %s, stopping", sig) ioloop.IOLoop.instance().stop() def _signal_info(self, sig, frame): print self.notebook_info() def init_components(self): """Check the components submodule, and warn if it's unclean""" status = submodule.check_submodule_status() if status == 'missing': self.log.warn( "components submodule missing, running `git submodule update`") submodule.update_submodules(submodule.ipython_parent()) elif status == 'unclean': self.log.warn( "components submodule unclean, you may see 404s on static/components" ) self.log.warn( "run `setup.py submodule` or `git submodule update` to update") @catch_config_error def initialize(self, argv=None): self.init_logging() super(NotebookApp, self).initialize(argv) self.init_kernel_argv() self.init_configurables() self.init_components() self.init_webapp() self.init_signal() def cleanup_kernels(self): """Shutdown all kernels. The kernels will shutdown themselves when this process no longer exists, but explicit shutdown allows the KernelManagers to cleanup the connection files. """ self.log.info('Shutting down kernels') self.kernel_manager.shutdown_all() def notebook_info(self): "Return the current working directory and the server url information" mgr_info = self.notebook_manager.info_string() + "\n" return mgr_info + "The IPython Notebook is running at: %s" % self._url def start(self): """ Start the IPython Notebook server app, after initialization This method takes no arguments so all configuration and initialization must be done prior to calling this method.""" ip = self.ip if self.ip else '[all ip addresses on your system]' proto = 'https' if self.certfile else 'http' info = self.log.info self._url = "%s://%s:%i%s" % (proto, ip, self.port, self.base_project_url) for line in self.notebook_info().split("\n"): info(line) info( "Use Control-C to stop this server and shut down all kernels (twice to skip confirmation)." ) if self.open_browser or self.file_to_run: ip = self.ip or LOCALHOST try: browser = webbrowser.get(self.browser or None) except webbrowser.Error as e: self.log.warn('No web browser found: %s.' % e) browser = None if self.file_to_run: name, _ = os.path.splitext(os.path.basename(self.file_to_run)) url = self.notebook_manager.rev_mapping.get(name, '') else: url = '' if browser: b = lambda: browser.open("%s://%s:%i%s%s" % ( proto, ip, self.port, self.base_project_url, url), new=2) threading.Thread(target=b).start() try: ioloop.IOLoop.instance().start() except KeyboardInterrupt: info("Interrupted...") finally: self.cleanup_kernels()
class NotebookApp(BaseIPythonApplication): name = 'ipython-notebook' description = """ The IPython HTML Notebook. This launches a Tornado based HTML Notebook Server that serves up an HTML5/Javascript Notebook client. """ examples = _examples classes = IPythonConsoleApp.classes + [MappingKernelManager, NotebookManager, FileNotebookManager] flags = Dict(flags) aliases = Dict(aliases) subcommands = dict( list=(NbserverListApp, NbserverListApp.description.splitlines()[0]), ) kernel_argv = List(Unicode) def _log_level_default(self): return logging.INFO def _log_format_default(self): """override default log format to include time""" return u"%(asctime)s.%(msecs).03d [%(name)s]%(highlevel)s %(message)s" # create requested profiles by default, if they don't exist: auto_create = Bool(True) # file to be opened in the notebook server file_to_run = Unicode('') # Network related information. ip = Unicode(config=True, help="The IP address the notebook server will listen on." ) def _ip_default(self): return localhost() def _ip_changed(self, name, old, new): if new == u'*': self.ip = u'' port = Integer(8888, config=True, help="The port the notebook server will listen on." ) port_retries = Integer(50, config=True, help="The number of additional ports to try if the specified port is not available." ) certfile = Unicode(u'', config=True, help="""The full path to an SSL/TLS certificate file.""" ) keyfile = Unicode(u'', config=True, help="""The full path to a private key file for usage with SSL/TLS.""" ) cookie_secret = Bytes(b'', config=True, help="""The random bytes used to secure cookies. By default this is a new random number every time you start the Notebook. Set it to a value in a config file to enable logins to persist across server sessions. Note: Cookie secrets should be kept private, do not share config files with cookie_secret stored in plaintext (you can read the value from a file). """ ) def _cookie_secret_default(self): return os.urandom(1024) password = Unicode(u'', config=True, help="""Hashed password to use for web authentication. To generate, type in a python/IPython shell: from IPython.lib import passwd; passwd() The string should be of the form type:salt:hashed-password. """ ) open_browser = Bool(True, config=True, help="""Whether to open in a browser after starting. The specific browser used is platform dependent and determined by the python standard library `webbrowser` module, unless it is overridden using the --browser (NotebookApp.browser) configuration option. """) browser = Unicode(u'', config=True, help="""Specify what command to use to invoke a web browser when opening the notebook. If not specified, the default browser will be determined by the `webbrowser` standard library module, which allows setting of the BROWSER environment variable to override it. """) webapp_settings = Dict(config=True, help="Supply overrides for the tornado.web.Application that the " "IPython notebook uses.") enable_mathjax = Bool(True, config=True, help="""Whether to enable MathJax for typesetting math/TeX MathJax is the javascript library IPython uses to render math/LaTeX. It is very large, so you may want to disable it if you have a slow internet connection, or for offline use of the notebook. When disabled, equations etc. will appear as their untransformed TeX source. """ ) def _enable_mathjax_changed(self, name, old, new): """set mathjax url to empty if mathjax is disabled""" if not new: self.mathjax_url = u'' base_project_url = Unicode('/', config=True, help='''The base URL for the notebook server. Leading and trailing slashes can be omitted, and will automatically be added. ''') def _base_project_url_changed(self, name, old, new): if not new.startswith('/'): self.base_project_url = '/'+new elif not new.endswith('/'): self.base_project_url = new+'/' base_kernel_url = Unicode('/', config=True, help='''The base URL for the kernel server Leading and trailing slashes can be omitted, and will automatically be added. ''') def _base_kernel_url_changed(self, name, old, new): if not new.startswith('/'): self.base_kernel_url = '/'+new elif not new.endswith('/'): self.base_kernel_url = new+'/' websocket_url = Unicode("", config=True, help="""The base URL for the websocket server, if it differs from the HTTP server (hint: it almost certainly doesn't). Should be in the form of an HTTP origin: ws[s]://hostname[:port] """ ) extra_static_paths = List(Unicode, config=True, help="""Extra paths to search for serving static files. This allows adding javascript/css to be available from the notebook server machine, or overriding individual files in the IPython""" ) def _extra_static_paths_default(self): return [os.path.join(self.profile_dir.location, 'static')] @property def static_file_path(self): """return extra paths + the default location""" return self.extra_static_paths + [DEFAULT_STATIC_FILES_PATH] nbextensions_path = List(Unicode, config=True, help="""paths for Javascript extensions. By default, this is just IPYTHONDIR/nbextensions""" ) def _nbextensions_path_default(self): return [os.path.join(get_ipython_dir(), 'nbextensions')] mathjax_url = Unicode("", config=True, help="""The url for MathJax.js.""" ) def _mathjax_url_default(self): if not self.enable_mathjax: return u'' static_url_prefix = self.webapp_settings.get("static_url_prefix", url_path_join(self.base_project_url, "static") ) # try local mathjax, either in nbextensions/mathjax or static/mathjax for (url_prefix, search_path) in [ (url_path_join(self.base_project_url, "nbextensions"), self.nbextensions_path), (static_url_prefix, self.static_file_path), ]: self.log.debug("searching for local mathjax in %s", search_path) try: mathjax = filefind(os.path.join('mathjax', 'MathJax.js'), search_path) except IOError: continue else: url = url_path_join(url_prefix, u"mathjax/MathJax.js") self.log.info("Serving local MathJax from %s at %s", mathjax, url) return url # no local mathjax, serve from CDN if self.certfile: # HTTPS: load from Rackspace CDN, because SSL certificate requires it host = u"https://c328740.ssl.cf1.rackcdn.com" else: host = u"http://cdn.mathjax.org" url = host + u"/mathjax/latest/MathJax.js" self.log.info("Using MathJax from CDN: %s", url) return url def _mathjax_url_changed(self, name, old, new): if new and not self.enable_mathjax: # enable_mathjax=False overrides mathjax_url self.mathjax_url = u'' else: self.log.info("Using MathJax: %s", new) notebook_manager_class = DottedObjectName('IPython.html.services.notebooks.filenbmanager.FileNotebookManager', config=True, help='The notebook manager class to use.') trust_xheaders = Bool(False, config=True, help=("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers" "sent by the upstream reverse proxy. Necessary if the proxy handles SSL") ) info_file = Unicode() def _info_file_default(self): info_file = "nbserver-%s.json"%os.getpid() return os.path.join(self.profile_dir.security_dir, info_file) def parse_command_line(self, argv=None): super(NotebookApp, self).parse_command_line(argv) if self.extra_args: arg0 = self.extra_args[0] f = os.path.abspath(arg0) self.argv.remove(arg0) if not os.path.exists(f): self.log.critical("No such file or directory: %s", f) self.exit(1) if os.path.isdir(f): self.config.FileNotebookManager.notebook_dir = f elif os.path.isfile(f): self.file_to_run = f def init_kernel_argv(self): """construct the kernel arguments""" # Scrub frontend-specific flags self.kernel_argv = swallow_argv(self.argv, notebook_aliases, notebook_flags) if any(arg.startswith(u'--pylab') for arg in self.kernel_argv): self.log.warn('\n '.join([ "Starting all kernels in pylab mode is not recommended,", "and will be disabled in a future release.", "Please use the %matplotlib magic to enable matplotlib instead.", "pylab implies many imports, which can have confusing side effects", "and harm the reproducibility of your notebooks.", ])) # Kernel should inherit default config file from frontend self.kernel_argv.append("--IPKernelApp.parent_appname='%s'" % self.name) # Kernel should get *absolute* path to profile directory self.kernel_argv.extend(["--profile-dir", self.profile_dir.location]) def init_configurables(self): # force Session default to be secure default_secure(self.config) self.kernel_manager = MappingKernelManager( parent=self, log=self.log, kernel_argv=self.kernel_argv, connection_dir = self.profile_dir.security_dir, ) kls = import_item(self.notebook_manager_class) self.notebook_manager = kls(parent=self, log=self.log) self.session_manager = SessionManager(parent=self, log=self.log) self.cluster_manager = ClusterManager(parent=self, log=self.log) self.cluster_manager.update_profiles() def init_logging(self): # This prevents double log messages because tornado use a root logger that # self.log is a child of. The logging module dipatches log messages to a log # and all of its ancenstors until propagate is set to False. self.log.propagate = False # hook up tornado 3's loggers to our app handlers for name in ('access', 'application', 'general'): logger = logging.getLogger('tornado.%s' % name) logger.parent = self.log logger.setLevel(self.log.level) def init_webapp(self): """initialize tornado webapp and httpserver""" self.web_app = NotebookWebApplication( self, self.kernel_manager, self.notebook_manager, self.cluster_manager, self.session_manager, self.log, self.base_project_url, self.webapp_settings ) if self.certfile: ssl_options = dict(certfile=self.certfile) if self.keyfile: ssl_options['keyfile'] = self.keyfile else: ssl_options = None self.web_app.password = self.password self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options, xheaders=self.trust_xheaders) if not self.ip: warning = "WARNING: The notebook server is listening on all IP addresses" if ssl_options is None: self.log.critical(warning + " and not using encryption. This " "is not recommended.") if not self.password: self.log.critical(warning + " and not using authentication. " "This is highly insecure and not recommended.") success = None for port in random_ports(self.port, self.port_retries+1): try: self.http_server.listen(port, self.ip) except socket.error as e: if e.errno == errno.EADDRINUSE: self.log.info('The port %i is already in use, trying another random port.' % port) continue elif e.errno in (errno.EACCES, getattr(errno, 'WSAEACCES', errno.EACCES)): self.log.warn("Permission to listen on port %i denied" % port) continue else: raise else: self.port = port success = True break if not success: self.log.critical('ERROR: the notebook server could not be started because ' 'no available port could be found.') self.exit(1) @property def display_url(self): ip = self.ip if self.ip else '[all ip addresses on your system]' return self._url(ip) @property def connection_url(self): ip = self.ip if self.ip else localhost() return self._url(ip) def _url(self, ip): proto = 'https' if self.certfile else 'http' return "%s://%s:%i%s" % (proto, ip, self.port, self.base_project_url) def init_signal(self): if not sys.platform.startswith('win'): signal.signal(signal.SIGINT, self._handle_sigint) signal.signal(signal.SIGTERM, self._signal_stop) if hasattr(signal, 'SIGUSR1'): # Windows doesn't support SIGUSR1 signal.signal(signal.SIGUSR1, self._signal_info) if hasattr(signal, 'SIGINFO'): # only on BSD-based systems signal.signal(signal.SIGINFO, self._signal_info) def _handle_sigint(self, sig, frame): """SIGINT handler spawns confirmation dialog""" # register more forceful signal handler for ^C^C case signal.signal(signal.SIGINT, self._signal_stop) # request confirmation dialog in bg thread, to avoid # blocking the App thread = threading.Thread(target=self._confirm_exit) thread.daemon = True thread.start() def _restore_sigint_handler(self): """callback for restoring original SIGINT handler""" signal.signal(signal.SIGINT, self._handle_sigint) def _confirm_exit(self): """confirm shutdown on ^C A second ^C, or answering 'y' within 5s will cause shutdown, otherwise original SIGINT handler will be restored. This doesn't work on Windows. """ # FIXME: remove this delay when pyzmq dependency is >= 2.1.11 time.sleep(0.1) info = self.log.info info('interrupted') print(self.notebook_info()) sys.stdout.write("Shutdown this notebook server (y/[n])? ") sys.stdout.flush() r,w,x = select.select([sys.stdin], [], [], 5) if r: line = sys.stdin.readline() if line.lower().startswith('y'): self.log.critical("Shutdown confirmed") ioloop.IOLoop.instance().stop() return else: print("No answer for 5s:", end=' ') print("resuming operation...") # no answer, or answer is no: # set it back to original SIGINT handler # use IOLoop.add_callback because signal.signal must be called # from main thread ioloop.IOLoop.instance().add_callback(self._restore_sigint_handler) def _signal_stop(self, sig, frame): self.log.critical("received signal %s, stopping", sig) ioloop.IOLoop.instance().stop() def _signal_info(self, sig, frame): print(self.notebook_info()) def init_components(self): """Check the components submodule, and warn if it's unclean""" status = submodule.check_submodule_status() if status == 'missing': self.log.warn("components submodule missing, running `git submodule update`") submodule.update_submodules(submodule.ipython_parent()) elif status == 'unclean': self.log.warn("components submodule unclean, you may see 404s on static/components") self.log.warn("run `setup.py submodule` or `git submodule update` to update") @catch_config_error def initialize(self, argv=None): super(NotebookApp, self).initialize(argv) self.init_logging() self.init_kernel_argv() self.init_configurables() self.init_components() self.init_webapp() self.init_signal() def cleanup_kernels(self): """Shutdown all kernels. The kernels will shutdown themselves when this process no longer exists, but explicit shutdown allows the KernelManagers to cleanup the connection files. """ self.log.info('Shutting down kernels') self.kernel_manager.shutdown_all() def notebook_info(self): "Return the current working directory and the server url information" info = self.notebook_manager.info_string() + "\n" info += "%d active kernels \n" % len(self.kernel_manager._kernels) return info + "The IPython Notebook is running at: %s" % self.display_url def server_info(self): """Return a JSONable dict of information about this server.""" return {'url': self.connection_url, 'hostname': self.ip if self.ip else 'localhost', 'port': self.port, 'secure': bool(self.certfile), 'base_project_url': self.base_project_url, 'notebook_dir': os.path.abspath(self.notebook_manager.notebook_dir), } def write_server_info_file(self): """Write the result of server_info() to the JSON file info_file.""" with open(self.info_file, 'w') as f: json.dump(self.server_info(), f, indent=2) def remove_server_info_file(self): """Remove the nbserver-<pid>.json file created for this server. Ignores the error raised when the file has already been removed. """ try: os.unlink(self.info_file) except OSError as e: if e.errno != errno.ENOENT: raise def start(self): """ Start the IPython Notebook server app, after initialization This method takes no arguments so all configuration and initialization must be done prior to calling this method.""" if self.subapp is not None: return self.subapp.start() info = self.log.info for line in self.notebook_info().split("\n"): info(line) info("Use Control-C to stop this server and shut down all kernels (twice to skip confirmation).") self.write_server_info_file() if self.open_browser or self.file_to_run: try: browser = webbrowser.get(self.browser or None) except webbrowser.Error as e: self.log.warn('No web browser found: %s.' % e) browser = None f = self.file_to_run if f: nbdir = os.path.abspath(self.notebook_manager.notebook_dir) if f.startswith(nbdir): f = f[len(nbdir):] else: self.log.warn( "Probably won't be able to open notebook %s " "because it is not in notebook_dir %s", f, nbdir, ) if os.path.isfile(self.file_to_run): url = url_path_join('notebooks', f) else: url = url_path_join('tree', f) if browser: b = lambda : browser.open("%s%s" % (self.connection_url, url), new=2) threading.Thread(target=b).start() try: ioloop.IOLoop.instance().start() except KeyboardInterrupt: info("Interrupted...") finally: self.cleanup_kernels() self.remove_server_info_file()
class ExecuteReplyOkay(Reference): payload = List(Dict) user_expressions = Dict()
class ZMQTerminalIPythonApp(TerminalIPythonApp, IPythonConsoleApp): name = "ipython-console" """Start a terminal frontend to the IPython zmq kernel.""" description = """ The IPython terminal-based Console. This launches a Console application inside a terminal. The Console supports various extra features beyond the traditional single-process Terminal IPython shell, such as connecting to an existing ipython session, via: ipython console --existing where the previous session could have been created by another ipython console, an ipython qtconsole, or by opening an ipython notebook. """ examples = _examples classes = [ZMQTerminalInteractiveShell] + IPythonConsoleApp.classes flags = Dict(flags) aliases = Dict(aliases) frontend_aliases = Any(frontend_aliases) frontend_flags = Any(frontend_flags) subcommands = Dict() def parse_command_line(self, argv=None): super(ZMQTerminalIPythonApp, self).parse_command_line(argv) self.build_kernel_argv(argv) def init_shell(self): IPythonConsoleApp.initialize(self) # relay sigint to kernel signal.signal(signal.SIGINT, self.handle_sigint) self.shell = ZMQTerminalInteractiveShell.instance( config=self.config, display_banner=False, profile_dir=self.profile_dir, ipython_dir=self.ipython_dir, kernel_manager=self.kernel_manager) def init_gui_pylab(self): # no-op, because we don't want to import matplotlib in the frontend. pass def handle_sigint(self, *args): if self.shell._executing: if self.kernel_manager.has_kernel: # interrupt already gets passed to subprocess by signal handler. # Only if we prevent that should we need to explicitly call # interrupt_kernel, until which time, this would result in a # double-interrupt: # self.kernel_manager.interrupt_kernel() pass else: self.shell.write_err('\n') error("Cannot interrupt kernels we didn't start.\n") else: # raise the KeyboardInterrupt if we aren't waiting for execution, # so that the interact loop advances, and prompt is redrawn, etc. raise KeyboardInterrupt def init_code(self): # no-op in the frontend, code gets run in the backend pass
class HighlightMagicsPreprocessor(Preprocessor): """ Detects and tags code cells that use a different languages than Python. """ # list of magic language extensions and their associated pygment lexers default_languages = Dict( default_value={ '%%R': 'r', '%%bash': 'bash', '%%cython': 'cython', '%%javascript': 'javascript', '%%julia': 'julia', '%%latex': 'latex', '%%octave': 'octave', '%%perl': 'perl', '%%ruby': 'ruby', '%%sh': 'sh'}) # user defined language extensions languages = Dict( config=True, help=("Syntax highlighting for magic's extension languages. " "Each item associates a language magic extension such as %%R, " "with a pygments lexer such as r.")) def __init__(self, config=None, **kw): """Public constructor""" super(HighlightMagicsPreprocessor, self).__init__(config=config, **kw) # Update the default languages dict with the user configured ones self.default_languages.update(self.languages) # build a regular expression to catch language extensions and choose # an adequate pygments lexer any_language = "|".join(self.default_languages.keys()) self.re_magic_language = re.compile( r'^\s*({0})\s+'.format(any_language)) def which_magic_language(self, source): """ When a cell uses another language through a magic extension, the other language is returned. If no language magic is detected, this function returns None. Parameters ---------- source: str Source code of the cell to highlight """ m = self.re_magic_language.match(source) if m: # By construction of the re, the matched language must be in the # languages dictionary return self.default_languages[m.group(1)] else: return None def preprocess_cell(self, cell, resources, cell_index): """ Tags cells using a magic extension language Parameters ---------- cell : NotebookNode cell Notebook cell being processed resources : dictionary Additional resources used in the conversion process. Allows preprocessors to pass variables into the Jinja engine. cell_index : int Index of the cell being processed (see base.py) """ # Only tag code cells if hasattr(cell, "input") and cell.cell_type == "code": magic_language = self.which_magic_language(cell.input) if magic_language: cell['metadata']['magics_language'] = magic_language return cell, resources
class DOMWidget(Widget): visible = Bool(True, help="Whether the widget is visible.", sync=True) _css = Dict(sync=True) # Internal CSS property dict def get_css(self, key, selector=""): """Get a CSS property of the widget. Note: This function does not actually request the CSS from the front-end; Only properties that have been set with set_css can be read. Parameters ---------- key: unicode CSS key selector: unicode (optional) JQuery selector used when the CSS key/value was set. """ if selector in self._css and key in self._css[selector]: return self._css[selector][key] else: return None def set_css(self, dict_or_key, value=None, selector=''): """Set one or more CSS properties of the widget. This function has two signatures: - set_css(css_dict, selector='') - set_css(key, value, selector='') Parameters ---------- css_dict : dict CSS key/value pairs to apply key: unicode CSS key value: CSS value selector: unicode (optional, kwarg only) JQuery selector to use to apply the CSS key/value. If no selector is provided, an empty selector is used. An empty selector makes the front-end try to apply the css to a default element. The default element is an attribute unique to each view, which is a DOM element of the view that should be styled with common CSS (see `$el_to_style` in the Javascript code). """ if not selector in self._css: self._css[selector] = {} my_css = self._css[selector] if value is None: css_dict = dict_or_key else: css_dict = {dict_or_key: value} for (key, value) in css_dict.items(): if not (key in my_css and value == my_css[key]): my_css[key] = value self.send_state('_css') def add_class(self, class_names, selector=""): """Add class[es] to a DOM element. Parameters ---------- class_names: unicode or list Class name(s) to add to the DOM element(s). selector: unicode (optional) JQuery selector to select the DOM element(s) that the class(es) will be added to. """ class_list = class_names if isinstance(class_list, list): class_list = ' '.join(class_list) self.send({ "msg_type" : "add_class", "class_list" : class_list, "selector" : selector }) def remove_class(self, class_names, selector=""): """Remove class[es] from a DOM element. Parameters ---------- class_names: unicode or list Class name(s) to remove from the DOM element(s). selector: unicode (optional) JQuery selector to select the DOM element(s) that the class(es) will be removed from. """ class_list = class_names if isinstance(class_list, list): class_list = ' '.join(class_list) self.send({ "msg_type" : "remove_class", "class_list" : class_list, "selector" : selector, })
class IPKernelApp(BaseIPythonApplication, InteractiveShellApp): name = 'ipkernel' aliases = Dict(kernel_aliases) flags = Dict(kernel_flags) classes = [Kernel, ZMQInteractiveShell, ProfileDir, Session] # the kernel class, as an importstring kernel_class = DottedObjectName('IPython.kernel.zmq.ipkernel.Kernel') kernel = Any() poller = Any( ) # don't restrict this even though current pollers are all Threads heartbeat = Instance(Heartbeat) session = Instance('IPython.kernel.zmq.session.Session') ports = Dict() # inherit config file name from parent: parent_appname = Unicode(config=True) def _parent_appname_changed(self, name, old, new): if self.config_file_specified: # it was manually specified, ignore return self.config_file_name = new.replace('-', '_') + u'_config.py' # don't let this count as specifying the config file self.config_file_specified = False # connection info: transport = CaselessStrEnum(['tcp', 'ipc'], default_value='tcp', config=True) ip = Unicode( config=True, help="Set the IP or interface on which the kernel will listen.") def _ip_default(self): if self.transport == 'ipc': if self.connection_file: return os.path.splitext(self.abs_connection_file)[0] + '-ipc' else: return 'kernel-ipc' else: return LOCALHOST hb_port = Integer(0, config=True, help="set the heartbeat port [default: random]") shell_port = Integer(0, config=True, help="set the shell (ROUTER) port [default: random]") iopub_port = Integer(0, config=True, help="set the iopub (PUB) port [default: random]") stdin_port = Integer(0, config=True, help="set the stdin (DEALER) port [default: random]") connection_file = Unicode( '', config=True, help= """JSON file in which to store connection info [default: kernel-<pid>.json] This file will contain the IP, ports, and authentication key needed to connect clients to this kernel. By default, this file will be created in the security dir of the current profile, but can be specified by absolute path. """) @property def abs_connection_file(self): if os.path.basename(self.connection_file) == self.connection_file: return os.path.join(self.profile_dir.security_dir, self.connection_file) else: return self.connection_file # streams, etc. no_stdout = Bool(False, config=True, help="redirect stdout to the null device") no_stderr = Bool(False, config=True, help="redirect stderr to the null device") outstream_class = DottedObjectName( 'IPython.kernel.zmq.iostream.OutStream', config=True, help="The importstring for the OutStream factory") displayhook_class = DottedObjectName( 'IPython.kernel.zmq.displayhook.ZMQDisplayHook', config=True, help="The importstring for the DisplayHook factory") # polling parent = Integer( 0, config=True, help="""kill this process if its parent dies. On Windows, the argument specifies the HANDLE of the parent process, otherwise it is simply boolean. """) interrupt = Integer(0, config=True, help="""ONLY USED ON WINDOWS Interrupt this process when the parent is signaled. """) def init_crash_handler(self): # Install minimal exception handling sys.excepthook = FormattedTB(mode='Verbose', color_scheme='NoColor', ostream=sys.__stdout__) def init_poller(self): if sys.platform == 'win32': if self.interrupt or self.parent: self.poller = ParentPollerWindows(self.interrupt, self.parent) elif self.parent: self.poller = ParentPollerUnix() def _bind_socket(self, s, port): iface = '%s://%s' % (self.transport, self.ip) if self.transport == 'tcp': if port <= 0: port = s.bind_to_random_port(iface) else: s.bind("tcp://%s:%i" % (self.ip, port)) elif self.transport == 'ipc': if port <= 0: port = 1 path = "%s-%i" % (self.ip, port) while os.path.exists(path): port = port + 1 path = "%s-%i" % (self.ip, port) else: path = "%s-%i" % (self.ip, port) s.bind("ipc://%s" % path) return port def load_connection_file(self): """load ip/port/hmac config from JSON connection file""" try: fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir]) except IOError: self.log.debug("Connection file not found: %s", self.connection_file) # This means I own it, so I will clean it up: atexit.register(self.cleanup_connection_file) return self.log.debug(u"Loading connection file %s", fname) with open(fname) as f: s = f.read() cfg = json.loads(s) self.transport = cfg.get('transport', self.transport) if self.ip == self._ip_default() and 'ip' in cfg: # not overridden by config or cl_args self.ip = cfg['ip'] for channel in ('hb', 'shell', 'iopub', 'stdin'): name = channel + '_port' if getattr(self, name) == 0 and name in cfg: # not overridden by config or cl_args setattr(self, name, cfg[name]) if 'key' in cfg: self.config.Session.key = str_to_bytes(cfg['key']) def write_connection_file(self): """write connection info to JSON file""" cf = self.abs_connection_file self.log.debug("Writing connection file: %s", cf) write_connection_file(cf, ip=self.ip, key=self.session.key, transport=self.transport, shell_port=self.shell_port, stdin_port=self.stdin_port, hb_port=self.hb_port, iopub_port=self.iopub_port) def cleanup_connection_file(self): cf = self.abs_connection_file self.log.debug("Cleaning up connection file: %s", cf) try: os.remove(cf) except (IOError, OSError): pass self.cleanup_ipc_files() def cleanup_ipc_files(self): """cleanup ipc files if we wrote them""" if self.transport != 'ipc': return for port in (self.shell_port, self.iopub_port, self.stdin_port, self.hb_port): ipcfile = "%s-%i" % (self.ip, port) try: os.remove(ipcfile) except (IOError, OSError): pass def init_connection_file(self): if not self.connection_file: self.connection_file = "kernel-%s.json" % os.getpid() try: self.load_connection_file() except Exception: self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True) self.exit(1) def init_sockets(self): # Create a context, a session, and the kernel sockets. self.log.info("Starting the kernel at pid: %i", os.getpid()) context = zmq.Context.instance() # Uncomment this to try closing the context. # atexit.register(context.term) self.shell_socket = context.socket(zmq.ROUTER) self.shell_port = self._bind_socket(self.shell_socket, self.shell_port) self.log.debug("shell ROUTER Channel on port: %i" % self.shell_port) self.iopub_socket = context.socket(zmq.PUB) self.iopub_port = self._bind_socket(self.iopub_socket, self.iopub_port) self.log.debug("iopub PUB Channel on port: %i" % self.iopub_port) self.stdin_socket = context.socket(zmq.ROUTER) self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port) self.log.debug("stdin ROUTER Channel on port: %i" % self.stdin_port) def init_heartbeat(self): """start the heart beating""" # heartbeat doesn't share context, because it mustn't be blocked # by the GIL, which is accessed by libzmq when freeing zero-copy messages hb_ctx = zmq.Context() self.heartbeat = Heartbeat(hb_ctx, (self.transport, self.ip, self.hb_port)) self.hb_port = self.heartbeat.port self.log.debug("Heartbeat REP Channel on port: %i" % self.hb_port) self.heartbeat.start() # Helper to make it easier to connect to an existing kernel. # set log-level to critical, to make sure it is output self.log.critical("To connect another client to this kernel, use:") def log_connection_info(self): """display connection info, and store ports""" basename = os.path.basename(self.connection_file) if basename == self.connection_file or \ os.path.dirname(self.connection_file) == self.profile_dir.security_dir: # use shortname tail = basename if self.profile != 'default': tail += " --profile %s" % self.profile else: tail = self.connection_file self.log.critical("--existing %s", tail) self.ports = dict(shell=self.shell_port, iopub=self.iopub_port, stdin=self.stdin_port, hb=self.hb_port) def init_session(self): """create our session object""" default_secure(self.config) self.session = Session(config=self.config, username=u'kernel') def init_blackhole(self): """redirects stdout/stderr to devnull if necessary""" if self.no_stdout or self.no_stderr: blackhole = open(os.devnull, 'w') if self.no_stdout: sys.stdout = sys.__stdout__ = blackhole if self.no_stderr: sys.stderr = sys.__stderr__ = blackhole def init_io(self): """Redirect input streams and set a display hook.""" if self.outstream_class: outstream_factory = import_item(str(self.outstream_class)) sys.stdout = outstream_factory(self.session, self.iopub_socket, u'stdout') sys.stderr = outstream_factory(self.session, self.iopub_socket, u'stderr') if self.displayhook_class: displayhook_factory = import_item(str(self.displayhook_class)) sys.displayhook = displayhook_factory(self.session, self.iopub_socket) def init_signal(self): signal.signal(signal.SIGINT, signal.SIG_IGN) def init_kernel(self): """Create the Kernel object itself""" shell_stream = ZMQStream(self.shell_socket) kernel = Kernel( config=self.config, session=self.session, shell_streams=[shell_stream], iopub_socket=self.iopub_socket, stdin_socket=self.stdin_socket, log=self.log, profile_dir=self.profile_dir, ) kernel.record_ports(self.ports) self.kernel = kernel def init_gui_pylab(self): """Enable GUI event loop integration, taking pylab into account.""" # Provide a wrapper for :meth:`InteractiveShellApp.init_gui_pylab` # to ensure that any exception is printed straight to stderr. # Normally _showtraceback associates the reply with an execution, # which means frontends will never draw it, as this exception # is not associated with any execute request. shell = self.shell _showtraceback = shell._showtraceback try: # replace pyerr-sending traceback with stderr def print_tb(etype, evalue, stb): print("GUI event loop or pylab initialization failed", file=io.stderr) print(shell.InteractiveTB.stb2text(stb), file=io.stderr) shell._showtraceback = print_tb InteractiveShellApp.init_gui_pylab(self) finally: shell._showtraceback = _showtraceback def init_shell(self): self.shell = self.kernel.shell self.shell.configurables.append(self) @catch_config_error def initialize(self, argv=None): super(IPKernelApp, self).initialize(argv) self.init_blackhole() self.init_connection_file() self.init_session() self.init_poller() self.init_sockets() self.init_heartbeat() # writing/displaying connection info must be *after* init_sockets/heartbeat self.log_connection_info() self.write_connection_file() self.init_io() self.init_signal() self.init_kernel() # shell init steps self.init_path() self.init_shell() self.init_gui_pylab() self.init_extensions() self.init_code() # flush stdout/stderr, so that anything written to these streams during # initialization do not get associated with the first execution request sys.stdout.flush() sys.stderr.flush() def start(self): if self.poller is not None: self.poller.start() self.kernel.start() try: ioloop.IOLoop.instance().start() except KeyboardInterrupt: pass
class CommManager(LoggingConfigurable): """Manager for Comms in the Kernel""" # If this is instantiated by a non-IPython kernel, shell will be None shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True) kernel = Instance('ipython_kernel.kernelbase.Kernel') iopub_socket = Any() def _iopub_socket_default(self): return self.kernel.iopub_socket session = Instance('ipython_kernel.session.Session') def _session_default(self): return self.kernel.session comms = Dict() targets = Dict() # Public APIs def register_target(self, target_name, f): """Register a callable f for a given target name f will be called with two arguments when a comm_open message is received with `target`: - the Comm instance - the `comm_open` message itself. f can be a Python callable or an import string for one. """ if isinstance(f, string_types): f = import_item(f) self.targets[target_name] = f def unregister_target(self, target_name, f): """Unregister a callable registered with register_target""" return self.targets.pop(target_name) def register_comm(self, comm): """Register a new comm""" comm_id = comm.comm_id comm.shell = self.shell comm.kernel = self.kernel comm.iopub_socket = self.iopub_socket self.comms[comm_id] = comm return comm_id def unregister_comm(self, comm): """Unregister a comm, and close its counterpart""" # unlike get_comm, this should raise a KeyError comm = self.comms.pop(comm.comm_id) def get_comm(self, comm_id): """Get a comm with a particular id Returns the comm if found, otherwise None. This will not raise an error, it will log messages if the comm cannot be found. """ if comm_id not in self.comms: self.log.error("No such comm: %s", comm_id) self.log.debug("Current comms: %s", lazy_keys(self.comms)) return # call, because we store weakrefs comm = self.comms[comm_id] return comm # Message handlers def comm_open(self, stream, ident, msg): """Handler for comm_open messages""" content = msg['content'] comm_id = content['comm_id'] target_name = content['target_name'] f = self.targets.get(target_name, None) comm = Comm( comm_id=comm_id, shell=self.shell, kernel=self.kernel, iopub_socket=self.iopub_socket, primary=False, ) self.register_comm(comm) if f is None: self.log.error("No such comm target registered: %s", target_name) else: try: f(comm, msg) return except Exception: self.log.error("Exception opening comm with target: %s", target_name, exc_info=True) # Failure. try: comm.close() except: self.log.error("""Could not close comm during `comm_open` failure clean-up. The comm may not have been opened yet.""", exc_info=True) def comm_msg(self, stream, ident, msg): """Handler for comm_msg messages""" content = msg['content'] comm_id = content['comm_id'] comm = self.get_comm(comm_id) if comm is None: # no such comm return try: comm.handle_msg(msg) except Exception: self.log.error("Exception in comm_msg for %s", comm_id, exc_info=True) def comm_close(self, stream, ident, msg): """Handler for comm_close messages""" content = msg['content'] comm_id = content['comm_id'] comm = self.get_comm(comm_id) if comm is None: # no such comm self.log.debug("No such comm to close: %s", comm_id) return del self.comms[comm_id] try: comm.handle_close(msg) except Exception: self.log.error("Exception handling comm_close for %s", comm_id, exc_info=True)
class TaskScheduler(SessionFactory): """Python TaskScheduler object. This is the simplest object that supports msg_id based DAG dependencies. *Only* task msg_ids are checked, not msg_ids of jobs submitted via the MUX queue. """ # input arguments: scheme = Instance( FunctionType, default=leastload) # function for determining the destination client_stream = Instance(zmqstream.ZMQStream) # client-facing stream engine_stream = Instance(zmqstream.ZMQStream) # engine-facing stream notifier_stream = Instance(zmqstream.ZMQStream) # hub-facing sub stream mon_stream = Instance(zmqstream.ZMQStream) # hub-facing pub stream # internals: graph = Dict() # dict by msg_id of [ msg_ids that depend on key ] depending = Dict() # dict by msg_id of (msg_id, raw_msg, after, follow) pending = Dict() # dict by engine_uuid of submitted tasks completed = Dict() # dict by engine_uuid of completed tasks failed = Dict() # dict by engine_uuid of failed tasks destinations = Dict( ) # dict by msg_id of engine_uuids where jobs ran (reverse of completed+failed) clients = Dict() # dict by msg_id for who submitted the task targets = List() # list of target IDENTs loads = List() # list of engine loads all_completed = Set() # set of all completed tasks all_failed = Set() # set of all failed tasks all_done = Set() # set of all finished tasks=union(completed,failed) all_ids = Set() # set of all submitted task IDs blacklist = Dict( ) # dict by msg_id of locations where a job has encountered UnmetDependency auditor = Instance('zmq.eventloop.ioloop.PeriodicCallback') def start(self): self.engine_stream.on_recv(self.dispatch_result, copy=False) self._notification_handlers = dict( registration_notification=self._register_engine, unregistration_notification=self._unregister_engine) self.notifier_stream.on_recv(self.dispatch_notification) self.auditor = ioloop.PeriodicCallback(self.audit_timeouts, 2e3, self.loop) # 1 Hz self.auditor.start() self.log.info("Scheduler started...%r" % self) def resume_receiving(self): """Resume accepting jobs.""" self.client_stream.on_recv(self.dispatch_submission, copy=False) def stop_receiving(self): """Stop accepting jobs while there are no engines. Leave them in the ZMQ queue.""" self.client_stream.on_recv(None) #----------------------------------------------------------------------- # [Un]Registration Handling #----------------------------------------------------------------------- def dispatch_notification(self, msg): """dispatch register/unregister events.""" idents, msg = self.session.feed_identities(msg) msg = self.session.unpack_message(msg) msg_type = msg['msg_type'] handler = self._notification_handlers.get(msg_type, None) if handler is None: raise Exception("Unhandled message type: %s" % msg_type) else: try: handler(str(msg['content']['queue'])) except KeyError: self.log.error("task::Invalid notification msg: %s" % msg) @logged def _register_engine(self, uid): """New engine with ident `uid` became available.""" # head of the line: self.targets.insert(0, uid) self.loads.insert(0, 0) # initialize sets self.completed[uid] = set() self.failed[uid] = set() self.pending[uid] = {} if len(self.targets) == 1: self.resume_receiving() def _unregister_engine(self, uid): """Existing engine with ident `uid` became unavailable.""" if len(self.targets) == 1: # this was our only engine self.stop_receiving() # handle any potentially finished tasks: self.engine_stream.flush() self.completed.pop(uid) self.failed.pop(uid) # don't pop destinations, because it might be used later # map(self.destinations.pop, self.completed.pop(uid)) # map(self.destinations.pop, self.failed.pop(uid)) idx = self.targets.index(uid) self.targets.pop(idx) self.loads.pop(idx) # wait 5 seconds before cleaning up pending jobs, since the results might # still be incoming if self.pending[uid]: dc = ioloop.DelayedCallback( lambda: self.handle_stranded_tasks(uid), 5000, self.loop) dc.start() @logged def handle_stranded_tasks(self, engine): """Deal with jobs resident in an engine that died.""" lost = self.pending.pop(engine) for msg_id, (raw_msg, targets, MET, follow, timeout) in lost.iteritems(): self.all_failed.add(msg_id) self.all_done.add(msg_id) idents, msg = self.session.feed_identities(raw_msg, copy=False) msg = self.session.unpack_message(msg, copy=False, content=False) parent = msg['header'] idents = [idents[0], engine] + idents[1:] # print (idents) try: raise error.EngineError( "Engine %r died while running task %r" % (engine, msg_id)) except: content = error.wrap_exception() msg = self.session.send(self.client_stream, 'apply_reply', content, parent=parent, ident=idents) self.session.send(self.mon_stream, msg, ident=['outtask'] + idents) self.update_graph(msg_id) #----------------------------------------------------------------------- # Job Submission #----------------------------------------------------------------------- @logged def dispatch_submission(self, raw_msg): """Dispatch job submission to appropriate handlers.""" # ensure targets up to date: self.notifier_stream.flush() try: idents, msg = self.session.feed_identities(raw_msg, copy=False) msg = self.session.unpack_message(msg, content=False, copy=False) except: self.log.error("task::Invaid task: %s" % raw_msg, exc_info=True) return # send to monitor self.mon_stream.send_multipart(['intask'] + raw_msg, copy=False) header = msg['header'] msg_id = header['msg_id'] self.all_ids.add(msg_id) # targets targets = set(header.get('targets', [])) # time dependencies after = Dependency(header.get('after', [])) if after.all: if after.success: after.difference_update(self.all_completed) if after.failure: after.difference_update(self.all_failed) if after.check(self.all_completed, self.all_failed): # recast as empty set, if `after` already met, # to prevent unnecessary set comparisons after = MET # location dependencies follow = Dependency(header.get('follow', [])) # turn timeouts into datetime objects: timeout = header.get('timeout', None) if timeout: timeout = datetime.now() + timedelta(0, timeout, 0) args = [raw_msg, targets, after, follow, timeout] # validate and reduce dependencies: for dep in after, follow: # check valid: if msg_id in dep or dep.difference(self.all_ids): self.depending[msg_id] = args return self.fail_unreachable(msg_id, error.InvalidDependency) # check if unreachable: if dep.unreachable(self.all_completed, self.all_failed): self.depending[msg_id] = args return self.fail_unreachable(msg_id) if after.check(self.all_completed, self.all_failed): # time deps already met, try to run if not self.maybe_run(msg_id, *args): # can't run yet self.save_unmet(msg_id, *args) else: self.save_unmet(msg_id, *args) # @logged def audit_timeouts(self): """Audit all waiting tasks for expired timeouts.""" now = datetime.now() for msg_id in self.depending.keys(): # must recheck, in case one failure cascaded to another: if msg_id in self.depending: raw, after, targets, follow, timeout = self.depending[msg_id] if timeout and timeout < now: self.fail_unreachable(msg_id, timeout=True) @logged def fail_unreachable(self, msg_id, why=error.ImpossibleDependency): """a task has become unreachable, send a reply with an ImpossibleDependency error.""" if msg_id not in self.depending: self.log.error("msg %r already failed!" % msg_id) return raw_msg, targets, after, follow, timeout = self.depending.pop(msg_id) for mid in follow.union(after): if mid in self.graph: self.graph[mid].remove(msg_id) # FIXME: unpacking a message I've already unpacked, but didn't save: idents, msg = self.session.feed_identities(raw_msg, copy=False) msg = self.session.unpack_message(msg, copy=False, content=False) header = msg['header'] try: raise why() except: content = error.wrap_exception() self.all_done.add(msg_id) self.all_failed.add(msg_id) msg = self.session.send(self.client_stream, 'apply_reply', content, parent=header, ident=idents) self.session.send(self.mon_stream, msg, ident=['outtask'] + idents) self.update_graph(msg_id, success=False) @logged def maybe_run(self, msg_id, raw_msg, targets, after, follow, timeout): """check location dependencies, and run if they are met.""" blacklist = self.blacklist.setdefault(msg_id, set()) if follow or targets or blacklist: # we need a can_run filter def can_run(idx): target = self.targets[idx] # check targets if targets and target not in targets: return False # check blacklist if target in blacklist: return False # check follow return follow.check(self.completed[target], self.failed[target]) indices = filter(can_run, range(len(self.targets))) if not indices: # couldn't run if follow.all: # check follow for impossibility dests = set() relevant = set() if follow.success: relevant = self.all_completed if follow.failure: relevant = relevant.union(self.all_failed) for m in follow.intersection(relevant): dests.add(self.destinations[m]) if len(dests) > 1: self.fail_unreachable(msg_id) return False if targets: # check blacklist+targets for impossibility targets.difference_update(blacklist) if not targets or not targets.intersection(self.targets): self.fail_unreachable(msg_id) return False return False else: indices = None self.submit_task(msg_id, raw_msg, targets, follow, timeout, indices) return True @logged def save_unmet(self, msg_id, raw_msg, targets, after, follow, timeout): """Save a message for later submission when its dependencies are met.""" self.depending[msg_id] = [raw_msg, targets, after, follow, timeout] # track the ids in follow or after, but not those already finished for dep_id in after.union(follow).difference(self.all_done): if dep_id not in self.graph: self.graph[dep_id] = set() self.graph[dep_id].add(msg_id) @logged def submit_task(self, msg_id, raw_msg, targets, follow, timeout, indices=None): """Submit a task to any of a subset of our targets.""" if indices: loads = [self.loads[i] for i in indices] else: loads = self.loads idx = self.scheme(loads) if indices: idx = indices[idx] target = self.targets[idx] # print (target, map(str, msg[:3])) self.engine_stream.send(target, flags=zmq.SNDMORE, copy=False) self.engine_stream.send_multipart(raw_msg, copy=False) self.add_job(idx) self.pending[target][msg_id] = (raw_msg, targets, MET, follow, timeout) content = dict(msg_id=msg_id, engine_id=target) self.session.send(self.mon_stream, 'task_destination', content=content, ident=['tracktask', self.session.session]) #----------------------------------------------------------------------- # Result Handling #----------------------------------------------------------------------- @logged def dispatch_result(self, raw_msg): """dispatch method for result replies""" try: idents, msg = self.session.feed_identities(raw_msg, copy=False) msg = self.session.unpack_message(msg, content=False, copy=False) except: self.log.error("task::Invaid result: %s" % raw_msg, exc_info=True) return header = msg['header'] if header.get('dependencies_met', True): success = (header['status'] == 'ok') self.handle_result(idents, msg['parent_header'], raw_msg, success) # send to Hub monitor self.mon_stream.send_multipart(['outtask'] + raw_msg, copy=False) else: self.handle_unmet_dependency(idents, msg['parent_header']) @logged def handle_result(self, idents, parent, raw_msg, success=True): """handle a real task result, either success or failure""" # first, relay result to client engine = idents[0] client = idents[1] # swap_ids for XREP-XREP mirror raw_msg[:2] = [client, engine] # print (map(str, raw_msg[:4])) self.client_stream.send_multipart(raw_msg, copy=False) # now, update our data structures msg_id = parent['msg_id'] self.blacklist.pop(msg_id, None) self.pending[engine].pop(msg_id) if success: self.completed[engine].add(msg_id) self.all_completed.add(msg_id) else: self.failed[engine].add(msg_id) self.all_failed.add(msg_id) self.all_done.add(msg_id) self.destinations[msg_id] = engine self.update_graph(msg_id, success) @logged def handle_unmet_dependency(self, idents, parent): """handle an unmet dependency""" engine = idents[0] msg_id = parent['msg_id'] if msg_id not in self.blacklist: self.blacklist[msg_id] = set() self.blacklist[msg_id].add(engine) args = self.pending[engine].pop(msg_id) raw, targets, after, follow, timeout = args if self.blacklist[msg_id] == targets: self.depending[msg_id] = args return self.fail_unreachable(msg_id) elif not self.maybe_run(msg_id, *args): # resubmit failed, put it back in our dependency tree self.save_unmet(msg_id, *args) @logged def update_graph(self, dep_id, success=True): """dep_id just finished. Update our dependency graph and submit any jobs that just became runable.""" # print ("\n\n***********") # pprint (dep_id) # pprint (self.graph) # pprint (self.depending) # pprint (self.all_completed) # pprint (self.all_failed) # print ("\n\n***********\n\n") if dep_id not in self.graph: return jobs = self.graph.pop(dep_id) for msg_id in jobs: raw_msg, targets, after, follow, timeout = self.depending[msg_id] if after.unreachable(self.all_completed, self.all_failed) or follow.unreachable( self.all_completed, self.all_failed): self.fail_unreachable(msg_id) elif after.check(self.all_completed, self.all_failed): # time deps met, maybe run if self.maybe_run(msg_id, raw_msg, targets, MET, follow, timeout): self.depending.pop(msg_id) for mid in follow.union(after): if mid in self.graph: self.graph[mid].remove(msg_id) #---------------------------------------------------------------------- # methods to be overridden by subclasses #---------------------------------------------------------------------- def add_job(self, idx): """Called after self.targets[idx] just got the job with header. Override with subclasses. The default ordering is simple LRU. The default loads are the number of outstanding jobs.""" self.loads[idx] += 1 for lis in (self.targets, self.loads): lis.append(lis.pop(idx)) def finish_job(self, idx): """Called after self.targets[idx] just finished a job. Override with subclasses.""" self.loads[idx] -= 1
class BaseIPythonApplication(Application): name = Unicode(u'ipython') description = Unicode(u'IPython: an enhanced interactive Python shell.') version = Unicode(release.version) aliases = Dict(base_aliases) flags = Dict(base_flags) classes = List([ProfileDir]) # Track whether the config_file has changed, # because some logic happens only if we aren't using the default. config_file_specified = Bool(False) config_file_name = Unicode(u'ipython_config.py') def _config_file_name_default(self): return self.name.replace('-', '_') + u'_config.py' def _config_file_name_changed(self, name, old, new): if new != old: self.config_file_specified = True # The directory that contains IPython's builtin profiles. builtin_profile_dir = Unicode( os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')) config_file_paths = List(Unicode) def _config_file_paths_default(self): return [os.getcwdu()] profile = Unicode(u'default', config=True, help="""The IPython profile to use.""") def _profile_changed(self, name, old, new): self.builtin_profile_dir = os.path.join(get_ipython_package_dir(), u'config', u'profile', new) ipython_dir = Unicode(get_ipython_dir(), config=True, help=""" The name of the IPython directory. This directory is used for logging configuration (through profiles), history storage, etc. The default is usually $HOME/.ipython. This options can also be specified through the environment variable IPYTHONDIR. """) overwrite = Bool( False, config=True, help="""Whether to overwrite existing config files when copying""") auto_create = Bool( False, config=True, help="""Whether to create profile dir if it doesn't exist""") config_files = List(Unicode) def _config_files_default(self): return [u'ipython_config.py'] copy_config_files = Bool( False, config=True, help="""Whether to install the default config files into the profile dir. If a new profile is being created, and IPython contains config files for that profile, then they will be staged into the new directory. Otherwise, default config files will be automatically generated. """) verbose_crash = Bool( False, config=True, help= """Create a massive crash report when IPython encounters what may be an internal error. The default is to append a short message to the usual traceback""") # The class to use as the crash handler. crash_handler_class = Type(crashhandler.CrashHandler) @catch_config_error def __init__(self, **kwargs): super(BaseIPythonApplication, self).__init__(**kwargs) # ensure current working directory exists try: directory = os.getcwdu() except: # raise exception self.log.error("Current working directory doesn't exist.") raise # ensure even default IPYTHONDIR exists if not os.path.exists(self.ipython_dir): self._ipython_dir_changed('ipython_dir', self.ipython_dir, self.ipython_dir) #------------------------------------------------------------------------- # Various stages of Application creation #------------------------------------------------------------------------- def init_crash_handler(self): """Create a crash handler, typically setting sys.excepthook to it.""" self.crash_handler = self.crash_handler_class(self) sys.excepthook = self.excepthook def unset_crashhandler(): sys.excepthook = sys.__excepthook__ atexit.register(unset_crashhandler) def excepthook(self, etype, evalue, tb): """this is sys.excepthook after init_crashhandler set self.verbose_crash=True to use our full crashhandler, instead of a regular traceback with a short message (crash_handler_lite) """ if self.verbose_crash: return self.crash_handler(etype, evalue, tb) else: return crashhandler.crash_handler_lite(etype, evalue, tb) def _ipython_dir_changed(self, name, old, new): if old in sys.path: sys.path.remove(old) sys.path.append(os.path.abspath(new)) if not os.path.isdir(new): os.makedirs(new, mode=0o777) readme = os.path.join(new, 'README') if not os.path.exists(readme): path = os.path.join(get_ipython_package_dir(), u'config', u'profile') shutil.copy(os.path.join(path, 'README'), readme) self.log.debug("IPYTHONDIR set to: %s" % new) def load_config_file(self, suppress_errors=True): """Load the config file. By default, errors in loading config are handled, and a warning printed on screen. For testing, the suppress_errors option is set to False, so errors will make tests fail. """ self.log.debug("Searching path %s for config files", self.config_file_paths) base_config = 'ipython_config.py' self.log.debug("Attempting to load config file: %s" % base_config) try: Application.load_config_file(self, base_config, path=self.config_file_paths) except ConfigFileNotFound: # ignore errors loading parent self.log.debug("Config file %s not found", base_config) pass if self.config_file_name == base_config: # don't load secondary config return self.log.debug("Attempting to load config file: %s" % self.config_file_name) try: Application.load_config_file(self, self.config_file_name, path=self.config_file_paths) except ConfigFileNotFound: # Only warn if the default config file was NOT being used. if self.config_file_specified: msg = self.log.warn else: msg = self.log.debug msg("Config file not found, skipping: %s", self.config_file_name) except: # For testing purposes. if not suppress_errors: raise self.log.warn("Error loading config file: %s" % self.config_file_name, exc_info=True) def init_profile_dir(self): """initialize the profile dir""" try: # location explicitly specified: location = self.config.ProfileDir.location except AttributeError: # location not specified, find by profile name try: p = ProfileDir.find_profile_dir_by_name( self.ipython_dir, self.profile, self.config) except ProfileDirError: # not found, maybe create it (always create default profile) if self.auto_create or self.profile == 'default': try: p = ProfileDir.create_profile_dir_by_name( self.ipython_dir, self.profile, self.config) except ProfileDirError: self.log.fatal("Could not create profile: %r" % self.profile) self.exit(1) else: self.log.info("Created profile dir: %r" % p.location) else: self.log.fatal("Profile %r not found." % self.profile) self.exit(1) else: self.log.info("Using existing profile dir: %r" % p.location) else: # location is fully specified try: p = ProfileDir.find_profile_dir(location, self.config) except ProfileDirError: # not found, maybe create it if self.auto_create: try: p = ProfileDir.create_profile_dir( location, self.config) except ProfileDirError: self.log.fatal( "Could not create profile directory: %r" % location) self.exit(1) else: self.log.info("Creating new profile dir: %r" % location) else: self.log.fatal("Profile directory %r not found." % location) self.exit(1) else: self.log.info("Using existing profile dir: %r" % location) self.profile_dir = p self.config_file_paths.append(p.location) def init_config_files(self): """[optionally] copy default config files into profile dir.""" # copy config files path = self.builtin_profile_dir if self.copy_config_files: src = self.profile cfg = self.config_file_name if path and os.path.exists(os.path.join(path, cfg)): self.log.warn( "Staging %r from %s into %r [overwrite=%s]" % (cfg, src, self.profile_dir.location, self.overwrite)) self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite) else: self.stage_default_config_file() else: # Still stage *bundled* config files, but not generated ones # This is necessary for `ipython profile=sympy` to load the profile # on the first go files = glob.glob(os.path.join(path, '*.py')) for fullpath in files: cfg = os.path.basename(fullpath) if self.profile_dir.copy_config_file(cfg, path=path, overwrite=False): # file was copied self.log.warn( "Staging bundled %s from %s into %r" % (cfg, self.profile, self.profile_dir.location)) def stage_default_config_file(self): """auto generate default config file, and stage it into the profile.""" s = self.generate_config_file() fname = os.path.join(self.profile_dir.location, self.config_file_name) if self.overwrite or not os.path.exists(fname): self.log.warn("Generating default config file: %r" % (fname)) with open(fname, 'w') as f: f.write(s) @catch_config_error def initialize(self, argv=None): # don't hook up crash handler before parsing command-line self.parse_command_line(argv) self.init_crash_handler() if self.subapp is not None: # stop here if subapp is taking over return cl_config = self.config self.init_profile_dir() self.init_config_files() self.load_config_file() # enforce cl-opts override configfile opts: self.update_config(cl_config)
class Exporter(LoggingConfigurable): """ Exports notebooks into other file formats. Uses Jinja 2 templating engine to output new formats. Inherit from this class if you are creating a new template type along with new filters/preprocessors. If the filters/ preprocessors provided by default suffice, there is no need to inherit from this class. Instead, override the template_file and file_extension traits via a config file. {filters} """ # finish the docstring __doc__ = __doc__.format(filters='- ' + '\n - '.join(default_filters.keys())) template_file = Unicode(u'default', config=True, help="Name of the template file to use") def _template_file_changed(self, name, old, new): if new == 'default': self.template_file = self.default_template else: self.template_file = new self.template = None self._load_template() default_template = Unicode(u'') template = Any() environment = Any() file_extension = Unicode( 'txt', config=True, help="Extension of the file that should be written to disk") template_path = List(['.'], config=True) def _template_path_changed(self, name, old, new): self._load_template() default_template_path = Unicode( os.path.join("..", "templates"), help="Path where the template files are located.") template_skeleton_path = Unicode( os.path.join("..", "templates", "skeleton"), help="Path where the template skeleton files are located.") #Jinja block definitions jinja_comment_block_start = Unicode("", config=True) jinja_comment_block_end = Unicode("", config=True) jinja_variable_block_start = Unicode("", config=True) jinja_variable_block_end = Unicode("", config=True) jinja_logic_block_start = Unicode("", config=True) jinja_logic_block_end = Unicode("", config=True) #Extension that the template files use. template_extension = Unicode(".tpl", config=True) #Configurability, allows the user to easily add filters and preprocessors. preprocessors = List( config=True, help="""List of preprocessors, by name or namespace, to enable.""") filters = Dict( config=True, help="""Dictionary of filters, by name and namespace, to add to the Jinja environment.""") default_preprocessors = List( [ nbpreprocessors.coalesce_streams, nbpreprocessors.SVG2PDFPreprocessor, nbpreprocessors.ExtractOutputPreprocessor, nbpreprocessors.CSSHTMLHeaderPreprocessor, nbpreprocessors.RevealHelpPreprocessor, nbpreprocessors.LatexPreprocessor, nbpreprocessors.SphinxPreprocessor ], config=True, help="""List of preprocessors available by default, by name, namespace, instance, or type.""") def __init__(self, config=None, extra_loaders=None, **kw): """ Public constructor Parameters ---------- config : config User configuration instance. extra_loaders : list[of Jinja Loaders] ordered list of Jinja loader to find templates. Will be tried in order before the default FileSystem ones. template : str (optional, kw arg) Template to use when exporting. """ if not config: config = self.default_config super(Exporter, self).__init__(config=config, **kw) #Init self._init_template() self._init_environment(extra_loaders=extra_loaders) self._init_preprocessors() self._init_filters() @property def default_config(self): return Config() def _config_changed(self, name, old, new): """When setting config, make sure to start with our default_config""" c = self.default_config if new: c.merge(new) if c != old: self.config = c super(Exporter, self)._config_changed(name, old, c) def _load_template(self): """Load the Jinja template object from the template file This is a no-op if the template attribute is already defined, or the Jinja environment is not setup yet. This is triggered by various trait changes that would change the template. """ if self.template is not None: return # called too early, do nothing if self.environment is None: return # Try different template names during conversion. First try to load the # template by name with extension added, then try loading the template # as if the name is explicitly specified, then try the name as a # 'flavor', and lastly just try to load the template by module name. module_name = self.__module__.rsplit('.', 1)[-1] try_names = [] if self.template_file: try_names.extend([ self.template_file + self.template_extension, self.template_file, module_name + '_' + self.template_file + self.template_extension, ]) try_names.append(module_name + self.template_extension) for try_name in try_names: self.log.debug("Attempting to load template %s", try_name) try: self.template = self.environment.get_template(try_name) except (TemplateNotFound, IOError): pass except Exception as e: self.log.warn("Unexpected exception loading template: %s", try_name, exc_info=True) else: self.log.info("Loaded template %s", try_name) break def from_notebook_node(self, nb, resources=None, **kw): """ Convert a notebook from a notebook node instance. Parameters ---------- nb : Notebook node resources : dict (**kw) of additional resources that can be accessed read/write by preprocessors and filters. """ nb_copy = copy.deepcopy(nb) resources = self._init_resources(resources) # Preprocess nb_copy, resources = self._preprocess(nb_copy, resources) self._load_template() if self.template is not None: output = self.template.render(nb=nb_copy, resources=resources) else: raise IOError('template file "%s" could not be found' % self.template_file) return output, resources def from_filename(self, filename, resources=None, **kw): """ Convert a notebook from a notebook file. Parameters ---------- filename : str Full filename of the notebook file to open and convert. """ #Pull the metadata from the filesystem. if resources is None: resources = ResourcesDict() if not 'metadata' in resources or resources['metadata'] == '': resources['metadata'] = ResourcesDict() basename = os.path.basename(filename) notebook_name = basename[:basename.rfind('.')] resources['metadata']['name'] = notebook_name modified_date = datetime.datetime.fromtimestamp( os.path.getmtime(filename)) resources['metadata']['modified_date'] = modified_date.strftime( text.date_format) with io.open(filename) as f: return self.from_notebook_node(nbformat.read(f, 'json'), resources=resources, **kw) def from_file(self, file_stream, resources=None, **kw): """ Convert a notebook from a notebook file. Parameters ---------- file_stream : file-like object Notebook file-like object to convert. """ return self.from_notebook_node(nbformat.read(file_stream, 'json'), resources=resources, **kw) def register_preprocessor(self, preprocessor, enabled=False): """ Register a preprocessor. Preprocessors are classes that act upon the notebook before it is passed into the Jinja templating engine. Preprocessors are also capable of passing additional information to the Jinja templating engine. Parameters ---------- preprocessor : preprocessor """ if preprocessor is None: raise TypeError('preprocessor') isclass = isinstance(preprocessor, type) constructed = not isclass #Handle preprocessor's registration based on it's type if constructed and isinstance(preprocessor, py3compat.string_types): #Preprocessor is a string, import the namespace and recursively call #this register_preprocessor method preprocessor_cls = import_item(preprocessor) return self.register_preprocessor(preprocessor_cls, enabled) if constructed and hasattr(preprocessor, '__call__'): #Preprocessor is a function, no need to construct it. #Register and return the preprocessor. if enabled: preprocessor.enabled = True self._preprocessors.append(preprocessor) return preprocessor elif isclass and isinstance(preprocessor, MetaHasTraits): #Preprocessor is configurable. Make sure to pass in new default for #the enabled flag if one was specified. self.register_preprocessor(preprocessor(parent=self), enabled) elif isclass: #Preprocessor is not configurable, construct it self.register_preprocessor(preprocessor(), enabled) else: #Preprocessor is an instance of something without a __call__ #attribute. raise TypeError('preprocessor') def register_filter(self, name, jinja_filter): """ Register a filter. A filter is a function that accepts and acts on one string. The filters are accesible within the Jinja templating engine. Parameters ---------- name : str name to give the filter in the Jinja engine filter : filter """ if jinja_filter is None: raise TypeError('filter') isclass = isinstance(jinja_filter, type) constructed = not isclass #Handle filter's registration based on it's type if constructed and isinstance(jinja_filter, py3compat.string_types): #filter is a string, import the namespace and recursively call #this register_filter method filter_cls = import_item(jinja_filter) return self.register_filter(name, filter_cls) if constructed and hasattr(jinja_filter, '__call__'): #filter is a function, no need to construct it. self.environment.filters[name] = jinja_filter return jinja_filter elif isclass and isinstance(jinja_filter, MetaHasTraits): #filter is configurable. Make sure to pass in new default for #the enabled flag if one was specified. filter_instance = jinja_filter(parent=self) self.register_filter(name, filter_instance) elif isclass: #filter is not configurable, construct it filter_instance = jinja_filter() self.register_filter(name, filter_instance) else: #filter is an instance of something without a __call__ #attribute. raise TypeError('filter') def _init_template(self): """ Make sure a template name is specified. If one isn't specified, try to build one from the information we know. """ self._template_file_changed('template_file', self.template_file, self.template_file) def _init_environment(self, extra_loaders=None): """ Create the Jinja templating environment. """ here = os.path.dirname(os.path.realpath(__file__)) loaders = [] if extra_loaders: loaders.extend(extra_loaders) paths = self.template_path paths.extend([ os.path.join(here, self.default_template_path), os.path.join(here, self.template_skeleton_path) ]) loaders.append(FileSystemLoader(paths)) self.environment = Environment(loader=ChoiceLoader(loaders), extensions=JINJA_EXTENSIONS) #Set special Jinja2 syntax that will not conflict with latex. if self.jinja_logic_block_start: self.environment.block_start_string = self.jinja_logic_block_start if self.jinja_logic_block_end: self.environment.block_end_string = self.jinja_logic_block_end if self.jinja_variable_block_start: self.environment.variable_start_string = self.jinja_variable_block_start if self.jinja_variable_block_end: self.environment.variable_end_string = self.jinja_variable_block_end if self.jinja_comment_block_start: self.environment.comment_start_string = self.jinja_comment_block_start if self.jinja_comment_block_end: self.environment.comment_end_string = self.jinja_comment_block_end def _init_preprocessors(self): """ Register all of the preprocessors needed for this exporter, disabled unless specified explicitly. """ self._preprocessors = [] #Load default preprocessors (not necessarly enabled by default). if self.default_preprocessors: for preprocessor in self.default_preprocessors: self.register_preprocessor(preprocessor) #Load user preprocessors. Enable by default. if self.preprocessors: for preprocessor in self.preprocessors: self.register_preprocessor(preprocessor, enabled=True) def _init_filters(self): """ Register all of the filters required for the exporter. """ #Add default filters to the Jinja2 environment for key, value in default_filters.items(): self.register_filter(key, value) #Load user filters. Overwrite existing filters if need be. if self.filters: for key, user_filter in self.filters.items(): self.register_filter(key, user_filter) def _init_resources(self, resources): #Make sure the resources dict is of ResourcesDict type. if resources is None: resources = ResourcesDict() if not isinstance(resources, ResourcesDict): new_resources = ResourcesDict() new_resources.update(resources) resources = new_resources #Make sure the metadata extension exists in resources if 'metadata' in resources: if not isinstance(resources['metadata'], ResourcesDict): resources['metadata'] = ResourcesDict(resources['metadata']) else: resources['metadata'] = ResourcesDict() if not resources['metadata']['name']: resources['metadata']['name'] = 'Notebook' #Set the output extension resources['output_extension'] = self.file_extension return resources def _preprocess(self, nb, resources): """ Preprocess the notebook before passing it into the Jinja engine. To preprocess the notebook is to apply all of the Parameters ---------- nb : notebook node notebook that is being exported. resources : a dict of additional resources that can be accessed read/write by preprocessors and filters. """ # Do a copy.deepcopy first, # we are never safe enough with what the preprocessors could do. nbc = copy.deepcopy(nb) resc = copy.deepcopy(resources) #Run each preprocessor on the notebook. Carry the output along #to each preprocessor for preprocessor in self._preprocessors: nbc, resc = preprocessor(nbc, resc) return nbc, resc
class TaskScheduler(SessionFactory): """Python TaskScheduler object. This is the simplest object that supports msg_id based DAG dependencies. *Only* task msg_ids are checked, not msg_ids of jobs submitted via the MUX queue. """ hwm = Integer(1, config=True, help="""specify the High Water Mark (HWM) for the downstream socket in the Task scheduler. This is the maximum number of allowed outstanding tasks on each engine. The default (1) means that only one task can be outstanding on each engine. Setting TaskScheduler.hwm=0 means there is no limit, and the engines continue to be assigned tasks while they are working, effectively hiding network latency behind computation, but can result in an imbalance of work when submitting many heterogenous tasks all at once. Any positive value greater than one is a compromise between the two. """) scheme_name = Enum( ('leastload', 'pure', 'lru', 'plainrandom', 'weighted', 'twobin'), 'leastload', config=True, allow_none=False, help="""select the task scheduler scheme [default: Python LRU] Options are: 'pure', 'lru', 'plainrandom', 'weighted', 'twobin','leastload'""" ) def _scheme_name_changed(self, old, new): self.log.debug("Using scheme %r" % new) self.scheme = globals()[new] # input arguments: scheme = Instance(FunctionType) # function for determining the destination def _scheme_default(self): return leastload client_stream = Instance(zmqstream.ZMQStream) # client-facing stream engine_stream = Instance(zmqstream.ZMQStream) # engine-facing stream notifier_stream = Instance(zmqstream.ZMQStream) # hub-facing sub stream mon_stream = Instance(zmqstream.ZMQStream) # hub-facing pub stream query_stream = Instance(zmqstream.ZMQStream) # hub-facing DEALER stream # internals: queue = Instance(deque) # sorted list of Jobs def _queue_default(self): return deque() queue_map = Dict() # dict by msg_id of Jobs (for O(1) access to the Queue) graph = Dict() # dict by msg_id of [ msg_ids that depend on key ] retries = Dict() # dict by msg_id of retries remaining (non-neg ints) # waiting = List() # list of msg_ids ready to run, but haven't due to HWM pending = Dict() # dict by engine_uuid of submitted tasks completed = Dict() # dict by engine_uuid of completed tasks failed = Dict() # dict by engine_uuid of failed tasks destinations = Dict( ) # dict by msg_id of engine_uuids where jobs ran (reverse of completed+failed) clients = Dict() # dict by msg_id for who submitted the task targets = List() # list of target IDENTs loads = List() # list of engine loads # full = Set() # set of IDENTs that have HWM outstanding tasks all_completed = Set() # set of all completed tasks all_failed = Set() # set of all failed tasks all_done = Set() # set of all finished tasks=union(completed,failed) all_ids = Set() # set of all submitted task IDs ident = CBytes() # ZMQ identity. This should just be self.session.session # but ensure Bytes def _ident_default(self): return self.session.bsession def start(self): self.query_stream.on_recv(self.dispatch_query_reply) self.session.send(self.query_stream, "connection_request", {}) self.engine_stream.on_recv(self.dispatch_result, copy=False) self.client_stream.on_recv(self.dispatch_submission, copy=False) self._notification_handlers = dict( registration_notification=self._register_engine, unregistration_notification=self._unregister_engine) self.notifier_stream.on_recv(self.dispatch_notification) self.log.info("Scheduler started [%s]" % self.scheme_name) def resume_receiving(self): """Resume accepting jobs.""" self.client_stream.on_recv(self.dispatch_submission, copy=False) def stop_receiving(self): """Stop accepting jobs while there are no engines. Leave them in the ZMQ queue.""" self.client_stream.on_recv(None) #----------------------------------------------------------------------- # [Un]Registration Handling #----------------------------------------------------------------------- def dispatch_query_reply(self, msg): """handle reply to our initial connection request""" try: idents, msg = self.session.feed_identities(msg) except ValueError: self.log.warn("task::Invalid Message: %r", msg) return try: msg = self.session.unserialize(msg) except ValueError: self.log.warn("task::Unauthorized message from: %r" % idents) return content = msg['content'] for uuid in content.get('engines', {}).values(): self._register_engine(cast_bytes(uuid)) @util.log_errors def dispatch_notification(self, msg): """dispatch register/unregister events.""" try: idents, msg = self.session.feed_identities(msg) except ValueError: self.log.warn("task::Invalid Message: %r", msg) return try: msg = self.session.unserialize(msg) except ValueError: self.log.warn("task::Unauthorized message from: %r" % idents) return msg_type = msg['header']['msg_type'] handler = self._notification_handlers.get(msg_type, None) if handler is None: self.log.error("Unhandled message type: %r" % msg_type) else: try: handler(cast_bytes(msg['content']['uuid'])) except Exception: self.log.error("task::Invalid notification msg: %r", msg, exc_info=True) def _register_engine(self, uid): """New engine with ident `uid` became available.""" # head of the line: self.targets.insert(0, uid) self.loads.insert(0, 0) # initialize sets self.completed[uid] = set() self.failed[uid] = set() self.pending[uid] = {} # rescan the graph: self.update_graph(None) def _unregister_engine(self, uid): """Existing engine with ident `uid` became unavailable.""" if len(self.targets) == 1: # this was our only engine pass # handle any potentially finished tasks: self.engine_stream.flush() # don't pop destinations, because they might be used later # map(self.destinations.pop, self.completed.pop(uid)) # map(self.destinations.pop, self.failed.pop(uid)) # prevent this engine from receiving work idx = self.targets.index(uid) self.targets.pop(idx) self.loads.pop(idx) # wait 5 seconds before cleaning up pending jobs, since the results might # still be incoming if self.pending[uid]: dc = ioloop.DelayedCallback( lambda: self.handle_stranded_tasks(uid), 5000, self.loop) dc.start() else: self.completed.pop(uid) self.failed.pop(uid) def handle_stranded_tasks(self, engine): """Deal with jobs resident in an engine that died.""" lost = self.pending[engine] for msg_id in lost.keys(): if msg_id not in self.pending[engine]: # prevent double-handling of messages continue raw_msg = lost[msg_id].raw_msg idents, msg = self.session.feed_identities(raw_msg, copy=False) parent = self.session.unpack(msg[1].bytes) idents = [engine, idents[0]] # build fake error reply try: raise error.EngineError( "Engine %r died while running task %r" % (engine, msg_id)) except: content = error.wrap_exception() # build fake metadata md = dict( status=u'error', engine=engine, date=datetime.now(), ) msg = self.session.msg('apply_reply', content, parent=parent, metadata=md) raw_reply = map(zmq.Message, self.session.serialize(msg, ident=idents)) # and dispatch it self.dispatch_result(raw_reply) # finally scrub completed/failed lists self.completed.pop(engine) self.failed.pop(engine) #----------------------------------------------------------------------- # Job Submission #----------------------------------------------------------------------- @util.log_errors def dispatch_submission(self, raw_msg): """Dispatch job submission to appropriate handlers.""" # ensure targets up to date: self.notifier_stream.flush() try: idents, msg = self.session.feed_identities(raw_msg, copy=False) msg = self.session.unserialize(msg, content=False, copy=False) except Exception: self.log.error("task::Invaid task msg: %r" % raw_msg, exc_info=True) return # send to monitor self.mon_stream.send_multipart([b'intask'] + raw_msg, copy=False) header = msg['header'] md = msg['metadata'] msg_id = header['msg_id'] self.all_ids.add(msg_id) # get targets as a set of bytes objects # from a list of unicode objects targets = md.get('targets', []) targets = map(cast_bytes, targets) targets = set(targets) retries = md.get('retries', 0) self.retries[msg_id] = retries # time dependencies after = md.get('after', None) if after: after = Dependency(after) if after.all: if after.success: after = Dependency( after.difference(self.all_completed), success=after.success, failure=after.failure, all=after.all, ) if after.failure: after = Dependency( after.difference(self.all_failed), success=after.success, failure=after.failure, all=after.all, ) if after.check(self.all_completed, self.all_failed): # recast as empty set, if `after` already met, # to prevent unnecessary set comparisons after = MET else: after = MET # location dependencies follow = Dependency(md.get('follow', [])) # turn timeouts into datetime objects: timeout = md.get('timeout', None) if timeout: timeout = time.time() + float(timeout) job = Job( msg_id=msg_id, raw_msg=raw_msg, idents=idents, msg=msg, header=header, targets=targets, after=after, follow=follow, timeout=timeout, metadata=md, ) if timeout: # schedule timeout callback self.loop.add_timeout(timeout, lambda: self.job_timeout(job)) # validate and reduce dependencies: for dep in after, follow: if not dep: # empty dependency continue # check valid: if msg_id in dep or dep.difference(self.all_ids): self.queue_map[msg_id] = job return self.fail_unreachable(msg_id, error.InvalidDependency) # check if unreachable: if dep.unreachable(self.all_completed, self.all_failed): self.queue_map[msg_id] = job return self.fail_unreachable(msg_id) if after.check(self.all_completed, self.all_failed): # time deps already met, try to run if not self.maybe_run(job): # can't run yet if msg_id not in self.all_failed: # could have failed as unreachable self.save_unmet(job) else: self.save_unmet(job) def job_timeout(self, job): """callback for a job's timeout. The job may or may not have been run at this point. """ now = time.time() if job.timeout >= (now + 1): self.log.warn("task %s timeout fired prematurely: %s > %s", job.msg_id, job.timeout, now) if job.msg_id in self.queue_map: # still waiting, but ran out of time self.log.info("task %r timed out", job.msg_id) self.fail_unreachable(job.msg_id, error.TaskTimeout) def fail_unreachable(self, msg_id, why=error.ImpossibleDependency): """a task has become unreachable, send a reply with an ImpossibleDependency error.""" if msg_id not in self.queue_map: self.log.error("task %r already failed!", msg_id) return job = self.queue_map.pop(msg_id) # lazy-delete from the queue job.removed = True for mid in job.dependents: if mid in self.graph: self.graph[mid].remove(msg_id) try: raise why() except: content = error.wrap_exception() self.log.debug("task %r failing as unreachable with: %s", msg_id, content['ename']) self.all_done.add(msg_id) self.all_failed.add(msg_id) msg = self.session.send(self.client_stream, 'apply_reply', content, parent=job.header, ident=job.idents) self.session.send(self.mon_stream, msg, ident=[b'outtask'] + job.idents) self.update_graph(msg_id, success=False) def available_engines(self): """return a list of available engine indices based on HWM""" if not self.hwm: return range(len(self.targets)) available = [] for idx in range(len(self.targets)): if self.loads[idx] < self.hwm: available.append(idx) return available def maybe_run(self, job): """check location dependencies, and run if they are met.""" msg_id = job.msg_id self.log.debug("Attempting to assign task %s", msg_id) available = self.available_engines() if not available: # no engines, definitely can't run return False if job.follow or job.targets or job.blacklist or self.hwm: # we need a can_run filter def can_run(idx): # check hwm if self.hwm and self.loads[idx] == self.hwm: return False target = self.targets[idx] # check blacklist if target in job.blacklist: return False # check targets if job.targets and target not in job.targets: return False # check follow return job.follow.check(self.completed[target], self.failed[target]) indices = filter(can_run, available) if not indices: # couldn't run if job.follow.all: # check follow for impossibility dests = set() relevant = set() if job.follow.success: relevant = self.all_completed if job.follow.failure: relevant = relevant.union(self.all_failed) for m in job.follow.intersection(relevant): dests.add(self.destinations[m]) if len(dests) > 1: self.queue_map[msg_id] = job self.fail_unreachable(msg_id) return False if job.targets: # check blacklist+targets for impossibility job.targets.difference_update(job.blacklist) if not job.targets or not job.targets.intersection( self.targets): self.queue_map[msg_id] = job self.fail_unreachable(msg_id) return False return False else: indices = None self.submit_task(job, indices) return True def save_unmet(self, job): """Save a message for later submission when its dependencies are met.""" msg_id = job.msg_id self.log.debug("Adding task %s to the queue", msg_id) self.queue_map[msg_id] = job self.queue.append(job) # track the ids in follow or after, but not those already finished for dep_id in job.after.union(job.follow).difference(self.all_done): if dep_id not in self.graph: self.graph[dep_id] = set() self.graph[dep_id].add(msg_id) def submit_task(self, job, indices=None): """Submit a task to any of a subset of our targets.""" if indices: loads = [self.loads[i] for i in indices] else: loads = self.loads idx = self.scheme(loads) if indices: idx = indices[idx] target = self.targets[idx] # print (target, map(str, msg[:3])) # send job to the engine self.engine_stream.send(target, flags=zmq.SNDMORE, copy=False) self.engine_stream.send_multipart(job.raw_msg, copy=False) # update load self.add_job(idx) self.pending[target][job.msg_id] = job # notify Hub content = dict(msg_id=job.msg_id, engine_id=target.decode('ascii')) self.session.send(self.mon_stream, 'task_destination', content=content, ident=[b'tracktask', self.ident]) #----------------------------------------------------------------------- # Result Handling #----------------------------------------------------------------------- @util.log_errors def dispatch_result(self, raw_msg): """dispatch method for result replies""" try: idents, msg = self.session.feed_identities(raw_msg, copy=False) msg = self.session.unserialize(msg, content=False, copy=False) engine = idents[0] try: idx = self.targets.index(engine) except ValueError: pass # skip load-update for dead engines else: self.finish_job(idx) except Exception: self.log.error("task::Invaid result: %r", raw_msg, exc_info=True) return md = msg['metadata'] parent = msg['parent_header'] if md.get('dependencies_met', True): success = (md['status'] == 'ok') msg_id = parent['msg_id'] retries = self.retries[msg_id] if not success and retries > 0: # failed self.retries[msg_id] = retries - 1 self.handle_unmet_dependency(idents, parent) else: del self.retries[msg_id] # relay to client and update graph self.handle_result(idents, parent, raw_msg, success) # send to Hub monitor self.mon_stream.send_multipart([b'outtask'] + raw_msg, copy=False) else: self.handle_unmet_dependency(idents, parent) def handle_result(self, idents, parent, raw_msg, success=True): """handle a real task result, either success or failure""" # first, relay result to client engine = idents[0] client = idents[1] # swap_ids for ROUTER-ROUTER mirror raw_msg[:2] = [client, engine] # print (map(str, raw_msg[:4])) self.client_stream.send_multipart(raw_msg, copy=False) # now, update our data structures msg_id = parent['msg_id'] self.pending[engine].pop(msg_id) if success: self.completed[engine].add(msg_id) self.all_completed.add(msg_id) else: self.failed[engine].add(msg_id) self.all_failed.add(msg_id) self.all_done.add(msg_id) self.destinations[msg_id] = engine self.update_graph(msg_id, success) def handle_unmet_dependency(self, idents, parent): """handle an unmet dependency""" engine = idents[0] msg_id = parent['msg_id'] job = self.pending[engine].pop(msg_id) job.blacklist.add(engine) if job.blacklist == job.targets: self.queue_map[msg_id] = job self.fail_unreachable(msg_id) elif not self.maybe_run(job): # resubmit failed if msg_id not in self.all_failed: # put it back in our dependency tree self.save_unmet(job) if self.hwm: try: idx = self.targets.index(engine) except ValueError: pass # skip load-update for dead engines else: if self.loads[idx] == self.hwm - 1: self.update_graph(None) def update_graph(self, dep_id=None, success=True): """dep_id just finished. Update our dependency graph and submit any jobs that just became runnable. Called with dep_id=None to update entire graph for hwm, but without finishing a task. """ # print ("\n\n***********") # pprint (dep_id) # pprint (self.graph) # pprint (self.queue_map) # pprint (self.all_completed) # pprint (self.all_failed) # print ("\n\n***********\n\n") # update any jobs that depended on the dependency msg_ids = self.graph.pop(dep_id, []) # recheck *all* jobs if # a) we have HWM and an engine just become no longer full # or b) dep_id was given as None if dep_id is None or self.hwm and any( [load == self.hwm - 1 for load in self.loads]): jobs = self.queue using_queue = True else: using_queue = False jobs = deque(sorted(self.queue_map[msg_id] for msg_id in msg_ids)) to_restore = [] while jobs: job = jobs.popleft() if job.removed: continue msg_id = job.msg_id put_it_back = True if job.after.unreachable(self.all_completed, self.all_failed)\ or job.follow.unreachable(self.all_completed, self.all_failed): self.fail_unreachable(msg_id) put_it_back = False elif job.after.check(self.all_completed, self.all_failed): # time deps met, maybe run if self.maybe_run(job): put_it_back = False self.queue_map.pop(msg_id) for mid in job.dependents: if mid in self.graph: self.graph[mid].remove(msg_id) # abort the loop if we just filled up all of our engines. # avoids an O(N) operation in situation of full queue, # where graph update is triggered as soon as an engine becomes # non-full, and all tasks after the first are checked, # even though they can't run. if not self.available_engines(): break if using_queue and put_it_back: # popped a job from the queue but it neither ran nor failed, # so we need to put it back when we are done # make sure to_restore preserves the same ordering to_restore.append(job) # put back any tasks we popped but didn't run if using_queue: self.queue.extendleft(to_restore) #---------------------------------------------------------------------- # methods to be overridden by subclasses #---------------------------------------------------------------------- def add_job(self, idx): """Called after self.targets[idx] just got the job with header. Override with subclasses. The default ordering is simple LRU. The default loads are the number of outstanding jobs.""" self.loads[idx] += 1 for lis in (self.targets, self.loads): lis.append(lis.pop(idx)) def finish_job(self, idx): """Called after self.targets[idx] just finished a job. Override with subclasses.""" self.loads[idx] -= 1
class ProfileList(Application): name = u'ipython-profile' description = list_help examples = _list_examples aliases = Dict({ 'ipython-dir': 'ProfileList.ipython_dir', 'log-level': 'Application.log_level', }) flags = Dict( dict(debug=({ 'Application': { 'log_level': 0 } }, "Set Application.log_level to 0, maximizing log output."))) ipython_dir = Unicode(get_ipython_dir(), config=True, help=""" The name of the IPython directory. This directory is used for logging configuration (through profiles), history storage, etc. The default is usually $HOME/.ipython. This options can also be specified through the environment variable IPYTHONDIR. """) def _print_profiles(self, profiles): """print list of profiles, indented.""" for profile in profiles: print(' %s' % profile) def list_profile_dirs(self): profiles = list_bundled_profiles() if profiles: print() print("Available profiles in IPython:") self._print_profiles(profiles) print() print(" The first request for a bundled profile will copy it") print(" into your IPython directory (%s)," % self.ipython_dir) print(" where you can customize it.") profiles = list_profiles_in(self.ipython_dir) if profiles: print() print("Available profiles in %s:" % self.ipython_dir) self._print_profiles(profiles) profiles = list_profiles_in(py3compat.getcwd()) if profiles: print() print("Available profiles in current directory (%s):" % py3compat.getcwd()) self._print_profiles(profiles) print() print("To use any of the above profiles, start IPython with:") print(" ipython --profile=<name>") print() def start(self): self.list_profile_dirs()
class ClusterManager(LoggingConfigurable): profiles = Dict() delay = CFloat( 1., config=True, help="delay (in s) between starting the controller and the engines") loop = Instance('zmq.eventloop.ioloop.IOLoop') def _loop_default(self): from zmq.eventloop.ioloop import IOLoop return IOLoop.instance() def build_launchers(self, profile_dir): starter = DummyIPClusterStart(log=self.log) starter.initialize(['--profile-dir', profile_dir]) cl = starter.controller_launcher esl = starter.engine_launcher n = starter.n return cl, esl, n def get_profile_dir(self, name, path): p = ProfileDir.find_profile_dir_by_name(path, name=name) return p.location def update_profiles(self): """List all profiles in the ipython_dir and cwd. """ for path in [get_ipython_dir(), os.getcwdu()]: for profile in list_profiles_in(path): pd = self.get_profile_dir(profile, path) if profile not in self.profiles: self.log.debug("Overwriting profile %s" % profile) self.profiles[profile] = { 'profile': profile, 'profile_dir': pd, 'status': 'stopped' } def list_profiles(self): self.update_profiles() result = [self.profile_info(p) for p in sorted(self.profiles.keys())] return result def check_profile(self, profile): if profile not in self.profiles: raise web.HTTPError(404, u'profile not found') def profile_info(self, profile): self.check_profile(profile) result = {} data = self.profiles.get(profile) result['profile'] = profile result['profile_dir'] = data['profile_dir'] result['status'] = data['status'] if 'n' in data: result['n'] = data['n'] return result def start_cluster(self, profile, n=None): """Start a cluster for a given profile.""" self.check_profile(profile) data = self.profiles[profile] if data['status'] == 'running': raise web.HTTPError(409, u'cluster already running') cl, esl, default_n = self.build_launchers(data['profile_dir']) n = n if n is not None else default_n def clean_data(): data.pop('controller_launcher', None) data.pop('engine_set_launcher', None) data.pop('n', None) data['status'] = 'stopped' def engines_stopped(r): self.log.debug('Engines stopped') if cl.running: cl.stop() clean_data() esl.on_stop(engines_stopped) def controller_stopped(r): self.log.debug('Controller stopped') if esl.running: esl.stop() clean_data() cl.on_stop(controller_stopped) dc = ioloop.DelayedCallback(lambda: cl.start(), 0, self.loop) dc.start() dc = ioloop.DelayedCallback(lambda: esl.start(n), 1000 * self.delay, self.loop) dc.start() self.log.debug('Cluster started') data['controller_launcher'] = cl data['engine_set_launcher'] = esl data['n'] = n data['status'] = 'running' return self.profile_info(profile) def stop_cluster(self, profile): """Stop a cluster for a given profile.""" self.check_profile(profile) data = self.profiles[profile] if data['status'] == 'stopped': raise web.HTTPError(409, u'cluster not running') data = self.profiles[profile] cl = data['controller_launcher'] esl = data['engine_set_launcher'] if cl.running: cl.stop() if esl.running: esl.stop() # Return a temp info dict, the real one is updated in the on_stop # logic above. result = { 'profile': data['profile'], 'profile_dir': data['profile_dir'], 'status': 'stopped' } return result def stop_all_clusters(self): for p in self.profiles.keys(): self.stop_cluster(profile)
class ProfileCreate(BaseIPythonApplication): name = u'ipython-profile' description = create_help examples = _create_examples auto_create = Bool(True, config=False) def _log_format_default(self): return "[%(name)s] %(message)s" def _copy_config_files_default(self): return True parallel = Bool(False, config=True, help="whether to include parallel computing config files") def _parallel_changed(self, name, old, new): parallel_files = [ 'ipcontroller_config.py', 'ipengine_config.py', 'ipcluster_config.py' ] if new: for cf in parallel_files: self.config_files.append(cf) else: for cf in parallel_files: if cf in self.config_files: self.config_files.remove(cf) def parse_command_line(self, argv): super(ProfileCreate, self).parse_command_line(argv) # accept positional arg as profile name if self.extra_args: self.profile = self.extra_args[0] flags = Dict(create_flags) classes = [ProfileDir] def _import_app(self, app_path): """import an app class""" app = None name = app_path.rsplit('.', 1)[-1] try: app = import_item(app_path) except ImportError: self.log.info("Couldn't import %s, config file will be excluded", name) except Exception: self.log.warn('Unexpected error importing %s', name, exc_info=True) return app def init_config_files(self): super(ProfileCreate, self).init_config_files() # use local imports, since these classes may import from here from IPython.terminal.ipapp import TerminalIPythonApp apps = [TerminalIPythonApp] for app_path in ( 'IPython.kernel.zmq.kernelapp.IPKernelApp', 'IPython.terminal.console.app.ZMQTerminalIPythonApp', 'IPython.qt.console.qtconsoleapp.IPythonQtConsoleApp', 'IPython.html.notebookapp.NotebookApp', 'IPython.nbconvert.nbconvertapp.NbConvertApp', ): app = self._import_app(app_path) if app is not None: apps.append(app) if self.parallel: from IPython.parallel.apps.ipcontrollerapp import IPControllerApp from IPython.parallel.apps.ipengineapp import IPEngineApp from IPython.parallel.apps.ipclusterapp import IPClusterStart from IPython.parallel.apps.iploggerapp import IPLoggerApp apps.extend([ IPControllerApp, IPEngineApp, IPClusterStart, IPLoggerApp, ]) for App in apps: app = App() app.config.update(self.config) app.log = self.log app.overwrite = self.overwrite app.copy_config_files = True app.ipython_dir = self.ipython_dir app.profile_dir = self.profile_dir app.init_config_files() def stage_default_config_file(self): pass
class ExtractFigureTransformer(ActivatableTransformer): """ Extracts all of the figures from the notebook file. The extracted figures are returned in the 'resources' dictionary. """ extra_extension_map = Dict({}, config=True, help="""Extra map to override extension based on type. Useful for latex where SVG will be converted to PDF before inclusion """) key_format_map = Dict({}, config=True,) figure_name_format_map = Dict({}, config=True) display_data_priority = List(['svg', 'png', 'latex', 'jpg', 'jpeg','text']) #TODO: Change this to .format {} syntax default_key_template = Unicode('_fig_{index:02d}.{ext}', config=True) def cell_transform(self, cell, resources, index): """ Apply a transformation on each cell, Parameters ---------- cell : NotebookNode cell Notebook cell being processed resources : dictionary Additional resources used in the conversion process. Allows transformers to pass variables into the Jinja engine. index : int Index of the cell being processed (see base.py) """ if resources.get(FIGURES_KEY, None) is None : resources[FIGURES_KEY] = {TEXT_KEY:{},BINARY_KEY:{}} for out in cell.get('outputs', []): for out_type in self.display_data_priority: if out.hasattr(out_type): figname, key, data, binary = self._new_figure(out[out_type], out_type) out['key_'+out_type] = figname if binary : resources[FIGURES_KEY][BINARY_KEY][key] = data else : resources[FIGURES_KEY][TEXT_KEY][key] = data index += 1 return cell, resources def _get_override_extension(self, extension): """Gets the overriden extension if it exists, else returns extension. Parameters ---------- extension : str File extension. """ if extension in self.extra_extension_map : return self.extra_extension_map[extension] return extension def _new_figure(self, data, format): """Create a new figure file in the given format. Parameters ---------- data : str Cell data (from Notebook node cell) format : str Figure format index : int Index of the figure being extracted """ figure_name_template = self.figure_name_format_map.get(format, self.default_key_template) key_template = self.key_format_map.get(format, self.default_key_template) #Make sure the figure counter has been initiated if self.index_generator is None: self.index_generator = self._index_generator() #TODO: option to pass the hash as data? index = self.index_generator.next() figure_name = figure_name_template.format(index=index, ext=self._get_override_extension(format)) key = key_template.format(index=index, ext=self._get_override_extension(format)) #Binary files are base64-encoded, SVG is already XML binary = False if format in ('png', 'jpg', 'pdf'): data = data.decode('base64') binary = True return figure_name, key, data, binary def _index_generator(self): """ Generates a unique index for association with extracted figures. """ index = 0 while True: yield index index+=1
class IPythonConsoleApp(Configurable): name = 'ipython-console-mixin' default_config_file_name='ipython_config.py' description = """ The IPython Mixin Console. This class contains the common portions of console client (QtConsole, ZMQ-based terminal console, etc). It is not a full console, in that launched terminal subprocesses will not be able to accept input. The Console using this mixing supports various extra features beyond the single-process Terminal IPython shell, such as connecting to existing kernel, via: ipython <appname> --existing as well as tunnel via SSH """ classes = classes flags = Dict(flags) aliases = Dict(aliases) kernel_manager_class = BlockingKernelManager kernel_argv = List(Unicode) # frontend flags&aliases to be stripped when building kernel_argv frontend_flags = Any(app_flags) frontend_aliases = Any(app_aliases) # create requested profiles by default, if they don't exist: auto_create = CBool(True) # connection info: sshserver = Unicode('', config=True, help="""The SSH server to use to connect to the kernel.""") sshkey = Unicode('', config=True, help="""Path to the ssh key to use for logging in to the ssh server.""") hb_port = Int(0, config=True, help="set the heartbeat port [default: random]") shell_port = Int(0, config=True, help="set the shell (ROUTER) port [default: random]") iopub_port = Int(0, config=True, help="set the iopub (PUB) port [default: random]") stdin_port = Int(0, config=True, help="set the stdin (DEALER) port [default: random]") connection_file = Unicode('', config=True, help="""JSON file in which to store connection info [default: kernel-<pid>.json] This file will contain the IP, ports, and authentication key needed to connect clients to this kernel. By default, this file will be created in the security-dir of the current profile, but can be specified by absolute path. """) def _connection_file_default(self): return 'kernel-%i.json' % os.getpid() existing = CUnicode('', config=True, help="""Connect to an already running kernel""") confirm_exit = CBool(True, config=True, help=""" Set to display confirmation dialog on exit. You can always use 'exit' or 'quit', to force a direct exit without any confirmation.""", ) def build_kernel_argv(self, argv=None): """build argv to be passed to kernel subprocess""" if argv is None: argv = sys.argv[1:] self.kernel_argv = swallow_argv(argv, self.frontend_aliases, self.frontend_flags) # kernel should inherit default config file from frontend self.kernel_argv.append("--IPKernelApp.parent_appname='%s'" % self.name) def init_connection_file(self): """find the connection file, and load the info if found. The current working directory and the current profile's security directory will be searched for the file if it is not given by absolute path. When attempting to connect to an existing kernel and the `--existing` argument does not match an existing file, it will be interpreted as a fileglob, and the matching file in the current profile's security dir with the latest access time will be used. After this method is called, self.connection_file contains the *full path* to the connection file, never just its name. """ if self.existing: try: cf = find_connection_file(self.existing) except Exception: self.log.critical("Could not find existing kernel connection file %s", self.existing) self.exit(1) self.log.info("Connecting to existing kernel: %s" % cf) self.connection_file = cf else: # not existing, check if we are going to write the file # and ensure that self.connection_file is a full path, not just the shortname try: cf = find_connection_file(self.connection_file) except Exception: # file might not exist if self.connection_file == os.path.basename(self.connection_file): # just shortname, put it in security dir cf = os.path.join(self.profile_dir.security_dir, self.connection_file) else: cf = self.connection_file self.connection_file = cf # should load_connection_file only be used for existing? # as it is now, this allows reusing ports if an existing # file is requested try: self.load_connection_file() except Exception: self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True) self.exit(1) def load_connection_file(self): """load ip/port/hmac config from JSON connection file""" # this is identical to IPKernelApp.load_connection_file # perhaps it can be centralized somewhere? try: fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir]) except IOError: self.log.debug("Connection File not found: %s", self.connection_file) return self.log.debug(u"Loading connection file %s", fname) with open(fname) as f: cfg = json.load(f) self.config.KernelManager.transport = cfg.get('transport', 'tcp') self.config.KernelManager.ip = cfg.get('ip', LOCALHOST) for channel in ('hb', 'shell', 'iopub', 'stdin'): name = channel + '_port' if getattr(self, name) == 0 and name in cfg: # not overridden by config or cl_args setattr(self, name, cfg[name]) if 'key' in cfg: self.config.Session.key = str_to_bytes(cfg['key']) def init_ssh(self): """set up ssh tunnels, if needed.""" if not self.existing or (not self.sshserver and not self.sshkey): return self.load_connection_file() transport = self.config.KernelManager.transport ip = self.config.KernelManager.ip if transport != 'tcp': self.log.error("Can only use ssh tunnels with TCP sockets, not %s", transport) sys.exit(-1) if self.sshkey and not self.sshserver: # specifying just the key implies that we are connecting directly self.sshserver = ip ip = LOCALHOST # build connection dict for tunnels: info = dict(ip=ip, shell_port=self.shell_port, iopub_port=self.iopub_port, stdin_port=self.stdin_port, hb_port=self.hb_port ) self.log.info("Forwarding connections to %s via %s"%(ip, self.sshserver)) # tunnels return a new set of ports, which will be on localhost: self.config.KernelManager.ip = LOCALHOST try: newports = tunnel_to_kernel(info, self.sshserver, self.sshkey) except: # even catch KeyboardInterrupt self.log.error("Could not setup tunnels", exc_info=True) self.exit(1) self.shell_port, self.iopub_port, self.stdin_port, self.hb_port = newports cf = self.connection_file base,ext = os.path.splitext(cf) base = os.path.basename(base) self.connection_file = os.path.basename(base)+'-ssh'+ext self.log.critical("To connect another client via this tunnel, use:") self.log.critical("--existing %s" % self.connection_file) def _new_connection_file(self): cf = '' while not cf: # we don't need a 128b id to distinguish kernels, use more readable # 48b node segment (12 hex chars). Users running more than 32k simultaneous # kernels can subclass. ident = str(uuid.uuid4()).split('-')[-1] cf = os.path.join(self.profile_dir.security_dir, 'kernel-%s.json' % ident) # only keep if it's actually new. Protect against unlikely collision # in 48b random search space cf = cf if not os.path.exists(cf) else '' return cf def init_kernel_manager(self): # Don't let Qt or ZMQ swallow KeyboardInterupts. signal.signal(signal.SIGINT, signal.SIG_DFL) # Create a KernelManager and start a kernel. self.kernel_manager = self.kernel_manager_class( shell_port=self.shell_port, iopub_port=self.iopub_port, stdin_port=self.stdin_port, hb_port=self.hb_port, connection_file=self.connection_file, config=self.config, ) # start the kernel if not self.existing: self.kernel_manager.start_kernel(extra_arguments=self.kernel_argv) atexit.register(self.kernel_manager.cleanup_ipc_files) elif self.sshserver: # ssh, write new connection file self.kernel_manager.write_connection_file() atexit.register(self.kernel_manager.cleanup_connection_file) self.kernel_manager.start_channels() def initialize(self, argv=None): """ Classes which mix this class in should call: IPythonConsoleApp.initialize(self,argv) """ self.init_connection_file() default_secure(self.config) self.init_ssh() self.init_kernel_manager()
class PromptManager(Configurable): """This is the primary interface for producing IPython's prompts.""" shell = Instance('IPython.core.interactiveshell.InteractiveShellABC') color_scheme_table = Instance(coloransi.ColorSchemeTable) color_scheme = Unicode('Linux', config=True) def _color_scheme_changed(self, name, new_value): self.color_scheme_table.set_active_scheme(new_value) for pname in ['in', 'in2', 'out', 'rewrite']: # We need to recalculate the number of invisible characters self.update_prompt(pname) lazy_evaluate_fields = Dict(help=""" This maps field names used in the prompt templates to functions which will be called when the prompt is rendered. This allows us to include things like the current time in the prompts. Functions are only called if they are used in the prompt. """) def _lazy_evaluate_fields_default(self): return lazily_evaluate.copy() in_template = Unicode( 'In [\\#]: ', config=True, help="Input prompt. '\\#' will be transformed to the prompt number") in2_template = Unicode(' .\\D.: ', config=True, help="Continuation prompt.") out_template = Unicode( 'Out[\\#]: ', config=True, help="Output prompt. '\\#' will be transformed to the prompt number") justify = Bool(True, config=True, help=""" If True (default), each prompt will be right-aligned with the preceding one. """) # We actually store the expanded templates here: templates = Dict() # The number of characters in the last prompt rendered, not including # colour characters. width = Int() txtwidth = Int() # Not including right-justification # The number of characters in each prompt which don't contribute to width invisible_chars = Dict() def _invisible_chars_default(self): return {'in': 0, 'in2': 0, 'out': 0, 'rewrite': 0} def __init__(self, shell, **kwargs): super(PromptManager, self).__init__(shell=shell, **kwargs) # Prepare colour scheme table self.color_scheme_table = coloransi.ColorSchemeTable( [PColNoColors, PColLinux, PColLightBG], self.color_scheme) self._formatter = UserNSFormatter(shell) # Prepare templates & numbers of invisible characters self.update_prompt('in', self.in_template) self.update_prompt('in2', self.in2_template) self.update_prompt('out', self.out_template) self.update_prompt('rewrite') self.on_trait_change(self._update_prompt_trait, ['in_template', 'in2_template', 'out_template']) def update_prompt(self, name, new_template=None): """This is called when a prompt template is updated. It processes abbreviations used in the prompt template (like \#) and calculates how many invisible characters (ANSI colour escapes) the resulting prompt contains. It is also called for each prompt on changing the colour scheme. In both cases, traitlets should take care of calling this automatically. """ if new_template is not None: self.templates[name] = multiple_replace(prompt_abbreviations, new_template) # We count invisible characters (colour escapes) on the last line of the # prompt, to calculate the width for lining up subsequent prompts. invis_chars = _lenlastline(self._render(name, color=True)) - \ _lenlastline(self._render(name, color=False)) self.invisible_chars[name] = invis_chars def _update_prompt_trait(self, traitname, new_template): name = traitname[:-9] # Cut off '_template' self.update_prompt(name, new_template) def _render(self, name, color=True, **kwargs): """Render but don't justify, or update the width or txtwidth attributes. """ if name == 'rewrite': return self._render_rewrite(color=color) if color: scheme = self.color_scheme_table.active_colors if name == 'out': colors = color_lists['normal'] colors.number, colors.prompt, colors.normal = \ scheme.out_number, scheme.out_prompt, scheme.normal else: colors = color_lists['inp'] colors.number, colors.prompt, colors.normal = \ scheme.in_number, scheme.in_prompt, scheme.in_normal if name == 'in2': colors.prompt = scheme.in_prompt2 else: # No color colors = color_lists['nocolor'] colors.number, colors.prompt, colors.normal = '', '', '' count = self.shell.execution_count # Shorthand # Build the dictionary to be passed to string formatting fmtargs = dict(color=colors, count=count, dots="." * len(str(count)), width=self.width, txtwidth=self.txtwidth) fmtargs.update(self.lazy_evaluate_fields) fmtargs.update(kwargs) # Prepare the prompt prompt = colors.prompt + self.templates[name] + colors.normal # Fill in required fields return self._formatter.format(prompt, **fmtargs) def _render_rewrite(self, color=True): """Render the ---> rewrite prompt.""" if color: scheme = self.color_scheme_table.active_colors # We need a non-input version of these escapes color_prompt = scheme.in_prompt.replace("\001", "").replace("\002", "") color_normal = scheme.normal else: color_prompt, color_normal = '', '' return color_prompt + "-> ".rjust(self.txtwidth, "-") + color_normal def render(self, name, color=True, just=None, **kwargs): """ Render the selected prompt. Parameters ---------- name : str Which prompt to render. One of 'in', 'in2', 'out', 'rewrite' color : bool If True (default), include ANSI escape sequences for a coloured prompt. just : bool If True, justify the prompt to the width of the last prompt. The default is stored in self.justify. **kwargs : Additional arguments will be passed to the string formatting operation, so they can override the values that would otherwise fill in the template. Returns ------- A string containing the rendered prompt. """ res = self._render(name, color=color, **kwargs) # Handle justification of prompt invis_chars = self.invisible_chars[name] if color else 0 self.txtwidth = _lenlastline(res) - invis_chars just = self.justify if (just is None) else just # If the prompt spans more than one line, don't try to justify it: if just and name != 'in' and ('\n' not in res) and ('\r' not in res): res = res.rjust(self.width + invis_chars) self.width = _lenlastline(res) - invis_chars return res
class BaseParallelApplication(BaseIPythonApplication): """The base Application for IPython.parallel apps Principle extensions to BaseIPyythonApplication: * work_dir * remote logging via pyzmq * IOLoop instance """ crash_handler_class = ParallelCrashHandler def _log_level_default(self): # temporarily override default_log_level to INFO return logging.INFO def _log_format_default(self): """override default log format to include time""" return u"%(asctime)s.%(msecs).03d [%(name)s] %(message)s" work_dir = Unicode(os.getcwdu(), config=True, help='Set the working dir for the process.') def _work_dir_changed(self, name, old, new): self.work_dir = unicode(expand_path(new)) log_to_file = Bool(config=True, help="whether to log to a file") clean_logs = Bool(False, config=True, help="whether to cleanup old logfiles before starting") log_url = Unicode('', config=True, help="The ZMQ URL of the iplogger to aggregate logging.") cluster_id = Unicode( '', config=True, help= """String id to add to runtime files, to prevent name collisions when using multiple clusters with a single profile simultaneously. When set, files will be named like: 'ipcontroller-<cluster_id>-engine.json' Since this is text inserted into filenames, typical recommendations apply: Simple character strings are ideal, and spaces are not recommended (but should generally work). """) def _cluster_id_changed(self, name, old, new): self.name = self.__class__.name if new: self.name += '-%s' % new def _config_files_default(self): return [ 'ipcontroller_config.py', 'ipengine_config.py', 'ipcluster_config.py' ] loop = Instance('zmq.eventloop.ioloop.IOLoop') def _loop_default(self): from zmq.eventloop.ioloop import IOLoop return IOLoop.instance() aliases = Dict(base_aliases) flags = Dict(base_flags) @catch_config_error def initialize(self, argv=None): """initialize the app""" super(BaseParallelApplication, self).initialize(argv) self.to_work_dir() self.reinit_logging() def to_work_dir(self): wd = self.work_dir if unicode(wd) != os.getcwdu(): os.chdir(wd) self.log.info("Changing to working dir: %s" % wd) # This is the working dir by now. sys.path.insert(0, '') def reinit_logging(self): # Remove old log files log_dir = self.profile_dir.log_dir if self.clean_logs: for f in os.listdir(log_dir): if re.match(r'%s-\d+\.(log|err|out)' % self.name, f): os.remove(os.path.join(log_dir, f)) if self.log_to_file: # Start logging to the new log file log_filename = self.name + u'-' + str(os.getpid()) + u'.log' logfile = os.path.join(log_dir, log_filename) open_log_file = open(logfile, 'w') else: open_log_file = None if open_log_file is not None: while self.log.handlers: self.log.removeHandler(self.log.handlers[0]) self._log_handler = logging.StreamHandler(open_log_file) self.log.addHandler(self._log_handler) else: self._log_handler = self.log.handlers[0] # Add timestamps to log format: self._log_formatter = logging.Formatter(self.log_format, datefmt="%Y-%m-%d %H:%M:%S") self._log_handler.setFormatter(self._log_formatter) # do not propagate log messages to root logger # ipcluster app will sometimes print duplicate messages during shutdown # if this is 1 (default): self.log.propagate = False def write_pid_file(self, overwrite=False): """Create a .pid file in the pid_dir with my pid. This must be called after pre_construct, which sets `self.pid_dir`. This raises :exc:`PIDFileError` if the pid file exists already. """ pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid') if os.path.isfile(pid_file): pid = self.get_pid_from_file() if not overwrite: raise PIDFileError( 'The pid file [%s] already exists. \nThis could mean that this ' 'server is already running with [pid=%s].' % (pid_file, pid)) with open(pid_file, 'w') as f: self.log.info("Creating pid file: %s" % pid_file) f.write(repr(os.getpid()) + '\n') def remove_pid_file(self): """Remove the pid file. This should be called at shutdown by registering a callback with :func:`reactor.addSystemEventTrigger`. This needs to return ``None``. """ pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid') if os.path.isfile(pid_file): try: self.log.info("Removing pid file: %s" % pid_file) os.remove(pid_file) except: self.log.warn("Error removing the pid file: %s" % pid_file) def get_pid_from_file(self): """Get the pid from the pid file. If the pid file doesn't exist a :exc:`PIDFileError` is raised. """ pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid') if os.path.isfile(pid_file): with open(pid_file, 'r') as f: s = f.read().strip() try: pid = int(s) except: raise PIDFileError("invalid pid file: %s (contents: %r)" % (pid_file, s)) return pid else: raise PIDFileError('pid file not found: %s' % pid_file) def check_pid(self, pid): if os.name == 'nt': try: import ctypes # returns 0 if no such process (of ours) exists # positive int otherwise p = ctypes.windll.kernel32.OpenProcess(1, 0, pid) except Exception: self.log.warn( "Could not determine whether pid %i is running via `OpenProcess`. " " Making the likely assumption that it is." % pid) return True return bool(p) else: try: p = Popen(['ps', 'x'], stdout=PIPE, stderr=PIPE) output, _ = p.communicate() except OSError: self.log.warn( "Could not determine whether pid %i is running via `ps x`. " " Making the likely assumption that it is." % pid) return True pids = map(int, re.findall(r'^\W*\d+', output, re.MULTILINE)) return pid in pids
class NotebookManager(LoggingConfigurable): notebook_dir = Unicode(os.getcwd(), config=True, help=""" The directory to use for notebooks. """) save_script = Bool( False, config=True, help="""Automatically create a Python script when saving the notebook. For easier use of import, %run and %loadpy across notebooks, a <notebook-name>.py script will be created next to any <notebook-name>.ipynb on each save. This can also be set with the short `--script` flag. """) filename_ext = Unicode(u'.ipynb') allowed_formats = List([u'json', u'py']) # Map notebook_ids to notebook names mapping = Dict() # Map notebook names to notebook_ids rev_mapping = Dict() def list_notebooks(self): """List all notebooks in the notebook dir. This returns a list of dicts of the form:: dict(notebook_id=notebook,name=name) """ names = glob.glob( os.path.join(self.notebook_dir, '*' + self.filename_ext)) names = [os.path.splitext(os.path.basename(name))[0] for name in names] data = [] for name in names: if name not in self.rev_mapping: notebook_id = self.new_notebook_id(name) else: notebook_id = self.rev_mapping[name] data.append(dict(notebook_id=notebook_id, name=name)) data = sorted(data, key=lambda item: item['name']) return data def new_notebook_id(self, name): """Generate a new notebook_id for a name and store its mappings.""" # TODO: the following will give stable urls for notebooks, but unless # the notebooks are immediately redirected to their new urls when their # filemname changes, nasty inconsistencies result. So for now it's # disabled and instead we use a random uuid4() call. But we leave the # logic here so that we can later reactivate it, whhen the necessary # url redirection code is written. #notebook_id = unicode(uuid.uuid5(uuid.NAMESPACE_URL, # 'file://'+self.get_path_by_name(name).encode('utf-8'))) notebook_id = unicode(uuid.uuid4()) self.mapping[notebook_id] = name self.rev_mapping[name] = notebook_id return notebook_id def delete_notebook_id(self, notebook_id): """Delete a notebook's id only. This doesn't delete the actual notebook.""" name = self.mapping[notebook_id] del self.mapping[notebook_id] del self.rev_mapping[name] def notebook_exists(self, notebook_id): """Does a notebook exist?""" if notebook_id not in self.mapping: return False path = self.get_path_by_name(self.mapping[notebook_id]) return os.path.isfile(path) def find_path(self, notebook_id): """Return a full path to a notebook given its notebook_id.""" try: name = self.mapping[notebook_id] except KeyError: raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id) return self.get_path_by_name(name) def get_path_by_name(self, name): """Return a full path to a notebook given its name.""" filename = name + self.filename_ext path = os.path.join(self.notebook_dir, filename) return path def get_notebook(self, notebook_id, format=u'json'): """Get the representation of a notebook in format by notebook_id.""" format = unicode(format) if format not in self.allowed_formats: raise web.HTTPError(415, u'Invalid notebook format: %s' % format) last_modified, nb = self.get_notebook_object(notebook_id) kwargs = {} if format == 'json': # don't split lines for sending over the wire, because it # should match the Python in-memory format. kwargs['split_lines'] = False data = current.writes(nb, format, **kwargs) name = nb.get('name', 'notebook') return last_modified, name, data def get_notebook_object(self, notebook_id): """Get the NotebookNode representation of a notebook by notebook_id.""" path = self.find_path(notebook_id) if not os.path.isfile(path): raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id) info = os.stat(path) last_modified = datetime.datetime.utcfromtimestamp(info.st_mtime) with open(path, 'r') as f: s = f.read() try: # v1 and v2 and json in the .ipynb files. nb = current.reads(s, u'json') except: raise web.HTTPError(500, u'Unreadable JSON notebook.') if 'name' not in nb: nb.name = os.path.split(path)[-1].split(u'.')[0] return last_modified, nb def save_new_notebook(self, data, name=None, format=u'json'): """Save a new notebook and return its notebook_id. If a name is passed in, it overrides any values in the notebook data and the value in the data is updated to use that value. """ if format not in self.allowed_formats: raise web.HTTPError(415, u'Invalid notebook format: %s' % format) try: nb = current.reads(data.decode('utf-8'), format) except: raise web.HTTPError(400, u'Invalid JSON data') if name is None: try: name = nb.metadata.name except AttributeError: raise web.HTTPError(400, u'Missing notebook name') nb.metadata.name = name notebook_id = self.new_notebook_id(name) self.save_notebook_object(notebook_id, nb) return notebook_id def save_notebook(self, notebook_id, data, name=None, format=u'json'): """Save an existing notebook by notebook_id.""" if format not in self.allowed_formats: raise web.HTTPError(415, u'Invalid notebook format: %s' % format) try: nb = current.reads(data.decode('utf-8'), format) except: raise web.HTTPError(400, u'Invalid JSON data') if name is not None: nb.metadata.name = name self.save_notebook_object(notebook_id, nb) def save_notebook_object(self, notebook_id, nb): """Save an existing notebook object by notebook_id.""" if notebook_id not in self.mapping: raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id) old_name = self.mapping[notebook_id] try: new_name = nb.metadata.name except AttributeError: raise web.HTTPError(400, u'Missing notebook name') path = self.get_path_by_name(new_name) try: with open(path, 'w') as f: current.write(nb, f, u'json') except Exception as e: raise web.HTTPError( 400, u'Unexpected error while saving notebook: %s' % e) # save .py script as well if self.save_script: pypath = os.path.splitext(path)[0] + '.py' try: with open(pypath, 'w') as f: current.write(nb, f, u'py') except Exception as e: raise web.HTTPError( 400, u'Unexpected error while saving notebook as script: %s' % e) if old_name != new_name: old_path = self.get_path_by_name(old_name) if os.path.isfile(old_path): os.unlink(old_path) if self.save_script: old_pypath = os.path.splitext(old_path)[0] + '.py' if os.path.isfile(old_pypath): os.unlink(old_pypath) self.mapping[notebook_id] = new_name self.rev_mapping[new_name] = notebook_id def delete_notebook(self, notebook_id): """Delete notebook by notebook_id.""" path = self.find_path(notebook_id) if not os.path.isfile(path): raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id) os.unlink(path) self.delete_notebook_id(notebook_id) def increment_filename(self, basename): """Return a non-used filename of the form basename<int>. This searches through the filenames (basename0, basename1, ...) until is find one that is not already being used. It is used to create Untitled and Copy names that are unique. """ i = 0 while True: name = u'%s%i' % (basename, i) path = self.get_path_by_name(name) if not os.path.isfile(path): break else: i = i + 1 return path, name def new_notebook(self): """Create a new notebook and return its notebook_id.""" path, name = self.increment_filename('Untitled') notebook_id = self.new_notebook_id(name) metadata = current.new_metadata(name=name) nb = current.new_notebook(metadata=metadata) with open(path, 'w') as f: current.write(nb, f, u'json') return notebook_id def copy_notebook(self, notebook_id): """Copy an existing notebook and return its notebook_id.""" last_mod, nb = self.get_notebook_object(notebook_id) name = nb.metadata.name + '-Copy' path, name = self.increment_filename(name) nb.metadata.name = name notebook_id = self.new_notebook_id(name) self.save_notebook_object(notebook_id, nb) return notebook_id
class Application(SingletonConfigurable): """A singleton application with full configuration support.""" # The name of the application, will usually match the name of the command # line application name = Unicode('application') # The description of the application that is printed at the beginning # of the help. description = Unicode('This is an application.') # default section descriptions option_description = Unicode(option_description) keyvalue_description = Unicode(keyvalue_description) subcommand_description = Unicode(subcommand_description) # The usage and example string that goes at the end of the help string. examples = Unicode() # A sequence of Configurable subclasses whose config=True attributes will # be exposed at the command line. classes = List([]) # The version string of this application. version = Unicode('0.0') # the argv used to initialize the application argv = List(Unicode) # The log level for the application log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'), default_value=logging.WARN, config=True, help="Set the log level by value or name.") def _log_level_changed(self, name, old, new): """Adjust the log level when log_level is set.""" if isinstance(new, str): new = getattr(logging, new) self.log_level = new self.log.setLevel(new) log_datefmt = Unicode("%Y-%m-%d %H:%M:%S", config=True, help="The date format used by logging formatters for %(asctime)s" ) def _log_datefmt_changed(self, name, old, new): self._log_format_changed() log_format = Unicode("[%(name)s]%(highlevel)s %(message)s", config=True, help="The Logging format template", ) def _log_format_changed(self, name, old, new): """Change the log formatter when log_format is set.""" _log_handler = self.log.handlers[0] _log_formatter = LevelFormatter(new, datefmt=self.log_datefmt) _log_handler.setFormatter(_log_formatter) log = Instance(logging.Logger) def _log_default(self): """Start logging for this application. The default is to log to stderr using a StreamHandler, if no default handler already exists. The log level starts at logging.WARN, but this can be adjusted by setting the ``log_level`` attribute. """ log = logging.getLogger(self.__class__.__name__) log.setLevel(self.log_level) log.propagate = False _log = log # copied from Logger.hasHandlers() (new in Python 3.2) while _log: if _log.handlers: return log if not _log.propagate: break else: _log = _log.parent if sys.executable.endswith('pythonw.exe'): # this should really go to a file, but file-logging is only # hooked up in parallel applications _log_handler = logging.StreamHandler(open(os.devnull, 'w')) else: _log_handler = logging.StreamHandler() _log_formatter = LevelFormatter(self.log_format, datefmt=self.log_datefmt) _log_handler.setFormatter(_log_formatter) log.addHandler(_log_handler) return log # the alias map for configurables aliases = Dict({'log-level' : 'Application.log_level'}) # flags for loading Configurables or store_const style flags # flags are loaded from this dict by '--key' flags # this must be a dict of two-tuples, the first element being the Config/dict # and the second being the help string for the flag flags = Dict() def _flags_changed(self, name, old, new): """ensure flags dict is valid""" for key,value in new.items(): assert len(value) == 2, "Bad flag: %r:%s"%(key,value) assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s"%(key,value) assert isinstance(value[1], str), "Bad flag: %r:%s"%(key,value) # subcommands for launching other applications # if this is not empty, this will be a parent Application # this must be a dict of two-tuples, # the first element being the application class/import string # and the second being the help string for the subcommand subcommands = Dict() # parse_command_line will initialize a subapp, if requested subapp = Instance('IPython.config.application.Application', allow_none=True) # extra command-line arguments that don't set config values extra_args = List(Unicode) def __init__(self, **kwargs): SingletonConfigurable.__init__(self, **kwargs) # Ensure my class is in self.classes, so my attributes appear in command line # options and config files. if self.__class__ not in self.classes: self.classes.insert(0, self.__class__) def _config_changed(self, name, old, new): SingletonConfigurable._config_changed(self, name, old, new) self.log.debug('Config changed:') self.log.debug(repr(new)) @catch_config_error def initialize(self, argv=None): """Do the basic steps to configure me. Override in subclasses. """ self.parse_command_line(argv) def start(self): """Start the app mainloop. Override in subclasses. """ if self.subapp is not None: return self.subapp.start() def print_alias_help(self): """Print the alias part of the help.""" if not self.aliases: return lines = [] classdict = {} for cls in self.classes: # include all parents (up to, but excluding Configurable) in available names for c in cls.mro()[:-3]: classdict[c.__name__] = c for alias, longname in self.aliases.items(): classname, traitname = longname.split('.',1) cls = classdict[classname] trait = cls.class_traits(config=True)[traitname] help = cls.class_get_trait_help(trait).splitlines() # reformat first line help[0] = help[0].replace(longname, alias) + ' (%s)'%longname if len(alias) == 1: help[0] = help[0].replace('--%s='%alias, '-%s '%alias) lines.extend(help) # lines.append('') print(os.linesep.join(lines)) def print_flag_help(self): """Print the flag part of the help.""" if not self.flags: return lines = [] for m, (cfg,help) in self.flags.items(): prefix = '--' if len(m) > 1 else '-' lines.append(prefix+m) lines.append(indent(dedent(help.strip()))) # lines.append('') print(os.linesep.join(lines)) def print_options(self): if not self.flags and not self.aliases: return lines = ['Options'] lines.append('-'*len(lines[0])) lines.append('') for p in wrap_paragraphs(self.option_description): lines.append(p) lines.append('') print(os.linesep.join(lines)) self.print_flag_help() self.print_alias_help() print() def print_subcommands(self): """Print the subcommand part of the help.""" if not self.subcommands: return lines = ["Subcommands"] lines.append('-'*len(lines[0])) lines.append('') for p in wrap_paragraphs(self.subcommand_description): lines.append(p) lines.append('') for subc, (cls, help) in self.subcommands.items(): lines.append(subc) if help: lines.append(indent(dedent(help.strip()))) lines.append('') print(os.linesep.join(lines)) def print_help(self, classes=False): """Print the help for each Configurable class in self.classes. If classes=False (the default), only flags and aliases are printed. """ self.print_description() self.print_subcommands() self.print_options() if classes: if self.classes: print("Class parameters") print("----------------") print() for p in wrap_paragraphs(self.keyvalue_description): print(p) print() for cls in self.classes: cls.class_print_help() print() else: print("To see all available configurables, use `--help-all`") print() self.print_examples() def print_description(self): """Print the application description.""" for p in wrap_paragraphs(self.description): print(p) print() def print_examples(self): """Print usage and examples. This usage string goes at the end of the command line help string and should contain examples of the application's usage. """ if self.examples: print("Examples") print("--------") print() print(indent(dedent(self.examples.strip()))) print() def print_version(self): """Print the version string.""" print(self.version) def update_config(self, config): """Fire the traits events when the config is updated.""" # Save a copy of the current config. newconfig = deepcopy(self.config) # Merge the new config into the current one. newconfig.merge(config) # Save the combined config as self.config, which triggers the traits # events. self.config = newconfig @catch_config_error def initialize_subcommand(self, subc, argv=None): """Initialize a subcommand with argv.""" subapp,help = self.subcommands.get(subc) if isinstance(subapp, str): subapp = import_item(subapp) # clear existing instances self.__class__.clear_instance() # instantiate self.subapp = subapp.instance(config=self.config) # and initialize subapp self.subapp.initialize(argv) def flatten_flags(self): """flatten flags and aliases, so cl-args override as expected. This prevents issues such as an alias pointing to InteractiveShell, but a config file setting the same trait in TerminalInteraciveShell getting inappropriate priority over the command-line arg. Only aliases with exactly one descendent in the class list will be promoted. """ # build a tree of classes in our list that inherit from a particular # it will be a dict by parent classname of classes in our list # that are descendents mro_tree = defaultdict(list) for cls in self.classes: clsname = cls.__name__ for parent in cls.mro()[1:-3]: # exclude cls itself and Configurable,HasTraits,object mro_tree[parent.__name__].append(clsname) # flatten aliases, which have the form: # { 'alias' : 'Class.trait' } aliases = {} for alias, cls_trait in self.aliases.items(): cls,trait = cls_trait.split('.',1) children = mro_tree[cls] if len(children) == 1: # exactly one descendent, promote alias cls = children[0] aliases[alias] = '.'.join([cls,trait]) # flatten flags, which are of the form: # { 'key' : ({'Cls' : {'trait' : value}}, 'help')} flags = {} for key, (flagdict, help) in self.flags.items(): newflag = {} for cls, subdict in flagdict.items(): children = mro_tree[cls] # exactly one descendent, promote flag section if len(children) == 1: cls = children[0] newflag[cls] = subdict flags[key] = (newflag, help) return flags, aliases @catch_config_error def parse_command_line(self, argv=None): """Parse the command line arguments.""" argv = sys.argv[1:] if argv is None else argv self.argv = list(argv) if argv and argv[0] == 'help': # turn `ipython help notebook` into `ipython notebook -h` argv = argv[1:] + ['-h'] if self.subcommands and len(argv) > 0: # we have subcommands, and one may have been specified subc, subargv = argv[0], argv[1:] if re.match(r'^\w(\-?\w)*$', subc) and subc in self.subcommands: # it's a subcommand, and *not* a flag or class parameter return self.initialize_subcommand(subc, subargv) # Arguments after a '--' argument are for the script IPython may be # about to run, not IPython iteslf. For arguments parsed here (help and # version), we want to only search the arguments up to the first # occurrence of '--', which we're calling interpreted_argv. try: interpreted_argv = argv[:argv.index('--')] except ValueError: interpreted_argv = argv if any(x in interpreted_argv for x in ('-h', '--help-all', '--help')): self.print_help('--help-all' in interpreted_argv) self.exit(0) if '--version' in interpreted_argv or '-V' in interpreted_argv: self.print_version() self.exit(0) # flatten flags&aliases, so cl-args get appropriate priority: flags,aliases = self.flatten_flags() loader = KVArgParseConfigLoader(argv=argv, aliases=aliases, flags=flags) config = loader.load_config() self.update_config(config) # store unparsed args in extra_args self.extra_args = loader.extra_args @catch_config_error def load_config_file(self, filename, path=None): """Load a .py based config file by filename and path.""" loader = PyFileConfigLoader(filename, path=path) try: config = loader.load_config() except ConfigFileNotFound: # problem finding the file, raise raise except Exception: # try to get the full filename, but it will be empty in the # unlikely event that the error raised before filefind finished filename = loader.full_filename or filename # problem while running the file self.log.error("Exception while loading config file %s", filename, exc_info=True) else: self.log.debug("Loaded config file: %s", loader.full_filename) self.update_config(config) def generate_config_file(self): """generate default config file from Configurables""" lines = ["# Configuration file for %s."%self.name] lines.append('') lines.append('c = get_config()') lines.append('') for cls in self.classes: lines.append(cls.class_config_section()) return '\n'.join(lines) def exit(self, exit_status=0): self.log.debug("Exiting application: %s" % self.name) sys.exit(exit_status) @classmethod def launch_instance(cls, argv=None, **kwargs): """Launch a global instance of this Application If a global instance already exists, this reinitializes and starts it """ app = cls.instance(**kwargs) app.initialize(argv) app.start()
class EngineFactory(RegistrationFactory): """IPython engine""" # configurables: out_stream_factory=Type('IPython.kernel.zmq.iostream.OutStream', config=True, help="""The OutStream for handling stdout/err. Typically 'IPython.kernel.zmq.iostream.OutStream'""") display_hook_factory=Type('IPython.kernel.zmq.displayhook.ZMQDisplayHook', config=True, help="""The class for handling displayhook. Typically 'IPython.kernel.zmq.displayhook.ZMQDisplayHook'""") location=Unicode(config=True, help="""The location (an IP address) of the controller. This is used for disambiguating URLs, to determine whether loopback should be used to connect or the public address.""") timeout=Float(5.0, config=True, help="""The time (in seconds) to wait for the Controller to respond to registration requests before giving up.""") max_heartbeat_misses=Integer(50, config=True, help="""The maximum number of times a check for the heartbeat ping of a controller can be missed before shutting down the engine. If set to 0, the check is disabled.""") sshserver=Unicode(config=True, help="""The SSH server to use for tunneling connections to the Controller.""") sshkey=Unicode(config=True, help="""The SSH private key file to use when tunneling connections to the Controller.""") paramiko=Bool(sys.platform == 'win32', config=True, help="""Whether to use paramiko instead of openssh for tunnels.""") @property def tunnel_mod(self): from zmq.ssh import tunnel return tunnel # not configurable: connection_info = Dict() user_ns = Dict() id = Integer(allow_none=True) registrar = Instance('zmq.eventloop.zmqstream.ZMQStream') kernel = Instance(Kernel) hb_check_period=Integer() # States for the heartbeat monitoring # Initial values for monitored and pinged must satisfy "monitored > pinged == False" so that # during the first check no "missed" ping is reported. Must be floats for Python 3 compatibility. _hb_last_pinged = 0.0 _hb_last_monitored = 0.0 _hb_missed_beats = 0 # The zmq Stream which receives the pings from the Heart _hb_listener = None bident = CBytes() ident = Unicode() def _ident_changed(self, name, old, new): self.bident = cast_bytes(new) using_ssh=Bool(False) def __init__(self, **kwargs): super(EngineFactory, self).__init__(**kwargs) self.ident = self.session.session def init_connector(self): """construct connection function, which handles tunnels.""" self.using_ssh = bool(self.sshkey or self.sshserver) if self.sshkey and not self.sshserver: # We are using ssh directly to the controller, tunneling localhost to localhost self.sshserver = self.url.split('://')[1].split(':')[0] if self.using_ssh: if self.tunnel_mod.try_passwordless_ssh(self.sshserver, self.sshkey, self.paramiko): password=False else: password = getpass("SSH Password for %s: "%self.sshserver) else: password = False def connect(s, url): url = disambiguate_url(url, self.location) if self.using_ssh: self.log.debug("Tunneling connection to %s via %s", url, self.sshserver) return self.tunnel_mod.tunnel_connection(s, url, self.sshserver, keyfile=self.sshkey, paramiko=self.paramiko, password=password, ) else: return s.connect(url) def maybe_tunnel(url): """like connect, but don't complete the connection (for use by heartbeat)""" url = disambiguate_url(url, self.location) if self.using_ssh: self.log.debug("Tunneling connection to %s via %s", url, self.sshserver) url, tunnelobj = self.tunnel_mod.open_tunnel(url, self.sshserver, keyfile=self.sshkey, paramiko=self.paramiko, password=password, ) return str(url) return connect, maybe_tunnel def register(self): """send the registration_request""" self.log.info("Registering with controller at %s"%self.url) ctx = self.context connect,maybe_tunnel = self.init_connector() reg = ctx.socket(zmq.DEALER) reg.setsockopt(zmq.IDENTITY, self.bident) connect(reg, self.url) self.registrar = zmqstream.ZMQStream(reg, self.loop) content = dict(uuid=self.ident) self.registrar.on_recv(lambda msg: self.complete_registration(msg, connect, maybe_tunnel)) # print (self.session.key) self.session.send(self.registrar, "registration_request", content=content) def _report_ping(self, msg): """Callback for when the heartmonitor.Heart receives a ping""" #self.log.debug("Received a ping: %s", msg) self._hb_last_pinged = time.time() def complete_registration(self, msg, connect, maybe_tunnel): # print msg self.loop.remove_timeout(self._abort_timeout) ctx = self.context loop = self.loop identity = self.bident idents,msg = self.session.feed_identities(msg) msg = self.session.deserialize(msg) content = msg['content'] info = self.connection_info def url(key): """get zmq url for given channel""" return str(info["interface"] + ":%i" % info[key]) if content['status'] == 'ok': self.id = int(content['id']) # launch heartbeat # possibly forward hb ports with tunnels hb_ping = maybe_tunnel(url('hb_ping')) hb_pong = maybe_tunnel(url('hb_pong')) hb_monitor = None if self.max_heartbeat_misses > 0: # Add a monitor socket which will record the last time a ping was seen mon = self.context.socket(zmq.SUB) mport = mon.bind_to_random_port('tcp://%s' % localhost()) mon.setsockopt(zmq.SUBSCRIBE, b"") self._hb_listener = zmqstream.ZMQStream(mon, self.loop) self._hb_listener.on_recv(self._report_ping) hb_monitor = "tcp://%s:%i" % (localhost(), mport) heart = Heart(hb_ping, hb_pong, hb_monitor , heart_id=identity) heart.start() # create Shell Connections (MUX, Task, etc.): shell_addrs = url('mux'), url('task') # Use only one shell stream for mux and tasks stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) stream.setsockopt(zmq.IDENTITY, identity) shell_streams = [stream] for addr in shell_addrs: connect(stream, addr) # control stream: control_addr = url('control') control_stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) control_stream.setsockopt(zmq.IDENTITY, identity) connect(control_stream, control_addr) # create iopub stream: iopub_addr = url('iopub') iopub_socket = ctx.socket(zmq.PUB) iopub_socket.setsockopt(zmq.IDENTITY, identity) connect(iopub_socket, iopub_addr) # disable history: self.config.HistoryManager.hist_file = ':memory:' # Redirect input streams and set a display hook. if self.out_stream_factory: sys.stdout = self.out_stream_factory(self.session, iopub_socket, u'stdout') sys.stdout.topic = cast_bytes('engine.%i.stdout' % self.id) sys.stderr = self.out_stream_factory(self.session, iopub_socket, u'stderr') sys.stderr.topic = cast_bytes('engine.%i.stderr' % self.id) if self.display_hook_factory: sys.displayhook = self.display_hook_factory(self.session, iopub_socket) sys.displayhook.topic = cast_bytes('engine.%i.execute_result' % self.id) self.kernel = Kernel(parent=self, int_id=self.id, ident=self.ident, session=self.session, control_stream=control_stream, shell_streams=shell_streams, iopub_socket=iopub_socket, loop=loop, user_ns=self.user_ns, log=self.log) self.kernel.shell.display_pub.topic = cast_bytes('engine.%i.displaypub' % self.id) # periodically check the heartbeat pings of the controller # Should be started here and not in "start()" so that the right period can be taken # from the hubs HeartBeatMonitor.period if self.max_heartbeat_misses > 0: # Use a slightly bigger check period than the hub signal period to not warn unnecessary self.hb_check_period = int(content['hb_period'])+10 self.log.info("Starting to monitor the heartbeat signal from the hub every %i ms." , self.hb_check_period) self._hb_reporter = ioloop.PeriodicCallback(self._hb_monitor, self.hb_check_period, self.loop) self._hb_reporter.start() else: self.log.info("Monitoring of the heartbeat signal from the hub is not enabled.") # FIXME: This is a hack until IPKernelApp and IPEngineApp can be fully merged app = IPKernelApp(parent=self, shell=self.kernel.shell, kernel=self.kernel, log=self.log) app.init_profile_dir() app.init_code() self.kernel.start() else: self.log.fatal("Registration Failed: %s"%msg) raise Exception("Registration Failed: %s"%msg) self.log.info("Completed registration with id %i"%self.id) def abort(self): self.log.fatal("Registration timed out after %.1f seconds"%self.timeout) if self.url.startswith('127.'): self.log.fatal(""" If the controller and engines are not on the same machine, you will have to instruct the controller to listen on an external IP (in ipcontroller_config.py): c.HubFactory.ip='*' # for all interfaces, internal and external c.HubFactory.ip='192.168.1.101' # or any interface that the engines can see or tunnel connections via ssh. """) self.session.send(self.registrar, "unregistration_request", content=dict(id=self.id)) time.sleep(1) sys.exit(255) def _hb_monitor(self): """Callback to monitor the heartbeat from the controller""" self._hb_listener.flush() if self._hb_last_monitored > self._hb_last_pinged: self._hb_missed_beats += 1 self.log.warn("No heartbeat in the last %s ms (%s time(s) in a row).", self.hb_check_period, self._hb_missed_beats) else: #self.log.debug("Heartbeat received (after missing %s beats).", self._hb_missed_beats) self._hb_missed_beats = 0 if self._hb_missed_beats >= self.max_heartbeat_misses: self.log.fatal("Maximum number of heartbeats misses reached (%s times %s ms), shutting down.", self.max_heartbeat_misses, self.hb_check_period) self.session.send(self.registrar, "unregistration_request", content=dict(id=self.id)) self.loop.stop() self._hb_last_monitored = time.time() def start(self): loop = self.loop def _start(): self.register() self._abort_timeout = loop.add_timeout(loop.time() + self.timeout, self.abort) self.loop.add_callback(_start)
class Kernel(Configurable): #--------------------------------------------------------------------------- # Kernel interface #--------------------------------------------------------------------------- # attribute to override with a GUI eventloop = Any(None) shell = Instance('IPython.core.interactiveshell.InteractiveShellABC') session = Instance(Session) shell_socket = Instance('zmq.Socket') iopub_socket = Instance('zmq.Socket') stdin_socket = Instance('zmq.Socket') log = Instance(logging.Logger) # Private interface # Time to sleep after flushing the stdout/err buffers in each execute # cycle. While this introduces a hard limit on the minimal latency of the # execute cycle, it helps prevent output synchronization problems for # clients. # Units are in seconds. The minimum zmq latency on local host is probably # ~150 microseconds, set this to 500us for now. We may need to increase it # a little if it's not enough after more interactive testing. _execute_sleep = Float(0.0005, config=True) # Frequency of the kernel's event loop. # Units are in seconds, kernel subclasses for GUI toolkits may need to # adapt to milliseconds. _poll_interval = Float(0.05, config=True) # If the shutdown was requested over the network, we leave here the # necessary reply message so it can be sent by our registered atexit # handler. This ensures that the reply is only sent to clients truly at # the end of our shutdown process (which happens after the underlying # IPython shell's own shutdown). _shutdown_message = None # This is a dict of port number that the kernel is listening on. It is set # by record_ports and used by connect_request. _recorded_ports = Dict() def __init__(self, **kwargs): super(Kernel, self).__init__(**kwargs) # Before we even start up the shell, register *first* our exit handlers # so they come before the shell's atexit.register(self._at_shutdown) # Initialize the InteractiveShell subclass self.shell = ZMQInteractiveShell.instance(config=self.config) self.shell.displayhook.session = self.session self.shell.displayhook.pub_socket = self.iopub_socket self.shell.display_pub.session = self.session self.shell.display_pub.pub_socket = self.iopub_socket # TMP - hack while developing self.shell._reply_content = None # Build dict of handlers for message types msg_types = [ 'execute_request', 'complete_request', 'object_info_request', 'history_request', 'connect_request', 'shutdown_request'] self.handlers = {} for msg_type in msg_types: self.handlers[msg_type] = getattr(self, msg_type) def do_one_iteration(self): """Do one iteration of the kernel's evaluation loop. """ try: ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK) except Exception: self.log.warn("Invalid Message:", exc_info=True) return if msg is None: return msg_type = msg['header']['msg_type'] # This assert will raise in versions of zeromq 2.0.7 and lesser. # We now require 2.0.8 or above, so we can uncomment for safety. # print(ident,msg, file=sys.__stdout__) assert ident is not None, "Missing message part." # Print some info about this message and leave a '--->' marker, so it's # easier to trace visually the message chain when debugging. Each # handler prints its message at the end. self.log.debug('\n*** MESSAGE TYPE:'+str(msg_type)+'***') self.log.debug(' Content: '+str(msg['content'])+'\n --->\n ') # Find and call actual handler for message handler = self.handlers.get(msg_type, None) if handler is None: self.log.error("UNKNOWN MESSAGE TYPE:" +str(msg)) else: handler(ident, msg) # Check whether we should exit, in case the incoming message set the # exit flag on if self.shell.exit_now: self.log.debug('\nExiting IPython kernel...') # We do a normal, clean exit, which allows any actions registered # via atexit (such as history saving) to take place. sys.exit(0) def start(self): """ Start the kernel main loop. """ poller = zmq.Poller() poller.register(self.shell_socket, zmq.POLLIN) # loop while self.eventloop has not been overridden while self.eventloop is None: try: # scale by extra factor of 10, because there is no # reason for this to be anything less than ~ 0.1s # since it is a real poller and will respond # to events immediately # double nested try/except, to properly catch KeyboardInterrupt # due to pyzmq Issue #130 try: poller.poll(10*1000*self._poll_interval) self.do_one_iteration() except: raise except KeyboardInterrupt: # Ctrl-C shouldn't crash the kernel io.raw_print("KeyboardInterrupt caught in kernel") if self.eventloop is not None: try: self.eventloop(self) except KeyboardInterrupt: # Ctrl-C shouldn't crash the kernel io.raw_print("KeyboardInterrupt caught in kernel") def record_ports(self, ports): """Record the ports that this kernel is using. The creator of the Kernel instance must call this methods if they want the :meth:`connect_request` method to return the port numbers. """ self._recorded_ports = ports #--------------------------------------------------------------------------- # Kernel request handlers #--------------------------------------------------------------------------- def _publish_pyin(self, code, parent): """Publish the code request on the pyin stream.""" pyin_msg = self.session.send(self.iopub_socket, u'pyin',{u'code':code}, parent=parent) def execute_request(self, ident, parent): status_msg = self.session.send(self.iopub_socket, u'status', {u'execution_state':u'busy'}, parent=parent ) try: content = parent[u'content'] code = content[u'code'] silent = content[u'silent'] except: self.log.error("Got bad msg: ") self.log.error(str(Message(parent))) return shell = self.shell # we'll need this a lot here # Replace raw_input. Note that is not sufficient to replace # raw_input in the user namespace. if content.get('allow_stdin', False): raw_input = lambda prompt='': self._raw_input(prompt, ident, parent) else: raw_input = lambda prompt='' : self._no_raw_input() if py3compat.PY3: __builtin__.input = raw_input else: __builtin__.raw_input = raw_input # Set the parent message of the display hook and out streams. shell.displayhook.set_parent(parent) shell.display_pub.set_parent(parent) sys.stdout.set_parent(parent) sys.stderr.set_parent(parent) # Re-broadcast our input for the benefit of listening clients, and # start computing output if not silent: self._publish_pyin(code, parent) reply_content = {} try: if silent: # run_code uses 'exec' mode, so no displayhook will fire, and it # doesn't call logging or history manipulations. Print # statements in that code will obviously still execute. shell.run_code(code) else: # FIXME: the shell calls the exception handler itself. shell.run_cell(code, store_history=True) except: status = u'error' # FIXME: this code right now isn't being used yet by default, # because the run_cell() call above directly fires off exception # reporting. This code, therefore, is only active in the scenario # where runlines itself has an unhandled exception. We need to # uniformize this, for all exception construction to come from a # single location in the codbase. etype, evalue, tb = sys.exc_info() tb_list = traceback.format_exception(etype, evalue, tb) reply_content.update(shell._showtraceback(etype, evalue, tb_list)) else: status = u'ok' reply_content[u'status'] = status # Return the execution counter so clients can display prompts reply_content['execution_count'] = shell.execution_count -1 # FIXME - fish exception info out of shell, possibly left there by # runlines. We'll need to clean up this logic later. if shell._reply_content is not None: reply_content.update(shell._reply_content) # reset after use shell._reply_content = None # At this point, we can tell whether the main code execution succeeded # or not. If it did, we proceed to evaluate user_variables/expressions if reply_content['status'] == 'ok': reply_content[u'user_variables'] = \ shell.user_variables(content[u'user_variables']) reply_content[u'user_expressions'] = \ shell.user_expressions(content[u'user_expressions']) else: # If there was an error, don't even try to compute variables or # expressions reply_content[u'user_variables'] = {} reply_content[u'user_expressions'] = {} # Payloads should be retrieved regardless of outcome, so we can both # recover partial output (that could have been generated early in a # block, before an error) and clear the payload system always. reply_content[u'payload'] = shell.payload_manager.read_payload() # Be agressive about clearing the payload because we don't want # it to sit in memory until the next execute_request comes in. shell.payload_manager.clear_payload() # Flush output before sending the reply. sys.stdout.flush() sys.stderr.flush() # FIXME: on rare occasions, the flush doesn't seem to make it to the # clients... This seems to mitigate the problem, but we definitely need # to better understand what's going on. if self._execute_sleep: time.sleep(self._execute_sleep) # Send the reply. reply_content = json_clean(reply_content) reply_msg = self.session.send(self.shell_socket, u'execute_reply', reply_content, parent, ident=ident) self.log.debug(str(reply_msg)) if reply_msg['content']['status'] == u'error': self._abort_queue() status_msg = self.session.send(self.iopub_socket, u'status', {u'execution_state':u'idle'}, parent=parent ) def complete_request(self, ident, parent): txt, matches = self._complete(parent) matches = {'matches' : matches, 'matched_text' : txt, 'status' : 'ok'} matches = json_clean(matches) completion_msg = self.session.send(self.shell_socket, 'complete_reply', matches, parent, ident) self.log.debug(str(completion_msg)) def object_info_request(self, ident, parent): object_info = self.shell.object_inspect(parent['content']['oname']) # Before we send this object over, we scrub it for JSON usage oinfo = json_clean(object_info) msg = self.session.send(self.shell_socket, 'object_info_reply', oinfo, parent, ident) self.log.debug(msg) def history_request(self, ident, parent): # We need to pull these out, as passing **kwargs doesn't work with # unicode keys before Python 2.6.5. hist_access_type = parent['content']['hist_access_type'] raw = parent['content']['raw'] output = parent['content']['output'] if hist_access_type == 'tail': n = parent['content']['n'] hist = self.shell.history_manager.get_tail(n, raw=raw, output=output, include_latest=True) elif hist_access_type == 'range': session = parent['content']['session'] start = parent['content']['start'] stop = parent['content']['stop'] hist = self.shell.history_manager.get_range(session, start, stop, raw=raw, output=output) elif hist_access_type == 'search': pattern = parent['content']['pattern'] hist = self.shell.history_manager.search(pattern, raw=raw, output=output) else: hist = [] content = {'history' : list(hist)} content = json_clean(content) msg = self.session.send(self.shell_socket, 'history_reply', content, parent, ident) self.log.debug(str(msg)) def connect_request(self, ident, parent): if self._recorded_ports is not None: content = self._recorded_ports.copy() else: content = {} msg = self.session.send(self.shell_socket, 'connect_reply', content, parent, ident) self.log.debug(msg) def shutdown_request(self, ident, parent): self.shell.exit_now = True self._shutdown_message = self.session.msg(u'shutdown_reply', parent['content'], parent) sys.exit(0) #--------------------------------------------------------------------------- # Protected interface #--------------------------------------------------------------------------- def _abort_queue(self): while True: try: ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK) except Exception: self.log.warn("Invalid Message:", exc_info=True) continue if msg is None: break else: assert ident is not None, \ "Unexpected missing message part." self.log.debug("Aborting:\n"+str(Message(msg))) msg_type = msg['header']['msg_type'] reply_type = msg_type.split('_')[0] + '_reply' reply_msg = self.session.send(self.shell_socket, reply_type, {'status' : 'aborted'}, msg, ident=ident) self.log.debug(reply_msg) # We need to wait a bit for requests to come in. This can probably # be set shorter for true asynchronous clients. time.sleep(0.1) def _no_raw_input(self): """Raise StdinNotImplentedError if active frontend doesn't support stdin.""" raise StdinNotImplementedError("raw_input was called, but this frontend does not support stdin.") def _raw_input(self, prompt, ident, parent): # Flush output before making the request. sys.stderr.flush() sys.stdout.flush() # Send the input request. content = json_clean(dict(prompt=prompt)) msg = self.session.send(self.stdin_socket, u'input_request', content, parent, ident=ident) # Await a response. while True: try: ident, reply = self.session.recv(self.stdin_socket, 0) except Exception: self.log.warn("Invalid Message:", exc_info=True) else: break try: value = reply['content']['value'] except: self.log.error("Got bad raw_input reply: ") self.log.error(str(Message(parent))) value = '' return value def _complete(self, msg): c = msg['content'] try: cpos = int(c['cursor_pos']) except: # If we don't get something that we can convert to an integer, at # least attempt the completion guessing the cursor is at the end of # the text, if there's any, and otherwise of the line cpos = len(c['text']) if cpos==0: cpos = len(c['line']) return self.shell.complete(c['text'], c['line'], cpos) def _object_info(self, context): symbol, leftover = self._symbol_from_context(context) if symbol is not None and not leftover: doc = getattr(symbol, '__doc__', '') else: doc = '' object_info = dict(docstring = doc) return object_info def _symbol_from_context(self, context): if not context: return None, context base_symbol_string = context[0] symbol = self.shell.user_ns.get(base_symbol_string, None) if symbol is None: symbol = __builtin__.__dict__.get(base_symbol_string, None) if symbol is None: return None, context context = context[1:] for i, name in enumerate(context): new_symbol = getattr(symbol, name, None) if new_symbol is None: return symbol, context[i:] else: symbol = new_symbol return symbol, [] def _at_shutdown(self): """Actions taken at shutdown by the kernel, called by python's atexit. """ # io.rprint("Kernel at_shutdown") # dbg if self._shutdown_message is not None: self.session.send(self.shell_socket, self._shutdown_message) self.session.send(self.iopub_socket, self._shutdown_message) self.log.debug(str(self._shutdown_message)) # A very short sleep to give zmq time to flush its message buffers # before Python truly shuts down. time.sleep(0.01)